diff --git a/.github/workflows/k3s-p1-plan.yml b/.github/workflows/k3s-p1-plan.yml new file mode 100644 index 0000000000000..ae068adff0e3c --- /dev/null +++ b/.github/workflows/k3s-p1-plan.yml @@ -0,0 +1,24 @@ +name: K3s P1 plan contract +on: + pull_request: + paths: + - "examples/k3s-p1/**" + - ".github/workflows/k3s-p1-plan.yml" + push: + branches-ignore: [main] + paths: + - "examples/k3s-p1/**" + - ".github/workflows/k3s-p1-plan.yml" + workflow_dispatch: +jobs: + offline: + runs-on: ubuntu-latest + permissions: + contents: read + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install "PyYAML==6.0.2" + - run: python -m unittest discover -s examples/k3s-p1 -p 'test_*.py' -v diff --git a/examples/k3s-p1/OBSERVATION.md b/examples/k3s-p1/OBSERVATION.md new file mode 100644 index 0000000000000..86eb79f23432a --- /dev/null +++ b/examples/k3s-p1/OBSERVATION.md @@ -0,0 +1,132 @@ +# P1 offline binding observation — UID + desired Deployment contract + +**READ-ONLY SOURCE CHECK; NO K3s EXECUTION OR WORKER AUTHORIZATION.** The +`observe.py` helper compares one immutable P1 plan with API JSON for its +Namespace, PVC and Deployment, then extracts their stable Kubernetes UIDs. A previous +binding can be replay-checked so a replaced Namespace, PVC, Deployment **or pinned image** +**fails closed** rather than looking like the same worker lease. The v4 receipt contains a SHA-256 fingerprint of the full immutable OCI image +reference, never the raw registry/image name, and checks the observed +Deployment's **desired Pod template** image equals the plan. It also requires the +planned non-root Pod/container security, resource bounds, probes and no unexpected +host-network/PID/IPC access, injected init containers, registry credentials, +environment secrets or shell overrides. An unexpected admission mutation fails +closed for human review; Kubernetes may default other fields. Old v1/v2 binding +or v3 baselines cannot be reused: recollect a trusted v4 baseline from one approved +context. This does **not** check the image actually running in any Pod. + +This is **not** a provisioner, cluster lease, service/Pod readiness probe, +T3 environment identity check, HTTPS proof, peer authorization, task +completion, automatic expiry, or deletion approval. The CLI contacts **no** +cluster. JSON files can be stale or fabricated, so source and context must +be independently verified; never promote a local fixture to live evidence. + +## Reproducible offline tests + +From the fork checkout (no kubeconfig or provider token needed): + +```sh +python3 -m unittest discover -s examples/k3s-p1 -p 'test_*.py' -v +python3 -m py_compile examples/k3s-p1/observe.py +``` + +The focused binding tests check stable UIDs, lease/namespace mismatch, +PVC replacement, invalid/unbound/deleting storage, malformed API JSON and +invalid lease calendar dates (without a raw CLI traceback), retention/annotation drift, +immutable image reference drift/format, rejection of a legacy v1 baseline, +desired Deployment image/ServiceAccount/volume/selector and privilege/credential/resource drift, and CLI failure +without printing private IDs. + +## Human-gated future Kubernetes observation (NOT RUN) + +**Only after P0 real disposable-cluster acceptance and separate approval to +create one P1 worker**, inspect the rendered plan and explicitly confirm the +selected kubeconfig context and physical machine. The following commands are +read-only; they never deploy, pair, cancel, renew, clean up or grant access. +Keep output in an owner-private directory; do not commit snapshots/receipts. + +```sh +umask 077 +mkdir -p /tmp/t3-p1-private +# Use the namespace printed by plan.py, not a guessed or existing namespace. +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT get namespace YOUR_P1_NAMESPACE \ + -o json > /tmp/t3-p1-private/namespace.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE \ + get pvc t3-worker-data -o json > /tmp/t3-p1-private/pvc.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE \ + get deployment t3-worker -o json > /tmp/t3-p1-private/deployment.json +python3 examples/k3s-p1/observe.py \ + --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ + --namespace /tmp/t3-p1-private/namespace.json \ + --pvc /tmp/t3-p1-private/pvc.json \ + --deployment /tmp/t3-p1-private/deployment.json \ + > /tmp/t3-p1-private/binding.json +``` + +For a subsequent observation, **refresh all three Kubernetes JSON snapshots from the same +approved context** and use the **same pinned image receipt**, then pass +`--baseline /tmp/t3-p1-private/binding.json`. A Namespace/PVC/Deployment UID, desired image or PV +binding or Deployment generation change now fails. A missing/unbound PVC or lease/retention mismatch +also fails; nothing is automatically repaired or deleted. Store the baseline +safely and never overwrite it merely to make a failing comparison pass. + +**Still required separately:** verify the server's persisted T3 +`environmentId` and actual live Pod UID/image/Service/EndpointSlice (independent from the desired +Deployment spec), +private TLS and negative ACL tests, consented MCP OAuth link, exact pinned +repository/delegated task result, cancel/restart/TTL behavior and deliberate +PVC retention. An old or expired plan is not a live reservation. Consult +[work order #45](https://github.com/kvnloo/t3code/issues/45) and +[stacked Draft #47](https://github.com/kvnloo/t3code/pull/47) for gates. + +## Optional read-only Pod runtime snapshot — NOT live proof + +After the desired-spec binding above, `pod_check.py` checks **saved JSON** +from one approved disposable K3s context. It binds Deployment UID → +ReplicaSet controller UID → the **only** Pod UID, verifies Deployment rollout +generation/ready replicas, Pod/container readiness as reported by Kubernetes, +pinned container image reference, required Pod security/storage, and hashes +the runtime image/container IDs into a private receipt. The v4 binding also +pins the Deployment's observed generation, so a changed spec on the same UID +cannot be mistaken for the originally reviewed Deployment. It rejects extra Pods, +stale rollout, wrong controller, changed image, extra containers/credentials and +unapproved Pod mutations. Older inactive ReplicaSets are permitted. + +These are **offline comparisons of supplied snapshots**, not authentication +of the Kubernetes API, live reachability, effective isolation, T3 persisted +`environmentId`, peer permission or accepted work. The imageID *hash* is +an observation, **not** equality proof against a multi-platform manifest's +pinned digest. Pod UID changing across restarts needs separate review; stable +Deployment/PVC identity cannot substitute for T3's persisted identity. + +Only after a separately approved P1 cluster run, save full namespace +inventories from the **same explicitly verified kubeconfig context**, close +together in time, into an owner-private directory: + +```sh +umask 077 +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE \ + get replicasets -o json > /tmp/t3-p1-private/replicasets.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE \ + get pods -o json > /tmp/t3-p1-private/pods.json +# Re-capture deployment.json alongside both inventories; keep the existing +# original, independently verified binding.json and lease plan. +python3 examples/k3s-p1/pod_check.py \ + --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ + --binding /tmp/t3-p1-private/binding.json \ + --deployment /tmp/t3-p1-private/deployment.json \ + --replicasets /tmp/t3-p1-private/replicasets.json \ + --pods /tmp/t3-p1-private/pods.json \ + > /tmp/t3-p1-private/pod-snapshot.json +``` + +If the Deployment generation changes, stop, review the new desired spec and +re-run `observe.py` against fresh same-context snapshots; do **not** overwrite +the original binding merely to make the check pass. Kubernetes increments +Deployment generation on spec changes, but offline snapshots can still be forged. + +Run `python3 -m unittest discover -s examples/k3s-p1 -p 'test_pod_check.py' -v` +for synthetic, credential-free regressions. Keep API snapshots private; they may +expose Pod labels and topology. No `kubectl apply`, no network calls from the +script and no automatic worker acceptance. After this check, **still NOT RUN**: +real T3 environment identity, private TLS/ACL, consented OAuth peer link, +verified delegated result and cancellation/restart behavior. diff --git a/examples/k3s-p1/README.md b/examples/k3s-p1/README.md new file mode 100644 index 0000000000000..9eb979caf50f3 --- /dev/null +++ b/examples/k3s-p1/README.md @@ -0,0 +1,227 @@ +# P1 preview — deterministic worker plans, no cluster actuation + +**LOCAL CONTRACT ONLY / NOT RUN ON K3s.** This is a stacked preparatory +slice against [P0 draft #46](https://github.com/kvnloo/t3code/pull/46). +No P1 deployment is authorized until the P0 persistent owner passes real +disposable-cluster storage/restart, private HTTPS/ACL, client pairing and +saved-thread acceptance. + +## Scope + +plan.py turns a stable opaque idempotency key, immutable OCI image digest and +explicit UTC expiration into a reproducible **per-worker environment +blueprint**. The key is hashed (full SHA-256 in a private receipt, truncated +only for DNS names). The raw key is neither logged nor written to Kubernetes. +The file outputs exactly eight ordinary resources: restricted Namespace, +ResourceQuota, no-token ServiceAccount, separate PVC, one Recreate Deployment +running a headless T3 server, ClusterIP Service, private Tailscale Ingress and +an ingress-only NetworkPolicy. + +The offline route checker also rejects unexpected Service annotations, including +`tailscale.com/expose`, on both the named object and its namespace inventory. +A `ClusterIP` Service with that annotation can create a separate Tailscale L3 +proxy without a second Ingress. A matching snapshot is still **not** network +isolation, TLS, ACL or peer-authorization proof. + +The separate [offline UID observation helper](OBSERVATION.md) checks a +reported Kubernetes Namespace/PVC/Deployment binding **and immutable +image-reference fingerprint** against the plan and an earlier v4 baseline. The v4 binding also pins +the Deployment generation, preventing a same-UID spec update from passing a +later Pod check without renewed human review. +It checks the Deployment's desired template image, PVC/SA mounting, +non-root Pod/container security, resources and probes, and rejects host access, +additional containers and injected credentials/command overrides, +not the image of a live Pod. Legacy v1/v2/v3 baselines are rejected. It never contacts a cluster or proves a live T3 worker. + +After the offline desired Deployment/PVC binding, the optional [Pod snapshot checker](OBSERVATION.md#optional-read-only-pod-runtime-snapshot--not-live-proof) can validate saved ReplicaSet→Pod controller UID, API-reported readiness and running image reference, and reject concurrent/mutated Pods, unexpected host ports, changed startup/probe behavior or injected post-start hooks. It also compares the active ReplicaSet's replacement-Pod spec, template labels/annotations and selector with the approved Deployment, rejecting injection or routing drift that could affect a future Pod after restart/eviction. This check also rejects unapproved **Pod-level annotations** (including Multus secondary-network attachments), injected runtime classes / GPU resource claims, changed DNS/placement controls, and mismatched termination/service-account fields while permitting normal Kubernetes defaults. If the cluster injects benign annotations, review them explicitly before changing the strict contract; never silently allow all annotations. This is still **NOT live K3s proof**: its hashed imageID does not attest a multi-platform OCI digest, and it cannot establish persisted T3 identity, TLS, peer consent or successful delegation. + +The local private mode-0600 receipt is create-once and atomically published only after a synced write. Concurrent identical requests reuse the complete receipt; a failed publish leaves no half-written final record. A retry with identical +arguments renders identical YAML. Reusing the key with changed image/expiry +fails closed rather than silently rebinding a purported lease. The receipt +is **not** a distributed lock or a cluster reservation. Expiration is advisory +metadata only; **no TTL cleanup or enforcement exists**. PVC retention always +requires a human decision. The CNI and tailnet ACLs still need negative tests; +the manifest does not restrict egress. + +## Offline usage + +Requires Python 3.10+ and PyYAML. In the fork checkout: + + python3 -m pip install 'PyYAML==6.0.2' + python3 -m unittest discover -s examples/k3s-p1 -p 'test_*.py' -v + +Create a private receipt directory locally: + + mkdir -m 700 /tmp/k3s-p1-receipts + +After replacing the placeholder with a **real** verified 64-character +lowercase image digest, run: + + python3 examples/k3s-p1/plan.py \ + --lease-key 'approved-request:example-001' \ + --image 'ghcr.io/OWNER/t3-worker@sha256:REPLACE_WITH_64_HEX' \ + --expires-at '2026-11-01T00:00:00Z' \ + --state-dir /tmp/k3s-p1-receipts > /tmp/k3s-p1-review.yaml + +The output is YAML for review, **not an automatic kubectl command**. Do not +use a credential or private text as a lease key. Directory mode must deny +group and other users. If a receipt exists with altered args, the command +fails instead of overwriting it. + +## Future acceptance — NOT RUN + +1. Complete P0 live acceptance first. Confirm the pinned worker image + actually supports the latest T3 linked-environment beta, provider/runtime + requirements, and a persistent environment ID. +2. Inspect the generated eight-object YAML and chosen *disposable* K3s + context. Verify the computed worker namespace is unused. Any dry run, + image build or cluster apply requires separate owner authorization. +3. The offline UID helper checks supplied Namespace/PVC snapshots, but does + not persist a runtime lease or authorize reuse. Develop an opt-in + reconcile/create/retry path recording verified live Pod/Service ownership, + stable T3 descriptor, lease identity, expiry and explicit retention/cancel/ + resume states; a lost response must never duplicate accepted completion. +4. Independently approve OAuth peer linking on both environments, then run + one real remote delegated task against a pinned repository revision. + Client-saved environment and T3 peer OAuth are not interchangeable. + Test restart, route loss, token revocation, cancellation and negative ACLs. +5. Never enroll temporary workers automatically in client load balancing or + assume worker and owner share a filesystem, provider secrets or GPUs. + +The original P0 image is pinned to T3 release v0.1.3; latest-main beta +compatibility is **unverified**. At upstream main `960d5761c13f68a2f10635bcd26489a5fa404ec1` +(2026-10-29), linked-environment `RemoteDelegation.ts` exists but still launches a +remote child before recording the owning task ([#17051](https://github.com/pingdotgg/t3code/issues/17051)); +its restart durability is not proven. This preview does not install tools, +clone repositories, contact Kubernetes, link agents, grant credentials, +launch workers or alter the current deployment. + +Canonical decisions: [fork #45](https://github.com/kvnloo/t3code/issues/45). +No upstream writes and no READY status. + + +## Offline private-route drift check (no cluster calls) + +`route_check.py` compares a private P1 plan receipt with saved Kubernetes +**Service, Ingress, named NetworkPolicy and complete namespace-wide ServiceList, IngressList and NetworkPolicyList JSON**. It rejects a public +LoadBalancer/NodePort, external IP, wrong backend, non-Tailscale IngressClass, +Funnel/unreviewed ingress annotations, extra ingress rules, changed tailnet +hostname, a widened NetworkPolicy, **an additional public Ingress or LoadBalancer/NodePort Service, or an additional wildcard-allow policy**. +The helper also inspects **advertised** `Ingress.status.loadBalancer.ingress` +and `Service.status.loadBalancer.ingress` when present, including both +named-object and namespace-inventory snapshots. A public IP, external hostname, +multiple advertised endpoints, inconsistent snapshots or malformed status +fails closed even when the Kubernetes `spec` still looks private. An absent +Ingress status is permitted only for an **unapplied desired-plan check**; it +does **not** mean the route exists. The matching `*.ts.net` name does not +establish tailnet ownership or ACL enforcement. Live HTTPS identity verification +and a separate deny-path probe remain mandatory. + +Kubernetes ingress policy rules are additive; checking the named policy alone +cannot exclude another policy that exposes the worker. The inventory must +contain exactly the named policy with the same UID and spec, and exactly the named private Ingress and ClusterIP Service in their respective inventories. The checked-in +`test_route_check.py` covers those regressions. This is a **static desired +spec check**: a matching snapshot does **not** prove a trusted API response, +correct CNI enforcement, private Tailscale TLS/ACLs, OAuth permission or +a working pod. Do not connect or deploy a worker based on this alone. + +Only **after P0 real disposable-cluster acceptance** and separate owner +approval for a P1 sandbox, save the read-only API snapshots privately: + +```sh +umask 077 +mkdir -p /tmp/t3-p1-private +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get service t3-worker -o json > /tmp/t3-p1-private/service.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get ingress t3-worker -o json > /tmp/t3-p1-private/ingress.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get networkpolicy worker-ingress -o json > /tmp/t3-p1-private/policy.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get networkpolicies -o json > /tmp/t3-p1-private/all-policies.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get ingresses -o json > /tmp/t3-p1-private/all-ingresses.json +kubectl --context YOUR_APPROVED_DISPOSABLE_CONTEXT -n YOUR_P1_NAMESPACE get services -o json > /tmp/t3-p1-private/all-services.json +python3 examples/k3s-p1/route_check.py \ + --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ + --service /tmp/t3-p1-private/service.json \ + --ingress /tmp/t3-p1-private/ingress.json \ + --network-policy /tmp/t3-p1-private/policy.json \ + --all-network-policies /tmp/t3-p1-private/all-policies.json \ + --all-ingresses /tmp/t3-p1-private/all-ingresses.json \ + --all-services /tmp/t3-p1-private/all-services.json +``` + +A failed check requires manual review, never automatic rollback or deletion. +Unexpected admission-controller annotations and any additional network +policy fail closed. A saved inventory can be stale or collected mid-change. It does not enumerate +routes in other namespaces or non-Kubernetes host networking. Repeat the +approved-context read and independently inspect effective CNI rules. +Also check actual DNS, HTTPS certificate, peer identity, private ACL negative test +and CNI policy enforcement independently; do not treat these API objects as +proof of the data plane. + + +## Optional read-only, revision-checked snapshot collection + +**NOT RUN on any cluster.** Only after a human has separately approved a +*disposable* P1 K3s worker and its exact context, the new `capture.py` +reduces mistakes in the manual observation commands above. It uses only +explicit-context `kubectl get -o json` calls with deadlines; it cannot +provision, mutate, link, cancel or delete a worker. It collects eight named +resources (Namespace, ResourceQuota, ServiceAccount, PVC, Deployment, Service, +Ingress, NetworkPolicy) and five complete namespace inventories (policies, ingresses, services, +ReplicaSets, Pods). Each is read twice and mismatches fail closed **before +any output is published**. Named resources must have the planned Kubernetes +kind/name/namespace and a UID; the named Service, Ingress and NetworkPolicy +must also exactly match their corresponding namespace-inventory entry. +This rejects mixed GET/LIST views even when each view is independently stable. +The capture now fail-closes if the named ResourceQuota has different +hard limits, any scopes/scopeSelector, or an unreconciled `status.hard`, +and if the named ServiceAccount is deleting, permits API-token automount, +or references token/pull secrets. Matching *snapshots* do **not** prove +Kubernetes quota enforcement, RBAC permissions or absence of admission +mutation; human approval, CNI/ACL negative tests and live verification remain +mandatory. +It catches only a subset of mid-capture drift: Kubernetes offers **no atomic +cross-resource snapshot** here, and matching objects do not prove freshness, +route integrity or runtime attestation. + +Create a new path inside a private, caller-owned 0700 directory. +Snapshot JSON may contain private addresses, labels, topology and paths, +so **never commit or share the output**. + +```sh +umask 077 +mkdir -m 700 /tmp/t3-p1-captures +python3 examples/k3s-p1/capture.py \ + --context YOUR_APPROVED_DISPOSABLE_CONTEXT \ + --plan /tmp/k3s-p1-receipts/YOUR_FULL_LEASE_SHA256.json \ + --output-dir /tmp/t3-p1-captures/first +``` + +The command refuses existing destination paths, non-private parents, +invalid plans or inconsistent repeated reads. It writes each JSON as 0600 +and atomically publishes the directory after all writes are synced. Linux +`renameat2(RENAME_NOREPLACE)` prevents a racing writer's empty destination +directory from being replaced; unsupported kernels fail closed, not silently +overwrite a previous snapshot. Run +existing `observe.py`, `pod_check.py` and `route_check.py` on these +**saved files**, using `first/namespace.json`, `first/quota.json`, +`first/serviceaccount.json`, `first/pvc.json`, +`first/deployment.json`, `first/service.json`, +`first/ingress.json`, `first/policy.json`, +`first/all-policies.json`, `first/all-ingresses.json`, +`first/all-services.json`, `first/replicasets.json`, and +`first/pods.json`. Never overwrite a previously accepted baseline. +Repeat capture into a *new* private directory when observing recovery. + +For credential-free development tests (does **not** contact kubectl): + +```sh +python3 -m unittest discover -s examples/k3s-p1 -p 'test_capture.py' -v +python3 -m py_compile examples/k3s-p1/capture.py examples/k3s-p1/test_capture.py +``` + +Upstream `main@08a463e29976617134de01e9341f898bc881d0eb` (2026-10-29) +documents linked-environment `delegate_task` and permissioned peer linking +in `docs/user/remote-access.md` and +`docs/internals/remote.md`; this is source-level availability on +`main`, **not** verified compatibility of the P0 `v0.1.3` image. +Upstream record-before-launch issue #17051 remains open. Human-approved +live P0 persistence/route/client proof and P1 OAuth/delegation are still HOLD. diff --git a/examples/k3s-p1/capture.py b/examples/k3s-p1/capture.py new file mode 100644 index 0000000000000..74f7bac08ed07 --- /dev/null +++ b/examples/k3s-p1/capture.py @@ -0,0 +1,253 @@ +#!/usr/bin/env python3 +"""Read-only, fail-closed P1 Kubernetes snapshot capture; never applies resources. + +Human-approved disposable K3s context required. Snapshots are not live +attestation or peer authorization. Private local output, not a lease. +""" + +import argparse +import ctypes +import errno +import json +import os +from pathlib import Path +import re +import shutil +import stat +import subprocess +import tempfile + + +class CaptureError(RuntimeError): + """Cannot safely capture a coherent P1 API snapshot.""" + + +NAMES = ( + ('namespace', 'namespace', None), + ('quota', 'resourcequota', 'worker-quota'), + ('serviceaccount', 'serviceaccount', 't3-worker'), + ('pvc', 'pvc', 't3-worker-data'), + ('deployment', 'deployment', 't3-worker'), + ('service', 'service', 't3-worker'), + ('ingress', 'ingress', 't3-worker'), + ('policy', 'networkpolicy', 'worker-ingress'), +) +LISTS = ( + ('all-policies', 'networkpolicies', 'NetworkPolicyList'), + ('all-ingresses', 'ingresses', 'IngressList'), + ('all-services', 'services', 'ServiceList'), + ('replicasets', 'replicasets', 'ReplicaSetList'), + ('pods', 'pods', 'PodList'), +) + +NAMED_KINDS = { + 'namespace': 'Namespace', + 'quota': 'ResourceQuota', + 'serviceaccount': 'ServiceAccount', + 'pvc': 'PersistentVolumeClaim', + 'deployment': 'Deployment', + 'service': 'Service', + 'ingress': 'Ingress', + 'policy': 'NetworkPolicy', +} + + + +# Exact P1 manifest controls from plan.py. ResourceQuota scopes can exempt a +# worker from limits; named-object identity alone is not enough for review. +# These API objects are snapshots, not proof of quota/RBAC enforcement. +EXPECTED_QUOTA_HARD = { + 'pods': '1', 'persistentvolumeclaims': '1', + 'requests.cpu': '2', 'requests.memory': '4Gi', + 'limits.cpu': '4', 'limits.memory': '8Gi', + 'requests.storage': '20Gi', + 'requests.ephemeral-storage': '2Gi', + 'limits.ephemeral-storage': '4Gi', +} + + +def _validate_controls(entries): + quota = entries['quota'] + service_account = entries['serviceaccount'] + qmeta = quota['metadata'] + smeta = service_account['metadata'] + if qmeta.get('deletionTimestamp') or smeta.get('deletionTimestamp'): + raise CaptureError('worker control resource is deleting') + spec = quota.get('spec') + status = quota.get('status') + if (not isinstance(spec, dict) or set(spec) != {'hard'} + or spec.get('hard') != EXPECTED_QUOTA_HARD + or not isinstance(status, dict) + or status.get('hard') != EXPECTED_QUOTA_HARD): + raise CaptureError('worker quota scope, bounds, or reconciliation differs from plan') + if (service_account.get('automountServiceAccountToken') is not False + or service_account.get('secrets') + or service_account.get('imagePullSecrets')): + raise CaptureError('worker ServiceAccount credential policy differs from plan') + + +def _read(context, namespace, resource, name, runner): + command = ['kubectl', '--context', context, '--request-timeout=10s'] + if resource != 'namespace': + command += ['-n', namespace] + command += ['get', resource] + if name: + command.append(name) + elif resource == 'namespace': + command.append(namespace) + command += ['-o', 'json'] + try: + response = runner(command, capture_output=True, text=True, timeout=20, check=False) + except (OSError, subprocess.TimeoutExpired) as exc: + raise CaptureError('read-only Kubernetes API request unavailable') from exc + if response.returncode: + raise CaptureError('read-only Kubernetes API request rejected') + try: + item = json.loads(response.stdout) + except (ValueError, TypeError) as exc: + raise CaptureError('malformed Kubernetes JSON') from exc + if not isinstance(item, dict): + raise CaptureError('unexpected Kubernetes API response') + return item + + +def capture(context, plan, runner=subprocess.run): + """Collect named objects and inventories, rejecting concurrent changes. + + Re-reading cannot make Kubernetes's multi-call snapshot atomic. Recheck + context and resource identities independently before trusting any result. + """ + if (not isinstance(context, str) or not context or context != context.strip() + or context.startswith('-') or any(c in context for c in '\r\n\x00')): + raise CaptureError('explicit kubectl context required') + if not isinstance(plan, dict) or plan.get('format') != 't3-k3s-p1-plan-v1': + raise CaptureError('invalid immutable plan') + lease = plan.get('leaseHash') + if not isinstance(lease, str) or not re.fullmatch('[0-9a-f]{64}', lease): + raise CaptureError('invalid lease hash') + namespace = 't3-p1-' + lease[:24] + if plan.get('namespace') != namespace: + raise CaptureError('namespace differs from lease plan') + entries = {} + for key, resource, name in NAMES: + item = _read(context, namespace, resource, name, runner) + meta = item.get('metadata') + expected_name = namespace if resource == 'namespace' else name + if (item.get('kind') != NAMED_KINDS[key] + or not isinstance(meta, dict) + or meta.get('name') != expected_name + or not isinstance(meta.get('uid'), str) or not meta['uid']): + raise CaptureError('named Kubernetes identity does not match the lease') + if resource == 'namespace': + if meta.get('namespace') not in (None, ''): + raise CaptureError('Namespace carries a foreign namespace') + elif meta.get('namespace') != namespace: + raise CaptureError('named resource belongs to a different namespace') + entries[key] = item + _validate_controls(entries) + for key, resource, kind in LISTS: + item = _read(context, namespace, resource, None, runner) + if item.get('kind') != kind or not isinstance(item.get('items'), list): + raise CaptureError('incomplete namespace inventory') + if any(not isinstance(obj, dict) or not isinstance(obj.get('metadata'), dict) + or obj['metadata'].get('namespace') != namespace for obj in item['items']): + raise CaptureError('inventory contains a foreign or malformed object') + entries[key] = item + # Separate named GET and LIST requests can observe different objects even + # when each individual request remains stable across both reads. Reject + # mixed snapshots before publishing a seemingly coherent private receipt. + for key, inventory in (('policy', 'all-policies'), + ('ingress', 'all-ingresses'), + ('service', 'all-services')): + named = entries[key] + matches = [obj for obj in entries[inventory]['items'] + if obj['metadata'].get('name') == named['metadata']['name']] + if len(matches) != 1 or matches[0] != named: + raise CaptureError('named route or policy disagrees with namespace inventory') + # Read everything again: reject mid-capture mutation of named endpoints, + # Deployments, Pods, ReplicaSets or extra namespace routes/policies. + for key, resource, name in NAMES: + if entries[key] != _read(context, namespace, resource, name, runner): + raise CaptureError('named resource changed during capture') + for key, resource, kind in LISTS: + if entries[key] != _read(context, namespace, resource, None, runner): + raise CaptureError('namespace inventory changed during capture') + return entries + + + +def _rename_directory_exclusive(source: Path, target: Path, parent_fd: int): + """Publish a directory atomically without clobbering another writer. + + This is a Linux-only acceptance helper. Without renameat2's + RENAME_NOREPLACE support, fail closed rather than overwriting evidence. + """ + renameat2 = getattr(ctypes.CDLL(None, use_errno=True), 'renameat2', None) + if renameat2 is None: + raise CaptureError('atomic no-replace directory publication unavailable') + renameat2.argtypes = (ctypes.c_int, ctypes.c_char_p, + ctypes.c_int, ctypes.c_char_p, ctypes.c_uint) + renameat2.restype = ctypes.c_int + if renameat2(parent_fd, os.fsencode(source.name), + parent_fd, os.fsencode(target.name), 1): # RENAME_NOREPLACE + if ctypes.get_errno() == errno.EEXIST: + raise CaptureError('output directory already exists') + raise CaptureError('atomic no-replace directory publication unavailable') + +def publish(entries, output): + """Atomically expose a new private directory only after all files sync.""" + if not output.is_absolute() or output.exists() or output.is_symlink(): + raise CaptureError('output must be a new absolute directory') + parent = output.parent + try: + parent_fd = os.open(parent, os.O_RDONLY | os.O_DIRECTORY | getattr(os, 'O_NOFOLLOW', 0)) + except OSError as exc: + raise CaptureError('private output parent unavailable') from exc + try: + meta = os.fstat(parent_fd) + if not stat.S_ISDIR(meta.st_mode) or meta.st_uid != os.getuid() or meta.st_mode & 0o077: + raise CaptureError('output parent must be caller-owned and private (0700)') + temp = Path(tempfile.mkdtemp(prefix='.p1-capture-', dir=parent)) + try: + for key, item in entries.items(): + path = temp / (key + '.json') + with path.open('x', encoding='utf-8') as file: + os.chmod(path, 0o600) + json.dump(item, file, sort_keys=True) + file.write('\n') + file.flush() + os.fsync(file.fileno()) + # Sync staging directory entries before the parent rename. File + # fsync alone does not durably commit the file names inside it. + temp_fd = os.open(temp, os.O_RDONLY | os.O_DIRECTORY | getattr(os, 'O_NOFOLLOW', 0)) + try: + os.fsync(temp_fd) + finally: + os.close(temp_fd) + _rename_directory_exclusive(temp, output, parent_fd) + os.fsync(parent_fd) + finally: + if temp.exists(): + shutil.rmtree(temp) + finally: + os.close(parent_fd) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--context', required=True) + parser.add_argument('--plan', required=True, type=Path) + parser.add_argument('--output-dir', required=True, type=Path) + args = parser.parse_args() + try: + plan = json.loads(args.plan.read_text(encoding='utf-8')) + entries = capture(args.context, plan) + publish(entries, args.output_dir) + except (OSError, ValueError, TypeError, AttributeError, CaptureError) as exc: + parser.exit(1, f'FAIL: P1 read-only snapshot capture ({type(exc).__name__})\n') + print('PASS: 13 API JSON snapshots captured to private directory (read-only Kubernetes calls)') + print('NOT RUN: live integrity attestation, HTTPS, OAuth, linked delegation, cancellation, TTL') + + +if __name__ == '__main__': + main() diff --git a/examples/k3s-p1/observe.py b/examples/k3s-p1/observe.py new file mode 100644 index 0000000000000..87ebdb6f93f48 --- /dev/null +++ b/examples/k3s-p1/observe.py @@ -0,0 +1,259 @@ +#!/usr/bin/env python3 +"""P1 offline binding check: Namespace, PVC and desired Deployment; no K8s calls. + +A matching Kubernetes snapshot is NOT worker readiness, T3 identity, peer +consent, a task completion, or a license to apply/delete any resources. +""" + +import argparse +import hashlib +import json +from pathlib import Path +from datetime import datetime +import re + + +class ObservationError(ValueError): + """Observation cannot be bound safely to the immutable lease plan.""" + + +def _require(value, expected, label): + if value != expected: + raise ObservationError(f'{label} does not match lease plan') + + +def _uid(obj, label): + value = obj.get('metadata', {}).get('uid') + if not isinstance(value, str) or not re.fullmatch(r'[a-zA-Z0-9-]{8,128}', value): + raise ObservationError(f'{label} lacks a valid Kubernetes UID') + return value + + +def _inspect(plan, namespace, pvc, deployment): + """Verify immutable lease identity and bound storage from API JSON objects. + + Inputs must be collected from one explicitly approved Kubernetes context. + This does not contact or attest that context; it also cannot prove PVC + contents, live Pod ownership, TLS, peer OAuth, or CNI enforcement. + """ + if not all(isinstance(x, dict) for x in (plan, namespace, pvc, deployment)): + raise ObservationError('expected JSON objects') + lease_hash = plan.get('leaseHash') + if not isinstance(lease_hash, str) or not re.fullmatch('[0-9a-f]{64}', lease_hash): + raise ObservationError('invalid lease hash') + name = 't3-p1-' + lease_hash[:24] + _require(plan.get('format'), 't3-k3s-p1-plan-v1', 'plan version') + _require(plan.get('namespace'), name, 'planned namespace') + _require(plan.get('retention'), 'retain', 'planned retention') + expiry = plan.get('expiresAt') + if not isinstance(expiry, str) or not re.fullmatch(r'\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}Z', expiry): + raise ObservationError('invalid lease expiry') + try: + expiry_date = datetime.strptime(expiry, '%Y-%m-%dT%H:%M:%SZ') + except ValueError as exc: + raise ObservationError('invalid lease expiration calendar date') from exc + if expiry_date.year < 2020 or expiry_date.year > 2100: + raise ObservationError('lease expiration is outside planning bounds') + # A stable Namespace/PVC is not enough: the same key must not silently + # rebind to a different image. Hash the *whole* pinned reference so the + # private binding receipt never exposes the registry or image name. + image = plan.get('image') + if (not isinstance(image, str) or not re.fullmatch( + r'(?=.{1,255}$)[a-z0-9][a-z0-9._:/-]*/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}', image) + or '..' in image or '//' in image): + raise ObservationError('invalid immutable image reference') + + # v3 binds the actual observed Deployment's *desired template* to the + # immutable plan. v2 hashed the plan alone and could miss image drift in + # Kubernetes. This still says nothing about which image a live Pod runs. + _require(deployment.get('kind'), 'Deployment', 'deployment kind') + dmeta = deployment.get('metadata', {}) + _require(dmeta.get('name'), 't3-worker', 'deployment name') + _require(dmeta.get('namespace'), name, 'deployment namespace') + if dmeta.get('deletionTimestamp'): + raise ObservationError('deployment is deleting') + deployment_uid = _uid(deployment, 'deployment') + generation = dmeta.get('generation') + if type(generation) is not int or generation < 1: + raise ObservationError('deployment lacks a valid generation') + dspec = deployment.get('spec', {}) + _require(dspec.get('replicas'), 1, 'deployment replicas') + _require(dspec.get('strategy', {}).get('type'), 'Recreate', 'deployment strategy') + expected_labels = {'app': 't3-worker'} + _require(dspec.get('selector', {}).get('matchLabels'), expected_labels, 'deployment selector') + template = dspec.get('template', {}) + template_meta = template.get('metadata', {}) + _require(template_meta.get('labels'), expected_labels, 'deployment Pod labels') + # Admission-triggering metadata can change a future replacement Pod even + # when the approved image, ServiceAccount and persistent volume match. + if (set(template_meta) - {'labels', 'annotations', 'creationTimestamp'} + or template_meta.get('creationTimestamp') is not None + or template_meta.get('annotations')): + raise ObservationError('deployment unexpected Pod template metadata') + pod = template.get('spec', {}) + _require(pod.get('serviceAccountName'), 't3-worker', 'deployment ServiceAccount') + _require(pod.get('automountServiceAccountToken'), False, 'deployment API token') + containers = pod.get('containers') + if not isinstance(containers, list) or len(containers) != 1 or not isinstance(containers[0], dict): + raise ObservationError('deployment must have exactly one T3 container') + container = containers[0] + _require(container.get('name'), 't3', 'deployment container') + _require(container.get('image'), image, 'observed deployment image') + # Fail closed if the observed desired Pod loosens its isolation while + # retaining the same Deployment UID, image, ServiceAccount and PVC. + # These checks cover spec drift, not admission/CNI enforcement or live Pods. + _require(pod.get('hostname'), 't3-worker', 'deployment hostname') + _require(pod.get('terminationGracePeriodSeconds'), 90, 'deployment termination grace') + _require(pod.get('securityContext'), { + 'runAsNonRoot': True, 'runAsUser': 10001, 'runAsGroup': 10001, + 'fsGroup': 10001, 'seccompProfile': {'type': 'RuntimeDefault'}, + }, 'deployment Pod security') + for field in ('hostNetwork', 'hostPID', 'hostIPC', 'shareProcessNamespace'): + if pod.get(field) not in (None, False): + raise ObservationError('deployment host isolation drift') + for field in ('initContainers', 'ephemeralContainers', 'imagePullSecrets', + 'hostAliases'): + if pod.get(field): + raise ObservationError('deployment unexpected Pod credential or container path') + _require(container.get('workingDir'), '/data', 'deployment working directory') + _require(container.get('imagePullPolicy'), 'IfNotPresent', 'deployment pull policy') + # Kubernetes defaults protocol=TCP in the observed object, even when + # omitted from the manifest. Accept that default but never hostPorts. + ports = container.get('ports') + if not isinstance(ports, list) or len(ports) != 1 or not isinstance(ports[0], dict): + raise ObservationError('deployment HTTP port shape changed') + _require(ports[0].get('name'), 'http', 'deployment HTTP port name') + _require(ports[0].get('containerPort'), 3773, 'deployment HTTP port') + _require(ports[0].get('protocol', 'TCP'), 'TCP', 'deployment HTTP protocol') + if ports[0].get('hostPort') not in (None, 0) or ports[0].get('hostIP'): + raise ObservationError('deployment host port is forbidden') + _require(container.get('resources'), { + 'requests': {'cpu': '500m', 'memory': '1Gi', 'ephemeral-storage': '512Mi'}, + 'limits': {'cpu': '2', 'memory': '4Gi', 'ephemeral-storage': '2Gi'}, + }, 'deployment resource bounds') + security = container.get('securityContext') + if not isinstance(security, dict): + raise ObservationError('deployment missing container security context') + for field, value in { + 'allowPrivilegeEscalation': False, 'readOnlyRootFilesystem': True, + 'capabilities': {'drop': ['ALL']}, + }.items(): + _require(security.get(field), value, 'deployment container security') + if security.get('privileged') not in (None, False): + raise ObservationError('deployment privileged container') + # Container-level overrides take precedence over Pod securityContext. + # Only allow safe equivalent defaults; unknown keys fail closed. + allowed = {'allowPrivilegeEscalation', 'readOnlyRootFilesystem', + 'capabilities', 'privileged', 'runAsNonRoot', 'runAsUser', + 'runAsGroup', 'seccompProfile'} + if set(security) - allowed: + raise ObservationError('deployment unexpected container security override') + for field, expected in { + 'runAsNonRoot': True, 'runAsUser': 10001, 'runAsGroup': 10001, + 'seccompProfile': {'type': 'RuntimeDefault'}, + }.items(): + if field in security: + _require(security[field], expected, 'deployment container security override') + for field in ('env', 'envFrom', 'command', 'args', 'lifecycle', 'volumeDevices'): + if container.get(field): + raise ObservationError('deployment unexpected injected container configuration') + # Kubernetes may add probe defaults to fetched API JSON. + for probe_name, failures in (('startupProbe', 60), ('readinessProbe', 3)): + probe = container.get(probe_name) + if not isinstance(probe, dict): + raise ObservationError('deployment probe missing') + _require(probe.get('tcpSocket'), {'port': 'http'}, 'deployment probe target') + _require(probe.get('periodSeconds'), 5, 'deployment probe frequency') + _require(probe.get('failureThreshold', 3), failures, 'deployment probe failures') + _require(probe.get('timeoutSeconds', 1), 1, 'deployment probe timeout') + _require(probe.get('successThreshold', 1), 1, 'deployment probe successes') + _require(probe.get('initialDelaySeconds', 0), 0, 'deployment probe initial delay') + if any(key not in { + 'tcpSocket', 'periodSeconds', 'failureThreshold', 'timeoutSeconds', + 'successThreshold', 'initialDelaySeconds', + } for key in probe): + raise ObservationError('deployment unexpected probe configuration') + _require(container.get('volumeMounts'), [ + {'name': 'data', 'mountPath': '/data'}, + {'name': 'tmp', 'mountPath': '/tmp'}, + ], 'deployment volume mounts') + _require(pod.get('volumes'), [ + {'name': 'data', 'persistentVolumeClaim': {'claimName': 't3-worker-data'}}, + {'name': 'tmp', 'emptyDir': {'sizeLimit': '1Gi'}}, + ], 'deployment volumes') + + _require(namespace.get('kind'), 'Namespace', 'namespace kind') + nmeta = namespace.get('metadata', {}) + _require(nmeta.get('name'), name, 'namespace name') + _require(namespace.get('status', {}).get('phase'), 'Active', 'namespace phase') + if nmeta.get('deletionTimestamp'): + raise ObservationError('namespace is deleting') + labels = nmeta.get('labels', {}) + _require(labels.get('pod-security.kubernetes.io/enforce'), 'restricted', 'Pod Security') + annotations = nmeta.get('annotations', {}) + _require(annotations.get('t3code.dev/lease-key-sha256'), lease_hash, 'namespace lease') + _require(annotations.get('t3code.dev/lease-expires-at'), expiry, 'namespace expiry') + _require(annotations.get('t3code.dev/expiry-enforced'), 'false', 'TTL disclaimer') + _require(annotations.get('t3code.dev/retention'), 'retain-until-owner-decision', 'PVC retention') + + _require(pvc.get('kind'), 'PersistentVolumeClaim', 'PVC kind') + pmeta = pvc.get('metadata', {}) + _require(pmeta.get('name'), 't3-worker-data', 'PVC name') + _require(pmeta.get('namespace'), name, 'PVC namespace') + if pmeta.get('deletionTimestamp'): + raise ObservationError('PVC is deleting') + _require(pvc.get('status', {}).get('phase'), 'Bound', 'PVC phase') + pspec = pvc.get('spec', {}) + _require(pspec.get('accessModes'), ['ReadWriteOnce'], 'PVC access mode') + _require(pspec.get('resources', {}).get('requests', {}).get('storage'), '10Gi', 'PVC request') + volume = pspec.get('volumeName') + if not isinstance(volume, str) or not volume or len(volume) > 253: + raise ObservationError('PVC has no bound PersistentVolume name') + + return {'format': 't3-k3s-p1-binding-v4', 'leaseHash': lease_hash, + 'namespace': name, 'namespaceUID': _uid(namespace, 'namespace'), + 'pvcUID': _uid(pvc, 'PVC'), 'deploymentUID': deployment_uid, + 'deploymentGeneration': generation, 'volumeName': volume, + 'imageRefSha256': hashlib.sha256(image.encode('ascii')).hexdigest(), + 'expiresAt': expiry, 'retention': 'retain'} + + +def inspect(plan, namespace, pvc, deployment): + """Treat malformed API JSON as a failed binding, without leaking details.""" + try: + return _inspect(plan, namespace, pvc, deployment) + except (AttributeError, KeyError, TypeError) as exc: + # Fetched Kubernetes objects may contain unexpected null/list values. + # Never let a malformed snapshot escape as an unhandled traceback. + raise ObservationError('malformed JSON object in P1 binding observation') from exc + + +def compare(current, baseline): + if not isinstance(baseline, dict) or current != baseline: + raise ObservationError('binding changed: refuse to resume or accept a worker') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--plan', type=Path, required=True, help='private immutable lease-plan receipt') + parser.add_argument('--namespace', type=Path, required=True, help='kubectl get namespace -o json') + parser.add_argument('--pvc', type=Path, required=True, help='kubectl get pvc -o json') + parser.add_argument('--deployment', type=Path, required=True, + help='kubectl get deployment t3-worker -o json') + parser.add_argument('--baseline', type=Path, help='previously verified private binding JSON') + args = parser.parse_args() + try: + documents = [json.loads(path.read_text(encoding='utf-8')) + for path in (args.plan, args.namespace, args.pvc, args.deployment)] + binding = inspect(*documents) + if args.baseline is not None: + compare(binding, json.loads(args.baseline.read_text(encoding='utf-8'))) + except (OSError, json.JSONDecodeError, ObservationError) as exc: + # Avoid echoing object contents or OS errors with private host paths. + parser.exit(1, f'FAIL: P1 lease/PVC binding could not be verified ({type(exc).__name__})\n') + print(json.dumps(binding, sort_keys=True)) + print('DESIRED-SPEC BINDING ONLY: NOT live Pod image, readiness, HTTPS, OAuth or delegation proof', file=__import__('sys').stderr) + + +if __name__ == '__main__': + main() diff --git a/examples/k3s-p1/plan.py b/examples/k3s-p1/plan.py new file mode 100644 index 0000000000000..c68f2b9072de2 --- /dev/null +++ b/examples/k3s-p1/plan.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""P1 preview: deterministic, fail-closed T3 worker plan, with NO cluster writes. + +A stored local lease-plan is NOT a running lease: there is no Kubernetes apply, +lease renewal, expiry controller, peer OAuth grant, or T3 task submission here. +P0 live acceptance remains a prerequisite to any P1 rollout. +""" + +import argparse +from datetime import datetime, timezone +import hashlib +import json +import os +from pathlib import Path +import re +import secrets +import stat +import sys + +import yaml + +KEY = re.compile(r"[A-Za-z0-9][A-Za-z0-9._:/-]{0,127}\Z") +IMAGE = re.compile(r"(?=.{1,255}$)[a-z0-9][a-z0-9._:/-]*/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}\Z") +UTC = re.compile(r"\d{4}-\d\d-\d\dT\d\d:\d\d:\d\dZ\Z") + + +class PlanError(ValueError): + """Invalid plan or conflicting immutable lease request.""" + + +def build(lease_key: str, image: str, expires_at: str) -> tuple[dict, list[dict]]: + if not isinstance(lease_key, str) or not KEY.fullmatch(lease_key) or '..' in lease_key: + raise PlanError('lease key must be a stable, bounded opaque ASCII identifier') + if not isinstance(image, str) or not IMAGE.fullmatch(image) or '..' in image or '//' in image: + raise PlanError('image must use an immutable registry/repository@sha256 digest') + if not isinstance(expires_at, str) or not UTC.fullmatch(expires_at): + raise PlanError('expires-at must be an exact UTC timestamp YYYY-MM-DDTHH:MM:SSZ') + try: + timestamp = datetime.strptime(expires_at, '%Y-%m-%dT%H:%M:%SZ').replace(tzinfo=timezone.utc) + except ValueError as exc: + raise PlanError('invalid UTC expiration timestamp') from exc + if timestamp.year < 2020 or timestamp.year > 2100: + raise PlanError('expiration outside supported planning bounds') + + full_hash = hashlib.sha256(lease_key.encode('ascii')).hexdigest() + suffix = full_hash[:24] + ns = f't3-p1-{suffix}' + ingress_host = f't3-p1-{suffix}' + labels = {'app': 't3-worker'} + annotations = {'t3code.dev/lease-key-sha256': full_hash, + 't3code.dev/lease-expires-at': expires_at, + 't3code.dev/expiry-enforced': 'false', + 't3code.dev/retention': 'retain-until-owner-decision'} + plan = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': full_hash, 'namespace': ns, + 'image': image, 'expiresAt': expires_at, 'retention': 'retain'} + + def obj(api: str, kind: str, name: str, spec: dict | None = None) -> dict: + doc = {'apiVersion': api, 'kind': kind, 'metadata': {'name': name, 'namespace': ns}} + if spec is not None: + doc['spec'] = spec + return doc + + namespace = {'apiVersion': 'v1', 'kind': 'Namespace', 'metadata': { + 'name': ns, 'annotations': annotations, + 'labels': {'pod-security.kubernetes.io/enforce': 'restricted', + 'pod-security.kubernetes.io/enforce-version': 'latest'}}} + quota = obj('v1', 'ResourceQuota', 'worker-quota', {'hard': { + 'pods': '1', 'persistentvolumeclaims': '1', 'requests.cpu': '2', + 'requests.memory': '4Gi', 'limits.cpu': '4', 'limits.memory': '8Gi', + 'requests.storage': '20Gi', 'requests.ephemeral-storage': '2Gi', + 'limits.ephemeral-storage': '4Gi'}}) + sa = obj('v1', 'ServiceAccount', 't3-worker') + sa['automountServiceAccountToken'] = False + pvc = obj('v1', 'PersistentVolumeClaim', 't3-worker-data', { + 'accessModes': ['ReadWriteOnce'], 'resources': {'requests': {'storage': '10Gi'}}}) + container = { + 'name': 't3', 'image': image, 'imagePullPolicy': 'IfNotPresent', + 'workingDir': '/data', 'ports': [{'name': 'http', 'containerPort': 3773}], + 'resources': {'requests': {'cpu': '500m', 'memory': '1Gi', 'ephemeral-storage': '512Mi'}, + 'limits': {'cpu': '2', 'memory': '4Gi', 'ephemeral-storage': '2Gi'}}, + 'securityContext': {'allowPrivilegeEscalation': False, 'readOnlyRootFilesystem': True, + 'capabilities': {'drop': ['ALL']}}, + 'startupProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5, 'failureThreshold': 60}, + 'readinessProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5}, + 'volumeMounts': [{'name': 'data', 'mountPath': '/data'}, + {'name': 'tmp', 'mountPath': '/tmp'}], + } + pod = { + 'serviceAccountName': 't3-worker', 'automountServiceAccountToken': False, + 'hostname': 't3-worker', 'terminationGracePeriodSeconds': 90, + 'securityContext': {'runAsNonRoot': True, 'runAsUser': 10001, + 'runAsGroup': 10001, 'fsGroup': 10001, + 'seccompProfile': {'type': 'RuntimeDefault'}}, + 'containers': [container], + 'volumes': [{'name': 'data', 'persistentVolumeClaim': {'claimName': 't3-worker-data'}}, + {'name': 'tmp', 'emptyDir': {'sizeLimit': '1Gi'}}], + } + deployment = obj('apps/v1', 'Deployment', 't3-worker', { + 'replicas': 1, 'strategy': {'type': 'Recreate'}, 'selector': {'matchLabels': labels}, + 'template': {'metadata': {'labels': labels}, 'spec': pod}}) + service = obj('v1', 'Service', 't3-worker', { + 'type': 'ClusterIP', 'selector': labels, + 'ports': [{'name': 'http', 'port': 80, 'targetPort': 'http', 'protocol': 'TCP'}]}) + ingress = obj('networking.k8s.io/v1', 'Ingress', 't3-worker', { + 'ingressClassName': 'tailscale', 'tls': [{'hosts': [ingress_host]}], + 'defaultBackend': {'service': {'name': 't3-worker', 'port': {'name': 'http'}}}}) + policy = obj('networking.k8s.io/v1', 'NetworkPolicy', 'worker-ingress', { + 'podSelector': {'matchLabels': labels}, 'policyTypes': ['Ingress'], + 'ingress': [{'from': [{'namespaceSelector': {'matchLabels': { + 'kubernetes.io/metadata.name': 'tailscale'}}}], + 'ports': [{'protocol': 'TCP', 'port': 3773}]}]}) + return plan, [namespace, quota, sa, pvc, deployment, service, ingress, policy] + + +def lock(state_dir: Path, plan: dict) -> bool: + """Create-once 0600 local receipt; retries match exactly or fail closed. + + Requires owner-created private absolute directory. Never overwrites a + conflicting or symlinked plan; never contacts the cluster. + """ + if not state_dir.is_absolute(): + raise PlanError('state directory must be an absolute owner-private path') + flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) | getattr(os, 'O_NOFOLLOW', 0) + try: + directory = os.open(state_dir, flags) + except OSError as exc: + raise PlanError('state directory unavailable') from exc + try: + st = os.fstat(directory) + if not stat.S_ISDIR(st.st_mode) or (st.st_mode & 0o077) or st.st_uid != os.getuid(): + raise PlanError('state directory must be owned by caller with mode 0700') + filename = plan['leaseHash'] + '.json' + data = (json.dumps(plan, sort_keys=True, separators=(',', ':')) + '\n').encode('utf-8') + # Never publish a receipt until its bytes are fully written and synced. + # O_EXCL on the final name exposes an empty/partial file to concurrent + # retries; a hard link from a private temporary inode publishes atomically. + pending = f'.{filename}.{secrets.token_hex(16)}.pending' + file_flags = os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, 'O_NOFOLLOW', 0) + created = False + try: + file = os.open(pending, file_flags, 0o600, dir_fd=directory) + with os.fdopen(file, 'wb') as output: + output.write(data) + output.flush() + os.fsync(output.fileno()) + try: + os.link(pending, filename, src_dir_fd=directory, + dst_dir_fd=directory, follow_symlinks=False) + except FileExistsError: + # Another caller published first. Verify its exact bytes below. + pass + else: + os.fsync(directory) + created = True + except OSError as exc: + raise PlanError('cannot establish local lease receipt') from exc + finally: + try: + os.unlink(pending, dir_fd=directory) + except FileNotFoundError: + pass + if created: + return True + read_flags = os.O_RDONLY | getattr(os, 'O_NOFOLLOW', 0) + try: + file = os.open(filename, read_flags, dir_fd=directory) + with os.fdopen(file, 'rb') as source: + info = os.fstat(source.fileno()) + if (not stat.S_ISREG(info.st_mode) or info.st_mode & 0o077 + or info.st_uid != os.getuid() or info.st_size > 4096): + raise PlanError('existing lease receipt is unsafe') + previous = source.read(4097) + except OSError as exc: + raise PlanError('existing lease receipt unavailable or unsafe') from exc + if previous != data: + raise PlanError('same lease key has conflicting image, expiry, or plan contract') + return False + finally: + os.close(directory) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('--lease-key', required=True, help='opaque stable idempotency key (not printed)') + parser.add_argument('--image', required=True, help='T3-compatible worker image@sha256 digest') + parser.add_argument('--expires-at', required=True, help='UTC expiry metadata; NOT enforced') + parser.add_argument('--state-dir', required=True, type=Path, help='pre-existing private (0700) directory') + args = parser.parse_args() + try: + plan, documents = build(args.lease_key, args.image, args.expires_at) + created = lock(args.state_dir, plan) + except PlanError as exc: + parser.exit(1, f'FAIL: {exc}\n') + # stdout contains declarative YAML only; operator must separately inspect, + # authorize, and apply. No automatic time-based retention/deletion. + print(yaml.safe_dump_all(documents, sort_keys=False, explicit_start=True), end='') + print('P1 PREVIEW ONLY: no cluster write, OAuth pairing, TTL action, or worker execution; ' + f'local receipt {"created" if created else "reused"}', file=sys.stderr) + + +if __name__ == '__main__': + main() diff --git a/examples/k3s-p1/pod_check.py b/examples/k3s-p1/pod_check.py new file mode 100644 index 0000000000000..db353a884f236 --- /dev/null +++ b/examples/k3s-p1/pod_check.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +"""Read-only P1 Pod ownership/runtime snapshot check; NEVER a live attestation.""" +import argparse +import hashlib +import json +from pathlib import Path +import re + + +class PodError(ValueError): + """An untrusted Kubernetes snapshot does not support the claimed worker.""" + + +def need(condition, message): + if not condition: + raise PodError(message) + + +def meta(obj, kind, namespace): + need(isinstance(obj, dict) and obj.get('kind') == kind, 'invalid object kind') + value = obj.get('metadata') + need(isinstance(value, dict) and value.get('namespace') == namespace + and not value.get('deletionTimestamp'), 'resource identity or deletion drift') + uid = value.get('uid') + need(isinstance(uid, str) and re.fullmatch(r'[A-Za-z0-9-]{8,128}', uid), 'missing resource UID') + return value + + +def controller(obj, kind, name, uid): + refs = obj.get('metadata', {}).get('ownerReferences') + need(isinstance(refs, list) and len(refs) == 1, 'missing exclusive controller reference') + ref = refs[0] + need(isinstance(ref, dict) and ref.get('controller') is True + and (ref.get('kind'), ref.get('name'), ref.get('uid')) == (kind, name, uid), + 'controller reference drift') + + +def inspect(plan, binding, deployment, replicasets, pods): + """Check offline snapshots from one approved context, not actual connectivity.""" + need(isinstance(plan, dict) and isinstance(binding, dict), 'invalid plan/binding') + lease = plan.get('leaseHash') + need(plan.get('format') == 't3-k3s-p1-plan-v1' and isinstance(lease, str) + and re.fullmatch(r'[0-9a-f]{64}', lease), 'invalid immutable plan') + ns = 't3-p1-' + lease[:24] + image = plan.get('image') + need(isinstance(image, str) and re.fullmatch( + r'(?=.{1,255}$)[a-z0-9][a-z0-9._:/-]*/[a-z0-9][a-z0-9._/-]*@sha256:[a-f0-9]{64}', image) + and '..' not in image and '//' not in image, 'image not pinned') + image_hash = hashlib.sha256(image.encode('ascii')).hexdigest() + need(plan.get('namespace') == ns and binding.get('format') == 't3-k3s-p1-binding-v4' + and binding.get('namespace') == ns and binding.get('leaseHash') == lease + and binding.get('imageRefSha256') == image_hash, 'lease binding mismatch') + dmeta = meta(deployment, 'Deployment', ns) + need(dmeta.get('name') == 't3-worker' and dmeta['uid'] == binding.get('deploymentUID'), + 'deployment binding drift') + # A Deployment may keep its UID while its desired spec is changed. The + # read-only binding must be refreshed and re-reviewed for any new generation. + need(type(dmeta.get('generation')) is int and dmeta['generation'] > 0 + and binding.get('deploymentGeneration') == dmeta['generation'], + 'deployment changed since verified binding') + ds = deployment.get('spec', {}) + template = ds.get('template', {}).get('spec', {}) + need(isinstance(template, dict) and isinstance(template.get('containers'), list) + and len(template['containers']) == 1 and template['containers'][0].get('image') == image, + 'deployment image drift') + status = deployment.get('status', {}) + need(isinstance(status, dict) and isinstance(dmeta.get('generation'), int) + and status.get('observedGeneration') == dmeta['generation'] + and all(status.get(k) == 1 for k in ('replicas', 'readyReplicas', 'availableReplicas', 'updatedReplicas')), + 'deployment rollout not observed ready') + need(isinstance(replicasets, dict) and replicasets.get('kind') == 'ReplicaSetList' + and isinstance(replicasets.get('items'), list), 'incomplete ReplicaSet inventory') + active_rs = [] + for rs in replicasets['items']: + rm = meta(rs, 'ReplicaSet', ns) + controller(rs, 'Deployment', 't3-worker', dmeta['uid']) + rs_spec = rs.get('spec', {}) + rs_status = rs.get('status', {}) + need(isinstance(rs_spec, dict) and isinstance(rs_status, dict), 'malformed ReplicaSet') + if rs_spec.get('replicas', 0) or rs_status.get('replicas', 0): + active_rs.append((rs, rm)) + need(len(active_rs) == 1, 'ambiguous live ReplicaSet') + rs, rm = active_rs[0] + need(rs.get('spec', {}).get('replicas') == 1 + and rs.get('status', {}).get('readyReplicas') == 1, 'ReplicaSet not ready') + # ReplicaSet is the template used for the NEXT Pod after restart/eviction. + # A clean, Ready Pod is insufficient if its controller was subsequently + # changed to launch a different image, credential or unsafe container. + # Compare spec only: Deployment adds pod-template-hash to RS template labels. + rs_template = rs.get('spec', {}).get('template') + need(isinstance(rs_template, dict) and isinstance(rs_template.get('spec'), dict) + and rs_template['spec'] == template, 'active ReplicaSet template drift') + # Spec equality alone misses an edited replacement template's labels, + # selector or annotations. Those can change routing or trigger admission + # sidecars when the ReplicaSet creates a replacement Pod. + desired_meta = ds.get('template', {}).get('metadata') + replacement_meta = rs_template.get('metadata') + need(isinstance(desired_meta, dict) and isinstance(replacement_meta, dict) + and set(desired_meta) <= {'labels', 'annotations', 'creationTimestamp'} + and set(replacement_meta) <= {'labels', 'annotations', 'creationTimestamp'} + and desired_meta.get('creationTimestamp') is None + and replacement_meta.get('creationTimestamp') is None + and desired_meta.get('labels') == {'app': 't3-worker'} + and not desired_meta.get('annotations') + and not replacement_meta.get('annotations'), + 'replacement Pod template metadata drift') + replacement_labels = replacement_meta.get('labels') + need(isinstance(replacement_labels, dict) + and set(replacement_labels) == {'app', 'pod-template-hash'} + and replacement_labels.get('app') == 't3-worker' + and isinstance(replacement_labels.get('pod-template-hash'), str) + and re.fullmatch(r'[a-z0-9-]{1,63}', replacement_labels['pod-template-hash']), + 'replacement Pod template labels drift') + selector = rs.get('spec', {}).get('selector') + need(isinstance(selector, dict) + and selector.get('matchLabels') == replacement_labels + and not selector.get('matchExpressions') + and set(selector) <= {'matchLabels', 'matchExpressions'}, + 'active ReplicaSet selector drift') + need(isinstance(pods, dict) and pods.get('kind') == 'PodList' + and isinstance(pods.get('items'), list) and len(pods['items']) == 1, + 'worker namespace does not have exactly one Pod') + pod = pods['items'][0] + pm = meta(pod, 'Pod', ns) + controller(pod, 'ReplicaSet', rm.get('name'), rm['uid']) + need(pm.get('labels') == replacement_labels, 'Pod selector drift') + # Pod-level CNI/admission annotations may attach additional interfaces even + # when the ReplicaSet template and container specification remain intact. + need(pm.get('annotations') in (None, {}), 'unapproved Pod admission annotations') + ps = pod.get('spec', {}) + need(isinstance(ps, dict) and ps.get('serviceAccountName') == 't3-worker' + and ps.get('automountServiceAccountToken') is False + and ps.get('securityContext') == template.get('securityContext') + and ps.get('volumes') == template.get('volumes'), 'admitted Pod security/storage drift') + for field in ('hostNetwork', 'hostPID', 'hostIPC', 'shareProcessNamespace'): + need(ps.get(field) in (None, False), 'Pod host isolation drift') + # Admission can change the Pod's runtime, GPU/DRA allocation, routing or + # scheduling without changing any field compared below. Only normal + # Kubernetes defaults are accepted; other overlays need new human review. + need(ps.get('runtimeClassName') in (None, '') + and not ps.get('overhead') and not ps.get('resourceClaims') + and ps.get('priorityClassName') in (None, ''), + 'unapproved Pod runtime or resource allocation') + need(not ps.get('nodeSelector') and not ps.get('affinity') + and not ps.get('topologySpreadConstraints') + and not ps.get('dnsConfig') + and ps.get('dnsPolicy', 'ClusterFirst') == 'ClusterFirst', + 'unapproved Pod DNS or placement policy') + need(ps.get('serviceAccount') in (None, 't3-worker') + and ps.get('terminationGracePeriodSeconds', 30) == + template.get('terminationGracePeriodSeconds', 30), + 'Pod service account or termination policy drift') + for field in ('initContainers', 'ephemeralContainers', 'imagePullSecrets', 'hostAliases'): + need(not ps.get(field), 'unexpected Pod configuration') + containers = ps.get('containers') + need(isinstance(containers, list) and len(containers) == 1 + and isinstance(containers[0], dict), 'unexpected Pod containers') + for field in ('name', 'image', 'securityContext', 'resources', 'volumeMounts'): + need(containers[0].get(field) == template['containers'][0].get(field), + 'admitted container configuration drift') + for field in ('env', 'envFrom', 'command', 'args', 'volumeDevices', + 'lifecycle', 'stdin', 'stdinOnce', 'tty'): + need(not containers[0].get(field), 'injected Pod command or lifecycle drift') + # The desired Deployment snapshot is inspected separately. Bind the admitted + # Pod's startup path and port/probes to it; matching only image/resources + # misses hostPort, postStart exec and probe-command injection. + need(ps.get('hostname') == template.get('hostname'), 'Pod hostname drift') + desired = template['containers'][0] + admitted = containers[0] + for field in ('workingDir', 'imagePullPolicy'): + need(admitted.get(field) == desired.get(field), + 'admitted container startup drift') + want_ports, got_ports = desired.get('ports'), admitted.get('ports') + need(isinstance(want_ports, list) and len(want_ports) == 1 + and isinstance(want_ports[0], dict) + and isinstance(got_ports, list) and len(got_ports) == 1 + and isinstance(got_ports[0], dict), 'admitted container port drift') + wp, gp = want_ports[0], got_ports[0] + need(gp.get('name') == wp.get('name') + and gp.get('containerPort') == wp.get('containerPort') + and gp.get('protocol', 'TCP') == wp.get('protocol', 'TCP') + and gp.get('hostPort') in (None, 0) and not gp.get('hostIP'), + 'admitted container port or host exposure drift') + probe_defaults = {'periodSeconds': 10, 'failureThreshold': 3, + 'timeoutSeconds': 1, 'successThreshold': 1, + 'initialDelaySeconds': 0} + for name in ('startupProbe', 'readinessProbe'): + expected_probe, actual_probe = desired.get(name), admitted.get(name) + need(isinstance(expected_probe, dict) and isinstance(actual_probe, dict) + and set(actual_probe) <= set(probe_defaults) | {'tcpSocket'} + and actual_probe.get('tcpSocket') == expected_probe.get('tcpSocket'), + 'admitted container probe drift') + for field, default in probe_defaults.items(): + need(actual_probe.get(field, default) == expected_probe.get(field, default), + 'admitted container probe timing drift') + pod_status = pod.get('status', {}) + need(isinstance(pod_status, dict) and pod_status.get('phase') == 'Running', 'Pod not running') + conditions = pod_status.get('conditions') + need(isinstance(conditions, list) and any(isinstance(c, dict) + and c.get('type') == 'Ready' and c.get('status') == 'True' for c in conditions), + 'Pod not reported ready') + cs = pod_status.get('containerStatuses') + need(isinstance(cs, list) and len(cs) == 1 and isinstance(cs[0], dict), + 'missing container status') + c = cs[0] + need(c.get('name') == 't3' and c.get('ready') is True + and c.get('started') is True and isinstance(c.get('state'), dict) + and isinstance(c['state'].get('running'), dict), 'T3 process not reported ready') + need(c.get('image') == image, 'running image reference differs from lease') + cid, iid = c.get('containerID'), c.get('imageID') + need(isinstance(cid, str) and len(cid) >= 16 + and isinstance(iid, str) and re.search(r'sha256:[a-f0-9]{64}', iid), + 'runtime container or image ID missing') + restarts = c.get('restartCount') + need(type(restarts) is int and restarts >= 0, 'invalid restart count') + return {'format': 't3-k3s-p1-pod-snapshot-v1', 'leaseHash': lease, + 'deploymentUID': dmeta['uid'], 'replicaSetUID': rm['uid'], + 'podUID': pm['uid'], 'imageRefSha256': image_hash, + 'imageIDSha256': hashlib.sha256(iid.encode()).hexdigest(), + 'containerIDSha256': hashlib.sha256(cid.encode()).hexdigest(), + 'restartCount': restarts} + + +def main(): + p = argparse.ArgumentParser(description=__doc__) + for flag in ('plan', 'binding', 'deployment', 'replicasets', 'pods'): + p.add_argument('--' + flag, required=True, type=Path) + a = p.parse_args() + try: + docs = [json.loads(getattr(a, key).read_text(encoding='utf-8')) + for key in ('plan', 'binding', 'deployment', 'replicasets', 'pods')] + receipt = inspect(*docs) + except (OSError, ValueError, TypeError, KeyError, AttributeError, IndexError) as exc: + p.exit(1, f'FAIL: P1 offline Pod snapshot not verified ({type(exc).__name__})\n') + print(json.dumps(receipt, sort_keys=True)) + print('NOT RUN: trusted live cluster, actual digest attestation, T3 identity, HTTPS, OAuth, delegation') + + +if __name__ == '__main__': + main() diff --git a/examples/k3s-p1/route_check.py b/examples/k3s-p1/route_check.py new file mode 100644 index 0000000000000..fa895e28345d6 --- /dev/null +++ b/examples/k3s-p1/route_check.py @@ -0,0 +1,188 @@ +#!/usr/bin/env python3 +"""Offline P1 Service/Ingress/NetworkPolicy route contract; never calls Kubernetes. + +It checks only desired API objects, not the effective CNI, operator, TLS, +DNS, tailnet ACL, peer OAuth grant, or a running T3 environment. +""" +import argparse +import json +from pathlib import Path +import re + + +class RouteError(ValueError): + """A saved Kubernetes route does not match the immutable P1 plan.""" + + +def _same(actual, expected, field): + if actual != expected: + raise RouteError(f'{field} differs from private P1 plan') + + +def _meta(obj, kind, name, namespace): + if not isinstance(obj, dict) or obj.get('kind') != kind: + raise RouteError(f'{kind} object is missing or malformed') + meta = obj.get('metadata') + if not isinstance(meta, dict) or meta.get('name') != name or meta.get('namespace') != namespace: + raise RouteError(f'{kind} identity differs from P1 plan') + if meta.get('deletionTimestamp'): + raise RouteError(f'{kind} is deleting') + return meta + + +def _safe_annotations(obj, kind): + # Controllers can act on Service annotations even for a ClusterIP. + # In particular, tailscale.com/expose enables an additional L3 proxy + # that does not appear in the ordinary Service/Ingress inventories. + annotations = obj['metadata'].get('annotations', {}) + if (not isinstance(annotations, dict) + or set(annotations) - {'kubectl.kubernetes.io/last-applied-configuration'}): + raise RouteError(f'{kind} has unexpected route-affecting annotations') + + +def _route_status(obj, kind, ns): + """Fail closed on an API-advertised public/extra route, even if spec is private. + + An empty status is valid for an unapplied desired spec, NOT evidence of + HTTPS reachability or a trustworthy tailnet identity. + """ + status = obj.get('status', {}) + if not isinstance(status, dict): + raise RouteError(f'{kind} has malformed route status') + lb = status.get('loadBalancer', {}) + if not isinstance(lb, dict): + raise RouteError(f'{kind} has malformed load-balancer status') + routes = lb.get('ingress', []) + if not isinstance(routes, list): + raise RouteError(f'{kind} advertises malformed routes') + if kind == 'Service': + if routes: + raise RouteError('ClusterIP Service advertises an unexpected external route') + return + if not routes: + return # Unscheduled ingress: desired spec only, not an HTTPS proof. + if len(routes) != 1 or not isinstance(routes[0], dict): + raise RouteError('Ingress advertises unexpected extra routes') + entry = routes[0] + if (set(entry) != {'hostname'} or not isinstance(entry['hostname'], str) + or not re.fullmatch(re.escape(ns) + r'\.[a-z0-9](?:[a-z0-9-]{0,61}[a-z0-9])?\.ts\.net', entry['hostname'])): + raise RouteError('Ingress does not advertise the private expected hostname') + + +def verify(plan, service, ingress, policy, all_policies, all_ingresses, all_services): + """Fail closed for route/spec drift; no cluster writes or readiness claims.""" + if not isinstance(plan, dict) or plan.get('format') != 't3-k3s-p1-plan-v1': + raise RouteError('invalid P1 plan format') + lease = plan.get('leaseHash') + if not isinstance(lease, str) or not re.fullmatch(r'[0-9a-f]{64}', lease): + raise RouteError('invalid P1 lease hash') + ns = 't3-p1-' + lease[:24] + _same(plan.get('namespace'), ns, 'namespace') + _meta(service, 'Service', 't3-worker', ns) + _meta(ingress, 'Ingress', 't3-worker', ns) + _meta(policy, 'NetworkPolicy', 'worker-ingress', ns) + _safe_annotations(service, 'Service') + _safe_annotations(ingress, 'Ingress') + + _route_status(service, 'Service', ns) + _route_status(ingress, 'Ingress', ns) + + spec = service.get('spec') + if not isinstance(spec, dict): + raise RouteError('invalid Service spec') + _same(spec.get('type', 'ClusterIP'), 'ClusterIP', 'Service type') + _same(spec.get('selector'), {'app': 't3-worker'}, 'Service selector') + if spec.get('externalIPs') or spec.get('externalName') or spec.get('loadBalancerIP'): + raise RouteError('Service must not expose public/external endpoints') + ports = spec.get('ports') + if not isinstance(ports, list) or len(ports) != 1 or not isinstance(ports[0], dict): + raise RouteError('Service ports differ from P1 plan') + port = ports[0] + for key, expected in {'name': 'http', 'port': 80, 'targetPort': 'http', 'protocol': 'TCP'}.items(): + _same(port.get(key, 'TCP' if key == 'protocol' else None), expected, 'Service port') + if port.get('nodePort') not in (None, 0): + raise RouteError('Service NodePort forbidden') + + ispec = ingress.get('spec') + if not isinstance(ispec, dict): + raise RouteError('invalid Ingress spec') + _same(ispec.get('ingressClassName'), 'tailscale', 'private IngressClass') + if ispec.get('rules') or ispec.get('defaultBackend') != { + 'service': {'name': 't3-worker', 'port': {'name': 'http'}} + }: + raise RouteError('Ingress backend or rules differ from private plan') + _same(ispec.get('tls'), [{'hosts': ['t3-p1-' + lease[:24]]}], 'private Ingress hostname') + + pspec = policy.get('spec') + if not isinstance(pspec, dict): + raise RouteError('invalid NetworkPolicy spec') + _same(pspec.get('podSelector'), {'matchLabels': {'app': 't3-worker'}}, 'policy selector') + _same(pspec.get('policyTypes'), ['Ingress'], 'policy types') + _same(pspec.get('ingress'), [{'from': [{'namespaceSelector': {'matchLabels': { + 'kubernetes.io/metadata.name': 'tailscale'}}}], + 'ports': [{'protocol': 'TCP', 'port': 3773}]}], 'private ingress allowlist') + if pspec.get('egress'): + raise RouteError('unexpected policy egress needs separate review') + + # Kubernetes ingress policy is additive. A single safe named policy says + # nothing about a second namespace-wide or wildcard allow policy. + if (not isinstance(all_policies, dict) + or all_policies.get('kind') != 'NetworkPolicyList' + or not isinstance(all_policies.get('items'), list) + or len(all_policies['items']) != 1): + raise RouteError('namespace-wide NetworkPolicy inventory is missing or ambiguous') + listed = all_policies['items'][0] + listed_meta = _meta(listed, 'NetworkPolicy', 'worker-ingress', ns) + uid = policy['metadata'].get('uid') + if (not isinstance(uid, str) or not re.fullmatch(r'[a-zA-Z0-9-]{8,128}', uid) + or listed_meta.get('uid') != uid): + raise RouteError('NetworkPolicy identity differs across snapshots') + _same(listed.get('spec'), pspec, 'inventory NetworkPolicy spec') + + # A second Ingress or LoadBalancer/NodePort Service in this namespace can + # advertise another route to the same worker. Only inspecting the named + # private Ingress and ClusterIP Service is not a namespace-wide inventory. + # This remains a desired-spec audit, NOT proof that traffic is denied. + for kind, expected, inventory in ( + ('Service', service, all_services), + ('Ingress', ingress, all_ingresses), + ): + if (not isinstance(inventory, dict) + or inventory.get('kind') != kind + 'List' + or not isinstance(inventory.get('items'), list) + or len(inventory['items']) != 1): + raise RouteError(f'{kind} namespace inventory is missing or ambiguous') + listed = inventory['items'][0] + listed_meta = _meta(listed, kind, 't3-worker', ns) + _safe_annotations(listed, kind) + expected_uid = expected['metadata'].get('uid') + if (not isinstance(expected_uid, str) + or not re.fullmatch(r'[a-zA-Z0-9-]{8,128}', expected_uid) + or listed_meta.get('uid') != expected_uid): + raise RouteError(f'{kind} identity differs across snapshots') + _same(listed.get('spec'), expected.get('spec'), f'inventory {kind} spec') + _route_status(listed, kind, ns) + _same(listed.get('status', {}), expected.get('status', {}), + f'inventory {kind} advertised route status') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + for flag in ('plan', 'service', 'ingress', 'network-policy', 'all-network-policies', + 'all-ingresses', 'all-services'): + parser.add_argument('--' + flag, type=Path, required=True) + args = parser.parse_args() + try: + values = [json.loads(getattr(args, x).read_text(encoding='utf-8')) + for x in ('plan', 'service', 'ingress', 'network_policy', 'all_network_policies', + 'all_ingresses', 'all_services')] + verify(*values) + except (OSError, ValueError, TypeError, AttributeError, KeyError) as exc: + # Do not repeat private kubeconfig paths, DNS names, or JSON in errors. + parser.exit(1, f'FAIL: P1 private route specification not verified ({type(exc).__name__})\n') + print('PASS: P1 desired private route and namespace route/policy inventories match (offline only)') + print('NOT RUN: DNS, private TLS, Tailscale ACL, NetworkPolicy enforcement, OAuth or live delegation') + + +if __name__ == '__main__': + main() diff --git a/examples/k3s-p1/test_capture.py b/examples/k3s-p1/test_capture.py new file mode 100644 index 0000000000000..defb690866476 --- /dev/null +++ b/examples/k3s-p1/test_capture.py @@ -0,0 +1,245 @@ +"""Local-only P1 capture tests: no kubectl or cluster is contacted.""" +import importlib.util +import json +import os +import stat +from pathlib import Path +import subprocess +import tempfile +import unittest +from unittest import mock + +spec = importlib.util.spec_from_file_location('capture', Path(__file__).with_name('capture.py')) +module = importlib.util.module_from_spec(spec) +spec.loader.exec_module(module) + +HASH = 'a' * 64 +NAMESPACE = 't3-p1-' + HASH[:24] +PLAN = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': HASH, 'namespace': NAMESPACE} + + +class Stub: + def __init__(self, drift=None, broken=None, tamper=None): + self.calls = [] + self.drift = drift + self.broken = broken + self.tamper = tamper + + def __call__(self, command, **kwargs): + self.calls.append(command) + assert command[:4] == ['kubectl', '--context', 'disposable-k3s', '--request-timeout=10s'] + assert command[-2:] == ['-o', 'json'] + assert kwargs['timeout'] == 20 and kwargs['check'] is False + resource = command[command.index('get') + 1] + remainder = command[command.index('get') + 2:-2] + name = remainder[0] if remainder else None + if self.broken == resource: + return subprocess.CompletedProcess(command, 1, '', 'private kubeconfig detail') + kinds = {'namespace': 'Namespace', 'resourcequota': 'ResourceQuota', + 'serviceaccount': 'ServiceAccount', 'pvc': 'PersistentVolumeClaim', + 'deployment': 'Deployment', 'service': 'Service', + 'ingress': 'Ingress', 'networkpolicy': 'NetworkPolicy'} + listed_names = {'networkpolicies': ('networkpolicy', 'worker-ingress'), + 'ingresses': ('ingress', 't3-worker'), + 'services': ('service', 't3-worker')} + if resource in ('networkpolicies', 'ingresses', 'services', 'replicasets', 'pods'): + items = [] + if resource in listed_names: + source, selected_name = listed_names[resource] + items = [{'kind': kinds[source], + 'metadata': {'name': selected_name, 'namespace': NAMESPACE, + 'uid': 'a1234567', 'resourceVersion': '2'}, + 'spec': {'replicas': 1}}] + obj = {'kind': dict(networkpolicies='NetworkPolicyList', ingresses='IngressList', + services='ServiceList', replicasets='ReplicaSetList', + pods='PodList')[resource], 'items': items} + else: + meta = {'name': name, 'uid': 'a1234567', 'resourceVersion': '2'} + if resource != 'namespace': + meta['namespace'] = NAMESPACE + obj = {'kind': kinds[resource], 'metadata': meta, 'spec': {'replicas': 1}} + if resource == 'resourcequota': + obj['spec'] = {'hard': module.EXPECTED_QUOTA_HARD.copy()} + obj['status'] = {'hard': module.EXPECTED_QUOTA_HARD.copy()} + if resource == 'serviceaccount': + obj.pop('spec') + obj['automountServiceAccountToken'] = False + if self.tamper is not None and self.tamper[0] == resource: + flaw = self.tamper[1] + if flaw == 'kind': + obj['kind'] = 'Other' + elif flaw == 'name': + obj['metadata']['name'] = 'unrelated' + elif flaw == 'namespace': + obj['metadata'].pop('namespace', None) + elif flaw == 'uid': + obj['metadata']['uid'] = None + elif flaw == 'inventory_missing': + obj['items'] = [] + elif flaw == 'inventory_drift': + obj['items'][0]['spec']['replicas'] = 2 + elif flaw == 'quota_scope': + obj['spec']['scopeSelector'] = {'matchExpressions': []} + elif flaw == 'quota_hard': + obj['spec']['hard']['pods'] = '3' + elif flaw == 'quota_unreconciled': + obj.pop('status', None) + elif flaw == 'sa_automount': + obj['automountServiceAccountToken'] = True + elif flaw == 'sa_secret': + obj['secrets'] = [{'name': 'unapproved-token'}] + elif flaw == 'sa_pull_secret': + obj['imagePullSecrets'] = [{'name': 'unapproved-pull'}] + elif flaw == 'deleting': + obj['metadata']['deletionTimestamp'] = '2026-10-29T00:00:00Z' + if self.drift == resource and len([c for c in self.calls if resource in c]) > 1: + obj['metadata'] = {'resourceVersion': '3'} + return subprocess.CompletedProcess(command, 0, json.dumps(obj), '') + + +class CaptureTests(unittest.TestCase): + def test_read_only_explicit_context_and_stable_snapshot(self): + stub = Stub() + records = module.capture('disposable-k3s', PLAN, runner=stub) + self.assertEqual(len(records), 13) + self.assertEqual(len(stub.calls), 26) + self.assertEqual(records['quota']['kind'], 'ResourceQuota') + self.assertEqual(records['serviceaccount']['kind'], 'ServiceAccount') + self.assertTrue(all('get' in c and 'apply' not in c and 'delete' not in c + and 'patch' not in c for c in stub.calls)) + self.assertTrue(all('--context' in c for c in stub.calls)) + + def test_reject_stale_or_unsafe_plan(self): + for ctx in ('', '-bad', 'disposable\ncluster', ' spaced '): + with self.subTest(ctx=ctx), self.assertRaises(module.CaptureError): + module.capture(ctx, PLAN, runner=Stub()) + for altered in ({}, {**PLAN, 'namespace': 'existing-home'}, + {**PLAN, 'leaseHash': 'bad'}): + with self.subTest(plan=altered), self.assertRaises(module.CaptureError): + module.capture('disposable-k3s', altered, runner=Stub()) + + def test_detect_change_in_named_resource_or_inventory(self): + for res in ('resourcequota', 'serviceaccount', 'deployment', 'ingresses', 'pods'): + with self.subTest(resource=res), self.assertRaises(module.CaptureError): + module.capture('disposable-k3s', PLAN, runner=Stub(drift=res)) + + def test_named_identity_and_mixed_inventory_fail_closed(self): + cases = ( + ('deployment', 'kind'), ('deployment', 'name'), + ('deployment', 'namespace'), ('deployment', 'uid'), + ('resourcequota', 'kind'), ('resourcequota', 'uid'), + ('serviceaccount', 'name'), ('serviceaccount', 'namespace'), + ('namespace', 'name'), ('services', 'inventory_missing'), + ('services', 'inventory_drift'), ('ingresses', 'inventory_missing'), + ('networkpolicies', 'inventory_drift'), + ) + for case in cases: + with self.subTest(mutation=case), self.assertRaises(module.CaptureError): + module.capture('disposable-k3s', PLAN, runner=Stub(tamper=case)) + + def test_quota_scope_and_service_account_credentials_fail_closed(self): + cases = ( + ('resourcequota', 'quota_scope'), + ('resourcequota', 'quota_hard'), + ('resourcequota', 'quota_unreconciled'), + ('resourcequota', 'deleting'), + ('serviceaccount', 'sa_automount'), + ('serviceaccount', 'sa_secret'), + ('serviceaccount', 'sa_pull_secret'), + ('serviceaccount', 'deleting'), + ) + for case in cases: + with self.subTest(mutation=case), self.assertRaises(module.CaptureError): + module.capture('disposable-k3s', PLAN, runner=Stub(tamper=case)) + + def test_missing_quota_or_service_account_fails_closed(self): + for res in ('resourcequota', 'serviceaccount'): + with self.subTest(resource=res), self.assertRaisesRegex( + module.CaptureError, 'request rejected'): + module.capture('disposable-k3s', PLAN, runner=Stub(broken=res)) + + def test_redacted_rejected_api_read(self): + with self.assertRaisesRegex(module.CaptureError, 'request rejected') as ctx: + module.capture('disposable-k3s', PLAN, runner=Stub(broken='pods')) + self.assertNotIn('private', str(ctx.exception)) + + def test_concurrent_empty_destination_cannot_replace_existing_evidence(self): + # Reproduce a writer creating the final directory after publish()'s + # initial absence check. Plain os.rename() would replace that empty + # directory; renameat2(RENAME_NOREPLACE) must preserve its inode. + with tempfile.TemporaryDirectory() as temp: + parent = Path(temp) / 'owner' + parent.mkdir(mode=0o700) + target = parent / 'snapshot' + make_temp = module.tempfile.mkdtemp + competing_inode = [] + + def race(*args, **kwargs): + staging = make_temp(*args, **kwargs) + target.mkdir() + competing_inode.append(target.stat().st_ino) + return staging + + with mock.patch.object(module.tempfile, 'mkdtemp', side_effect=race): + with self.assertRaises(module.CaptureError): + module.publish({'pods': {'kind': 'PodList', 'items': []}}, target) + self.assertEqual(target.stat().st_ino, competing_inode[0]) + self.assertEqual(list(parent.iterdir()), [target]) + + def test_staging_directory_fsync_precedes_parent_publication(self): + # A file's fsync does not make its *directory entry* crash durable. + # The staging directory must be synced before renameat2 exposes it. + with tempfile.TemporaryDirectory() as temp: + parent = Path(temp) / 'owner' + parent.mkdir(mode=0o700) + target = parent / 'snapshot' + events = [] + original_fsync = os.fsync + + def record_fsync(fd): + events.append('directory' if stat.S_ISDIR(os.fstat(fd).st_mode) + else 'file') + return original_fsync(fd) + + with mock.patch.object(module.os, 'fsync', side_effect=record_fsync): + module.publish({'pods': {'kind': 'PodList', 'items': []}}, target) + self.assertEqual(events, ['file', 'directory', 'directory']) + self.assertEqual(json.loads((target / 'pods.json').read_text())['kind'], + 'PodList') + + def test_staging_directory_sync_failure_cannot_publish(self): + with tempfile.TemporaryDirectory() as temp: + parent = Path(temp) / 'owner' + parent.mkdir(mode=0o700) + target = parent / 'snapshot' + original_fsync = os.fsync + + def reject_directory_fsync(fd): + if stat.S_ISDIR(os.fstat(fd).st_mode): + raise OSError('injected staging-directory fsync failure') + return original_fsync(fd) + + with mock.patch.object(module.os, 'fsync', + side_effect=reject_directory_fsync): + with self.assertRaises(OSError): + module.publish({'pods': {'kind': 'PodList', 'items': []}}, target) + self.assertFalse(target.exists()) + self.assertEqual(list(parent.iterdir()), []) + + def test_publish_private_atomic_and_no_overwrite(self): + with tempfile.TemporaryDirectory() as temp: + parent = Path(temp) / 'owner' + parent.mkdir(mode=0o700) + target = parent / 'snapshot' + module.publish({'pods': {'kind': 'PodList', 'items': []}}, target) + self.assertEqual(json.loads((target / 'pods.json').read_text())['kind'], 'PodList') + self.assertEqual((target / 'pods.json').stat().st_mode & 0o777, 0o600) + with self.assertRaises(module.CaptureError): + module.publish({'pods': {}}, target) + os.chmod(parent, 0o755) + with self.assertRaises(module.CaptureError): + module.publish({'pods': {}}, parent / 'another') + + +if __name__ == '__main__': + unittest.main() diff --git a/examples/k3s-p1/test_observe.py b/examples/k3s-p1/test_observe.py new file mode 100644 index 0000000000000..fc69ae3b1297d --- /dev/null +++ b/examples/k3s-p1/test_observe.py @@ -0,0 +1,375 @@ +"""Offline P1 lease/storage binding checks; no Kubernetes/network/secret access.""" +import importlib.util +from pathlib import Path +import copy +import json +import subprocess +import sys +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('p1_observe', HERE / 'observe.py') +assert spec and spec.loader +p = importlib.util.module_from_spec(spec) +spec.loader.exec_module(p) + +HASH = 'a' * 64 +NAMESPACE = 't3-p1-' + HASH[:24] +EXPIRY = '2026-11-01T00:00:00Z' +PLAN = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': HASH, 'namespace': NAMESPACE, + 'image': 'ghcr.io/owned/t3-worker@sha256:' + 'b' * 64, + 'expiresAt': EXPIRY, 'retention': 'retain'} +NS = {'kind': 'Namespace', 'metadata': {'name': NAMESPACE, + 'uid': '11111111-aaaa-bbbb-cccc-111111111111', + 'labels': {'pod-security.kubernetes.io/enforce': 'restricted'}, + 'annotations': {'t3code.dev/lease-key-sha256': HASH, + 't3code.dev/lease-expires-at': EXPIRY, + 't3code.dev/expiry-enforced': 'false', + 't3code.dev/retention': 'retain-until-owner-decision'}}, + 'status': {'phase': 'Active'}} +PVC = {'kind': 'PersistentVolumeClaim', 'metadata': { + 'name': 't3-worker-data', 'namespace': NAMESPACE, + 'uid': '22222222-aaaa-bbbb-cccc-222222222222'}, + 'spec': {'accessModes': ['ReadWriteOnce'], + 'resources': {'requests': {'storage': '10Gi'}}, + 'volumeName': 'pvc-22222222-aaaa-bbbb-cccc-222222222222'}, + 'status': {'phase': 'Bound'}} + +DEPLOYMENT = {'kind': 'Deployment', 'metadata': { + 'name': 't3-worker', 'namespace': NAMESPACE, + 'uid': '44444444-aaaa-bbbb-cccc-444444444444', 'generation': 2}, + 'spec': {'replicas': 1, 'strategy': {'type': 'Recreate'}, + 'selector': {'matchLabels': {'app': 't3-worker'}}, + 'template': {'metadata': {'labels': {'app': 't3-worker'}}, + 'spec': {'serviceAccountName': 't3-worker', + 'automountServiceAccountToken': False, + 'containers': [{'name': 't3', 'image': PLAN['image'], + 'volumeMounts': [ + {'name': 'data', 'mountPath': '/data'}, + {'name': 'tmp', 'mountPath': '/tmp'}]}], + 'volumes': [ + {'name': 'data', 'persistentVolumeClaim': { + 'claimName': 't3-worker-data'}}, + {'name': 'tmp', 'emptyDir': {'sizeLimit': '1Gi'}}]}}}} + + +_POD = DEPLOYMENT['spec']['template']['spec'] +_POD.update({ + 'hostname': 't3-worker', 'terminationGracePeriodSeconds': 90, + 'securityContext': { + 'runAsNonRoot': True, 'runAsUser': 10001, 'runAsGroup': 10001, + 'fsGroup': 10001, 'seccompProfile': {'type': 'RuntimeDefault'}, + }, +}) +_CONTAINER = _POD['containers'][0] +_CONTAINER.update({ + 'workingDir': '/data', 'imagePullPolicy': 'IfNotPresent', + 'ports': [{'name': 'http', 'containerPort': 3773}], + 'resources': { + 'requests': {'cpu': '500m', 'memory': '1Gi', 'ephemeral-storage': '512Mi'}, + 'limits': {'cpu': '2', 'memory': '4Gi', 'ephemeral-storage': '2Gi'}, + }, + 'securityContext': {'allowPrivilegeEscalation': False, 'readOnlyRootFilesystem': True, + 'capabilities': {'drop': ['ALL']}}, + 'startupProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5, + 'failureThreshold': 60}, + 'readinessProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5}, +}) + + + +class ObserveTests(unittest.TestCase): + def test_stable_namespace_and_pvc_uid(self): + initial = p.inspect(PLAN, NS, PVC, DEPLOYMENT) + self.assertEqual(initial['pvcUID'], PVC['metadata']['uid']) + p.compare(p.inspect(copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC), copy.deepcopy(DEPLOYMENT)), initial) + self.assertNotIn('image', initial) + self.assertNotIn('credentials', json.dumps(initial)) + + def test_generation_blocks_stale_same_uid_binding(self): + original = p.inspect(PLAN, NS, PVC, DEPLOYMENT) + self.assertEqual(original['deploymentGeneration'], 2) + changed = copy.deepcopy(DEPLOYMENT) + changed['metadata']['generation'] = 3 # Kubernetes spec mutation preserves UID. + with self.assertRaisesRegex(p.ObservationError, 'binding changed'): + p.compare(p.inspect(PLAN, NS, PVC, changed), original) + for invalid in (None, 0, -1, True, '2'): + bad = copy.deepcopy(DEPLOYMENT) + bad['metadata']['generation'] = invalid + with self.subTest(invalid=invalid), self.assertRaisesRegex( + p.ObservationError, 'generation'): + p.inspect(PLAN, NS, PVC, bad) + + def test_wrong_lease_identity_or_namespace_is_rejected(self): + for item, change in [('ns', {'metadata': {'name': 'other'}}), + ('plan', {'namespace': 'different'}), + ('plan', {'leaseHash': 'x' * 64})]: + with self.subTest(item=item), self.assertRaises(p.ObservationError): + parts = [copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC), copy.deepcopy(DEPLOYMENT)] + target = {'plan': 0, 'ns': 1}[item] + parts[target].update(change) + p.inspect(*parts) + + def test_missing_uid_and_binding_replacement_fail(self): + for obj in ('ns', 'pvc'): + parts = [copy.deepcopy(PLAN), copy.deepcopy(NS), copy.deepcopy(PVC), copy.deepcopy(DEPLOYMENT)] + parts[1 if obj == 'ns' else 2]['metadata']['uid'] = '' + with self.subTest(obj=obj), self.assertRaises(p.ObservationError): + p.inspect(*parts) + baseline = p.inspect(PLAN, NS, PVC, DEPLOYMENT) + replaced = copy.deepcopy(PVC) + replaced['metadata']['uid'] = '33333333-aaaa-bbbb-cccc-333333333333' + with self.assertRaisesRegex(p.ObservationError, 'binding changed'): + p.compare(p.inspect(PLAN, NS, replaced, DEPLOYMENT), baseline) + + def test_unbound_deleting_or_unsafe_claim_is_rejected(self): + for path, value in [(('status', 'phase'), 'Pending'), + (('metadata', 'deletionTimestamp'), '2026-10-28T01:00:00Z'), + (('spec', 'accessModes'), ['ReadWriteMany']), + (('spec', 'volumeName'), '')]: + with self.subTest(path=path): + bad = copy.deepcopy(PVC) + bad[path[0]][path[1]] = value + with self.assertRaises(p.ObservationError): + p.inspect(PLAN, NS, bad, DEPLOYMENT) + + def test_namespace_annotation_or_policy_drift_fails(self): + for key, value in [('t3code.dev/lease-key-sha256', 'b' * 64), + ('t3code.dev/expiry-enforced', 'true'), + ('t3code.dev/retention', 'delete')]: + with self.subTest(key=key): + bad = copy.deepcopy(NS) + bad['metadata']['annotations'][key] = value + with self.assertRaises(p.ObservationError): + p.inspect(PLAN, bad, PVC, DEPLOYMENT) + deleted = copy.deepcopy(NS) + deleted['metadata']['deletionTimestamp'] = '2026-10-28T01:00:00Z' + with self.assertRaises(p.ObservationError): + p.inspect(PLAN, deleted, PVC, DEPLOYMENT) + + def test_changed_pinned_image_invalidates_existing_binding(self): + first = p.inspect(PLAN, NS, PVC, DEPLOYMENT) + self.assertEqual(first['format'], 't3-k3s-p1-binding-v4') + self.assertEqual(len(first['imageRefSha256']), 64) + self.assertNotIn(PLAN['image'], json.dumps(first)) + changed = copy.deepcopy(PLAN) + changed['image'] = 'ghcr.io/owned/other@sha256:' + 'c' * 64 + redeployed = copy.deepcopy(DEPLOYMENT) + redeployed['spec']['template']['spec']['containers'][0]['image'] = changed['image'] + later = p.inspect(changed, NS, PVC, redeployed) + self.assertNotEqual(first['imageRefSha256'], later['imageRefSha256']) + with self.assertRaisesRegex(p.ObservationError, 'binding changed'): + p.compare(later, first) + + def test_invalid_image_and_legacy_binding_are_rejected(self): + for image in (None, 't3:latest', 'ghcr.io/owned/t3-worker:latest', + 'ghcr.io/owned/t3-worker@sha256:' + 'Z' * 64, + 'ghcr.io/owned/../other@sha256:' + 'a' * 64, + 'ghcr.io//owned/t3@sha256:' + 'a' * 64): + with self.subTest(image=image), self.assertRaisesRegex( + p.ObservationError, 'immutable image'): + bad = copy.deepcopy(PLAN) + bad['image'] = image + p.inspect(bad, NS, PVC, DEPLOYMENT) + current = p.inspect(PLAN, NS, PVC, DEPLOYMENT) + legacy = dict(current, format='t3-k3s-p1-binding-v1') + legacy.pop('imageRefSha256') + with self.assertRaisesRegex(p.ObservationError, 'binding changed'): + p.compare(current, legacy) + + + def test_observed_deployment_image_must_match_plan(self): + first = p.inspect(PLAN, NS, PVC, DEPLOYMENT) + self.assertEqual(first['deploymentUID'], DEPLOYMENT['metadata']['uid']) + for image in ('ghcr.io/other/t3@sha256:' + 'c' * 64, 't3:latest'): + bad = copy.deepcopy(DEPLOYMENT) + bad['spec']['template']['spec']['containers'][0]['image'] = image + with self.subTest(image=image), self.assertRaisesRegex( + p.ObservationError, 'observed deployment image'): + p.inspect(PLAN, NS, PVC, bad) + + def test_deployment_identity_and_pvc_mount_drift_fail_closed(self): + mutations = [ + ('namespace', ('metadata', 'namespace'), 'another'), + ('uid', ('metadata', 'uid'), ''), + ('replicas', ('spec', 'replicas'), 2), + ('strategy', ('spec', 'strategy'), {'type': 'RollingUpdate'}), + ('selector', ('spec', 'selector'), {'matchLabels': {'app': 'other'}}), + ('labels', ('spec', 'template', 'metadata', 'labels'), {'app': 'other'}), + ('sa', ('spec', 'template', 'spec', 'serviceAccountName'), 'default'), + ('token', ('spec', 'template', 'spec', 'automountServiceAccountToken'), True), + ('volume', ('spec', 'template', 'spec', 'volumes'), [ + {'name': 'data', 'hostPath': {'path': '/host'}}, + {'name': 'tmp', 'emptyDir': {'sizeLimit': '1Gi'}}]), + ('mount', ('spec', 'template', 'spec', 'containers'), [ + {'name': 't3', 'image': PLAN['image'], 'volumeMounts': [ + {'name': 'data', 'mountPath': '/home'}, {'name': 'tmp', 'mountPath': '/tmp'}]}]), + ] + for label, path, value in mutations: + bad = copy.deepcopy(DEPLOYMENT) + target = bad + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + with self.subTest(label=label), self.assertRaises(p.ObservationError): + p.inspect(PLAN, NS, PVC, bad) + replaced = copy.deepcopy(DEPLOYMENT) + replaced['metadata']['uid'] = '55555555-aaaa-bbbb-cccc-555555555555' + with self.assertRaisesRegex(p.ObservationError, 'binding changed'): + p.compare(p.inspect(PLAN, NS, PVC, replaced), + p.inspect(PLAN, NS, PVC, DEPLOYMENT)) + + def test_template_metadata_cannot_trigger_admission_on_restart(self): + for change in ('annotation', 'extra-field', 'timestamp'): + bad = copy.deepcopy(DEPLOYMENT) + meta = bad['spec']['template']['metadata'] + if change == 'annotation': + meta['annotations'] = {'sidecar.istio.io/inject': 'true'} + elif change == 'extra-field': + meta['finalizers'] = ['unexpected.example/finalizer'] + else: + meta['creationTimestamp'] = '2026-10-29T00:00:00Z' + with self.subTest(change=change), self.assertRaisesRegex( + p.ObservationError, 'template metadata'): + p.inspect(PLAN, NS, PVC, bad) + defaulted = copy.deepcopy(DEPLOYMENT) + defaulted['spec']['template']['metadata'].update( + {'annotations': {}, 'creationTimestamp': None}) + p.inspect(PLAN, NS, PVC, defaulted) + + def test_expected_kubernetes_api_defaults_do_not_false_reject(self): + defaulted = copy.deepcopy(DEPLOYMENT) + container = defaulted['spec']['template']['spec']['containers'][0] + container['ports'][0]['protocol'] = 'TCP' + for probe_name, failure_threshold in (('startupProbe', 60), ('readinessProbe', 3)): + container[probe_name].update({ + 'failureThreshold': failure_threshold, 'timeoutSeconds': 1, + 'successThreshold': 1, 'initialDelaySeconds': 0, + }) + p.compare(p.inspect(PLAN, NS, PVC, defaulted), + p.inspect(PLAN, NS, PVC, DEPLOYMENT)) + container['ports'][0]['hostPort'] = 3773 + with self.assertRaises(p.ObservationError): + p.inspect(PLAN, NS, PVC, defaulted) + + def test_privilege_network_and_credential_injection_fail_closed(self): + changes = [ + ('host-network', ('hostNetwork',), True), + ('host-pid', ('hostPID',), True), + ('host-ipc', ('hostIPC',), True), + ('sidecar', ('initContainers',), [{'name': 'other', 'image': PLAN['image']}]), + ('registry-secret', ('imagePullSecrets',), [{'name': 'secret'}]), + ('hostname', ('hostname',), 'other'), + ('pod-security', ('securityContext', 'runAsNonRoot'), False), + ('pod-seccomp', ('securityContext', 'seccompProfile'), {'type': 'Unconfined'}), + ('env-secret', ('containers', 0, 'envFrom'), [{'secretRef': {'name': 'secret'}}]), + ('shell-override', ('containers', 0, 'command'), ['/bin/sh']), + ('privileged', ('containers', 0, 'securityContext', 'privileged'), True), + ('container-root', ('containers', 0, 'securityContext', 'runAsUser'), 0), + ('container-seccomp', ('containers', 0, 'securityContext', 'seccompProfile'), + {'type': 'Unconfined'}), + ('container-procmount', ('containers', 0, 'securityContext', 'procMount'), + 'Unmasked'), + ('escalation', ('containers', 0, 'securityContext', 'allowPrivilegeEscalation'), True), + ('filesystem', ('containers', 0, 'securityContext', 'readOnlyRootFilesystem'), False), + ('caps', ('containers', 0, 'securityContext', 'capabilities'), {'drop': []}), + ('cpu', ('containers', 0, 'resources', 'limits', 'cpu'), '8'), + ('readiness', ('containers', 0, 'readinessProbe', 'tcpSocket', 'port'), 9999), + ] + for label, path, value in changes: + bad = copy.deepcopy(DEPLOYMENT) + target = bad['spec']['template']['spec'] + for segment in path[:-1]: + target = target[segment] + target[path[-1]] = value + with self.subTest(label=label), self.assertRaises(p.ObservationError): + p.inspect(PLAN, NS, PVC, bad) + + def test_missing_required_security_fields_fail_closed(self): + for path in [ + ('securityContext',), + ('containers', 0, 'securityContext'), + ('containers', 0, 'resources'), + ('containers', 0, 'startupProbe'), + ]: + bad = copy.deepcopy(DEPLOYMENT) + target = bad['spec']['template']['spec'] + for segment in path[:-1]: + target = target[segment] + del target[path[-1]] + with self.subTest(path=path), self.assertRaises(p.ObservationError): + p.inspect(PLAN, NS, PVC, bad) + + def test_cli_rejects_invalid_binding_without_emitting_ids(self): + with tempfile.TemporaryDirectory() as directory: + paths = [Path(directory) / name for name in ('plan.json', 'ns.json', 'pvc.json', 'deployment.json')] + for path, value in zip(paths, (PLAN, NS, PVC, DEPLOYMENT)): + path.write_text(json.dumps(value)) + argv = [sys.executable, str(HERE / 'observe.py'), '--plan', str(paths[0]), + '--namespace', str(paths[1]), '--pvc', str(paths[2]), + '--deployment', str(paths[3])] + first = subprocess.run(argv, capture_output=True, text=True, check=False) + self.assertEqual(first.returncode, 0, first.stderr) + baseline = Path(directory) / 'binding.json' + baseline.write_text(first.stdout) + okay = subprocess.run(argv + ['--baseline', str(baseline)], capture_output=True, + text=True, check=False) + self.assertEqual(okay.returncode, 0) + altered = copy.deepcopy(PVC) + altered['metadata']['uid'] = '33333333-aaaa-bbbb-cccc-333333333333' + paths[2].write_text(json.dumps(altered)) + rejected = subprocess.run(argv + ['--baseline', str(baseline)], capture_output=True, + text=True, check=False) + self.assertNotEqual(rejected.returncode, 0) + self.assertEqual(rejected.stdout, '') + self.assertNotIn(altered['metadata']['uid'], rejected.stderr) + + + def test_invalid_calendar_dates_and_planning_bounds_fail_closed(self): + for expiry in ('2026-02-30T00:00:00Z', '2026-13-01T00:00:00Z', + '2019-12-31T23:59:59Z', '2101-01-01T00:00:00Z'): + bad = copy.deepcopy(PLAN) + bad['expiresAt'] = expiry + with self.subTest(expiry=expiry), self.assertRaises(p.ObservationError): + p.inspect(bad, NS, PVC, DEPLOYMENT) + + def test_malformed_nested_api_json_fails_without_raw_traceback(self): + cases = [ + ('namespace-metadata', 1, ('metadata',), None), + ('namespace-status', 1, ('status',), []), + ('pvc-spec', 2, ('spec',), None), + ('pvc-requests', 2, ('spec', 'resources', 'requests'), []), + ('deployment-spec', 3, ('spec',), None), + ('deployment-selector', 3, ('spec', 'selector'), None), + ('deployment-pod', 3, ('spec', 'template', 'spec'), []), + ] + for label, index, path, value in cases: + parts = [copy.deepcopy(x) for x in (PLAN, NS, PVC, DEPLOYMENT)] + target = parts[index] + for key in path[:-1]: + target = target[key] + target[path[-1]] = value + with self.subTest(label=label), self.assertRaisesRegex( + p.ObservationError, 'malformed JSON object'): + p.inspect(*parts) + + with tempfile.TemporaryDirectory() as directory: + documents = [copy.deepcopy(x) for x in (PLAN, NS, PVC, DEPLOYMENT)] + documents[1]['metadata'] = None + paths = [Path(directory) / name for name in + ('plan.json', 'namespace.json', 'pvc.json', 'deployment.json')] + for path, doc in zip(paths, documents): + path.write_text(json.dumps(doc), encoding='utf-8') + args = [sys.executable, str(HERE / 'observe.py')] + for key, path in zip(('plan', 'namespace', 'pvc', 'deployment'), paths): + args.extend((f'--{key}', str(path))) + result = subprocess.run(args, capture_output=True, text=True, check=False) + self.assertNotEqual(result.returncode, 0) + self.assertEqual(result.stdout, '') + self.assertNotIn('Traceback', result.stderr) + self.assertNotIn(str(paths[1]), result.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/examples/k3s-p1/test_plan.py b/examples/k3s-p1/test_plan.py new file mode 100644 index 0000000000000..0254e06adaa01 --- /dev/null +++ b/examples/k3s-p1/test_plan.py @@ -0,0 +1,154 @@ +"""P1 manifest/lease contract: deterministic, no network, kubectl or scheduler.""" +import importlib.util +from pathlib import Path +import json +import os +import tempfile +import unittest +from unittest import mock + +import yaml + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('p1_plan', HERE / 'plan.py') +assert spec and spec.loader +p = importlib.util.module_from_spec(spec) +spec.loader.exec_module(p) + +IMAGE = 'ghcr.io/owned/t3-worker@sha256:' + 'a' * 64 +EXPIRY = '2026-11-01T00:00:00Z' + + +class P1PlanTests(unittest.TestCase): + def test_deterministic_names_and_same_input(self): + a, objs = p.build('work-item:abc-23', IMAGE, EXPIRY) + b, again = p.build('work-item:abc-23', IMAGE, EXPIRY) + self.assertEqual(a, b) + self.assertEqual(yaml.safe_dump_all(objs), yaml.safe_dump_all(again)) + c, _ = p.build('work-item:abc-24', IMAGE, EXPIRY) + self.assertNotEqual(a['namespace'], c['namespace']) + self.assertNotIn('work-item:abc', yaml.safe_dump_all(objs)) + self.assertNotIn('work-item:abc', json.dumps(a)) + + def test_only_bounded_private_k8s_resources(self): + plan, docs = p.build('work:1', IMAGE, EXPIRY) + self.assertEqual([d['kind'] for d in docs], ['Namespace', 'ResourceQuota', + 'ServiceAccount', 'PersistentVolumeClaim', 'Deployment', + 'Service', 'Ingress', 'NetworkPolicy']) + self.assertTrue(all(d['metadata']['namespace'] == plan['namespace'] + for d in docs[1:])) + self.assertEqual(docs[0]['metadata']['labels']['pod-security.kubernetes.io/enforce'], 'restricted') + self.assertEqual(docs[0]['metadata']['annotations']['t3code.dev/expiry-enforced'], 'false') + self.assertEqual(docs[1]['spec']['hard']['pods'], '1') + self.assertFalse(docs[2]['automountServiceAccountToken']) + pod = docs[4]['spec']['template']['spec'] + self.assertFalse(pod['automountServiceAccountToken']) + self.assertTrue(pod['securityContext']['runAsNonRoot']) + self.assertEqual(docs[4]['spec']['strategy']['type'], 'Recreate') + self.assertEqual(pod['containers'][0]['image'], IMAGE) + self.assertEqual(pod['containers'][0]['securityContext']['capabilities']['drop'], ['ALL']) + self.assertTrue(pod['containers'][0]['securityContext']['readOnlyRootFilesystem']) + self.assertTrue(any(v.get('persistentVolumeClaim', {}).get('claimName') == 't3-worker-data' + for v in pod['volumes'])) + self.assertFalse(any('hostPath' in v for v in pod['volumes'])) + self.assertEqual(docs[5]['spec']['type'], 'ClusterIP') + self.assertEqual(docs[6]['spec']['ingressClassName'], 'tailscale') + self.assertNotIn('rules', docs[6]['spec']) + self.assertNotIn('tailscale.com/funnel', docs[6]['metadata'].get('annotations', {})) + self.assertEqual(docs[7]['spec']['policyTypes'], ['Ingress']) + self.assertEqual(docs[7]['spec']['ingress'][0]['ports'][0]['port'], 3773) + self.assertEqual(docs[7]['spec']['ingress'][0]['from'][0]['namespaceSelector'] + ['matchLabels']['kubernetes.io/metadata.name'], 'tailscale') + self.assertEqual([d['kind'] for d in yaml.safe_load_all(yaml.safe_dump_all(docs))], + [d['kind'] for d in docs]) + + def test_bad_keys_images_and_expiries_fail_closed(self): + for key in ('', '/bad', '-bad', '../steal', 'a\nkind:Pod', 'x' * 129): + with self.subTest(key=key), self.assertRaises(p.PlanError): + p.build(key, IMAGE, EXPIRY) + for image in ('t3:latest', 't3@sha256:' + 'a' * 64, + IMAGE.upper(), IMAGE + '\n---\nkind: Pod', + 'ghcr.io/x/../y@sha256:' + 'a' * 64): + with self.subTest(image=image), self.assertRaises(p.PlanError): + p.build('work:1', image, EXPIRY) + for date in ('', 'tomorrow', '2026-02-30T00:00:00Z', + '2026-11-01T00:00:00+01:00', '2000-01-01T00:00:00Z'): + with self.subTest(date=date), self.assertRaises(p.PlanError): + p.build('work:1', IMAGE, date) + + def test_local_receipt_is_0600_and_conflicts_fail(self): + with tempfile.TemporaryDirectory() as temp: + base = Path(temp) / 'private' + base.mkdir(mode=0o700) + plan, docs = p.build('work:1', IMAGE, EXPIRY) + self.assertTrue(p.lock(base, plan)) + self.assertFalse(p.lock(base, plan)) + path = base / (plan['leaseHash'] + '.json') + self.assertEqual(os.stat(path).st_mode & 0o777, 0o600) + self.assertNotIn('work:1', path.read_text()) + changed = dict(plan, image='ghcr.io/other/w@sha256:' + 'b' * 64) + with self.assertRaisesRegex(p.PlanError, 'conflicting'): + p.lock(base, changed) + path.chmod(0o644) + with self.assertRaisesRegex(p.PlanError, 'unsafe'): + p.lock(base, plan) + + def test_concurrent_identical_receipt_publish_is_atomic(self): + with tempfile.TemporaryDirectory() as temp: + base = Path(temp) / 'private' + base.mkdir(mode=0o700) + plan, _ = p.build('race:1', IMAGE, EXPIRY) + original_link = os.link + entered = False + + def publish_with_competitor(*args, **kwargs): + nonlocal entered + if not entered: + entered = True + # The first publisher has written its private temp file + # but has not exposed a final receipt. A competing retry + # can publish a whole receipt without seeing partial bytes. + self.assertTrue(p.lock(base, plan)) + return original_link(*args, **kwargs) + + with mock.patch.object(p.os, 'link', side_effect=publish_with_competitor): + self.assertFalse(p.lock(base, plan)) + self.assertTrue(entered) + self.assertEqual(len(list(base.iterdir())), 1) + self.assertEqual(len(list(base.glob('*.json'))), 1) + self.assertFalse(p.lock(base, plan)) + + def test_failed_atomic_publish_leaves_no_conflicting_receipt(self): + with tempfile.TemporaryDirectory() as temp: + base = Path(temp) / 'private' + base.mkdir(mode=0o700) + plan, _ = p.build('write:1', IMAGE, EXPIRY) + with mock.patch.object(p.os, 'link', side_effect=OSError('simulated I/O failure')): + with self.assertRaisesRegex(p.PlanError, 'cannot establish'): + p.lock(base, plan) + self.assertEqual(list(base.iterdir()), []) + self.assertTrue(p.lock(base, plan)) + + def test_rejects_symlinked_or_public_plan_directory(self): + with tempfile.TemporaryDirectory() as temp: + private = Path(temp) / 'private' + private.mkdir(mode=0o700) + public = Path(temp) / 'public' + public.mkdir(mode=0o755) + symlink = Path(temp) / 'alias' + symlink.symlink_to(private) + plan, _ = p.build('work:2', IMAGE, EXPIRY) + with self.assertRaises(p.PlanError): + p.lock(public, plan) + with self.assertRaises(p.PlanError): + p.lock(symlink, plan) + with self.assertRaises(p.PlanError): + p.lock(Path('relative'), plan) + path = private / (plan['leaseHash'] + '.json') + path.symlink_to(public) + with self.assertRaises(p.PlanError): + p.lock(private, plan) + + +if __name__ == '__main__': + unittest.main() diff --git a/examples/k3s-p1/test_pod_check.py b/examples/k3s-p1/test_pod_check.py new file mode 100644 index 0000000000000..b69d0411c192b --- /dev/null +++ b/examples/k3s-p1/test_pod_check.py @@ -0,0 +1,300 @@ +"""Read-only adversarial snapshot regressions; no kubeconfig or credentials.""" +import copy +import hashlib +import importlib.util +from pathlib import Path +import json +import subprocess +import sys +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('pod_check', HERE / 'pod_check.py') +pc = importlib.util.module_from_spec(spec) +spec.loader.exec_module(pc) + +HASH = 'a' * 64 +NS = 't3-p1-' + HASH[:24] +IMAGE = 'ghcr.io/example/t3@sha256:' + 'b' * 64 +DUID = '11111111-1111-4111-8111-111111111111' +RUID = '22222222-2222-4222-8222-222222222222' +PUID = '33333333-3333-4333-8333-333333333333' +PLAN = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': HASH, 'namespace': NS, 'image': IMAGE} +BINDING = {'format': 't3-k3s-p1-binding-v4', 'leaseHash': HASH, 'namespace': NS, + 'deploymentUID': DUID, 'deploymentGeneration': 2, 'imageRefSha256': hashlib.sha256(IMAGE.encode()).hexdigest()} +CONTAINER = {'name': 't3', 'image': IMAGE, 'workingDir': '/data', + 'imagePullPolicy': 'IfNotPresent', + 'ports': [{'name': 'http', 'containerPort': 3773}], + 'startupProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5, + 'failureThreshold': 60}, + 'readinessProbe': {'tcpSocket': {'port': 'http'}, 'periodSeconds': 5}, + 'securityContext': {'readOnlyRootFilesystem': True}, + 'resources': {'limits': {'memory': '4Gi'}}, + 'volumeMounts': [{'name': 'data', 'mountPath': '/data'}]} +PODSPEC = {'serviceAccountName': 't3-worker', 'automountServiceAccountToken': False, + 'securityContext': {'runAsNonRoot': True}, + 'volumes': [{'name': 'data', 'persistentVolumeClaim': {'claimName': 't3-worker-data'}}], + 'containers': [copy.deepcopy(CONTAINER)]} +D = {'kind': 'Deployment', 'metadata': {'name': 't3-worker', 'uid': DUID, + 'namespace': NS, 'generation': 2}, + 'spec': {'template': {'metadata': {'labels': {'app': 't3-worker'}}, + 'spec': copy.deepcopy(PODSPEC)}}, + 'status': {'observedGeneration': 2, 'replicas': 1, 'readyReplicas': 1, + 'availableReplicas': 1, 'updatedReplicas': 1}} +RS = {'kind': 'ReplicaSet', 'metadata': {'name': 't3-worker-abcdef', 'uid': RUID, + 'namespace': NS, 'ownerReferences': [{'kind': 'Deployment', 'name': 't3-worker', + 'uid': DUID, 'controller': True}]}, + 'spec': {'replicas': 1, + 'selector': {'matchLabels': {'app': 't3-worker', 'pod-template-hash': 'abcdef'}}, + 'template': {'metadata': {'labels': {'app': 't3-worker', + 'pod-template-hash': 'abcdef'}}, + 'spec': copy.deepcopy(PODSPEC)}}, + 'status': {'readyReplicas': 1, 'replicas': 1}} +POD = {'kind': 'Pod', 'metadata': {'name': 't3-worker-abcdef-0123', 'namespace': NS, + 'uid': PUID, 'labels': {'app': 't3-worker', 'pod-template-hash': 'abcdef'}, 'ownerReferences': [ + {'kind': 'ReplicaSet', 'name': RS['metadata']['name'], 'uid': RUID, + 'controller': True}]}, 'spec': copy.deepcopy(PODSPEC), + 'status': {'phase': 'Running', 'conditions': [{'type': 'Ready', 'status': 'True'}], + 'containerStatuses': [{'name': 't3', 'ready': True, 'started': True, + 'state': {'running': {'startedAt': '2026-10-29T00:00:00Z'}}, + 'image': IMAGE, 'imageID': 'docker-pullable://example@sha256:' + 'c' * 64, + 'containerID': 'containerd://' + 'd' * 64, 'restartCount': 0}]}} +RS_LIST = {'kind': 'ReplicaSetList', 'items': [RS]} +PODS = {'kind': 'PodList', 'items': [POD]} + + +def fixture(): + return copy.deepcopy([PLAN, BINDING, D, RS_LIST, PODS]) + + +class RuntimeTests(unittest.TestCase): + def test_binds_single_running_pod_without_exposing_image(self): + result = pc.inspect(*fixture()) + self.assertEqual(result['podUID'], PUID) + self.assertEqual(result['deploymentUID'], DUID) + self.assertEqual(result['restartCount'], 0) + self.assertNotIn(IMAGE, json.dumps(result)) + old = fixture() + old[3]['items'].append({'kind': 'ReplicaSet', 'metadata': {'name': 't3-worker-old', + 'uid': '44444444-4444-4444-4444-444444444444', 'namespace': NS, + 'ownerReferences': RS['metadata']['ownerReferences']}, + 'spec': {'replicas': 0}, 'status': {'replicas': 0}}) + pc.inspect(*old) # Kubernetes can retain an inactive revision. + + def test_reject_ownership_or_lease_rebinding(self): + for idx, path, value in [ + (1, ('deploymentUID',), 'ffffffff-ffff-ffff-ffff-ffffffffffff'), + (1, ('deploymentGeneration',), 1), + (1, ('imageRefSha256',), '0' * 64), + (2, ('metadata', 'uid'), 'ffffffff-ffff-ffff-ffff-ffffffffffff'), + (3, ('items', 0, 'metadata', 'ownerReferences', 0, 'controller'), False), + (4, ('items', 0, 'metadata', 'ownerReferences', 0, 'uid'), 'fake'), + (4, ('items', 0, 'metadata', 'namespace'), 'other'), + ]: + args = fixture() + v = args[idx] + for key in path[:-1]: + v = v[key] + v[path[-1]] = value + with self.subTest(idx=idx, path=path), self.assertRaises(pc.PodError): + pc.inspect(*args) + + def test_reject_spec_change_between_binding_and_pod_read(self): + args = fixture() + args[2]['metadata']['generation'] = 3 + args[2]['status']['observedGeneration'] = 3 + # Same UID, image and ready replicas do not imply unchanged spec. + with self.assertRaisesRegex(pc.PodError, 'deployment changed'): + pc.inspect(*args) + for bad in (None, 0, True, '2'): + args = fixture() + args[1]['deploymentGeneration'] = bad + with self.subTest(bad=bad), self.assertRaises(pc.PodError): + pc.inspect(*args) + + def test_reject_active_replicaset_template_drift(self): + # A clean Pod can be replaced from an altered ReplicaSet template. + baseline = fixture() + baseline[2]['spec']['template']['metadata']['creationTimestamp'] = None + baseline[3]['items'][0]['spec']['template']['metadata']['creationTimestamp'] = None + pc.inspect(*baseline) # Kubernetes's harmless null timestamp is expected. + for change in ('missing', 'sidecar', 'image', 'service-account', 'volume'): + args = fixture() + spec = args[3]['items'][0]['spec'] + if change == 'missing': + del spec['template'] + elif change == 'sidecar': + spec['template']['spec']['containers'].append( + {'name': 'injected', 'image': IMAGE}) + elif change == 'image': + spec['template']['spec']['containers'][0]['image'] = 't3:latest' + elif change == 'service-account': + spec['template']['spec']['serviceAccountName'] = 'default' + elif change == 'volume': + spec['template']['spec']['volumes'] = [{'name': 'data', 'hostPath': {'path': '/host'}}] + with self.subTest(change=change), self.assertRaisesRegex( + pc.PodError, 'ReplicaSet template'): + pc.inspect(*args) + + def test_reject_replacement_metadata_or_selector_drift(self): + for change in ('sidecar-annotation', 'extra-label', 'wrong-hash', + 'wrong-selector', 'extra-selector', 'pod-label', 'desired-annotation'): + args = fixture() + desired = args[2]['spec']['template']['metadata'] + rs_spec = args[3]['items'][0]['spec'] + rs_meta = rs_spec['template']['metadata'] + if change == 'sidecar-annotation': + rs_meta['annotations'] = {'sidecar.istio.io/inject': 'true'} + elif change == 'extra-label': + rs_meta['labels']['expose'] = 'public' + elif change == 'wrong-hash': + rs_meta['labels']['pod-template-hash'] = 'wrong' + elif change == 'wrong-selector': + rs_spec['selector']['matchLabels']['app'] = 'another' + elif change == 'extra-selector': + rs_spec['selector']['matchExpressions'] = [ + {'key': 'tier', 'operator': 'Exists'}] + elif change == 'pod-label': + args[4]['items'][0]['metadata']['labels']['expose'] = 'public' + else: + desired['annotations'] = {'sidecar.istio.io/inject': 'true'} + with self.subTest(change=change), self.assertRaises(pc.PodError): + pc.inspect(*args) + + def test_reject_pod_admission_network_and_runtime_overlays(self): + # A Pod can gain a Multus interface without an altered Pod spec or + # replacement ReplicaSet template, and a RuntimeClass can change GPU + # access. Neither is in the approved one-worker P1 plan. + cases = [ + (('metadata', 'annotations'), {'k8s.v1.cni.cncf.io/networks': 'extra-net'}), + (('metadata', 'annotations'), {'v1.multus-cni.io/default-network': 'extra-net'}), + (('spec', 'runtimeClassName'), 'nvidia'), + (('spec', 'overhead'), {'cpu': '100m'}), + (('spec', 'resourceClaims'), [{'name': 'gpu'}]), + (('spec', 'priorityClassName'), 'system-node-critical'), + (('spec', 'nodeSelector'), {'gpu': 'true'}), + (('spec', 'affinity'), {'nodeAffinity': {'requiredDuringSchedulingIgnoredDuringExecution': {}}}), + (('spec', 'topologySpreadConstraints'), [{'maxSkew': 1}]), + (('spec', 'dnsConfig'), {'nameservers': ['8.8.8.8']}), + (('spec', 'dnsPolicy'), 'None'), + (('spec', 'serviceAccount'), 'default'), + (('spec', 'terminationGracePeriodSeconds'), 1), + ] + for path, value in cases: + args = fixture() + obj = args[4]['items'][0] + for key in path[:-1]: + obj = obj[key] + obj[path[-1]] = value + with self.subTest(path=path), self.assertRaises(pc.PodError): + pc.inspect(*args) + args = fixture() + pod = args[4]['items'][0] + pod['metadata']['annotations'] = {} + pod['spec']['dnsPolicy'] = 'ClusterFirst' + pod['spec']['serviceAccount'] = 't3-worker' + pod['spec']['terminationGracePeriodSeconds'] = 30 + pc.inspect(*args) # Empty annotations and Kubernetes defaults remain valid. + + def test_reject_false_readiness_and_concurrent_pods(self): + cases = [ + (2, ('status', 'observedGeneration'), 1), + (2, ('status', 'readyReplicas'), 0), + (3, ('items', 0, 'status', 'readyReplicas'), 0), + (4, ('items', 0, 'status', 'phase'), 'Pending'), + (4, ('items', 0, 'status', 'conditions', 0, 'status'), 'False'), + (4, ('items', 0, 'status', 'containerStatuses', 0, 'ready'), False), + (4, ('items', 0, 'status', 'containerStatuses', 0, 'state'), {'waiting': {}}), + (4, ('items', 0, 'status', 'containerStatuses', 0, 'image'), + 'ghcr.io/example/other@sha256:' + 'b' * 64), + ] + for idx, path, value in cases: + args = fixture() + obj = args[idx] + for key in path[:-1]: + obj = obj[key] + obj[path[-1]] = value + with self.subTest(idx=idx, path=path), self.assertRaises(pc.PodError): + pc.inspect(*args) + args = fixture() + args[4]['items'].append(copy.deepcopy(POD)) + with self.assertRaises(pc.PodError): + pc.inspect(*args) + + def test_reject_mutated_pod_and_untrusted_snapshots(self): + for path, value in [ + (('spec', 'automountServiceAccountToken'), True), + (('spec', 'containers', 0, 'securityContext'), {'privileged': True}), + (('spec', 'containers', 0, 'image'), 'different'), + (('spec', 'volumes'), []), + (('spec', 'initContainers'), [{'name': 'sidecar'}]), + (('metadata', 'deletionTimestamp'), '2026-10-29T00:00:00Z'), + ]: + args = fixture() + pod = args[4]['items'][0] + for key in path[:-1]: + pod = pod[key] + pod[path[-1]] = value + with self.subTest(path=path), self.assertRaises(pc.PodError): + pc.inspect(*args) + for idx in range(5): + args = fixture() + args[idx] = None + with self.subTest(idx=idx), self.assertRaises((pc.PodError, TypeError, AttributeError)): + pc.inspect(*args) + + def test_reject_admitted_startup_and_network_drift(self): + for path, value in [ + (('containers', 0, 'lifecycle'), + {'postStart': {'exec': {'command': ['/bin/sh', '-c', 'true']}}}), + (('containers', 0, 'ports', 0, 'hostPort'), 3773), + (('containers', 0, 'ports', 0, 'containerPort'), 9876), + (('containers', 0, 'startupProbe'), + {'exec': {'command': ['/bin/sh']}, 'periodSeconds': 5, + 'failureThreshold': 60}), + (('containers', 0, 'readinessProbe', 'tcpSocket', 'port'), 9876), + (('containers', 0, 'workingDir'), '/tmp'), + (('containers', 0, 'imagePullPolicy'), 'Always'), + (('hostname',), 'unexpected-host'), + ]: + args = fixture() + obj = args[4]['items'][0]['spec'] + for key in path[:-1]: + obj = obj[key] + obj[path[-1]] = value + with self.subTest(path=path), self.assertRaises(pc.PodError): + pc.inspect(*args) + + args = fixture() + admitted = args[4]['items'][0]['spec']['containers'][0] + # Kubernetes fills omitted probe defaults and TCP protocol. + admitted['ports'][0]['protocol'] = 'TCP' + for probe in ('startupProbe', 'readinessProbe'): + admitted[probe].update({'timeoutSeconds': 1, 'successThreshold': 1, + 'initialDelaySeconds': 0}) + admitted['readinessProbe']['failureThreshold'] = 3 + pc.inspect(*args) + + def test_cli_sanitizes_private_details(self): + with tempfile.TemporaryDirectory() as d: + paths = [Path(d) / name for name in ('plan', 'binding', 'deployment', 'replicasets', 'pods')] + for f, obj in zip(paths, fixture()): + f.write_text(json.dumps(obj)) + cmd = [sys.executable, str(HERE / 'pod_check.py')] + for flag, path in zip(('plan', 'binding', 'deployment', 'replicasets', 'pods'), paths): + cmd += ['--' + flag, str(path)] + good = subprocess.run(cmd, capture_output=True, text=True, check=False) + self.assertEqual(good.returncode, 0, good.stderr) + self.assertIn('NOT RUN:', good.stdout) + bad = copy.deepcopy(POD) + bad['status']['containerStatuses'][0]['image'] = 'private.example/secret' + paths[4].write_text(json.dumps({'kind': 'PodList', 'items': [bad]})) + failed = subprocess.run(cmd, capture_output=True, text=True, check=False) + self.assertNotEqual(failed.returncode, 0) + self.assertNotIn('private.example', failed.stderr) + self.assertNotIn(NS, failed.stderr) + + +if __name__ == '__main__': + unittest.main() diff --git a/examples/k3s-p1/test_route_check.py b/examples/k3s-p1/test_route_check.py new file mode 100644 index 0000000000000..0ea5454ee1f27 --- /dev/null +++ b/examples/k3s-p1/test_route_check.py @@ -0,0 +1,260 @@ +"""Offline malicious-route regression cases; no cluster/network required.""" +import copy +import importlib.util +from pathlib import Path +import json +import subprocess +import sys +import tempfile +import unittest + +HERE = Path(__file__).resolve().parent +spec = importlib.util.spec_from_file_location('route_check', HERE / 'route_check.py') +assert spec and spec.loader +rc = importlib.util.module_from_spec(spec) +spec.loader.exec_module(rc) + +H = 'a' * 64 +N = 't3-p1-' + H[:24] +PLAN = {'format': 't3-k3s-p1-plan-v1', 'leaseHash': H, 'namespace': N} +S = {'kind': 'Service', 'metadata': {'name': 't3-worker', 'namespace': N, + 'uid': 'aaaaaaaa-1111-2222-3333-aaaaaaaaaaaa'}, + 'spec': {'type': 'ClusterIP', 'selector': {'app': 't3-worker'}, + 'ports': [{'name': 'http', 'port': 80, 'targetPort': 'http'}]}} +I = {'kind': 'Ingress', 'metadata': {'name': 't3-worker', 'namespace': N, + 'uid': 'bbbbbbbb-1111-2222-3333-bbbbbbbbbbbb'}, + 'spec': {'ingressClassName': 'tailscale', 'tls': [{'hosts': [N]}], + 'defaultBackend': {'service': {'name': 't3-worker', 'port': {'name': 'http'}}}}} +P = {'kind': 'NetworkPolicy', 'metadata': {'name': 'worker-ingress', 'namespace': N, + 'uid': '11111111-aaaa-bbbb-cccc-111111111111'}, + 'spec': {'podSelector': {'matchLabels': {'app': 't3-worker'}}, 'policyTypes': ['Ingress'], + 'ingress': [{'from': [{'namespaceSelector': {'matchLabels': { + 'kubernetes.io/metadata.name': 'tailscale'}}}], + 'ports': [{'protocol': 'TCP', 'port': 3773}]}]}} + +ALL = {'kind': 'NetworkPolicyList', 'items': [copy.deepcopy(P)]} +ALL_INGRESSES = {'kind': 'IngressList', 'items': [copy.deepcopy(I)]} +ALL_SERVICES = {'kind': 'ServiceList', 'items': [copy.deepcopy(S)]} + + +class RouteTests(unittest.TestCase): + def test_matching_private_route(self): + rc.verify(PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES) + s = copy.deepcopy(S) + s['spec']['ports'][0]['protocol'] = 'TCP' # normal API default + i = copy.deepcopy(I) + i['metadata']['annotations'] = {'kubectl.kubernetes.io/last-applied-configuration': '...'} + rc.verify(PLAN, s, i, P, ALL, + {'kind': 'IngressList', 'items': [copy.deepcopy(i)]}, + {'kind': 'ServiceList', 'items': [copy.deepcopy(s)]}) + + def test_reject_public_service_and_route_drift(self): + changes = [ + (1, ('spec', 'type'), 'LoadBalancer'), + (1, ('spec', 'type'), 'NodePort'), + (1, ('spec', 'externalIPs'), ['8.8.8.8']), + (1, ('spec', 'ports'), [{'name': 'http', 'port': 80, 'targetPort': 'http', 'nodePort': 30080}]), + (1, ('spec', 'selector'), {'app': 'other'}), + (2, ('spec', 'ingressClassName'), 'nginx'), + (2, ('spec', 'rules'), [{'host': 'public.example.net'}]), + (2, ('spec', 'tls'), [{'hosts': ['public.example.net']}]), + (2, ('metadata', 'annotations'), {'tailscale.com/funnel': 'true'}), + (2, ('spec', 'defaultBackend'), {'service': {'name': 'other', 'port': {'name': 'http'}}}), + (3, ('spec', 'ingress'), [{'from': [{'ipBlock': {'cidr': '0.0.0.0/0'}}]}]), + (3, ('spec', 'policyTypes'), ['Ingress', 'Egress']), + (3, ('spec', 'podSelector'), {}), + ] + for idx, keys, value in changes: + with self.subTest(idx=idx, key=keys): + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + cursor = args[idx] + for key in keys[:-1]: + cursor = cursor[key] + cursor[keys[-1]] = value + with self.assertRaises(rc.RouteError): + rc.verify(*args) + + def test_reject_invalid_namespace_or_malformed_api_snapshots(self): + for plan in ({**PLAN, 'namespace': 'other'}, {**PLAN, 'leaseHash': 'not-hex'}, + {**PLAN, 'format': 'other'}): + with self.subTest(plan=plan), self.assertRaises(rc.RouteError): + rc.verify(plan, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES) + for idx in (1, 2, 3): + for invalid in (None, [], {'kind': 'other'}, {'kind': 'Service', 'metadata': None}): + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + args[idx] = invalid + with self.subTest(idx=idx, invalid=invalid), self.assertRaises(rc.RouteError): + rc.verify(*args) + + def test_reject_additive_or_inconsistent_policy_inventory(self): + rc.verify(PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES) + wildcard = {'kind': 'NetworkPolicy', + 'metadata': {'name': 'allow-anywhere', 'namespace': N, + 'uid': '22222222-aaaa-bbbb-cccc-222222222222'}, + 'spec': {'podSelector': {}, 'policyTypes': ['Ingress'], + 'ingress': [{}]}} + inventory_cases = [ + None, [], {}, + {'kind': 'List', 'items': [copy.deepcopy(P)]}, + {'kind': 'NetworkPolicyList', 'items': []}, + {'kind': 'NetworkPolicyList', 'items': [copy.deepcopy(P), wildcard]}, + ] + for bad in inventory_cases: + with self.subTest(bad=bad), self.assertRaises(rc.RouteError): + rc.verify(PLAN, S, I, P, bad, ALL_INGRESSES, ALL_SERVICES) + for key, value in [ + ('uid', 'different-uid-xxxxxxxxx'), + ('name', 'allow-anywhere'), + ('namespace', 'other-namespace'), + ('deletionTimestamp', '2026-10-29T00:00:00Z'), + ]: + bad = copy.deepcopy(ALL) + bad['items'][0]['metadata'][key] = value + with self.subTest(field=key), self.assertRaises(rc.RouteError): + rc.verify(PLAN, S, I, P, bad, ALL_INGRESSES, ALL_SERVICES) + bad = copy.deepcopy(ALL) + bad['items'][0]['spec']['ingress'] = [{}] + with self.assertRaises(rc.RouteError): + rc.verify(PLAN, S, I, P, bad, ALL_INGRESSES, ALL_SERVICES) + bad = copy.deepcopy(P) + bad['metadata']['uid'] = 'different-uid-xxxxxxxxx' + with self.assertRaises(rc.RouteError): + rc.verify(PLAN, S, I, bad, ALL, ALL_INGRESSES, ALL_SERVICES) + + def test_reject_additional_namespace_public_routes(self): + # Each named resource is still private; extra objects can advertise + # the same worker on another controller or through a public Service. + public_ingress = {'kind': 'Ingress', 'metadata': { + 'name': 'expose-worker', 'namespace': N, 'uid': 'cccccccc-1111-2222-3333-cccccccccccc'}, + 'spec': {'ingressClassName': 'nginx', + 'defaultBackend': {'service': {'name': 't3-worker', 'port': {'name': 'http'}}}}} + public_service = {'kind': 'Service', 'metadata': { + 'name': 'expose-worker', 'namespace': N, 'uid': 'dddddddd-1111-2222-3333-dddddddddddd'}, + 'spec': {'type': 'LoadBalancer', 'selector': {'app': 't3-worker'}, + 'ports': [{'port': 80, 'targetPort': 3773}]}} + for index, extra in [(5, public_ingress), (6, public_service)]: + with self.subTest(index=index): + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + args[index]['items'].append(extra) + with self.assertRaises(rc.RouteError): + rc.verify(*args) + + for index in (5, 6): + for bad in (None, {}, [], {'kind': 'List', 'items': []}, + {'kind': 'IngressList' if index == 5 else 'ServiceList', 'items': []}): + with self.subTest(index=index, bad=bad): + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + args[index] = bad + with self.assertRaises(rc.RouteError): + rc.verify(*args) + for field, value in [ + ('uid', 'different-uid-xxxxxxxxx'), ('namespace', 'wrong'), + ('deletionTimestamp', '2026-10-29T00:00:00Z'), + ]: + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + args[index]['items'][0]['metadata'][field] = value + with self.subTest(index=index, field=field), self.assertRaises(rc.RouteError): + rc.verify(*args) + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + args[index]['items'][0]['spec']['type' if index == 6 else 'ingressClassName'] = 'LoadBalancer' if index == 6 else 'nginx' + with self.subTest(index=index, field='spec'), self.assertRaises(rc.RouteError): + rc.verify(*args) + + def test_private_advertised_ingress_status_and_empty_status(self): + # An unscheduled Ingress has no status; an observed status must be private. + ingress = copy.deepcopy(I) + ingress['status'] = {'loadBalancer': {'ingress': [{ + 'hostname': N + '.tailxxx.ts.net'}]}} + service = copy.deepcopy(S) + service['status'] = {'loadBalancer': {}} + rc.verify(PLAN, service, ingress, P, ALL, + {'kind': 'IngressList', 'items': [copy.deepcopy(ingress)]}, + {'kind': 'ServiceList', 'items': [copy.deepcopy(service)]}) + + def test_reject_public_or_inconsistent_advertised_status(self): + # Public ingress, extra routes and stale mismatched inventory can + # otherwise pass when only desired .spec is validated. + cases = [ + (1, [{'ip': '203.0.113.10'}]), + (2, [{'hostname': 'public.example.com'}]), + (2, [{'ip': '203.0.113.10'}]), + (2, [{'hostname': N + '.tailxxx.ts.net', 'ip': '203.0.113.10'}]), + (2, [{'hostname': N + '.tailxxx.ts.net'}] * 2), + (5, [{'hostname': 'public.example.com'}]), + (6, [{'ip': '203.0.113.10'}]), + ] + for idx, routes in cases: + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + obj = args[idx]['items'][0] if idx in (5, 6) else args[idx] + obj['status'] = {'loadBalancer': {'ingress': routes}} + with self.subTest(idx=idx, routes=routes), self.assertRaises(rc.RouteError): + rc.verify(*args) + for idx in (1, 2, 5, 6): + for invalid in (None, {'loadBalancer': None}, {'loadBalancer': {'ingress': {}}}): + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + obj = args[idx]['items'][0] if idx in (5, 6) else args[idx] + obj['status'] = invalid + with self.subTest(idx=idx, invalid=invalid), self.assertRaises(rc.RouteError): + rc.verify(*args) + + def test_reject_controller_annotation_exposure_on_named_and_inventory_routes(self): + # Tailscale creates an L3 ingress proxy for annotated ClusterIP + # Services. It need not appear as another Ingress/Service here. + for idx, annotation in ( + (1, {'tailscale.com/expose': 'true'}), + (6, {'tailscale.com/expose': 'true'}), + (1, {'external-dns.kubernetes.io/hostname': 'public.example.org'}), + (6, {'external-dns.kubernetes.io/hostname': 'public.example.org'}), + (2, {'tailscale.com/funnel': 'true'}), + (5, {'tailscale.com/funnel': 'true'}), + (1, None), + (6, []), + ): + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + obj = args[idx]['items'][0] if idx in (5, 6) else args[idx] + obj['metadata']['annotations'] = annotation + with self.subTest(idx=idx, annotation=annotation), self.assertRaises(rc.RouteError): + rc.verify(*args) + + # kubectl apply metadata is not an alternate route. + args = copy.deepcopy([PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES]) + for idx in (1, 2, 5, 6): + obj = args[idx]['items'][0] if idx in (5, 6) else args[idx] + obj['metadata']['annotations'] = { + 'kubectl.kubernetes.io/last-applied-configuration': '{}'} + rc.verify(*args) + + def test_cli_no_network_and_no_private_details(self): + with tempfile.TemporaryDirectory() as d: + paths = [Path(d) / x for x in ('plan.json', 'service.json', 'ingress.json', 'policy.json', + 'all-policies.json', 'all-ingresses.json', 'all-services.json')] + for path, obj in zip(paths, (PLAN, S, I, P, ALL, ALL_INGRESSES, ALL_SERVICES)): + path.write_text(json.dumps(obj)) + args = [sys.executable, str(HERE / 'route_check.py'), '--plan', str(paths[0]), + '--service', str(paths[1]), '--ingress', str(paths[2]), + '--network-policy', str(paths[3]), + '--all-network-policies', str(paths[4]), + '--all-ingresses', str(paths[5]), '--all-services', str(paths[6])] + good = subprocess.run(args, capture_output=True, text=True, check=False) + self.assertEqual(good.returncode, 0, good.stderr) + self.assertIn('offline only', good.stdout) + bad_i = copy.deepcopy(I) + bad_i['metadata']['annotations'] = {'tailscale.com/funnel': 'true'} + paths[2].write_text(json.dumps(bad_i)) + bad = subprocess.run(args, capture_output=True, text=True, check=False) + self.assertNotEqual(bad.returncode, 0) + self.assertNotIn(N, bad.stderr) + self.assertNotIn('tailscale.com/funnel', bad.stderr) + paths[2].write_text(json.dumps(I)) + wide = copy.deepcopy(ALL) + wide['items'].append({ + 'kind': 'NetworkPolicy', 'metadata': {'name': 'allow-all', 'namespace': N}, + 'spec': {'podSelector': {}, 'policyTypes': ['Ingress'], 'ingress': [{}]}}) + paths[4].write_text(json.dumps(wide)) + missing_inventory = subprocess.run(args, capture_output=True, text=True, check=False) + self.assertNotEqual(missing_inventory.returncode, 0) + self.assertNotIn(N, missing_inventory.stderr) + + +if __name__ == '__main__': + unittest.main()