diff --git a/deploy/README.md b/deploy/README.md index 699e0d5f..37e7342a 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -723,3 +723,11 @@ never in an inventory and is removed with the run. ```sh deploy/test/host-key-verification.sh ``` + +### Promote an existing signed release + +For a canary-first serial update of existing executors without rebuilding, +use the [signed release rollout and rollback procedure](../docs/operations/release-rollout.md). +It requires a complete retained signed bundle and separately provisioned trust. +It refuses schema changes, preserves a stopped backup, and requires a fresh +measurement before progressing to the next host. diff --git a/deploy/ansible/roles/payload/tasks/main.yml b/deploy/ansible/roles/payload/tasks/main.yml index 0fbc2639..5baa5b3c 100644 --- a/deploy/ansible/roles/payload/tasks/main.yml +++ b/deploy/ansible/roles/payload/tasks/main.yml @@ -46,6 +46,51 @@ debuglet_release: "{{ lookup('ansible.builtin.file', dist_dir ~ '/release.json') | from_json }}" run_once: true +- name: Require a signed release for promotion + ansible.builtin.assert: + that: debuglet_release.files is defined + fail_msg: "Serial promotion requires a complete retained signed release bundle." + quiet: true + when: payload_require_signature | default(false) | bool + run_once: true + +- name: Verify a signed release before copying or executing its payload + when: debuglet_release.files is defined + run_once: true + block: + - name: Audit the independently trusted release and its complete inventory + ansible.builtin.command: + argv: + - "{{ ansible_playbook_python }}" + - "{{ playbook_dir }}/../../tools/release.py" + - audit + - --directory + - "{{ dist_dir }}" + - --version + - "{{ deploy_version }}" + - --source + - "{{ release_source_sha }}" + - --trust + - "{{ release_trust_file }}" + - --signer + - "{{ release_signer }}" + delegate_to: localhost + become: false + changed_when: false + + - name: Select the authenticated full package + vars: + archive: "debuglet-{{ debuglet_release.version }}-linux-amd64.tar.gz" + ansible.builtin.set_fact: + debuglet_release: + schema_version: 1 + version: "{{ debuglet_release.version }}" + source_sha: "{{ debuglet_release.source_sha }}" + archive: "{{ archive }}" + archive_sha256: "{{ debuglet_release.files[archive].sha256 }}" + installer_sha256: "{{ debuglet_release.files['install.sh'].sha256 }}" + go_version: "{{ (lookup('ansible.builtin.file', dist_dir ~ '/build-inputs.json') | from_json).build_record.metadata.go_version }}" + - name: Require a complete release record ansible.builtin.assert: that: @@ -147,8 +192,9 @@ {{ '' if payload_activate | bool else '--stage-only' }} creates: >- {{ payload_prefix ~ '/lib/debuglet/' ~ debuglet_release.version ~ '/share/debuglet/manifest.json' - if not payload_activate | bool or payload_cli_link.stat.lnk_source | default('') == - payload_prefix ~ '/lib/debuglet/' ~ debuglet_release.version ~ '/bin/dbl' + if not (payload_require_signature | default(false) | bool) + and (not payload_activate | bool or payload_cli_link.stat.lnk_source | default('') == + payload_prefix ~ '/lib/debuglet/' ~ debuglet_release.version ~ '/bin/dbl') else omit }} notify: Restart debuglet services diff --git a/deploy/ansible/rollout-executors.yml b/deploy/ansible/rollout-executors.yml new file mode 100644 index 00000000..f37f08c5 --- /dev/null +++ b/deploy/ansible/rollout-executors.yml @@ -0,0 +1,240 @@ +--- +# Promote one retained release to existing managed executors. A failed host +# ends the rollout; later inventory hosts are never touched. +- import_playbook: preflight.yml + +- name: Confirm the selected release and canary + hosts: executors + connection: local + gather_facts: false + become: false + any_errors_fatal: true + tasks: + - name: Require an explicit environment and release confirmation + ansible.builtin.assert: + that: + - rollout_confirm | default('') == debuglet_env ~ ':' ~ deploy_version ~ ':' ~ release_source_sha + - release_source_sha is match('^[0-9a-f]{40}$') + - rollout_canary | default('') == ansible_play_hosts_all[0] + - rollout_operator | default('') | trim | length > 0 + - rollout_cli is match('^/') + - rollout_client_config is match('^/') + - debuglet_manage_services | bool + - inventory_hostname not in (groups['dispatcher'] | default([])) + fail_msg: >- + Select the environment, exact release/source, and first inventory host + as rollout_canary. Set rollout_confirm=ENV:VERSION:SOURCE, name the + authorizing operator in rollout_operator, and provide + the controller's full installed CLI and authenticated connections file. + Co-located dispatchers need a separate maintenance plan. + quiet: true + +- name: Promote the canary and then each remaining executor + hosts: executors + order: inventory + serial: 1 + any_errors_fatal: true + gather_facts: true + become: true + vars: + payload_require_signature: true + payload_activate: false + payload_restart_services: false + tasks: + - name: Inspect the existing executor service + ansible.builtin.command: + argv: [systemctl, show, "{{ executor_service }}", --property=LoadState, --property=ActiveState] + register: rollout_initial + changed_when: false + + - name: Require an installed executor outside a failed or transitional state + ansible.builtin.assert: + that: + - "'LoadState=loaded' in rollout_initial.stdout_lines" + - "'ActiveState=active' in rollout_initial.stdout_lines or 'ActiveState=inactive' in rollout_initial.stdout_lines" + quiet: true + + - name: Read the previous deployment identity + ansible.builtin.slurp: + src: "{{ config_dir }}/deployment-{{ debuglet_env }}.json" + register: rollout_previous + + - name: Stage the exact signed package without changing command links + ansible.builtin.include_role: + name: payload + + - name: Preserve the executor's required kernel capabilities + community.general.capabilities: + path: "{{ payload_prefix }}/lib/debuglet/{{ deploy_version }}/bin/debuglet-executor" + capability: cap_net_admin,cap_bpf+ep + state: present + when: executor_enable_bpf | default(true) | bool and not (executor_ambient_caps | default(false) | bool) + + - name: Refuse a release incompatible with the current schema + ansible.builtin.command: + argv: + - runuser + - -u + - "{{ executor_user }}" + - -- + - "{{ payload_prefix }}/lib/debuglet/{{ deploy_version }}/bin/debuglet-executor" + - -config + - "{{ executor_config_dir }}/executor.toml" + - -check-database + changed_when: false + + - name: Stop admission and join the executor + ansible.builtin.systemd: + name: "{{ executor_service }}" + state: stopped + + - name: Inspect the completed shutdown + ansible.builtin.command: + argv: [systemctl, show, "{{ executor_service }}", --property=Result, --property=ExecMainStatus, --property=ActiveState] + register: rollout_stop + changed_when: false + + - name: Refuse backup or activation after an incomplete drain + ansible.builtin.assert: + that: + - "'Result=success' in rollout_stop.stdout_lines" + - "'ExecMainStatus=0' in rollout_stop.stdout_lines" + - "'ActiveState=inactive' in rollout_stop.stdout_lines" + fail_msg: "The executor did not stop cleanly. Keep its state intact and inspect the service before continuing." + quiet: true + + - name: Create a private rollback directory outside the live state + ansible.builtin.tempfile: + state: directory + path: "{{ executor_state_dir | dirname }}" + prefix: rollout-backup- + register: rollout_backup + + - name: Preserve the stopped state and deployment configuration + ansible.builtin.command: + argv: [tar, --create, --file, "{{ rollout_backup.path }}/{{ item.name }}.tar", --directory, "{{ item.source }}", .] + loop: + - {source: "{{ executor_state_dir }}", name: state} + - {source: "{{ config_dir }}", name: config} + loop_control: + label: "{{ item.name }}" + + - name: Verify the stopped backup against its sources + ansible.builtin.command: + argv: [tar, --compare, --file, "{{ rollout_backup.path }}/{{ item.name }}.tar", --directory, "{{ item.source }}"] + changed_when: false + loop: + - {source: "{{ executor_state_dir }}", name: state} + - {source: "{{ config_dir }}", name: config} + loop_control: + label: "{{ item.name }}" + + - name: Record backup digests for a later restore + ansible.builtin.stat: + path: "{{ rollout_backup.path }}/{{ item }}.tar" + checksum_algorithm: sha256 + loop: [state, config] + register: rollout_backup_files + + - name: Record the authorization, backup and prepared release + ansible.builtin.copy: + content: | + {{ { + 'host': inventory_hostname, 'executor_id': executor_id, + 'environment': debuglet_env, 'confirmation': rollout_confirm, 'operator': rollout_operator, + 'canary': rollout_canary, 'previous': rollout_previous.content | b64decode | from_json, + 'candidate': debuglet_release, 'drain_joined': true, + 'schema': 'compatible; no migration performed', + 'backup': rollout_backup.path, + 'backup_sha256': dict(rollout_backup_files.results | map(attribute='item') | zip(rollout_backup_files.results | map(attribute='stat.checksum'))), + 'status': 'prepared' + } | to_nice_json }} + dest: "{{ rollout_backup.path }}/rollout.json" + mode: "0600" + + - name: Activate the verified package without rebuilding + ansible.builtin.include_role: + name: payload + vars: + payload_activate: true + + - name: Render the active executor version without changing its identity + ansible.builtin.template: + src: roles/executor/templates/executor.toml.j2 + dest: "{{ executor_config_dir }}/executor.toml" + owner: "{{ executor_user }}" + group: "{{ executor_group }}" + mode: "0600" + no_log: true + + - name: Start the selected release + ansible.builtin.systemd: + name: "{{ executor_service }}" + state: started + + - name: Check service health before submitting a measurement + ansible.builtin.command: + argv: [systemctl, is-active, "{{ executor_service }}"] + changed_when: false + + - name: Wait for the selected executor to reconnect with the expected version + ansible.builtin.command: + argv: ["{{ rollout_cli }}", --config, "{{ rollout_client_config }}", --output, json, --timeout, 10s, nodes] + delegate_to: localhost + become: false + changed_when: false + register: rollout_nodes + until: >- + rollout_nodes.rc == 0 and + rollout_nodes.stdout | from_json | selectattr('id', 'equalto', executor_id) + | selectattr('ready', 'equalto', true) | selectattr('version', 'equalto', deploy_version) + | list | length == 1 + retries: 12 + delay: 5 + + - name: Require a fresh completed measurement on this executor + ansible.builtin.command: + argv: + - "{{ rollout_cli }}" + - --config + - "{{ rollout_client_config }}" + - --output + - json + - --timeout + - 60s + - run + - --sample + - hello + - --executor + - "{{ executor_id }}" + - --wait + - --allow-remote-test + delegate_to: localhost + become: false + register: rollout_measurement + changed_when: false + + - name: Confirm the measurement's executor and terminal success + ansible.builtin.assert: + that: + - (rollout_measurement.stdout | from_json).executor_id == executor_id + - (rollout_measurement.stdout | from_json).state == 'RunStateExited' + - (rollout_measurement.stdout | from_json).id | length > 0 + quiet: true + + - name: Record the accepted measurement before advancing to the next host + ansible.builtin.copy: + content: | + {{ { + 'host': inventory_hostname, 'executor_id': executor_id, + 'environment': debuglet_env, 'confirmation': rollout_confirm, 'operator': rollout_operator, + 'canary': rollout_canary, 'previous': rollout_previous.content | b64decode | from_json, + 'candidate': debuglet_release, 'drain_joined': true, + 'schema': 'compatible; no migration performed', + 'backup': rollout_backup.path, + 'backup_sha256': dict(rollout_backup_files.results | map(attribute='item') | zip(rollout_backup_files.results | map(attribute='stat.checksum'))), + 'status': 'verified', + 'measurement': rollout_measurement.stdout | from_json + } | to_nice_json }} + dest: "{{ rollout_backup.path }}/rollout.json" + mode: "0600" diff --git a/deploy/test/provisioner-check.sh b/deploy/test/provisioner-check.sh index ff86e1a9..45bfdc3f 100755 --- a/deploy/test/provisioner-check.sh +++ b/deploy/test/provisioner-check.sh @@ -16,7 +16,8 @@ # 4. a changed dependency digest stops a deployment in the preflight, # 5. a provisioner without a record stops a deployment in the preflight, # 6. deploy/test/ansible-render.sh passes inside it, which renders and -# applies the roles and checks that a repeat run changes nothing. +# applies the roles and checks that a repeat run changes nothing, +# 7. signed serial-promotion fixtures stop at failed canaries and unsafe state. # # It needs a built release package in deploy/dist: # @@ -190,6 +191,12 @@ else check 'the deployment render checks pass in the provisioner' fail fi +if provision "$DEBUGLET_PROVISIONER_IMAGE" python3 -m unittest deploy.test.test_rollout -v; then + check 'signed serial promotion and refusal checks pass' pass +else + check 'signed serial promotion and refusal checks pass' fail +fi + if [ "$failures" -ne 0 ]; then printf '%s check(s) failed\n' "$failures" >&2 exit 1 diff --git a/deploy/test/test_rollout.py b/deploy/test/test_rollout.py new file mode 100644 index 00000000..b50c30f7 --- /dev/null +++ b/deploy/test/test_rollout.py @@ -0,0 +1,196 @@ +"""Serial-promotion checks with real Ansible/signatures and fixture services. + +Run inside the pinned provisioner, without network access or managed hosts: + python3 -m unittest deploy.test.test_rollout -v +The service manager, package payload and measurement client are local fixtures; +this suite does not establish deployment or real-release acceptance. +""" +import hashlib +import json +import os +from pathlib import Path +import shutil +import subprocess +import sys +import unittest + +import yaml +from tools import test_release_retention as retention_tests +from tools.test_release_inventory import VERSION, SOURCE + +ROOT = Path(__file__).resolve().parents[2] +SYSTEMCTL = r'''#!PYTHON +import json, os, sys +from pathlib import Path +args = sys.argv[1:] +name = next((x for x in args if x.startswith('fixture-')), 'fixture-canary') +root = Path(os.environ['ROLLOUT_FIXTURE']) +state = root / (name + '.active') +active = not state.exists() or state.read_text() == 'yes' +if '--version' in args: + print('systemd 252'); sys.exit(0) +if 'stop' in args or 'start' in args: + action = 'stop' if 'stop' in args else 'start' + state.write_text('yes' if action == 'start' else 'no') + with (root / 'trace').open('a') as f: f.write(action + ' ' + name + '\n') + sys.exit(0) +if 'is-active' in args: + print('active' if active else 'inactive'); sys.exit(0 if active else 3) +if 'is-enabled' in args: + print('enabled'); sys.exit(0) +print('LoadState=loaded\nUnitFileState=enabled\nNeedDaemonReload=no\nResult=success\nExecMainStatus=0') +print('ActiveState=' + ('active' if active else 'inactive')) +print('SubState=' + ('running' if active else 'dead')) +''' +INSTALLER = r'''#!/bin/sh +set -eu +while [ "$#" -gt 0 ]; do + case "$1" in + --prefix) prefix=$2; shift 2;; + --version) version=$2; shift 2;; + --stage-only) shift;; + *) shift 2;; + esac +done +printf '%s\n' "$prefix" >> "$ROLLOUT_FIXTURE/installers" +[ ! -f "$prefix/lib/debuglet/$version/bad-tree" ] || exit 18 +mkdir -p "$prefix/lib/debuglet/$version/bin" "$prefix/lib/debuglet/$version/share/debuglet" "$prefix/bin" +printf '{}\n' > "$prefix/lib/debuglet/$version/share/debuglet/manifest.json" +printf '#!/bin/sh\n[ ! -f "%s/incompatible" ]\n' "$ROLLOUT_FIXTURE" > "$prefix/lib/debuglet/$version/bin/debuglet-executor" +chmod 755 "$prefix/lib/debuglet/$version/bin/debuglet-executor" +cp "$prefix/lib/debuglet/$version/bin/debuglet-executor" "$prefix/lib/debuglet/$version/bin/debuglet-dispatcher" +''' +CLIENT = r'''#!PYTHON +import json, os, sys +from pathlib import Path +root = Path(os.environ['ROLLOUT_FIXTURE']) +args = sys.argv[1:] +if args[-1] == 'nodes': + print(json.dumps([{'id': n, 'ready': True, 'version': os.environ['ROLLOUT_VERSION']} for n in ('11111111-1111-4111-8111-111111111111', '22222222-2222-4222-8222-222222222222')])) +else: + node = args[args.index('--executor') + 1] + with (root / 'trace').open('a') as f: f.write('measure ' + node + '\n') + if (root / 'fail-measurement').exists(): sys.exit(3) + print(json.dumps({'id': '33333333-3333-4333-8333-333333333333', 'executor_id': node, 'state': 'RunStateExited'})) +''' + + +class RolloutTests(unittest.TestCase): + def setUp(self): + self.assertIsNotNone(shutil.which('ansible-playbook'), 'run in the pinned provisioner') + self.fixture = retention_tests.RetentionTests('test_evidence_is_signed_and_survives_source_artifact_removal') + self.fixture.setUp() + self.addCleanup(self.fixture.doCleanups) + self.work = self.fixture.archive / 'rollout' + self.work.mkdir() + self.playbooks = self.work / 'ansible' + shutil.copytree(ROOT / 'deploy/ansible', self.playbooks) + # Only local service behavior is substituted. The complete preflight, + # real payload role, release audit, SSH signatures and backup tasks run. + shutil.copytree(ROOT / 'tools', self.work.parent / 'tools', ignore=shutil.ignore_patterns('__pycache__')) + shutil.copyfile(ROOT / 'deploy/provisioner.env', self.work / 'provisioner.env') + # payload's audit path is relative to playbook_dir/../../tools. + for name, code in [('systemctl', SYSTEMCTL), ('client', CLIENT)]: + p = self.work / name + p.write_text(code.replace('PYTHON', sys.executable, 1)); p.chmod(0o755) + Path('/run/systemd/system').mkdir(parents=True, exist_ok=True) + (self.fixture.directory / 'install.sh').write_text(INSTALLER) + sums = self.fixture.directory / 'SHA256SUMS' + lines = sums.read_text().splitlines() + lines[1] = hashlib.sha256(INSTALLER.encode()).hexdigest() + ' install.sh' + sums.write_text('\n'.join(lines) + '\n') + self.fixture.inventory.generate() + self.fixture.signed() + (self.work / 'known_hosts').write_text('fixture ssh-ed25519 AAAAFIXTURE\n') + (self.work / 'client.json').write_text('{}') + hosts = {} + for name, ident in [('canary', '11111111-1111-4111-8111-111111111111'), + ('second', '22222222-2222-4222-8222-222222222222')]: + host = self.work / name + (host / 'state/executor-dev').mkdir(parents=True) + (host / 'config/executor-dev').mkdir(parents=True) + (host / 'state/executor-dev/executor.db').write_text('unchanged fixture state') + (host / 'config/executor-dev/executor.toml').write_text('[identity]\nversion = "previous"\n') + (host / 'config/deployment-dev.json').write_text(json.dumps({'application': {'version': 'previous'}})) + hosts[name] = {'executor_id': ident, 'executor_service': 'fixture-' + name, + 'payload_prefix': str(host / 'prefix'), 'config_dir': str(host / 'config'), + 'state_dir': str(host / 'state'), 'log_dir': str(host / 'log'), + 'payload_staging_dir': str(host / 'staging'), 'systemd_unit_dir': str(host / 'units')} + (self.work / 'inventory.yml').write_text(yaml.safe_dump({'all': {'children': {'executors': {'hosts': hosts}}}}, sort_keys=False)) + settings = {'ansible_connection': 'local', 'ansible_become': False, + 'ansible_python_interpreter': sys.executable, 'debuglet_env': 'dev', + 'known_hosts_file': str(self.work / 'known_hosts'), 'dispatcher_addr': '127.0.0.1', 'executor_disable_tls': True, 'dispatcher_disable_tls': True, + 'executor_user': 'root', 'executor_group': 'root', 'debuglet_group': 'root', 'executor_interface': 'lo', + 'executor_enable_bpf': False, + 'dist_dir': str(self.fixture.directory), 'deploy_version': VERSION, 'release_source_sha': SOURCE, + 'release_trust_file': str(self.fixture.trust), 'release_signer': 'fixture', + 'rollout_confirm': 'dev:' + VERSION + ':' + SOURCE, 'rollout_canary': 'canary', + 'rollout_operator': 'fixture operator', 'rollout_cli': str(self.work / 'client'), + 'rollout_client_config': str(self.work / 'client.json')} + (self.work / 'settings.yml').write_text(yaml.safe_dump(settings)) + self.env = dict(os.environ, PATH=str(self.work) + os.pathsep + os.environ['PATH'], + ROLLOUT_FIXTURE=str(self.work), ROLLOUT_VERSION=VERSION, ANSIBLE_NOCOLOR='1') + + def run_rollout(self): + p = subprocess.run(['ansible-playbook', '-i', str(self.work / 'inventory.yml'), + '-e', '@' + str(self.work / 'settings.yml'), 'rollout-executors.yml'], + cwd=self.playbooks, env=self.env, text=True, capture_output=True, timeout=180) + return p, (self.work / 'trace').read_text().splitlines() if (self.work / 'trace').exists() else [] + + def test_serial_success_records_verified_backups_and_measurements(self): + p, trace = self.run_rollout() + self.assertEqual(p.returncode, 0, p.stdout + p.stderr) + self.assertEqual(trace, ['stop fixture-canary', 'start fixture-canary', + 'measure 11111111-1111-4111-8111-111111111111', + 'stop fixture-second', 'start fixture-second', + 'measure 22222222-2222-4222-8222-222222222222']) + for host in ('canary', 'second'): + record = next((self.work / host / 'state').glob('rollout-backup-*/rollout.json')) + row = json.loads(record.read_text()) + self.assertEqual(row['status'], 'verified') + self.assertTrue(row['drain_joined']) + for name, digest in row['backup_sha256'].items(): + self.assertEqual(hashlib.sha256((record.parent / (name + '.tar')).read_bytes()).hexdigest(), digest) + + def test_failed_canary_measurement_never_touches_second_host(self): + (self.work / 'fail-measurement').touch() + p, trace = self.run_rollout() + self.assertNotEqual(p.returncode, 0, p.stdout + p.stderr) + self.assertIn('measure 11111111-1111-4111-8111-111111111111', trace, p.stdout + p.stderr) + self.assertFalse((self.work / 'second/prefix').exists()) + self.assertEqual(json.loads((self.work / 'second/config/deployment-dev.json').read_text())['application']['version'], 'previous') + + def test_incompatible_schema_is_refused_before_stopping_current_service(self): + (self.work / 'incompatible').touch() + p, trace = self.run_rollout() + self.assertNotEqual(p.returncode, 0, p.stdout + p.stderr) + self.assertIn('Refuse a release incompatible with the current schema', p.stdout) + self.assertEqual(trace, []) + self.assertFalse((self.work / 'second/prefix').exists()) + + def test_existing_candidate_is_reverified_before_execution(self): + candidate = self.work / 'canary/prefix/lib/debuglet' / VERSION + (candidate / 'share/debuglet').mkdir(parents=True) + (candidate / 'share/debuglet/manifest.json').write_text('{}') + (candidate / 'bad-tree').touch() + p, trace = self.run_rollout() + self.assertNotEqual(p.returncode, 0, p.stdout + p.stderr) + self.assertEqual((self.work / 'installers').read_text().splitlines(), [str(self.work / 'canary/prefix')]) + self.assertEqual(trace, []) + self.assertFalse((self.work / 'second/prefix').exists()) + + def test_missing_required_capabilities_stop_before_drain(self): + settings = self.work / 'settings.yml' + value = yaml.safe_load(settings.read_text()); value['executor_enable_bpf'] = True + settings.write_text(yaml.safe_dump(value)) + for name, code in [('getcap', '#!/bin/sh\nexit 0\n'), ('setcap', '#!/bin/sh\nexit 1\n')]: + path = self.work / name; path.write_text(code); path.chmod(0o755) + p, trace = self.run_rollout() + self.assertNotEqual(p.returncode, 0, p.stdout + p.stderr) + self.assertIn("Preserve the executor's required kernel capabilities", p.stdout) + self.assertEqual(trace, []) + self.assertFalse((self.work / 'second/prefix').exists()) + + +if __name__ == '__main__': + unittest.main() diff --git a/docs/operations/release-rollout.md b/docs/operations/release-rollout.md new file mode 100644 index 00000000..8fc6d046 --- /dev/null +++ b/docs/operations/release-rollout.md @@ -0,0 +1,121 @@ +# Promote or roll back an executor release + +`deploy/ansible/rollout-executors.yml` promotes one complete retained signed +release through existing Ansible-managed executors. It installs identical bytes +on every host, starts with an explicit canary, and stops at the first failure. +It does not build packages, create accounts, change certificates, deploy a +dispatcher, apply migrations, or automatically restore state. +Hosts also listed as dispatchers are refused: their shared command links need +a separate maintenance plan. Required file capabilities are applied to the +staged executor before stopping the old service; failure stops the rollout. +Previously installed candidates are reverified by the signed installer before +their executables run. + +Use the inventory, pinned SSH identities and pinned provisioner from +[the deployment guide](../../deploy/README.md). The inventory remains the source +of the executor configuration. Review it before rollout: activation renders it +with the selected version. The controller needs a full installed CLI, its +connections file, and an account allowed to submit a small TEST-funded hello +measurement to every selected executor. No wallet or chain payment is involved. + +## Prepare and authorize + +Obtain the complete bundle from the approved immutable archive, including its +signed inventory and retained evidence. Provision release trust independently +as described in [signed releases](releases.md). Preserve the previous supported +bundle and its approved trust for rollback. Paths below are inside the pinned +provisioner; put private inputs under untracked `deploy/` paths available through +its read-only repository mount. Never commit connections, trust or inventory +files containing private deployment information. + +Create a private variables file with the exact approved release and source: + +```yaml +# Illustrative values: select actual retained artifacts and an authorized host. +deploy_version: v1.2.3 +release_source_sha: "0123456789abcdef0123456789abcdef01234567" +dist_dir: /repository/deploy/dist +release_trust_file: /repository/deploy/private/release-allowed-signers +release_signer: releases@example.org +rollout_operator: "operator/change-reference" +rollout_confirm: "dev:v1.2.3:0123456789abcdef0123456789abcdef01234567" +rollout_canary: executor1.example.org +rollout_cli: /repository/deploy/operator-client/bin/dbl +rollout_client_config: /repository/deploy/private/connections.json +``` + +The confirmation records an operator's explicit choice; it is not an identity +provider or an authorization-server decision. Access is controlled by the +operator's SSH and dispatcher credentials. Put the canary first in the selected +inventory order. Select the environment and inventory explicitly: + +```sh +deploy/scripts/provisioner.sh ansible-playbook -i hosts.dev.yml \ + -e @vars/dev.yml -e @../private/rollout.yml \ + -e 'known_hosts_file={{ playbook_dir }}/known_hosts.dev' \ + rollout-executors.yml +``` + +Each host first verifies the signature and complete inventory, stages the +selected package, and asks its daemon to check the current database schema. +An incompatible schema fails while the old executor still runs. Plan an +explicit, separately authorized database upgrade using `upgrade-database.yml` +when necessary; this rollout does not treat a migration as a reversible update. + +For a compatible host, the playbook stops the executor and requires a successful +joined shutdown. It archives and compares the stopped state and configuration, +then records their SHA-256 digests in a private `rollout-backup-*` directory next +to the state directory. Backups contain credentials: keep them private. The +record identifies the host, previous and candidate artifacts, operator +confirmation, drain and schema result. + +Only then does it activate the verified bytes. It requires service health, +reconnection with the selected version, and a fresh completed hello measurement +on that exact executor before advancing. An unreachable host, failed health +check or failed measurement stops progression. Hosts later in inventory order +keep their previous version. A failed canary can remain on the candidate; the +playbook preserves its prepared record and backup and does not conceal the +failure by automatically restarting an older binary. + +## Roll back deliberately + +For a bad release with compatible state, select the previous retained signed +bundle and its exact version/source in the same variables file. Keep the +failed executor stopped while inspecting it. A service that is already cleanly +stopped can enter the rollout; a failed or transitional service state requires +operator inspection before continuing. +It verifies the old candidate's schema before changing active links, then uses +the same serial health and measurement gates. Nothing is rebuilt. + +An old binary that cannot read the current schema is refused. Do not force its +startup or run an automatic downgrade. Keep the original instance stopped, +verify the recorded backup digests, and restore the matching state and complete +configuration into separate private paths. Review embedded database paths and +external credential references before using a copied configuration; they must +not point back into the original live state. Never run two instances with the +same executor identity. A managed deployment needs an explicit restore plan; +`dbl backup`/`restore` supports foreground TEST state only. + +Inspect the dispatcher-visible run states and the executor's retained-work +report around the rollback, then submit a fresh controlled measurement. +Interrupted work has an unknown or quarantined outcome; rollback does not +replay it. Work accepted after the backup may be unavailable after restore. +A forward-only migration or discarded old trust can make the old release +unusable; this procedure does not promise recovery beyond retained compatible +artifacts and verified backups. + +## Local validation scope + +The pinned provisioner can run the no-network two-host orchestration fixture: + +```sh +python3 -m unittest deploy.test.test_rollout -v +``` + +It exercises real Ansible, release signature/audit and backup operations, with +small package, service-manager and measurement-client fixtures. It verifies +serial progress, canary failure stopping the untouched host, and incompatible +schema refusal before drain. It does not establish a real signed release, +production service behavior, private archive access policy, or independent +operator acceptance. Rehearse those against the actual supported release pair +and authorized hosts before activating a deployment.