From 4cc5d38f4944854b613e10c0191bd13048a60d6e Mon Sep 17 00:00:00 2001 From: LUNO Date: Sat, 24 Oct 2026 12:44:05 +0800 Subject: [PATCH 01/12] ci: split heavy suites into parallel parts Run the motd, Cloudflare Tunnel and push worker registration suites as four independent parts each, sharing a definitions lib, and execute a shard's files with a bounded worker pool (TEST_JOBS, serial by default). Serialize no behaviour change: the parts are contiguous slices of the original files that each parse standalone, so every assertion still runs. Also map modules/system-optimize.sh to its suite, lint tests/lib, fetch only one commit for the lint job, and run Debian's whole-tree bash -n once instead of per shard. --- .github/workflows/shell-tests.yml | 7 +- README.md | 3 +- tests/lib/test-cloudflare-tunnel-common.sh | 181 ++ tests/lib/test-motd-common.sh | 62 + .../test-push-worker-registration-common.sh | 86 + tests/run.sh | 80 +- tests/test-cloudflare-tunnel-01.sh | 349 ++++ tests/test-cloudflare-tunnel-02.sh | 380 ++++ tests/test-cloudflare-tunnel-03.sh | 272 +++ tests/test-cloudflare-tunnel-04.sh | 375 ++++ tests/test-cloudflare-tunnel.sh | 1544 ---------------- tests/test-motd-01.sh | 555 ++++++ tests/test-motd-02.sh | 267 +++ tests/test-motd-03.sh | 233 +++ tests/test-motd-04.sh | 155 ++ tests/test-motd.sh | 1259 ------------- tests/test-push-worker-registration-01.sh | 576 ++++++ tests/test-push-worker-registration-02.sh | 358 ++++ tests/test-push-worker-registration-03.sh | 491 ++++++ tests/test-push-worker-registration-04.sh | 57 + tests/test-push-worker-registration.sh | 1553 ----------------- 21 files changed, 4472 insertions(+), 4371 deletions(-) create mode 100644 tests/lib/test-cloudflare-tunnel-common.sh create mode 100644 tests/lib/test-motd-common.sh create mode 100644 tests/lib/test-push-worker-registration-common.sh create mode 100644 tests/test-cloudflare-tunnel-01.sh create mode 100644 tests/test-cloudflare-tunnel-02.sh create mode 100644 tests/test-cloudflare-tunnel-03.sh create mode 100644 tests/test-cloudflare-tunnel-04.sh delete mode 100644 tests/test-cloudflare-tunnel.sh create mode 100644 tests/test-motd-01.sh create mode 100644 tests/test-motd-02.sh create mode 100644 tests/test-motd-03.sh create mode 100644 tests/test-motd-04.sh delete mode 100644 tests/test-motd.sh create mode 100644 tests/test-push-worker-registration-01.sh create mode 100644 tests/test-push-worker-registration-02.sh create mode 100644 tests/test-push-worker-registration-03.sh create mode 100644 tests/test-push-worker-registration-04.sh delete mode 100755 tests/test-push-worker-registration.sh diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index 9a0299b..173fae9 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v7 with: - fetch-depth: 0 + fetch-depth: 1 persist-credentials: false - name: Install pinned ShellCheck @@ -48,7 +48,7 @@ jobs: # Check both groups even when production lint fails. status=0 git ls-files -z -- "*.sh" ":!:tests/**" | xargs -0 -r shellcheck -x --severity=warning -- || status=1 - git ls-files -z -- "tests/*.sh" | xargs -0 -r shellcheck -x --severity=error -- || status=1 + git ls-files -z -- "tests/*.sh" "tests/lib/*.sh" | xargs -0 -r shellcheck -x --severity=error -- || status=1 exit "$status" ubuntu-tests: @@ -85,6 +85,7 @@ jobs: TEST_BASE_SHA: ${{ github.event.pull_request.base.sha }} TEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }} TEST_SHARD: ${{ matrix.shard }} + TEST_JOBS: 4 run: bash tests/run.sh "$TEST_SHARD" debian-tests: @@ -118,6 +119,7 @@ jobs: bash tests/test-selection-git.sh - name: Check Bash syntax + if: matrix.shard == 'other' run: find . -type f -name '*.sh' -print0 | xargs -0 -n1 bash -n - name: Run tests @@ -125,6 +127,7 @@ jobs: TEST_BASE_SHA: ${{ github.event.pull_request.base.sha }} TEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }} TEST_SHARD: ${{ matrix.shard }} + TEST_JOBS: 4 run: bash tests/run.sh "$TEST_SHARD" # Preserve existing required-check names; skipped/cancelled dependencies fail closed. diff --git a/README.md b/README.md index 078d5e5..ac278f9 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,8 @@ 生产宿主机（包括 Netcup）仅执行差异检查及改动脚本的 `bash -n`，不运行全量 ShellCheck 或完整测试集； 分析和测试交由 CI：PR 按完整差异及依赖映射选择套件，README/许可证修改无需业务测试； 公共设施或未知依赖改动回退全量，main 推送仍执行全量。两个必需 Job 始终运行，输出选择和跳过记录。 -Ubuntu 使用固定 ShellCheck v0.10.0：所有受 Git 跟踪、位于 tests/ 之外的 .sh 文件执行 warning 门槛，tests/ 内的 .sh 文件执行 error 门槛；新目录中的生产 .sh 文件自动纳入，不维护生产文件白名单。Zsh 配置 p10k-config.zsh 不交给 Bash ShellCheck。两组 lint 都会执行并累计失败，Debian 不重复 lint。 +motd、Cloudflare Tunnel 与 push worker 注册三个套件按用例族拆成 `-NN.sh` 分片，共用 `tests/lib/-common.sh` 的定义；同一分片内的文件由 `tests/run.sh` 以 `TEST_JOBS` 限定的并发执行（CI 取 4，本机默认串行）。 +Ubuntu 使用固定 ShellCheck v0.10.0：所有受 Git 跟踪、位于 tests/ 之外的 .sh 文件执行 warning 门槛，tests/ 与 tests/lib/ 内的 .sh 文件执行 error 门槛；新目录中的生产 .sh 文件自动纳入，不维护生产文件白名单。Zsh 配置 p10k-config.zsh 不交给 Bash ShellCheck。两组 lint 都会执行并累计失败，Debian 不重复 lint。 例外仅允许局部明确的 SC 规则编号和邻近理由；禁止文件级或全局 warning 抑制。测试 warning 不阻断不等于被 disable，禁止用抑制隐藏行为缺陷。CI 工作流改动继续触发全量测试回退；warning 门槛失败时后续 Ubuntu 行为套件不会执行，不能以 Debian 通过代替双环境验收。 语法检查仍覆盖全部 .sh 脚本；文档修改无需本地全套测试，但仍须通过必需 CI。 本机重测试必须使用原生资源限制及权限隔离；无法隔离或超限时停止，不自动提高额度或转宿主机执行。 diff --git a/tests/lib/test-cloudflare-tunnel-common.sh b/tests/lib/test-cloudflare-tunnel-common.sh new file mode 100644 index 0000000..cea083f --- /dev/null +++ b/tests/lib/test-cloudflare-tunnel-common.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (( EUID != 0 )); then + exec sudo --preserve-env=PATH bash "$0" "$@" +fi + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +readonly ROOT_DIR +TEST_DIR=$(mktemp -d -p "$ROOT_DIR") +readonly TEST_DIR +trap 'rm -rf "$TEST_DIR"' EXIT + +pass_count=0 +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } +pass() { pass_count=$((pass_count + 1)); printf 'PASS: %s\n' "$*"; } +assert_same() { cmp -s -- "$1" "$2" || fail "$3"; } +assert_absent() { [[ ! -e "$1" && ! -L "$1" ]] || fail "$2"; } + +make_fake_commands() { + local bin="$1" + mkdir -p "$bin" + cat > "$bin/curl" <<'FAKE' +#!/usr/bin/env bash +set -eu +printf 'curl:%s\n' "$*" >> "$FAKE_LOG" +[[ "${FAKE_CURL_FAIL:-}" != 1 ]] || exit 22 +output="" +while (( $# )); do + if [[ "$1" == -o ]]; then output="$2"; shift 2; continue; fi + shift +done +[[ -n "$output" ]] +printf 'fake-cloudflare-key\n' > "$output" +FAKE + cat > "$bin/gpg" <<'FAKE' +#!/usr/bin/env bash +set -eu +printf 'gpg:%s\n' "$*" >> "$FAKE_LOG" +[[ "${FAKE_GPG_FAIL:-}" != 1 ]] || exit 2 +case "${FAKE_GPG_CASE:-good}" in + good) + cat <<'EOF' +pub:-:4096:1:8A682D308D4E5E73:1761229688:::-:::scESC::::::23::0: +fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: +uid:-::::1761229688::HASH::CloudFlare Software Packaging 2025 ::::::::::0: +sub:-:4096:1:029E1444B7D9F50F:1761229688::::::e::::::23: +fpr:::::::::06C89DB3B80A8F4349697C76029E1444B7D9F50F: +EOF + ;; + fingerprint) + cat <<'EOF' +pub:-:4096:1:BAD:0:::-:::scESC: +fpr:::::::::0000000000000000000000000000000000000000: +uid:-::::0::HASH::CloudFlare Software Packaging 2025 ::: +EOF + ;; + uid) + cat <<'EOF' +pub:-:4096:1:8A682D308D4E5E73:0:::-:::scESC: +fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: +uid:-::::0::HASH::Attacker ::: +EOF + ;; + multi) + cat <<'EOF' +pub:-:4096:1:8A682D308D4E5E73:0:::-:::scESC: +fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: +uid:-::::0::HASH::CloudFlare Software Packaging 2025 ::: +pub:-:4096:1:BAD:0:::-:::scESC: +fpr:::::::::0000000000000000000000000000000000000000: +uid:-::::0::HASH::Other ::: +EOF + ;; +esac +FAKE + cat > "$bin/apt-get" <<'FAKE' +#!/usr/bin/env bash +set -eu +printf 'apt-get:%s\n' "$*" >> "$FAKE_LOG" +case "${1:-}" in + update) [[ "${FAKE_APT_UPDATE_FAIL:-}" != 1 ]] ;; + install) [[ "${FAKE_APT_INSTALL_FAIL:-}" != 1 ]] ;; + *) exit 0 ;; +esac +FAKE + cat > "$bin/systemctl" <<'FAKE' +#!/usr/bin/env bash +printf 'systemctl:%s\n' "$*" >> "$FAKE_LOG" +exit 0 +FAKE + cat > "$bin/cloudflared" <<'FAKE' +#!/usr/bin/env bash +printf 'cloudflared:%s\n' "$*" >> "$FAKE_LOG" +exit 0 +FAKE + chmod 0755 "$bin"/* +} + +new_case() { + CASE_DIR="$TEST_DIR/case" + rm -rf "$CASE_DIR" + mkdir -p "$CASE_DIR" + export CASE_DIR FAKE_LOG="$CASE_DIR/fake.log" + : > "$FAKE_LOG" + mkdir -p "$CASE_DIR/root/usr/share/keyrings" "$CASE_DIR/root/etc/apt/sources.list.d" \ + "$CASE_DIR/root/var/lib" + chmod 0755 "$CASE_DIR/root" "$CASE_DIR/root/usr" "$CASE_DIR/root/usr/share" \ + "$CASE_DIR/root/usr/share/keyrings" "$CASE_DIR/root/etc" "$CASE_DIR/root/etc/apt" \ + "$CASE_DIR/root/etc/apt/sources.list.d" "$CASE_DIR/root/var" "$CASE_DIR/root/var/lib" + make_fake_commands "$CASE_DIR/bin" + export PATH="$CASE_DIR/bin:$ORIGINAL_PATH" + export CLOUDFLARED_TRUST_ANCHOR="$CASE_DIR" + export CLOUDFLARED_KEYRING="$CASE_DIR/root/usr/share/keyrings/cloudflare-main.gpg" + export CLOUDFLARED_SOURCE_FILE="$CASE_DIR/root/etc/apt/sources.list.d/cloudflared.list" + export CLOUDFLARED_APT_SOURCE_ROOT="$CASE_DIR/root/etc/apt" + export CLOUDFLARED_STATE_DIR="$CASE_DIR/root/var/lib/cloudflared-wrapper" + export CLOUDFLARED_REPOSITORY_STATE_DIR="$CLOUDFLARED_STATE_DIR/repository" + export CLOUDFLARED_LEGACY_BIN="$CASE_DIR/legacy-cloudflared" + export CLOUDFLARED_APT_BIN="$CASE_DIR/usr-bin-cloudflared" + export CLOUDFLARED_LEGACY_UPDATER="$CASE_DIR/cloudflared-update" + export CLOUDFLARED_LEGACY_SERVICE="$CASE_DIR/cloudflared-updater.service" + export CLOUDFLARED_LEGACY_TIMER="$CASE_DIR/cloudflared-updater.timer" + export CLOUDFLARED_AUTO_UPDATE_SCRIPT="$CASE_DIR/cloudflared-apt-update" + export CLOUDFLARED_AUTO_UPDATE_SERVICE="$CASE_DIR/cloudflared-apt-update.service" + export CLOUDFLARED_AUTO_UPDATE_TIMER="$CASE_DIR/cloudflared-apt-update.timer" + export CLOUDFLARED_SERVICE_FILE="$CASE_DIR/cloudflared.service" + export CLOUDFLARED_BINARY_UPDATE_SERVICE="$CASE_DIR/cloudflared-update.service" + export CLOUDFLARED_BINARY_UPDATE_TIMER="$CASE_DIR/cloudflared-update.timer" + export CLOUDFLARED_TEST_INTERNALS=1 + unset FAKE_CURL_FAIL FAKE_GPG_FAIL FAKE_GPG_CASE FAKE_APT_UPDATE_FAIL FAKE_APT_INSTALL_FAIL + unset FAIL_INSTALL_CALL FAIL_RENAME_CALL INSTALL_CALL RENAME_CALL SIGNAL_PHASE + if [[ "${SCRIPT_SOURCED:-}" != 1 ]]; then + # shellcheck source=../tools/cloudflare_tunnel.sh + source "$ROOT_DIR/tools/cloudflare_tunnel.sh" + SCRIPT_SOURCED=1 + fi + init_runtime_config +} + +set_old_generation() { + printf 'old-key\n' > "$CLOUDFLARED_KEYRING" + repository_legacy_source_content > "$CLOUDFLARED_SOURCE_FILE" + chmod 0644 "$CLOUDFLARED_KEYRING" "$CLOUDFLARED_SOURCE_FILE" + cp "$CLOUDFLARED_KEYRING" "$CASE_DIR/expected-key" + cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" +} + +assert_old_generation() { + assert_same "$CLOUDFLARED_KEYRING" "$CASE_DIR/expected-key" "old key was not restored" + assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" "old source was not restored" +} + +run_failure_case() { + local name="$1" setup="$2" action="${3:-configure_repository}" + new_case + set_old_generation + eval "$setup" + set +e + eval "$action" >"$CASE_DIR/out" 2>"$CASE_DIR/err" + status=$? + set -e + if (( status == 0 )); then + fail "$name unexpectedly succeeded" + fi + assert_old_generation + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "$name left lock" + find "$CLOUDFLARED_REPOSITORY_STATE_DIR" -maxdepth 1 -type d -name 'failure-*' -print -quit | grep -q . || + fail "$name did not preserve failure evidence" + pass "$name" +} + +ORIGINAL_PATH=$PATH + +new_case +before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) +repository_source_content > "$CASE_DIR/source-output" +after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) +[[ "$before" == "$after" ]] || fail "source rendering had side effects" +[[ "$(cat "$CASE_DIR/source-output")" == "deb [signed-by=$CLOUDFLARED_KEYRING] https://pkg.cloudflare.com/cloudflared any main" ]] || + fail "source is not exact official definition" diff --git a/tests/lib/test-motd-common.sh b/tests/lib/test-motd-common.sh new file mode 100644 index 0000000..8e681ba --- /dev/null +++ b/tests/lib/test-motd-common.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."&&pwd); +readonly ROOT_DIR +TEST_DIR=$(mktemp -d); +readonly TEST_DIR +fail(){ echo "FAIL: $*" >&2; +exit 1; +}; +pass(){ echo "PASS: $*"; +}; +assert_eq(){ [[ "$1" == "$2" ]]||fail "$3: $1 != $2"; +pass "$3"; +}; +assert_fail(){ local l="$1"; +shift; +if "$@"; +then fail "$l"; +fi; +pass "$l"; +} +extract_payload(){ awk '/install -m 0755 \/dev\/stdin .*<<.SCRIPT./{c=1;next}c&&$0=="SCRIPT"{exit}c{print}' "$1"; +} +load_motd() { + . "$ROOT_DIR/tools/setup-motd.sh" +} + +create_fixture(){ local r="$1" e + e="$r/e"; +e+="tc"; +mkdir -m700 "$r" "$e" "$e/update-motd.d" "$r/lock"; +printf original>"$e/motd"; +chmod 640 "$e/motd"; +printf issue>"$e/issue"; +chmod 644 "$e/issue"; +printf old>"$e/update-motd.d/00-custom-welcome"; +chmod 700 "$e/update-motd.d/00-custom-welcome"; +printf uname>"$e/update-motd.d/10-uname"; +chmod 754 "$e/update-motd.d/10-uname"; +} +export_paths(){ local r="$1" e + e="$r/e"; +e+="tc"; +export MOTD_TEST_MODE=1 MOTD_ETC_ROOT="$e" MOTD_STATE_DIR="$r/state" MOTD_LOCK_FILE="$r/lock/motd.lock" MOTD_TRANSACTION_PARENT="$r/state/transactions"; +} +target_hash(){ local r="$1" e p + e="$r/e"; +e+="tc"; +for p in "$e/motd" "$e/issue" "$e/issue.net" "$e/update-motd.d/00-custom-welcome" "$e/update-motd.d/10-uname" "$e/update-motd.d/50-motd-news"; +do if [[ -L "$p" ]]; +then echo "L|$p|$(readlink "$p")"; +elif [[ -f "$p" ]]; +then echo "F|$p|$(stat -c %u:%g:%a "$p")|$(sha256sum "$p"|awk '{print $1}')"; +elif [[ ! -e "$p" ]]; +then echo "A|$p"; +else echo "O|$p|$(stat -c %F "$p")"; +fi; +done; +} +extract_payload "$ROOT_DIR/tools/setup-motd.sh">"$TEST_DIR/t"; +extract_payload "$ROOT_DIR/modules/system-customize.sh">"$TEST_DIR/m"; +cmp -s "$TEST_DIR/t" "$TEST_DIR/m"||fail payload; diff --git a/tests/lib/test-push-worker-registration-common.sh b/tests/lib/test-push-worker-registration-common.sh new file mode 100644 index 0000000..05cfeab --- /dev/null +++ b/tests/lib/test-push-worker-registration-common.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env bash +set -euo pipefail +# Safe runtime identity only: no environment dump, credentials, or xtrace. +printf "DIAG: bash=%s kernel=%s\n" "$BASH_VERSION" "$(uname -r)" +dpkg-query -W bash libc6 2>/dev/null || true + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) +readonly ROOT_DIR +readonly SCRIPT="$ROOT_DIR/tools/push.sh" +TEST_DIR=$(mktemp -d) +readonly TEST_DIR +CURRENT_FIXTURE_ROOT="" +trap 'rm -rf "$TEST_DIR"' EXIT + +failure_diagnostics() { + local root=${CURRENT_FIXTURE_ROOT:-} file pid + [[ -n "$root" && -d "$root" ]] || return 0 + printf '%s\n' '=== worker registration diagnostics ===' >&2 + printf '%s\n' '--- coordinator/status ---' >&2 + for file in "$root"/capture/coordinator* "$root"/capture/watchdog* "$root"/capture/timeout*; do + [[ -e "$file" || -L "$file" ]] || continue + printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true + done + printf '%s\n' '--- registration/release/stages ---' >&2 + find "$root" \( -name 'worker-registration.*' -o -name '.worker-registration.*' \) -print 2>/dev/null | sort >&2 || true + for file in "$root"/runtime/push-runtime.*/worker-registration.*; do + [[ -e "$file" || -L "$file" ]] || continue + printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true + done + printf '%s\n' '--- worker/session states ---' >&2 + for file in "$root"/runtime/push-runtime.*/worker-session.*.state; do + [[ -e "$file" || -L "$file" ]] || continue + printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true + done + printf '%s\n' '--- jobs ---' >&2; jobs -l >&2 || true + printf '%s\n' '--- runtime tree ---' >&2 + find "$root/runtime" -mindepth 1 -maxdepth 4 -printf '%y %m %p\n' 2>/dev/null | sort >&2 || true + printf '%s\n' '--- process output ---' >&2 + [[ -f "$root/output.log" ]] && cat "$root/output.log" >&2 2>/dev/null || true + printf '%s\n' '--- fake command calls ---' >&2 + for file in "$root"/capture/calls "$root"/capture/transfer-calls; do + [[ -e "$file" ]] && { printf '[%s]\n' "$file" >&2; cat "$file" >&2 || true; } + done + printf '%s\n' '--- recorded proc identities ---' >&2 + for file in "$root"/capture/*.pid; do + [[ -f "$file" && ! -L "$file" ]] || continue + pid=$(<"$file") + [[ "$pid" =~ ^[1-9][0-9]*$ && -r "/proc/$pid/stat" ]] && printf 'pid=%s stat=%s\n' "$pid" "$(<"/proc/$pid/stat")" >&2 + done +} +fail() { failure_diagnostics; printf 'FAIL: %s\n' "$*" >&2; exit 1; } +pass() { printf 'PASS: %s\n' "$*"; } +assert_eq() { [[ "$1" == "$2" ]] || fail "$3: expected '$1', got '$2'"; pass "$3"; } +process_is_running() { local pid="$1" state; [[ -r "/proc/$pid/stat" ]] || return 1; state=$(awk '{print $3}' "/proc/$pid/stat" 2>/dev/null) || return 1; [[ "$state" != Z ]]; } +test_process_identity_exists() { + local pid="$1" start="$2" + read_process_record "$pid" || return 1 + [[ "$PROC_UID" == "$EUID" && "$PROC_START" == "$start" ]] +} +wait_test_process_identity_gone() { + local pid="$1" start="$2" _ + for _ in {1..100}; do test_process_identity_exists "$pid" "$start" || return 0; sleep 0.05; done + return 1 +} +wait_test_process_identity_present() { + local pid="$1" start="$2" _ + for _ in {1..30}; do test_process_identity_exists "$pid" "$start" && return 0; [[ -r "/proc/$pid/stat" ]] || return 1; sleep 0.01; done + return 1 +} +wait_test_process_start() { + local pid="$1" value="" _ + for _ in {1..100}; do value=$(get_process_start_time "$pid" 2>/dev/null || true); [[ -n "$value" ]] && { printf '%s\n' "$value"; return 0; }; sleep 0.005; done + return 1 +} + +source_root="$TEST_DIR/source"; mkdir -m 0700 "$source_root" +source_result=$(env TMPDIR="$source_root" SCRIPT="$SCRIPT" bash -c ' +set -euo pipefail +before=$(find "$TMPDIR" -mindepth 1 -print -quit) +source "$SCRIPT" +[[ -z "$TEMP_DIR" && -z "$SUCCESS_FILE" && -z "$FAILED_FILE" ]] +[[ "$before" == "$(find "$TMPDIR" -mindepth 1 -print -quit)" ]] +printf source-safe +') +# shellcheck source=../tools/push.sh +source "$SCRIPT" diff --git a/tests/run.sh b/tests/run.sh index dbb2aa4..48f7830 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -14,6 +14,14 @@ case "$shard" in *) printf 'Invalid test shard: %s\n' "$shard" >&2; exit 2 ;; esac +# Suites split across several files are named -NN.sh; every part belongs +# to the family's shard and every part is selected by a change to the family. +family_of() { + local name=${1##*/} + name=${name%.sh} + printf '%s.sh\n' "${name%-[0-9][0-9]}" +} + declare -A selected=() full=true if [[ -n "${TEST_BASE_SHA:-}" && -n "${TEST_HEAD_SHA:-}" ]]; then @@ -29,10 +37,11 @@ if [[ -n "${TEST_BASE_SHA:-}" && -n "${TEST_HEAD_SHA:-}" ]]; then tools/push.sh) selected[test-push.sh]=1; selected[test-push-worker-registration.sh]=1 ;; tools/cloudflare_tunnel.sh) selected[test-cloudflare-tunnel.sh]=1 ;; modules/ssh-security.sh) selected[test-ssh-security.sh]=1 ;; + modules/system-optimize.sh) selected[test-system-optimize.sh]=1 ;; linux_setup.sh) selected[test-linux-setup.sh]=1; selected[test-setup-exec.sh]=1 ;; modules/zsh-setup.sh|p10k-config.zsh) selected[test-linux-setup.sh]=1 ;; - tests/test-*.sh) - if [[ -f "$ROOT_DIR/$path" ]]; then selected["${path##*/}"]=1; else full=true; fi ;; + tests/test-*.sh|tests/lib/*.sh) + if [[ -f "$ROOT_DIR/$path" ]]; then selected["$(family_of "$path")"]=1; else full=true; fi ;; README.md|LICENSE) ;; *) full=true ;; esac @@ -43,13 +52,19 @@ if [[ -n "${TEST_BASE_SHA:-}" && -n "${TEST_HEAD_SHA:-}" ]]; then fi printf 'Test selection: full=%s\n' "$full" +# Bounded parallelism: 1 keeps the original serial output byte for byte. +jobs=${TEST_JOBS:-1} +[[ "$jobs" =~ ^[1-9][0-9]*$ ]] || { printf 'Invalid TEST_JOBS: %s\n' "$jobs" >&2; exit 2; } + +plan=() for test_file in "$ROOT_DIR"/tests/test-*.sh; do - if [[ "$full" != true && -z "${selected[${test_file##*/}]:-}" ]]; then - printf 'SKIP: %s (unaffected by PR diff)\n' "${test_file##*/}" + test_name=$(basename "$test_file") + family=$(family_of "$test_name") + if [[ "$full" != true && -z "${selected[${family}]:-}" ]]; then + printf 'SKIP: %s (unaffected by PR diff)\n' "$test_name" continue fi - test_name=$(basename "$test_file") - case "$test_name" in + case "$family" in test-motd.sh) suite_shard=motd ;; test-push.sh|test-push-worker-registration.sh) suite_shard=push ;; *) suite_shard=other ;; @@ -58,21 +73,62 @@ for test_file in "$ROOT_DIR"/tests/test-*.sh; do printf 'SKIP: %s (assigned to %s shard)\n' "$test_name" "$suite_shard" continue fi - printf '\n==> %s\n' "$test_name" + plan+=("$test_file") +done + +run_suite() { + local test_file=$1 test_name started status + test_name=$(basename "$test_file") started=$SECONDS + printf '\n==> %s\n' "$test_name" # Keep the child shell standalone: an if/|| wrapper can change errexit behavior. set +e bash "$test_file" status=$? set -e - elapsed=$((SECONDS - started)) + local elapsed=$((SECONDS - started)) printf 'TIMING: %s elapsed=%ss exit=%s\n' "$test_name" "$elapsed" "$status" if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then printf -- '- `%s`: %ss, exit %s\n' "$test_name" "$elapsed" "$status" >> "$GITHUB_STEP_SUMMARY" fi - if (( status != 0 )); then - exit "$status" - fi -done + return "$status" +} + +if (( ${#plan[@]} == 0 )); then + printf '\nAll selected shell tests passed.\n' + exit 0 +fi + +if (( jobs == 1 )); then + for test_file in "${plan[@]}"; do + run_suite "$test_file" || exit "$?" + done +else + log_dir=$(mktemp -d) + trap 'rm -rf "$log_dir"' EXIT + running=0 + for test_file in "${plan[@]}"; do + test_name=$(basename "$test_file") + ( + rc=0 + run_suite "$test_file" > "$log_dir/$test_name.log" 2>&1 || rc=$? + printf '%s' "$rc" > "$log_dir/$test_name.rc" + ) & + running=$((running + 1)) + if (( running >= jobs )); then + wait -n || true + running=$((running - 1)) + fi + done + wait || true + failed=0 + for test_file in "${plan[@]}"; do + test_name=$(basename "$test_file") + cat "$log_dir/$test_name.log" + rc=$(cat "$log_dir/$test_name.rc" 2>/dev/null || printf 1) + (( rc == 0 )) || failed=1 + done + (( failed == 0 )) || exit 1 +fi printf '\nAll selected shell tests passed.\n' diff --git a/tests/test-cloudflare-tunnel-01.sh b/tests/test-cloudflare-tunnel-01.sh new file mode 100644 index 0000000..c1e38b5 --- /dev/null +++ b/tests/test-cloudflare-tunnel-01.sh @@ -0,0 +1,349 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-cloudflare-tunnel-common.sh" +pass "source rendering has zero side effects and exact fields" + +( + new_case + source_cwd=$PWD + source_before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) + cloudflare_tunnel_source() { printf 'sentinel:%s\n' "$*"; } + sentinel_before=$(declare -f cloudflare_tunnel_source) + trap 'sentinel-trap' HUP + set -o noclobber + umask 027 + unset CLOUDFLARED_TEST_INTERNALS + source "$ROOT_DIR/tools/cloudflare_tunnel.sh" + sentinel_after=$(declare -f cloudflare_tunnel_source) + [[ "$sentinel_before" == "$sentinel_after" ]] || fail "ordinary source replaced sentinel function" + [[ "$(cloudflare_tunnel_source probe)" == sentinel:probe ]] || fail "ordinary source changed sentinel behavior" + [[ "$(trap -p HUP)" == *sentinel-trap* ]] || fail "ordinary source changed caller trap" + [[ "$-" == *C* ]] || fail "ordinary source changed caller shell options" + [[ "$(umask)" == 0027 ]] || fail "ordinary source changed caller umask" + [[ "$PWD" == "$source_cwd" ]] || fail "ordinary source changed caller cwd" + [[ "${CLOUDFLARED_STATE_DIR+x}" == x ]] || fail "test setup lost state variable" + after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) + [[ "$source_before" == "$after" ]] || fail "ordinary source changed filesystem" +) +pass "ordinary source preserves sentinels, shell state, and filesystem" + +new_case +before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\\n' | sort) +bash "$ROOT_DIR/tools/cloudflare_tunnel.sh" status >/dev/null 2>&1 || fail "standalone status failed in fake environment" +after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\\n' | sort) +[[ "$before" == "$after" ]] || fail "ordinary status created state, lock, or temp files" +pass "ordinary help/status paths have zero state and lock side effects" + +cat > "$TEST_DIR/source-contract.sh" </dev/null' _ \ + "$ROOT_DIR/tools/cloudflare_tunnel.sh" || fail "internal source mode unavailable" +pass "explicit internal source mode exposes test functions" + +new_case +configure_repository || fail "configure before real uninstall failed" +require_root() { :; } +check_platform() { :; } +uninstall_cloudflared --confirmed || fail "uninstall normal path failed" +[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall left repository lock" +grep -Fxq 'apt-get:remove -y cloudflared' "$FAKE_LOG" || fail "uninstall skipped package removal" +assert_absent "$CLOUDFLARED_SOURCE_FILE" "real uninstall retained source" +assert_absent "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "real uninstall retained current" +[[ -f "$CLOUDFLARED_KEYRING" ]] || fail "real uninstall removed keyring" +pass "configure then real uninstall preserves keyring and releases lock" +init_runtime_config + +new_case +mkdir -p "$CLOUDFLARED_STATE_DIR.lock" +mkdir -p "$CLOUDFLARED_AUTO_UPDATE_SCRIPT" +printf marker > "$CASE_DIR/marker" +require_root() { :; } +check_platform() { :; } +if uninstall_cloudflared --confirmed >/dev/null 2>&1; then fail "uninstall lock competition unexpectedly succeeded"; fi +[[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall lock competition removed lock" +[[ -d "$CLOUDFLARED_AUTO_UPDATE_SCRIPT" ]] || fail "lock competition touched auto-update path" +! grep -Eq 'remove|disable|daemon-reload|apt-get:' "$FAKE_LOG" || fail "lock competition performed mutation" +pass "uninstall lock competition fails before auto-update, systemctl, and APT" + +new_case +mkdir -p "$CLOUDFLARED_STATE_DIR.lock" +if disable_auto_update --confirmed >/dev/null 2>&1; then fail "disable-auto-update lock competition unexpectedly succeeded"; fi +[[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "disable-auto-update removed competing lock" +pass "auto-update lock competition fails before state mutation" +chmod 0700 "$CLOUDFLARED_STATE_DIR.lock" +if configure_repository >/dev/null 2>&1; then fail "lock competition unexpectedly succeeded"; fi +pass "key/source lock competition" + +run_failure_case "key URL download failure" 'export FAKE_CURL_FAIL=1' +run_failure_case "gpg parser failure" 'export FAKE_GPG_FAIL=1' +run_failure_case "fingerprint mismatch" 'export FAKE_GPG_CASE=fingerprint' +run_failure_case "UID mismatch" 'export FAKE_GPG_CASE=uid' +run_failure_case "multiple primary keys" 'export FAKE_GPG_CASE=multi' + +for target in key source; do + for kind in symlink directory; do + new_case + case "$target" in key) path=$CLOUDFLARED_KEYRING ;; source) path=$CLOUDFLARED_SOURCE_FILE ;; esac + case "$kind" in + symlink) ln -s "$CASE_DIR/missing" "$path" ;; + directory) mkdir "$path" ;; + esac + if configure_repository >/dev/null 2>&1; then fail "$target $kind accepted"; fi + [[ -L "$path" || -d "$path" ]] || fail "$target $kind was changed" + pass "$target rejects $kind" + done + for bad in owner gid mode; do + new_case + case "$target" in key) path=$CLOUDFLARED_KEYRING ;; source) path=$CLOUDFLARED_SOURCE_FILE ;; esac + if [[ "$target" == source ]]; then + repository_source_content > "$path" + else + printf key > "$path" + fi + chmod 0644 "$path" + case "$bad" in + owner) chown 65534:0 "$path" ;; + gid) chown 0:65534 "$path" ;; + mode) chmod 0664 "$path" ;; + esac + if configure_repository >/dev/null 2>&1; then fail "$target wrong $bad accepted"; fi + pass "$target rejects wrong $bad" + done +done + +new_case +printf 'deb https://example.invalid stable main\n' > "$CASE_DIR/root/etc/apt/sources.list.d/extra.list" +chmod 0644 "$CASE_DIR/root/etc/apt/sources.list.d/extra.list" +configure_repository +pass "unrelated APT source remains untouched" + +new_case +printf 'deb https://pkg.cloudflare.com/cloudflared any main\n' > "$CASE_DIR/root/etc/apt/sources.list.d/duplicate.list" +chmod 0644 "$CASE_DIR/root/etc/apt/sources.list.d/duplicate.list" +if configure_repository >/dev/null 2>&1; then fail "duplicate Cloudflare source accepted"; fi +assert_absent "$CLOUDFLARED_KEYRING" "duplicate source changed key" +assert_absent "$CLOUDFLARED_SOURCE_FILE" "duplicate source wrote managed source" +pass "reject duplicate or extra Cloudflare source before commit" + +new_case +printf 'deb [signed-by=%s trusted=yes] https://pkg.cloudflare.com/cloudflared any main\n' \ + "$CLOUDFLARED_KEYRING" > "$CLOUDFLARED_SOURCE_FILE" +chmod 0644 "$CLOUDFLARED_SOURCE_FILE" +if configure_repository >/dev/null 2>&1; then fail "source injection accepted"; fi +pass "reject unknown source fields and injection content" + +( +new_case +set_old_generation +INSTALL_CALL=0 +FAIL_INSTALL_CALL=1 +repository_install_file() { + INSTALL_CALL=$((INSTALL_CALL + 1)) + (( INSTALL_CALL != FAIL_INSTALL_CALL )) || return 1 + command install -o 0 -g 0 -m "$1" -- "$2" "$3" +} +if configure_repository >/dev/null 2>&1; then fail "key stage write failure succeeded"; fi +assert_old_generation +) +pass "formal key write failure rolls back" + +( +new_case +set_old_generation +INSTALL_CALL=0 +FAIL_INSTALL_CALL=2 +repository_install_file() { + INSTALL_CALL=$((INSTALL_CALL + 1)) + (( INSTALL_CALL != FAIL_INSTALL_CALL )) || return 1 + command install -o 0 -g 0 -m "$1" -- "$2" "$3" +} +if configure_repository >/dev/null 2>&1; then fail "source stage write failure succeeded"; fi +assert_old_generation +) +pass "formal source write failure rolls back" + +( +new_case +set_old_generation +RENAME_CALL=0 +FAIL_RENAME_CALL=2 +repository_rename() { + RENAME_CALL=$((RENAME_CALL + 1)) + (( RENAME_CALL != FAIL_RENAME_CALL )) || return 1 + command mv -fT -- "$1" "$2" +} +if configure_repository >/dev/null 2>&1; then fail "source commit failure succeeded"; fi +assert_old_generation +) +pass "key success/source failure rolls back" + +( +new_case +set_old_generation +RENAME_CALL=0 +FAIL_RENAME_CALL=1 +repository_rename() { + RENAME_CALL=$((RENAME_CALL + 1)) + (( RENAME_CALL != FAIL_RENAME_CALL )) || return 1 + command mv -fT -- "$1" "$2" +} +if configure_repository >/dev/null 2>&1; then fail "key commit failure succeeded"; fi +assert_old_generation +) +pass "source remains old when key commit fails" + +run_failure_case "APT probe failure rollback" 'export FAKE_APT_UPDATE_FAIL=1' 'run_repository_apt_transaction install' +run_failure_case "APT install failure rollback" 'export FAKE_APT_INSTALL_FAIL=1' 'run_repository_apt_transaction install' + +for signal in HUP INT TERM; do + case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + for phase in download validate stage key-commit source-commit apt-probe apt-install; do + new_case + set_old_generation + cat > "$CASE_DIR/signal-runner.sh" <"$CASE_DIR/out" 2>"$CASE_DIR/err" + status=$? + set -e + [[ "$status" == "$expected" ]] || fail "$signal at $phase returned $status, expected $expected" + assert_old_generation + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "$signal at $phase left lock" + pass "$signal=$expected rolls back at $phase" + done +done + +( +new_case +set_old_generation +RENAME_CALL=0 +repository_rename() { + RENAME_CALL=$((RENAME_CALL + 1)) + case "$RENAME_CALL" in 2|3) return 1 ;; esac + command mv -fT -- "$1" "$2" +} +if configure_repository >/dev/null 2>&1; then fail "rollback failure unexpectedly succeeded"; fi +find "$CLOUDFLARED_REPOSITORY_STATE_DIR" -maxdepth 2 -type f -name rollback.log -print -quit | grep -q . || + fail "rollback failure did not preserve evidence" +) +pass "rollback failure returns nonzero and preserves evidence" + +new_case +set_old_generation +configure_repository +current="$CLOUDFLARED_REPOSITORY_STATE_DIR/current" +[[ -f "$current" ]] || fail "generation marker missing" +key_hash=$(sha256sum "$CLOUDFLARED_KEYRING" | awk '{print $1}') +printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflared any main\n' "$CLOUDFLARED_KEYRING" > "$CASE_DIR/independent-source" +assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/independent-source" "source differs from official literal" +source_hash=$(sha256sum "$CASE_DIR/independent-source" | awk '{print $1}') +grep -Fxq "key_sha256=$key_hash" "$current" || fail "key generation hash mismatch" +grep -Fxq "source_sha256=$source_hash" "$current" || fail "source generation hash mismatch" +pass "key/source commit as one recorded generation" + +new_case +set_old_generation +configure_repository +validate_current_repository_manifest || fail "trusted current manifest validation failed" +remove_managed_repository +init_runtime_config +assert_absent "$CLOUDFLARED_SOURCE_FILE" "trusted current did not remove managed source" +[[ -f "$CLOUDFLARED_KEYRING" ]] || fail "managed repository removal deleted keyring" +[[ ! -f "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" ]] || fail "managed repository removal retained current" +find "$CLOUDFLARED_STATE_DIR" -maxdepth 1 -type d -name 'uninstall-*' -print -quit | grep -q . || + fail "managed source backup missing" +[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "managed repository removal left lock" +pass "configure then real repository uninstall removes source and preserves keyring" + +for signal in HUP INT TERM; do + case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + for phase in before-snapshot-create capture-auto_update_script capture-auto_update_timer capture-legacy_marker lock-acquired before-disable-auto-update after-disable-auto-update before-apt-remove after-apt-remove before-source-remove after-source-remove final-apt-update before-lock-release; do + new_case + set_old_generation + configure_repository + cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" + cp "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "$CASE_DIR/expected-current" + set +e + CLOUDFLARED_TEST_INTERNALS=1 SIGNAL_PHASE="$phase" SIGNAL_NAME="$signal" \ + bash -c ' + source "$1" + require_root() { :; } + check_platform() { :; } + uninstall_transaction_hook() { + [[ "$1" != "$SIGNAL_PHASE" ]] || kill -s "$SIGNAL_NAME" "$BASHPID" + } + uninstall_cloudflared --confirmed + ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" >"$CASE_DIR/out" 2>"$CASE_DIR/err" + status=$? + set -e + [[ "$status" == "$expected" ]] || fail "uninstall $signal at $phase returned $status" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall $signal at $phase left lock" + assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" "uninstall $signal at $phase failed source restore" + assert_same "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "$CASE_DIR/expected-current" "uninstall $signal at $phase failed current restore" + acquire_repository_lock || fail "uninstall $signal at $phase lock not reusable" + release_repository_lock || fail "uninstall $signal at $phase lock cleanup failed" + pass "uninstall $signal=$expected cleans $phase" + done +done + +new_case +set_old_generation +configure_repository +uninstall_transaction_hook() { [[ "$1" != after-disable-auto-update ]] || exit 0; } +set +e +( uninstall_cloudflared --confirmed ) >"$CASE_DIR/out" 2>"$CASE_DIR/err" +status=$? +set -e +[[ "$status" == 1 ]] || fail "active uninstall exit 0 returned $status" +[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "active uninstall exit 0 left lock" +grep -R -Fq '活动卸载事务异常退出' "$CLOUDFLARED_REPOSITORY_STATE_DIR" || fail "active uninstall exit 0 lacked evidence" \ No newline at end of file diff --git a/tests/test-cloudflare-tunnel-02.sh b/tests/test-cloudflare-tunnel-02.sh new file mode 100644 index 0000000..1b0cec1 --- /dev/null +++ b/tests/test-cloudflare-tunnel-02.sh @@ -0,0 +1,380 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-cloudflare-tunnel-common.sh" + +pass "active uninstall exit 0 returns 1 and cleans transaction" + +for manifest_case in missing duplicate unknown generation key-digest source-digest source-content symlink directory fifo owner gid mode; do + new_case + set_old_generation + configure_repository + current="$CLOUDFLARED_REPOSITORY_STATE_DIR/current" + case "$manifest_case" in + missing) sed -i '/^source_sha256=/d' "$current" ;; + duplicate) cat "$current" >> "$current.copy"; cat "$current.copy" >> "$current"; rm "$current.copy" ;; + unknown) printf 'unknown=value\n' >> "$current" ;; + generation) sed -i 's/^generation=.*/generation=invalid/' "$current" ;; + key-digest) sed -i 's/^key_sha256=.*/key_sha256=0000000000000000000000000000000000000000000000000000000000000000/' "$current" ;; + source-digest) sed -i 's/^source_sha256=.*/source_sha256=0000000000000000000000000000000000000000000000000000000000000000/' "$current" ;; + source-content) printf '# tampered\n' >> "$CLOUDFLARED_SOURCE_FILE" ;; + symlink) rm "$current"; ln -s "$CASE_DIR/external" "$current"; printf external > "$CASE_DIR/external" ;; + directory) rm "$current"; mkdir "$current" ;; + fifo) rm "$current"; mkfifo "$current" ;; + owner) chown 65534:0 "$current" ;; + gid) chown 0:65534 "$current" ;; + mode) chmod 0644 "$current" ;; + esac + if remove_managed_repository >/dev/null 2>&1; then fail "current $manifest_case unexpectedly trusted"; fi + [[ -f "$CLOUDFLARED_KEYRING" ]] || fail "current $manifest_case deleted keyring" + [[ -e "$CLOUDFLARED_SOURCE_FILE" ]] || fail "current $manifest_case deleted source" + [[ ! -e "$CASE_DIR/external" || "$(cat "$CASE_DIR/external")" == external ]] || fail "current symlink target changed" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "current $manifest_case left lock" + pass "current manifest rejects $manifest_case" +done + +new_case +set_old_generation +export FAKE_APT_INSTALL_FAIL=1 +if run_repository_apt_transaction install >/dev/null 2>&1; then fail "old generation rollback test succeeded"; fi +assert_old_generation +pass "old key/source preserved on transaction failure" + +new_case +run_repository_apt_transaction install +grep -Fxq 'apt-get:update' "$FAKE_LOG" || fail "APT probe not called" +grep -Fxq 'apt-get:install -y cloudflared' "$FAKE_LOG" || fail "cloudflared package selection changed" +if grep -Eqi 'remove|purge|autoremove|linux-image|linux-headers' "$FAKE_LOG"; then + fail "repository transaction performed unrelated APT changes" +fi +pass "no kernel deletion or unrelated APT change" + +script_hashes_before=$(sha256sum "$ROOT_DIR/linux_setup.sh" "$ROOT_DIR"/modules/*.sh "$ROOT_DIR"/tools/push.sh \ + "$ROOT_DIR"/tools/setup-motd.sh "$ROOT_DIR"/tools/xanmod-install.sh) +script_hashes_after=$(sha256sum "$ROOT_DIR/linux_setup.sh" "$ROOT_DIR"/modules/*.sh "$ROOT_DIR"/tools/push.sh \ + "$ROOT_DIR"/tools/setup-motd.sh "$ROOT_DIR"/tools/xanmod-install.sh) +[[ "$script_hashes_before" == "$script_hashes_after" ]] || fail "payload or unrelated scripts changed during tests" +pass "payload and unrelated scripts remain unchanged" + +# Keep prior lifecycle coverage with fake systemctl only. +new_case +cat > "$CLOUDFLARED_SERVICE_FILE" < "$CLOUDFLARED_LEGACY_BIN" <<'EOF' +#!/usr/bin/env bash +echo 'cloudflared version 2025.1.0' +EOF +chmod 0755 "$CLOUDFLARED_LEGACY_BIN" +migrate_legacy_binary +assert_absent "$CLOUDFLARED_LEGACY_BIN" "recognized legacy binary was not migrated" +grep -Fq "ExecStart=$CLOUDFLARED_APT_BIN --no-autoupdate --token-file /etc/cloudflared/token" "$CLOUDFLARED_SERVICE_FILE" || + fail "legacy service path was not preserved" +pass "legacy binary and service migration behavior remains" + + +# Each malformed snapshot runs in a separate process: transaction traps and +# intentional corrupt fixtures must not leak into subsequent cases. +for schema_case in version generation duplicate unknown order state mode uid gid digest created payload-name payload-missing payload-extra payload-fifo payload-symlink manifest-fifo ancestor; do + ( + new_case + configure_repository || fail "schema fixture configure" + acquire_repository_lock || fail "schema fixture lock" + begin_uninstall_transaction || fail "schema fixture capture" + manifest="$UNINSTALL_SNAPSHOT_DIR/manifest" + case "$schema_case" in + version) sed -i '1d' "$manifest" ;; + generation) sed -i '2s/.*/generation=20000101T000000Z-1-1/' "$manifest" ;; + duplicate) sed -i '3s/.*/version=1/' "$manifest" ;; + unknown) sed -i '10s/.*/path=\/tmp\/forbidden/' "$manifest" ;; + order) sed -i '3s/auto_update_script/auto_update_timer/' "$manifest" ;; + state) sed -i '4s/.*/state=invalid/' "$manifest" ;; + mode) sed -i '5s/.*/mode=777/' "$manifest" ;; + uid) sed -i '6s/.*/uid=1/' "$manifest" ;; + gid) sed -i '7s/.*/gid=1/' "$manifest" ;; + digest) sed -i '8s/.*/sha256=invalid/' "$manifest" ;; + created) sed -i '0,/^transaction_created=false$/s//transaction_created=invalid/' "$manifest" ;; + payload-name) sed -i 's/^snapshot=target-source$/snapshot=target-current/' "$manifest" ;; + payload-missing) rm -- "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ;; + payload-extra) printf extra > "$UNINSTALL_SNAPSHOT_DIR/files/target-extra"; chmod 600 "$UNINSTALL_SNAPSHOT_DIR/files/target-extra" ;; + payload-fifo) mkfifo "$UNINSTALL_SNAPSHOT_DIR/files/.fifo" ;; + payload-symlink) ln -s target-source "$UNINSTALL_SNAPSHOT_DIR/files/.link" ;; + manifest-fifo) rm -- "$manifest"; mkfifo "$manifest" ;; + ancestor) chmod 0777 "$UNINSTALL_SNAPSHOT_DIR/files" ;; + esac + if uninstall_manifest_valid "$manifest"; then fail "schema accepted $schema_case"; fi + # No rollback is appropriate for this read-only parser test. + UNINSTALL_TRANSACTION_ACTIVE=false + UNINSTALL_SNAPSHOT_BUILDING=false + UNINSTALL_TRANSACTION_STATE=NONE + restore_repository_traps + # Parent alone owns TEST_DIR cleanup; do not run its EXIT trap here. + trap - EXIT + release_repository_lock || fail "schema fixture release" + ) > "$TEST_DIR/schema-$schema_case.log" 2>&1 || fail "schema $schema_case (see $TEST_DIR/schema-$schema_case.log)" + pass "uninstall manifest rejects $schema_case" +done + + +for ownership_case in owned foreign inode mode owner gid symlink directory fifo; do + ( + new_case + uninstall_transaction_hook() { :; } + trap - EXIT + configure_repository || fail "ownership configure" + acquire_repository_lock || fail "ownership lock" + begin_uninstall_transaction || fail "ownership capture" + target="$AUTO_UPDATE_SCRIPT" + printf 'owned-content\n' > "$CASE_DIR/payload" + chmod 0600 "$CASE_DIR/payload" + if [[ "$ownership_case" == foreign ]]; then + printf foreign > "$target" + chmod 0755 "$target" + else + uninstall_create_absent_target auto_update_script "$CASE_DIR/payload" || fail "exclusive creation" + case "$ownership_case" in + inode) mv "$target" "$target.original"; cp "$target.original" "$target" ;; + mode) chmod 0644 "$target" ;; + owner) chown 65534:0 "$target" ;; + gid) chown 0:65534 "$target" ;; + symlink) rm "$target"; ln -s "$CASE_DIR/payload" "$target" ;; + directory) rm "$target"; mkdir "$target" ;; + fifo) rm "$target"; mkfifo "$target" ;; + esac + fi + if [[ "$ownership_case" == owned ]]; then + uninstall_cleanup ownership-test || fail "owned rollback" + assert_absent "$target" "owned file survived" + else + if uninstall_cleanup ownership-test; then fail "foreign identity accepted"; fi + [[ -e "$target" || -L "$target" ]] || fail "foreign object deleted" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" ]] || fail "manifest lost" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ]] || fail "payload lost" + fi + trap - EXIT + ) > "$TEST_DIR/ownership-$ownership_case.log" 2>&1 || { cat "$TEST_DIR/ownership-$ownership_case.log"; fail "ownership $ownership_case"; } + pass "absent ownership $ownership_case" +done + +for journal_case in generation duplicate state unknown reason; do + ( + new_case + trap - EXIT + uninstall_transaction_hook() { :; } + configure_repository || fail "journal configure" + acquire_repository_lock || fail "journal lock" + begin_uninstall_transaction || fail "journal capture" + journal="$UNINSTALL_SNAPSHOT_DIR/journal" + case "$journal_case" in + generation) sed -i '2s/.*/generation=wrong/' "$journal" ;; + duplicate) printf 'state=ACTIVE\n' >> "$journal" ;; + state) sed -i '3s/.*/state=COMMITTED/' "$journal" ;; + unknown) sed -i '1s/.*/unknown=1/' "$journal" ;; + reason) sed -i '4s/.*/path=invalid/' "$journal" ;; + esac + if uninstall_journal_valid "$journal" ACTIVE; then fail "bad journal accepted"; fi + UNINSTALL_TRANSACTION_STATE=FAILED + UNINSTALL_TRANSACTION_ACTIVE=false + UNINSTALL_SNAPSHOT_BUILDING=false + restore_repository_traps + release_repository_lock || fail "journal release" + ) > "$TEST_DIR/journal-$journal_case.log" 2>&1 || fail "journal $journal_case" + pass "journal rejects $journal_case" +done + +for finalization_case in journal archive release restore; do + ( + new_case + trap - EXIT + uninstall_transaction_hook() { :; } + configure_repository || fail "finalization configure" + acquire_repository_lock || fail "finalization lock" + begin_uninstall_transaction || fail "finalization capture" + snapshot_before="$UNINSTALL_SNAPSHOT_DIR" + case "$finalization_case" in + journal|archive) + repository_rename() { + if [[ "$finalization_case" == journal && "$2" == */journal ]] || + [[ "$finalization_case" == archive && "$1" == "$snapshot_before" ]]; then return 1; fi + command mv -fT -- "$1" "$2" + } + ;; + release) + # A nonempty lock is an actual rmdir failure, not a core stub. + printf retained > "$REPOSITORY_LOCK_DIR/injected" + ;; + restore) + repository_install_file() { + [[ "$2" != */target-source ]] || return 1 + command install -o 0 -g 0 -m "$1" -- "$2" "$3" + } + ;; + esac + if uninstall_cleanup "injected-$finalization_case"; then fail "finalization failure accepted"; fi + [[ "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "failed guard state" + [[ "$UNINSTALL_TRANSACTION_ACTIVE" == false ]] || fail "active guard survived" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" ]] || fail "manifest deleted" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ]] || fail "source payload deleted" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-current" ]] || fail "current payload deleted" + cmp "$SOURCE_FILE" "$UNINSTALL_SNAPSHOT_DIR/files/target-source" || fail "source changed" + [[ -f "$KEYRING" ]] || fail "keyring deleted" + if [[ "$finalization_case" == release ]]; then + [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "lock evidence missing" + fi + ) > "$TEST_DIR/finalization-$finalization_case.log" 2>&1 || { + cat "$TEST_DIR/finalization-$finalization_case.log" + fail "finalization $finalization_case" + } + pass "finalization $finalization_case failure preserves complete evidence" +done + +for capture_id in auto_update_script auto_update_timer legacy_marker; do + ( + new_case + trap - EXIT + configure_repository || fail "capture configure" + cp "$SOURCE_FILE" "$CASE_DIR/source-before" + cp "$KEYRING" "$CASE_DIR/key-before" + acquire_repository_lock || fail "capture lock" + uninstall_transaction_hook() { [[ "$1" != "capture-$capture_id" ]]; } + if begin_uninstall_transaction; then fail "capture fault ignored"; fi + [[ "$UNINSTALL_TRANSACTION_STATE" == BUILDING ]] || fail "partial capture activated" + if uninstall_cleanup capture-failed; then fail "partial capture reported recovery"; fi + cmp "$SOURCE_FILE" "$CASE_DIR/source-before" || fail "partial source changed" + cmp "$KEYRING" "$CASE_DIR/key-before" || fail "partial key changed" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/journal" ]] || fail "partial journal missing" + [[ "$UNINSTALL_TRANSACTION_ACTIVE" == false ]] || fail "partial guard active" + ) > "$TEST_DIR/capture-$capture_id.log" 2>&1 || { + cat "$TEST_DIR/capture-$capture_id.log" + fail "capture $capture_id" + } + pass "capture failure $capture_id retains partial evidence without restoration" +done + +for archive_case in intact payload journal; do + ( + new_case + trap - EXIT + uninstall_transaction_hook() { :; } + configure_repository || fail "archive configure" + acquire_repository_lock || fail "archive lock" + begin_uninstall_transaction || fail "archive capture" + uninstall_transaction_hook() { + [[ "$1" == after-evidence-rename ]] || return 0 + case "$archive_case" in + payload) printf corrupt >> "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ;; + journal) printf 'unknown=1\n' >> "$UNINSTALL_SNAPSHOT_DIR/journal" ;; + esac + return 0 + } + rc=0 + uninstall_cleanup archive-verification || rc=$? + if [[ "$archive_case" == intact ]]; then + [[ "$rc" == 0 && "$UNINSTALL_TRANSACTION_STATE" == ROLLED_BACK ]] || fail "valid archive rejected" + else + [[ "$rc" != 0 && "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "corrupt archive accepted" + fi + [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" && -f "$UNINSTALL_SNAPSHOT_DIR/journal" ]] || fail "archive evidence lost" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-current" ]] || fail "archive payload lost" + ) > "$TEST_DIR/archive-$archive_case.log" 2>&1 || { + cat "$TEST_DIR/archive-$archive_case.log" + fail "archive $archive_case" + } + pass "post-rename archive verification $archive_case" +done + +for interrupted_phase in before-snapshot-create after-snapshot-active after-evidence-rename after-evidence-seal before-pending-clear; do + ( + new_case + trap - EXIT + configure_repository || fail "kill configure" + cp "$SOURCE_FILE" "$CASE_DIR/source-before" + ( + trap - EXIT + acquire_repository_lock || exit 1 + uninstall_transaction_hook() { + if [[ "$1" == "$interrupted_phase" ]]; then kill -KILL "$BASHPID"; fi + } + begin_uninstall_transaction || exit 1 + uninstall_cleanup kill-test + ) > "$CASE_DIR/killed.log" 2>&1 & + child=$! + rc=0 + wait "$child" || rc=$? + [[ "$rc" == 137 ]] || fail "SIGKILL did not reach $interrupted_phase" + # Simulate operator removing only the stale lock; pending must still block. + rmdir "$CLOUDFLARED_STATE_DIR.lock" || fail "stale lock remove" + : > "$FAKE_LOG" + for operation in install_cloudflared upgrade_cloudflared uninstall_cloudflared disable_auto_update; do + ( + trap - EXIT + "$operation" --confirmed + ) > "$CASE_DIR/retry-$operation.log" 2>&1 && fail "pending allowed $operation" + done + [[ ! -s "$FAKE_LOG" ]] || fail "pending executed external command" + cmp "$SOURCE_FILE" "$CASE_DIR/source-before" || fail "pending changed source" + compgen -G "$CLOUDFLARED_STATE_DIR/repository/pending-uninstall-*" >/dev/null || fail "pending evidence missing" + ) > "$TEST_DIR/kill-$interrupted_phase.log" 2>&1 || { + cat "$TEST_DIR/kill-$interrupted_phase.log" + fail "SIGKILL $interrupted_phase" + } + pass "SIGKILL $interrupted_phase fails closed in independent entrypoints" +done + +for stage_case in owned foreign; do + ( + new_case + trap - EXIT + configure_repository || fail "stage configure" + acquire_repository_lock || fail "stage lock" + begin_uninstall_transaction || fail "stage capture" + stage=$(mktemp "$CASE_DIR/root/.cloudflared-create.XXXXXX") + uninstall_register_stage "$stage" || fail "stage register" + if [[ "$stage_case" == foreign ]]; then + mv "$stage" "$stage.original" + printf foreign > "$stage" + fi + rc=0 + uninstall_cleanup stage-test || rc=$? + if [[ "$stage_case" == owned ]]; then + [[ "$rc" == 0 && ! -e "$stage" ]] || fail "owned stage not cleaned" + [[ "$(stat -c %a "$UNINSTALL_SNAPSHOT_DIR")" == 500 ]] || fail "archive writable" + [[ "$(stat -c %a "$UNINSTALL_SNAPSHOT_DIR/manifest")" == 400 ]] || fail "manifest writable" + validate_uninstall_archive "$UNINSTALL_SNAPSHOT_DIR" "$UNINSTALL_GENERATION" || fail "sealed archive invalid" + else + [[ "$rc" != 0 && "$(cat "$stage")" == foreign ]] || fail "foreign stage removed" + [[ -f "$UNINSTALL_PENDING" ]] || fail "stage failure lost pending" + fi + ) > "$TEST_DIR/stage-$stage_case.log" 2>&1 || { + cat "$TEST_DIR/stage-$stage_case.log" + fail "stage $stage_case" + } + pass "stage cleanup verifies $stage_case identity" +done + +for terminal_signal in HUP INT TERM; do + case "$terminal_signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + ( + new_case + trap - EXIT + configure_repository || fail "terminal signal configure" + rc=0 + ( + trap - EXIT + acquire_repository_lock || exit 1 + uninstall_transaction_hook() { + if [[ "$1" == after-evidence-seal ]]; then kill -s "$terminal_signal" "$BASHPID"; fi + } + begin_uninstall_transaction || exit 1 + uninstall_cleanup terminal-signal + ) > "$CASE_DIR/signal.log" 2>&1 || rc=$? + [[ "$rc" == "$expected" ]] || fail "terminal signal status $rc" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "terminal signal lock retained" + compgen -G "$CLOUDFLARED_STATE_DIR/repository/pending-uninstall-*" >/dev/null || fail "terminal signal pending missing" + [[ -f "$SOURCE_FILE" ]] || fail "terminal signal source lost" + ) > "$TEST_DIR/terminal-$terminal_signal.log" 2>&1 || { + cat "$TEST_DIR/terminal-$terminal_signal.log" + fail "terminal signal $terminal_signal" + } + pass "terminal $terminal_signal=$expected retains evidence without repeated rollback" +done diff --git a/tests/test-cloudflare-tunnel-03.sh b/tests/test-cloudflare-tunnel-03.sh new file mode 100644 index 0000000..0ec9437 --- /dev/null +++ b/tests/test-cloudflare-tunnel-03.sh @@ -0,0 +1,272 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-cloudflare-tunnel-common.sh" + +for restore_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do + ( + new_case + trap - EXIT + configure_repository || fail "six target configure" + mkdir -p "$(dirname "$AUTO_UPDATE_SCRIPT")" "$(dirname "$AUTO_UPDATE_SERVICE")" + write_auto_update_files || fail "six target auto update" + printf managed > "$STATE_DIR/repository-managed" + chmod 0600 "$STATE_DIR/repository-managed" + acquire_repository_lock || fail "six target lock" + begin_uninstall_transaction || fail "six target capture" + snapshot=$UNINSTALL_SNAPSHOT_DIR + while IFS= read -r id; do rm -- "$(uninstall_target_path "$id")"; done < <(uninstall_snapshot_targets) + repository_install_file() { + [[ "$2" != "$snapshot/files/target-$restore_id" ]] || return 1 + command install -o 0 -g 0 -m "$1" -- "$2" "$3" + } + if uninstall_cleanup six-target-failure; then fail "restore fault accepted"; fi + while IFS= read -r id; do + [[ -f "$snapshot/files/target-$id" ]] || fail "snapshot payload lost $id" + if [[ "$id" != "$restore_id" ]]; then + cmp "$(uninstall_target_path "$id")" "$snapshot/files/target-$id" || fail "restore skipped $id" + validate_secure_file "$(uninstall_target_path "$id")" "$(uninstall_target_mode "$id")" || fail "restore metadata $id" + fi + done < <(uninstall_snapshot_targets) + [[ -f "$UNINSTALL_PENDING" ]] || fail "restore pending lost" + uninstall_journal_valid "$snapshot/journal" ACTIVE || fail "failure overwrote original journal" + ) > "$TEST_DIR/restore-$restore_id.log" 2>&1 || { + cat "$TEST_DIR/restore-$restore_id.log" + fail "restore failure $restore_id" + } + pass "restore failure $restore_id continues remaining five targets and preserves original evidence" +done + +for target_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do + for bad_metadata in mode owner gid symlink directory fifo; do + ( + new_case + trap - EXIT + configure_repository || fail "metadata configure" + mkdir -p "$(dirname "$AUTO_UPDATE_SCRIPT")" "$(dirname "$AUTO_UPDATE_SERVICE")" + write_auto_update_files || fail "metadata auto update" + printf managed > "$STATE_DIR/repository-managed" + chmod 0600 "$STATE_DIR/repository-managed" + target=$(uninstall_target_path "$target_id") + case "$bad_metadata" in + mode) chmod 0666 "$target" ;; + owner) chown 1:0 "$target" ;; + gid) chown 0:1 "$target" ;; + symlink) mv "$target" "$target.external"; ln -s "$target.external" "$target" ;; + directory) rm "$target"; mkdir "$target" ;; + fifo) rm "$target"; mkfifo "$target" ;; + esac + acquire_repository_lock || fail "metadata lock" + if begin_uninstall_transaction; then fail "bad target captured"; fi + uninstall_cleanup bad-metadata >/dev/null 2>&1 || : + [[ -e "$target" || -L "$target" ]] || fail "untrusted target deleted" + [[ "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "capture failure state lost" + ) > "$TEST_DIR/meta-$target_id-$bad_metadata.log" 2>&1 || { + cat "$TEST_DIR/meta-$target_id-$bad_metadata.log" + fail "metadata $target_id $bad_metadata" + } + pass "capture rejects $target_id $bad_metadata without deleting target" + done +done + +( + unset CLOUDFLARED_TRUST_ANCHOR + init_runtime_config + [[ "$TRUST_ANCHOR" == / ]] || exit 1 + path_is_beneath_anchor / || exit 1 + path_is_beneath_anchor /var/lib || exit 1 + CLOUDFLARED_TRUST_ANCHOR=/var + path_is_beneath_anchor /var || exit 1 + path_is_beneath_anchor /var/lib || exit 1 + if path_is_beneath_anchor /variable; then exit 1; fi + if path_is_beneath_anchor /var/../etc; then exit 1; fi + path_is_beneath_anchor /var/lib/../lib || exit 1 +) || fail "pure anchor boundaries" +pass "pure paths validate default root anchor and normalized non-root boundaries" + +for creation_phase in before-create-marker before-create-manifest-stage before-create-manifest-rename before-create-publish after-create-publish; do + for creation_fault in failure HUP INT TERM; do + ( + new_case + trap - EXIT + configure_repository || fail "creation configure" + printf payload > "$CASE_DIR/payload" + chmod 0600 "$CASE_DIR/payload" + rc=0 + ( + trap - EXIT + acquire_repository_lock || exit 1 + begin_uninstall_transaction || exit 1 + uninstall_transaction_hook() { + [[ "$1" == "$creation_phase" ]] || return 0 + if [[ "$creation_fault" == failure ]]; then return 1; fi + kill -s "$creation_fault" "$BASHPID" + } + if uninstall_create_absent_target auto_update_script "$CASE_DIR/payload"; then exit 90; fi + uninstall_cleanup creation-failure || : + exit 1 + ) > "$CASE_DIR/creation.log" 2>&1 || rc=$? + case "$creation_fault" in failure) expected=1 ;; HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + [[ "$rc" == "$expected" ]] || fail "creation status $rc" + [[ ! -e "$AUTO_UPDATE_SCRIPT" && ! -L "$AUTO_UPDATE_SCRIPT" ]] || fail "owned publication not recovered" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "creation lock leaked" + ) > "$TEST_DIR/create-$creation_phase-$creation_fault.log" 2>&1 || { + cat "$TEST_DIR/create-$creation_phase-$creation_fault.log" + fail "creation $creation_phase $creation_fault" + } + pass "creation $creation_phase $creation_fault preserves ownership across publication" + done +done + +for bookkeeping_fault in marker stage rename; do + ( + new_case + trap - EXIT + configure_repository || fail "bookkeeping configure" + printf payload > "$CASE_DIR/payload"; chmod 0600 "$CASE_DIR/payload" + acquire_repository_lock || fail "bookkeeping lock" + begin_uninstall_transaction || fail "bookkeeping capture" + uninstall_transaction_hook() { + if [[ "$bookkeeping_fault" == marker && "$1" == before-create-marker ]]; then + mkdir "$UNINSTALL_SNAPSHOT_DIR/created-auto_update_script" + fi + return 0 + } + mktemp() { + if [[ "$bookkeeping_fault" == stage && "$1" == */manifest.stage.* ]]; then return 1; fi + command mktemp "$@" + } + repository_rename() { + if [[ "$bookkeeping_fault" == rename && "$1" == */manifest.stage.* ]]; then return 1; fi + command mv -fT -- "$1" "$2" + } + if uninstall_create_absent_target auto_update_script "$CASE_DIR/payload"; then fail "bookkeeping fault ignored"; fi + [[ ! -e "$AUTO_UPDATE_SCRIPT" ]] || fail "published before durable bookkeeping" + uninstall_cleanup bookkeeping-failure || : + ) > "$TEST_DIR/bookkeeping-$bookkeeping_fault.log" 2>&1 || { + cat "$TEST_DIR/bookkeeping-$bookkeeping_fault.log" + fail "bookkeeping $bookkeeping_fault" + } + pass "actual $bookkeeping_fault failure precedes ln publication" +done + +new_case +configure_repository || fail "URL configure" +grep '^curl:' "$FAKE_LOG" | grep -Fq 'https://pkg.cloudflare.com/cloudflare-main.gpg' || fail "key download URL changed" +printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflared any main\n' "$CLOUDFLARED_KEYRING" > "$CASE_DIR/official-source" +assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/official-source" "APT repository URL incorrect" +validate_current_repository_manifest || fail "official current rejected" +remove_managed_repository || fail "official source removal failed" +assert_absent "$CLOUDFLARED_SOURCE_FILE" "official source not removed" +pass "literal key URL and repository source validate current and real removal" + +new_case +printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflare-main.gpg any main\n' "$CLOUDFLARED_KEYRING" > "$CLOUDFLARED_SOURCE_FILE" +chmod 0644 "$CLOUDFLARED_SOURCE_FILE" +cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/wrong-source" +if configure_repository > "$CASE_DIR/rejected.log" 2>&1; then fail "gpg repository accepted"; fi +assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/wrong-source" "unowned wrong source overwritten" +[[ ! -s "$FAKE_LOG" ]] || fail "invalid source triggered external operation" +pass "gpg repository URL rejected without overwrite or download" + +for capture_file in old-key old-source old-current; do + for capture_fault in copy partial chmod HUP INT TERM exit0; do + ( + new_case + trap - EXIT + configure_repository || fail "capture fixture" + for object in "$KEYRING" "$SOURCE_FILE" "$REPOSITORY_STATE_DIR/current"; do + cp -p "$object" "$CASE_DIR/$(basename "$object").before" + done + : > "$FAKE_LOG" + rc=0 + ( + trap - EXIT + repository_copy_file() { + if [[ "$2" == */"$capture_file" ]]; then + case "$capture_fault" in + copy) return 1 ;; + partial) printf partial > "$2"; return 1 ;; + HUP|INT|TERM) kill -s "$capture_fault" "$BASHPID" ;; + exit0) exit 0 ;; + esac + fi + command cp -- "$1" "$2" + } + chmod() { + if [[ "$capture_fault" == chmod && "$2" == */"$capture_file" ]]; then return 1; fi + command chmod "$@" + } + configure_repository + ) > "$CASE_DIR/capture.log" 2>&1 || rc=$? + case "$capture_fault" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; *) expected=1 ;; esac + [[ "$rc" == "$expected" ]] || fail "capture status $rc" + for object in "$KEYRING" "$SOURCE_FILE" "$REPOSITORY_STATE_DIR/current"; do + cmp "$object" "$CASE_DIR/$(basename "$object").before" || fail "capture modified formal bytes" + [[ "$(stat -c '%u:%g:%a' "$object")" == "$(stat -c '%u:%g:%a' "$CASE_DIR/$(basename "$object").before")" ]] || fail "capture modified metadata" + done + [[ ! -s "$FAKE_LOG" ]] || fail "capture downloaded or called APT" + grep -q '未恢复或删除正式文件' "$CASE_DIR/capture.log" || fail "capture diagnostic missing" + acquire_repository_lock || fail "capture lock retained" + release_repository_lock || fail "capture lock release" + ) > "$TEST_DIR/capture-boundary-$capture_file-$capture_fault.log" 2>&1 || { + cat "$TEST_DIR/capture-boundary-$capture_file-$capture_fault.log" + fail "capture boundary $capture_file $capture_fault" + } + pass "repository capture $capture_file $capture_fault preserves formal bytes and metadata" + done +done + +( + new_case + trap - EXIT + set_old_generation + rc=0 + ( + trap - EXIT + begin_repository_transaction || exit 1 + exit 0 + ) > "$CASE_DIR/active-exit.log" 2>&1 || rc=$? + [[ "$rc" == 1 ]] || fail "active exit zero accepted" + assert_old_generation + [[ ! -e "$REPOSITORY_STATE_DIR/current" ]] || fail "absent current restored incorrectly" + acquire_repository_lock || fail "active exit lock retained" + release_repository_lock || fail "active exit release" +) || fail "active exit zero" +pass "complete repository transaction exit zero returns nonzero and restores absent current" + +for build_case in absent archive-failure; do + ( + new_case + trap - EXIT + if [[ "$build_case" == archive-failure ]]; then set_old_generation; fi + trap ':' HUP + before_traps=$(trap -p HUP INT TERM EXIT) + repository_copy_file() { printf partial > "$2"; return 1; } + if [[ "$build_case" == absent ]]; then + # No formal object exists; fail before any download via snapshot wrapper. + chmod() { + [[ "$2" != */transaction-* ]] || return 1 + command chmod "$@" + } + repository_transaction_hook() { exit 0; } + rc=0 + (begin_repository_transaction) > "$CASE_DIR/absent.log" 2>&1 || rc=$? + [[ "$rc" != 0 ]] || fail "absent abnormal exit accepted" + [[ ! -e "$KEYRING" && ! -e "$SOURCE_FILE" && ! -e "$REPOSITORY_STATE_DIR/current" ]] || fail "absent formal object created" + else + repository_rename() { return 1; } + if configure_repository > "$CASE_DIR/build.log" 2>&1; then fail "failed capture accepted"; fi + assert_old_generation + [[ -f "$REPOSITORY_TRANSACTION_DIR/old-key" ]] || fail "partial evidence lost" + [[ -f "$REPOSITORY_TRANSACTION_DIR/rollback.log" ]] || fail "failure diagnostic lost" + grep -q '回滚不完整' "$CASE_DIR/build.log" || fail "cleanup error not reported" + fi + [[ "$(trap -p HUP INT TERM EXIT)" == "$before_traps" ]] || fail "build traps changed" + acquire_repository_lock || fail "build cleanup lock retained" + release_repository_lock || fail "build cleanup lock release" + ) > "$TEST_DIR/build-$build_case.log" 2>&1 || { + cat "$TEST_DIR/build-$build_case.log" + fail "build $build_case" + } + pass "repository building $build_case preserves objects traps and evidence" +done diff --git a/tests/test-cloudflare-tunnel-04.sh b/tests/test-cloudflare-tunnel-04.sh new file mode 100644 index 0000000..bc9ed84 --- /dev/null +++ b/tests/test-cloudflare-tunnel-04.sh @@ -0,0 +1,375 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-cloudflare-tunnel-common.sh" + +for restore_object in key source current; do + for restore_fault in partial wrong-content wrong-mode rename post-content post-mode; do + ( + new_case + trap - EXIT + configure_repository || fail "restore fixture" + begin_repository_transaction || fail "restore transaction" + snapshot=$REPOSITORY_TRANSACTION_DIR + printf historical > "$REPOSITORY_STATE_DIR/current.rollback" + cp "$REPOSITORY_STATE_DIR/current.rollback" "$CASE_DIR/historical" + case "$restore_object" in key) target=$KEYRING ;; source) target=$SOURCE_FILE ;; current) target=$REPOSITORY_STATE_DIR/current ;; esac + cp "$target" "$CASE_DIR/pre-restore" + : > "$CASE_DIR/attempts" + repository_install_file() { + printf '%s\n' "$2" >> "$CASE_DIR/attempts" + if [[ "$2" == "$snapshot/old-$restore_object" ]]; then + case "$restore_fault" in + partial) printf partial > "$3"; return 1 ;; + wrong-content) command install -o 0 -g 0 -m "$1" "$2" "$3"; printf wrong > "$3"; return 0 ;; + wrong-mode) command install -o 0 -g 0 -m 0777 "$2" "$3"; return 0 ;; + esac + fi + command install -o 0 -g 0 -m "$1" "$2" "$3" + } + repository_rename() { + if [[ "$2" == "$target" && "$1" == */.cloudflared-rollback.* ]]; then + [[ "$restore_fault" != rename ]] || return 1 + command mv -fT -- "$1" "$2" || return 1 + case "$restore_fault" in post-content) printf corrupt > "$2" ;; post-mode) chmod 0777 "$2" ;; esac + return 0 + fi + command mv -fT -- "$1" "$2" + } + if rollback_repository_transaction injected > "$CASE_DIR/restore.log" 2>&1; then fail "restore fault accepted"; fi + for item in key source current; do + [[ -f "$REPOSITORY_TRANSACTION_DIR/old-$item" ]] || fail "backup lost $item" + grep -Fxq "$snapshot/old-$item" "$CASE_DIR/attempts" || fail "restore skipped $item" + done + case "$restore_fault" in partial|wrong-content|wrong-mode|rename) cmp "$target" "$CASE_DIR/pre-restore" || fail "invalid stage published" ;; esac + cmp "$REPOSITORY_STATE_DIR/current.rollback" "$CASE_DIR/historical" || fail "historical rollback touched" + if grep -q '旧 key/source 已恢复' "$CASE_DIR/restore.log"; then fail "false recovery success"; fi + grep -q '回滚不完整' "$CASE_DIR/restore.log" || fail "missing failure diagnosis" + acquire_repository_lock || fail "restore lock retained" + release_repository_lock || fail "restore lock release" + ) > "$TEST_DIR/publish-$restore_object-$restore_fault.log" 2>&1 || { + cat "$TEST_DIR/publish-$restore_object-$restore_fault.log" + fail "restore publish $restore_object $restore_fault" + } + pass "repository restore $restore_object $restore_fault rejects corruption and retains all backups" + done +done + +for operation in install_cloudflared upgrade_cloudflared; do + ( + new_case + trap - EXIT + cp "$CASE_DIR/bin/cloudflared" "$APT_BIN" + dpkg-query() { [[ "$*" == *'db:Status-Status'* ]] && { printf installed; return 0; }; return 1; } + timer_enabled=true + systemctl() { + printf 'systemctl:%s\n' "$*" >> "$FAKE_LOG" + if [[ "$*" == *cloudflared-apt-update.timer* && ( "$1" == is-enabled || "$1" == is-active ) ]]; then + [[ "$timer_enabled" == true ]] + else + return 0 + fi + } + write_auto_update_files || fail "timer fixture" + auto_update_file_is_managed "$AUTO_UPDATE_SCRIPT" || fail "main script template rejected" + auto_update_file_is_managed "$AUTO_UPDATE_SERVICE" || fail "main service template rejected" + auto_update_file_is_managed "$AUTO_UPDATE_TIMER" || fail "main timer template rejected" + "$operation" || fail "enabled operation" + [[ "$PRESERVE_AUTO_UPDATE" == true ]] || fail "strategy reset by init" + grep -Fxq 'systemctl:enable --now cloudflared-apt-update.timer' "$FAKE_LOG" || fail "timer not enabled" + timer_enabled=false + : > "$FAKE_LOG" + "$operation" || fail "disabled operation" + [[ "$PRESERVE_AUTO_UPDATE" == false ]] || fail "strategy leaked between operations" + if grep -Fxq 'systemctl:enable --now cloudflared-apt-update.timer' "$FAKE_LOG"; then fail "disabled timer enabled"; fi + timer_enabled=true + export FAKE_APT_UPDATE_FAIL=1 + : > "$FAKE_LOG" + if "$operation"; then fail "repository failure ignored"; fi + if grep -Eq 'systemctl:(disable|stop|enable|daemon-reload)' "$FAKE_LOG"; then fail "repository failure continued lifecycle"; fi + ) > "$TEST_DIR/policy-$operation.log" 2>&1 || { + cat "$TEST_DIR/policy-$operation.log" + fail "policy $operation" + } + pass "policy $operation preserves enabled main timer without cross-operation leakage" +done + +( + new_case + trap - EXIT + write_auto_update_files || fail "template fixture" + printf '# changed\n' >> "$AUTO_UPDATE_TIMER" + if auto_update_file_is_managed "$AUTO_UPDATE_TIMER"; then fail "partial marker trusted"; fi + cp "$AUTO_UPDATE_TIMER" "$CASE_DIR/foreign-timer" + if disable_auto_update --confirmed; then fail "unknown timer deleted"; fi + cmp "$AUTO_UPDATE_TIMER" "$CASE_DIR/foreign-timer" || fail "unknown timer changed" +) || fail "strict main template" +pass "main updater templates reject extra content without deleting unknown files" + +( + new_case + trap - EXIT + begin_repository_transaction || fail "absent transaction" + mv "$SOURCE_FILE" "$CASE_DIR/owned-source" + printf foreign > "$SOURCE_FILE"; chmod 0644 "$SOURCE_FILE" + if rollback_repository_transaction foreign; then fail "foreign absent accepted"; fi + [[ "$(cat "$SOURCE_FILE")" == foreign ]] || fail "foreign absent deleted" + [[ -f "$REPOSITORY_TRANSACTION_DIR/pending" ]] || fail "rollback pending missing" + : > "$FAKE_LOG" + if configure_repository; then fail "pending allowed restart"; fi + [[ ! -s "$FAKE_LOG" ]] || fail "pending performed external work" +) || fail "absent ownership" +pass "repository absent foreign object survives rollback and blocks restart" + +( + new_case + trap - EXIT + acquire_repository_lock || fail "lock fixture" + release_repository_lock || fail "first release" + mkdir "$CLOUDFLARED_STATE_DIR.lock" + release_repository_lock || fail "idempotent release" + [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "released another operation lock" +) || fail "lock identity" +pass "repeated lock release preserves next operation lock" + +( + new_case + trap - EXIT + set_old_generation + mkdir -p "$STATE_DIR" + chmod 0700 "$STATE_DIR" + printf unknown > "$STATE_DIR/repository-managed" + chmod 0600 "$STATE_DIR/repository-managed" + : > "$FAKE_LOG" + if uninstall_cloudflared --confirmed; then fail "bad legacy marker accepted"; fi + [[ ! -s "$FAKE_LOG" ]] || fail "legacy ownership checked after destructive operations" +) || fail "legacy preflight" +pass "legacy uninstall rejects unknown ownership before systemctl or APT" + +for review_fault in timer-enable stage-identity trap-restore; do + ( + new_case + trap - EXIT + case "$review_fault" in + timer-enable) + dpkg-query() { printf installed; } + systemctl() { + printf '%s\n' "$*" >> "$FAKE_LOG" + [[ "$*" != 'enable --now cloudflared-apt-update.timer' ]] + } + if enable_auto_update > "$CASE_DIR/error.log" 2>&1; then fail "timer enable failure hidden"; fi + grep -Fxq 'enable --now cloudflared-apt-update.timer' "$FAKE_LOG" || fail "timer injection not hit" + ;; + stage-identity) + begin_repository_transaction || fail "stage fixture" + stage=$(mktemp "$CASE_DIR/.stage.XXXXXX") + repository_track_stage "$stage" || fail "track stage" + mv "$stage" "$stage.owned" + printf foreign > "$stage" + if rollback_repository_transaction injected; then fail "foreign stage ignored"; fi + [[ "$(cat "$stage")" == foreign ]] || fail "foreign stage deleted" + ;; + trap-restore) + begin_repository_transaction || fail "trap fixture" + restore_one_trap() { printf '%s\n' "$1" >> "$CASE_DIR/trap-attempts"; return 1; } + if rollback_repository_transaction injected; then fail "trap failure hidden"; fi + [[ "$(wc -l < "$CASE_DIR/trap-attempts")" == 4 ]] || fail "trap attempts skipped" + [[ -f "$REPOSITORY_TRANSACTION_DIR/pending" ]] || fail "trap failure pending missing" + ;; + esac + ) > "$TEST_DIR/review-$review_fault.log" 2>&1 || { + cat "$TEST_DIR/review-$review_fault.log" + fail "review $review_fault" + } + pass "review $review_fault injection reached and failure preserved" +done + +for phase in disable-systemctl disable-backup disable-delete; do + for fault in HUP INT TERM zero failure; do + ( + new_case + trap - EXIT + configure_repository || fail "disable fixture" + write_auto_update_files || fail "disable files" + rc=0 + ( + trap - EXIT + updater_transaction_hook() { + [[ "$1" == "$phase" ]] || return 0 + case "$fault" in zero) exit 0 ;; failure) return 1 ;; *) kill -s "$fault" "$BASHPID" ;; esac + } + disable_auto_update --confirmed + ) > "$CASE_DIR/disable.log" 2>&1 || rc=$? + case "$fault" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; *) expected=1 ;; esac + [[ "$rc" == "$expected" ]] || fail "disable exit $rc" + [[ -f "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "premature file deletion" + compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "disable pending missing" + grep -q '自动更新操作未完成' "$CASE_DIR/disable.log" || fail "disable diagnostic missing" + acquire_repository_lock || fail "disable lock retained" + release_repository_lock || fail "disable lock release" + ) > "$TEST_DIR/disable-$phase-$fault.log" 2>&1 || { + cat "$TEST_DIR/disable-$phase-$fault.log" + fail "disable $phase $fault" + } + pass "independent disable $phase $fault preserves pending and exit contract" + done +done + +for phase in enable-validate enable-write enable-reload enable-timer disable-systemctl; do + ( + new_case + trap - EXIT + configure_repository || fail "barrier configure" + write_auto_update_files || fail "barrier files" + mkfifo "$CASE_DIR/ready" "$CASE_DIR/release" + ( + trap - EXIT + dpkg-query() { printf installed; } + updater_transaction_hook() { + [[ "$1" == "$phase" ]] || return 0 + printf ready > "$CASE_DIR/ready" + read -r _ < "$CASE_DIR/release" + } + if [[ "$phase" == disable-* ]]; then disable_auto_update --confirmed; else enable_auto_update; fi + ) > "$CASE_DIR/holder.log" 2>&1 & + holder=$! + read -r ready < "$CASE_DIR/ready" || [[ "$ready" == ready ]] || fail "barrier readiness" + for contender in enable_auto_update disable_auto_update uninstall_cloudflared; do + if ( "$contender" --confirmed ) > "$CASE_DIR/contender.log" 2>&1; then fail "contender bypassed lock"; fi + done + printf 'release\n' > "$CASE_DIR/release" + wait "$holder" || { cat "$CASE_DIR/holder.log"; fail "holder failed"; } + ) > "$TEST_DIR/barrier-$phase.log" 2>&1 || { cat "$TEST_DIR/barrier-$phase.log"; fail "barrier $phase"; } + pass "real updater barrier $phase excludes all competing mutation entrypoints" +done + +for fault in backup delete release; do + ( + new_case + trap - EXIT + configure_repository || fail "file fault fixture" + write_auto_update_files || fail "file fault files" + hit=false + cp() { + if [[ "$fault" == backup && "$*" == *auto-update-* ]]; then hit=true; return 1; fi + command cp "$@" + } + rm() { + if [[ "$fault" == delete && "$*" == *"$AUTO_UPDATE_SERVICE"* ]]; then hit=true; return 1; fi + command rm "$@" + } + rmdir() { + if [[ "$fault" == release ]]; then hit=true; return 1; fi + command rmdir "$@" + } + before=$(trap -p HUP INT TERM EXIT) + if disable_auto_update --confirmed > "$CASE_DIR/error.log" 2>&1; then fail "disable file fault accepted"; fi + [[ "$hit" == true ]] || fail "file fault not hit" + [[ "$(trap -p HUP INT TERM EXIT)" == "$before" ]] || fail "disable traps not restored" + compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "file fault pending missing" + case "$fault" in + backup) [[ -f "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "backup failure deleted files" ;; + delete) [[ ! -e "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "partial deletion group wrong" ;; + release) [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "failed release lost lock" ;; + esac + ) > "$TEST_DIR/disable-file-$fault.log" 2>&1 || { cat "$TEST_DIR/disable-file-$fault.log"; fail "disable file $fault"; } + pass "independent disable actual $fault failure retains group evidence and traps" +done + +# Separate shells load the implementation independently; only file/FIFO state is shared. +( + new_case + trap - EXIT + configure_repository || fail "uninstall barrier configure" + write_auto_update_files || fail "uninstall barrier updater" + mkfifo "$CASE_DIR/uninstall-ready" "$CASE_DIR/uninstall-release" + bash -c ' + source "$1" + check_platform() { :; } + uninstall_transaction_hook() { + [[ "$1" == after-snapshot-active ]] || return 0 + printf "%s\n" "$UNINSTALL_SNAPSHOT_DIR" > "$CASE_DIR/snapshot-path" + printf "ready\n" > "$CASE_DIR/uninstall-ready" + read -r _ < "$CASE_DIR/uninstall-release" + } + uninstall_cloudflared --confirmed + ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" > "$CASE_DIR/holder.log" 2>&1 & + holder=$! + # Timeout bounds a broken readiness handshake; no scheduling sleeps. + timeout 30 bash -c 'read -r ready < "$CASE_DIR/uninstall-ready"; [[ "$ready" == ready ]]' || fail "uninstall never reached active snapshot" + snapshot=$(cat "$CASE_DIR/snapshot-path") + [[ -f "$snapshot/manifest" ]] || fail "snapshot not ready" + lock_before=$(stat -c '%d:%i:%u:%g:%a' "$CLOUDFLARED_STATE_DIR.lock") + find "$REPOSITORY_STATE_DIR" -type f -exec sha256sum {} + | sort > "$CASE_DIR/state-before" + sha256sum "$AUTO_UPDATE_SCRIPT" "$AUTO_UPDATE_SERVICE" "$AUTO_UPDATE_TIMER" > "$CASE_DIR/updater-before" + for contender in enable_auto_update disable_auto_update; do + : > "$CASE_DIR/contender-external.log" + rc=0 + FAKE_LOG="$CASE_DIR/contender-external.log" bash -c ' + source "$1" + check_platform() { :; } + "$2" --confirmed + ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" "$contender" > "$CASE_DIR/contender.log" 2>&1 || rc=$? + [[ "$rc" != 0 ]] || fail "contender entered uninstall lock" + grep -q '无法取得 Cloudflare 仓库事务锁' "$CASE_DIR/contender.log" || fail "competition not reached" + [[ ! -s "$CASE_DIR/contender-external.log" ]] || fail "contender executed external command" + [[ "$(stat -c '%d:%i:%u:%g:%a' "$CLOUDFLARED_STATE_DIR.lock")" == "$lock_before" ]] || fail "holder lock changed" + find "$REPOSITORY_STATE_DIR" -type f -exec sha256sum {} + | sort > "$CASE_DIR/state-after" + cmp "$CASE_DIR/state-before" "$CASE_DIR/state-after" || fail "snapshot or pending changed" + sha256sum -c "$CASE_DIR/updater-before" >/dev/null || fail "contender changed updater" + done + printf 'release\n' > "$CASE_DIR/uninstall-release" + wait "$holder" || { cat "$CASE_DIR/holder.log"; fail "uninstall holder failed"; } + [[ ! -e "$SOURCE_FILE" && ! -e "$AUTO_UPDATE_TIMER" ]] || fail "uninstall did not complete" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall retained lock" +) > "$TEST_DIR/uninstall-holder.log" 2>&1 || { cat "$TEST_DIR/uninstall-holder.log"; fail "uninstall holder barrier"; } +pass "independent uninstall holder excludes enable and disable without changing snapshot pending or lock" + +for dependency_fault in update install; do + ( + new_case + trap - EXIT + write_auto_update_files || fail "dependency updater fixture" + sha256sum "$AUTO_UPDATE_SCRIPT" "$AUTO_UPDATE_SERVICE" "$AUTO_UPDATE_TIMER" > "$CASE_DIR/updater-before" + command() { + if [[ "$*" == '-v flock' ]]; then + printf 'flock-missing\n' >> "$CASE_DIR/dependency-hits" + return 1 + fi + builtin command "$@" + } + dpkg-query() { printf installed; } + apt-get() { + printf 'apt-get:%s\n' "$*" >> "$FAKE_LOG" + if [[ "$1" == update && "$dependency_fault" == update ]]; then return 41; fi + if [[ "$*" == 'install -y util-linux' ]]; then return 42; fi + return 0 + } + trap ':' HUP + before_traps=$(trap -p HUP INT TERM EXIT) + rc=0 + enable_auto_update > "$CASE_DIR/dependency.log" 2>&1 || rc=$? + case "$dependency_fault" in update) expected=41 ;; install) expected=42 ;; esac + [[ "$rc" == "$expected" ]] || fail "dependency status $rc" + grep -Fxq flock-missing "$CASE_DIR/dependency-hits" || fail "flock branch not reached" + grep -Fxq apt-get:update "$FAKE_LOG" || fail "dependency update not reached" + if [[ "$dependency_fault" == update ]]; then + if grep -q 'apt-get:install' "$FAKE_LOG"; then fail "install continued after update failure"; fi + else + grep -Fxq 'apt-get:install -y util-linux' "$FAKE_LOG" || fail "util install not reached" + fi + if grep -Eq 'systemctl:.*(reload|enable)' "$FAKE_LOG"; then fail "dependency failure continued timer work"; fi + sha256sum -c "$CASE_DIR/updater-before" >/dev/null || fail "dependency failure changed updater" + validate_current_repository_manifest || fail "prior repository configuration not recorded" + compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "dependency pending missing" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "dependency lock retained" + [[ "$(trap -p HUP INT TERM EXIT)" == "$before_traps" ]] || fail "dependency traps changed" + grep -q '自动更新操作未完成' "$CASE_DIR/dependency.log" || fail "dependency diagnostic missing" + ) > "$TEST_DIR/dependency-$dependency_fault.log" 2>&1 || { cat "$TEST_DIR/dependency-$dependency_fault.log"; fail "dependency $dependency_fault"; } + pass "missing flock $dependency_fault failure stops updater after recorded repository commit" +done + +entrypoint_output=$(bash -c "$(cat "$ROOT_DIR/tools/cloudflare_tunnel.sh")" cloudflare_tunnel.sh help) +grep -Fq 'cloudflare_tunnel.sh install' <<< "$entrypoint_output" || fail "bash -c entrypoint broken" +pass "bash -c entrypoint remains compatible" + +printf 'All cloudflare wrapper tests passed. PASS=%d\n' "$pass_count" diff --git a/tests/test-cloudflare-tunnel.sh b/tests/test-cloudflare-tunnel.sh deleted file mode 100644 index c23689a..0000000 --- a/tests/test-cloudflare-tunnel.sh +++ /dev/null @@ -1,1544 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if (( EUID != 0 )); then - exec sudo --preserve-env=PATH bash "$0" "$@" -fi - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -readonly ROOT_DIR -TEST_DIR=$(mktemp -d -p "$ROOT_DIR") -readonly TEST_DIR -trap 'rm -rf "$TEST_DIR"' EXIT - -pass_count=0 -fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } -pass() { pass_count=$((pass_count + 1)); printf 'PASS: %s\n' "$*"; } -assert_same() { cmp -s -- "$1" "$2" || fail "$3"; } -assert_absent() { [[ ! -e "$1" && ! -L "$1" ]] || fail "$2"; } - -make_fake_commands() { - local bin="$1" - mkdir -p "$bin" - cat > "$bin/curl" <<'FAKE' -#!/usr/bin/env bash -set -eu -printf 'curl:%s\n' "$*" >> "$FAKE_LOG" -[[ "${FAKE_CURL_FAIL:-}" != 1 ]] || exit 22 -output="" -while (( $# )); do - if [[ "$1" == -o ]]; then output="$2"; shift 2; continue; fi - shift -done -[[ -n "$output" ]] -printf 'fake-cloudflare-key\n' > "$output" -FAKE - cat > "$bin/gpg" <<'FAKE' -#!/usr/bin/env bash -set -eu -printf 'gpg:%s\n' "$*" >> "$FAKE_LOG" -[[ "${FAKE_GPG_FAIL:-}" != 1 ]] || exit 2 -case "${FAKE_GPG_CASE:-good}" in - good) - cat <<'EOF' -pub:-:4096:1:8A682D308D4E5E73:1761229688:::-:::scESC::::::23::0: -fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: -uid:-::::1761229688::HASH::CloudFlare Software Packaging 2025 ::::::::::0: -sub:-:4096:1:029E1444B7D9F50F:1761229688::::::e::::::23: -fpr:::::::::06C89DB3B80A8F4349697C76029E1444B7D9F50F: -EOF - ;; - fingerprint) - cat <<'EOF' -pub:-:4096:1:BAD:0:::-:::scESC: -fpr:::::::::0000000000000000000000000000000000000000: -uid:-::::0::HASH::CloudFlare Software Packaging 2025 ::: -EOF - ;; - uid) - cat <<'EOF' -pub:-:4096:1:8A682D308D4E5E73:0:::-:::scESC: -fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: -uid:-::::0::HASH::Attacker ::: -EOF - ;; - multi) - cat <<'EOF' -pub:-:4096:1:8A682D308D4E5E73:0:::-:::scESC: -fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: -uid:-::::0::HASH::CloudFlare Software Packaging 2025 ::: -pub:-:4096:1:BAD:0:::-:::scESC: -fpr:::::::::0000000000000000000000000000000000000000: -uid:-::::0::HASH::Other ::: -EOF - ;; -esac -FAKE - cat > "$bin/apt-get" <<'FAKE' -#!/usr/bin/env bash -set -eu -printf 'apt-get:%s\n' "$*" >> "$FAKE_LOG" -case "${1:-}" in - update) [[ "${FAKE_APT_UPDATE_FAIL:-}" != 1 ]] ;; - install) [[ "${FAKE_APT_INSTALL_FAIL:-}" != 1 ]] ;; - *) exit 0 ;; -esac -FAKE - cat > "$bin/systemctl" <<'FAKE' -#!/usr/bin/env bash -printf 'systemctl:%s\n' "$*" >> "$FAKE_LOG" -exit 0 -FAKE - cat > "$bin/cloudflared" <<'FAKE' -#!/usr/bin/env bash -printf 'cloudflared:%s\n' "$*" >> "$FAKE_LOG" -exit 0 -FAKE - chmod 0755 "$bin"/* -} - -new_case() { - CASE_DIR="$TEST_DIR/case" - rm -rf "$CASE_DIR" - mkdir -p "$CASE_DIR" - export CASE_DIR FAKE_LOG="$CASE_DIR/fake.log" - : > "$FAKE_LOG" - mkdir -p "$CASE_DIR/root/usr/share/keyrings" "$CASE_DIR/root/etc/apt/sources.list.d" \ - "$CASE_DIR/root/var/lib" - chmod 0755 "$CASE_DIR/root" "$CASE_DIR/root/usr" "$CASE_DIR/root/usr/share" \ - "$CASE_DIR/root/usr/share/keyrings" "$CASE_DIR/root/etc" "$CASE_DIR/root/etc/apt" \ - "$CASE_DIR/root/etc/apt/sources.list.d" "$CASE_DIR/root/var" "$CASE_DIR/root/var/lib" - make_fake_commands "$CASE_DIR/bin" - export PATH="$CASE_DIR/bin:$ORIGINAL_PATH" - export CLOUDFLARED_TRUST_ANCHOR="$CASE_DIR" - export CLOUDFLARED_KEYRING="$CASE_DIR/root/usr/share/keyrings/cloudflare-main.gpg" - export CLOUDFLARED_SOURCE_FILE="$CASE_DIR/root/etc/apt/sources.list.d/cloudflared.list" - export CLOUDFLARED_APT_SOURCE_ROOT="$CASE_DIR/root/etc/apt" - export CLOUDFLARED_STATE_DIR="$CASE_DIR/root/var/lib/cloudflared-wrapper" - export CLOUDFLARED_REPOSITORY_STATE_DIR="$CLOUDFLARED_STATE_DIR/repository" - export CLOUDFLARED_LEGACY_BIN="$CASE_DIR/legacy-cloudflared" - export CLOUDFLARED_APT_BIN="$CASE_DIR/usr-bin-cloudflared" - export CLOUDFLARED_LEGACY_UPDATER="$CASE_DIR/cloudflared-update" - export CLOUDFLARED_LEGACY_SERVICE="$CASE_DIR/cloudflared-updater.service" - export CLOUDFLARED_LEGACY_TIMER="$CASE_DIR/cloudflared-updater.timer" - export CLOUDFLARED_AUTO_UPDATE_SCRIPT="$CASE_DIR/cloudflared-apt-update" - export CLOUDFLARED_AUTO_UPDATE_SERVICE="$CASE_DIR/cloudflared-apt-update.service" - export CLOUDFLARED_AUTO_UPDATE_TIMER="$CASE_DIR/cloudflared-apt-update.timer" - export CLOUDFLARED_SERVICE_FILE="$CASE_DIR/cloudflared.service" - export CLOUDFLARED_BINARY_UPDATE_SERVICE="$CASE_DIR/cloudflared-update.service" - export CLOUDFLARED_BINARY_UPDATE_TIMER="$CASE_DIR/cloudflared-update.timer" - export CLOUDFLARED_TEST_INTERNALS=1 - unset FAKE_CURL_FAIL FAKE_GPG_FAIL FAKE_GPG_CASE FAKE_APT_UPDATE_FAIL FAKE_APT_INSTALL_FAIL - unset FAIL_INSTALL_CALL FAIL_RENAME_CALL INSTALL_CALL RENAME_CALL SIGNAL_PHASE - if [[ "${SCRIPT_SOURCED:-}" != 1 ]]; then - # shellcheck source=../tools/cloudflare_tunnel.sh - source "$ROOT_DIR/tools/cloudflare_tunnel.sh" - SCRIPT_SOURCED=1 - fi - init_runtime_config -} - -set_old_generation() { - printf 'old-key\n' > "$CLOUDFLARED_KEYRING" - repository_legacy_source_content > "$CLOUDFLARED_SOURCE_FILE" - chmod 0644 "$CLOUDFLARED_KEYRING" "$CLOUDFLARED_SOURCE_FILE" - cp "$CLOUDFLARED_KEYRING" "$CASE_DIR/expected-key" - cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" -} - -assert_old_generation() { - assert_same "$CLOUDFLARED_KEYRING" "$CASE_DIR/expected-key" "old key was not restored" - assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" "old source was not restored" -} - -run_failure_case() { - local name="$1" setup="$2" action="${3:-configure_repository}" - new_case - set_old_generation - eval "$setup" - set +e - eval "$action" >"$CASE_DIR/out" 2>"$CASE_DIR/err" - status=$? - set -e - if (( status == 0 )); then - fail "$name unexpectedly succeeded" - fi - assert_old_generation - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "$name left lock" - find "$CLOUDFLARED_REPOSITORY_STATE_DIR" -maxdepth 1 -type d -name 'failure-*' -print -quit | grep -q . || - fail "$name did not preserve failure evidence" - pass "$name" -} - -ORIGINAL_PATH=$PATH - -new_case -before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) -repository_source_content > "$CASE_DIR/source-output" -after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) -[[ "$before" == "$after" ]] || fail "source rendering had side effects" -[[ "$(cat "$CASE_DIR/source-output")" == "deb [signed-by=$CLOUDFLARED_KEYRING] https://pkg.cloudflare.com/cloudflared any main" ]] || - fail "source is not exact official definition" -pass "source rendering has zero side effects and exact fields" - -( - new_case - source_cwd=$PWD - source_before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) - cloudflare_tunnel_source() { printf 'sentinel:%s\n' "$*"; } - sentinel_before=$(declare -f cloudflare_tunnel_source) - trap 'sentinel-trap' HUP - set -o noclobber - umask 027 - unset CLOUDFLARED_TEST_INTERNALS - source "$ROOT_DIR/tools/cloudflare_tunnel.sh" - sentinel_after=$(declare -f cloudflare_tunnel_source) - [[ "$sentinel_before" == "$sentinel_after" ]] || fail "ordinary source replaced sentinel function" - [[ "$(cloudflare_tunnel_source probe)" == sentinel:probe ]] || fail "ordinary source changed sentinel behavior" - [[ "$(trap -p HUP)" == *sentinel-trap* ]] || fail "ordinary source changed caller trap" - [[ "$-" == *C* ]] || fail "ordinary source changed caller shell options" - [[ "$(umask)" == 0027 ]] || fail "ordinary source changed caller umask" - [[ "$PWD" == "$source_cwd" ]] || fail "ordinary source changed caller cwd" - [[ "${CLOUDFLARED_STATE_DIR+x}" == x ]] || fail "test setup lost state variable" - after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) - [[ "$source_before" == "$after" ]] || fail "ordinary source changed filesystem" -) -pass "ordinary source preserves sentinels, shell state, and filesystem" - -new_case -before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\\n' | sort) -bash "$ROOT_DIR/tools/cloudflare_tunnel.sh" status >/dev/null 2>&1 || fail "standalone status failed in fake environment" -after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\\n' | sort) -[[ "$before" == "$after" ]] || fail "ordinary status created state, lock, or temp files" -pass "ordinary help/status paths have zero state and lock side effects" - -cat > "$TEST_DIR/source-contract.sh" </dev/null' _ \ - "$ROOT_DIR/tools/cloudflare_tunnel.sh" || fail "internal source mode unavailable" -pass "explicit internal source mode exposes test functions" - -new_case -configure_repository || fail "configure before real uninstall failed" -require_root() { :; } -check_platform() { :; } -uninstall_cloudflared --confirmed || fail "uninstall normal path failed" -[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall left repository lock" -grep -Fxq 'apt-get:remove -y cloudflared' "$FAKE_LOG" || fail "uninstall skipped package removal" -assert_absent "$CLOUDFLARED_SOURCE_FILE" "real uninstall retained source" -assert_absent "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "real uninstall retained current" -[[ -f "$CLOUDFLARED_KEYRING" ]] || fail "real uninstall removed keyring" -pass "configure then real uninstall preserves keyring and releases lock" -init_runtime_config - -new_case -mkdir -p "$CLOUDFLARED_STATE_DIR.lock" -mkdir -p "$CLOUDFLARED_AUTO_UPDATE_SCRIPT" -printf marker > "$CASE_DIR/marker" -require_root() { :; } -check_platform() { :; } -if uninstall_cloudflared --confirmed >/dev/null 2>&1; then fail "uninstall lock competition unexpectedly succeeded"; fi -[[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall lock competition removed lock" -[[ -d "$CLOUDFLARED_AUTO_UPDATE_SCRIPT" ]] || fail "lock competition touched auto-update path" -! grep -Eq 'remove|disable|daemon-reload|apt-get:' "$FAKE_LOG" || fail "lock competition performed mutation" -pass "uninstall lock competition fails before auto-update, systemctl, and APT" - -new_case -mkdir -p "$CLOUDFLARED_STATE_DIR.lock" -if disable_auto_update --confirmed >/dev/null 2>&1; then fail "disable-auto-update lock competition unexpectedly succeeded"; fi -[[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "disable-auto-update removed competing lock" -pass "auto-update lock competition fails before state mutation" -chmod 0700 "$CLOUDFLARED_STATE_DIR.lock" -if configure_repository >/dev/null 2>&1; then fail "lock competition unexpectedly succeeded"; fi -pass "key/source lock competition" - -run_failure_case "key URL download failure" 'export FAKE_CURL_FAIL=1' -run_failure_case "gpg parser failure" 'export FAKE_GPG_FAIL=1' -run_failure_case "fingerprint mismatch" 'export FAKE_GPG_CASE=fingerprint' -run_failure_case "UID mismatch" 'export FAKE_GPG_CASE=uid' -run_failure_case "multiple primary keys" 'export FAKE_GPG_CASE=multi' - -for target in key source; do - for kind in symlink directory; do - new_case - case "$target" in key) path=$CLOUDFLARED_KEYRING ;; source) path=$CLOUDFLARED_SOURCE_FILE ;; esac - case "$kind" in - symlink) ln -s "$CASE_DIR/missing" "$path" ;; - directory) mkdir "$path" ;; - esac - if configure_repository >/dev/null 2>&1; then fail "$target $kind accepted"; fi - [[ -L "$path" || -d "$path" ]] || fail "$target $kind was changed" - pass "$target rejects $kind" - done - for bad in owner gid mode; do - new_case - case "$target" in key) path=$CLOUDFLARED_KEYRING ;; source) path=$CLOUDFLARED_SOURCE_FILE ;; esac - if [[ "$target" == source ]]; then - repository_source_content > "$path" - else - printf key > "$path" - fi - chmod 0644 "$path" - case "$bad" in - owner) chown 65534:0 "$path" ;; - gid) chown 0:65534 "$path" ;; - mode) chmod 0664 "$path" ;; - esac - if configure_repository >/dev/null 2>&1; then fail "$target wrong $bad accepted"; fi - pass "$target rejects wrong $bad" - done -done - -new_case -printf 'deb https://example.invalid stable main\n' > "$CASE_DIR/root/etc/apt/sources.list.d/extra.list" -chmod 0644 "$CASE_DIR/root/etc/apt/sources.list.d/extra.list" -configure_repository -pass "unrelated APT source remains untouched" - -new_case -printf 'deb https://pkg.cloudflare.com/cloudflared any main\n' > "$CASE_DIR/root/etc/apt/sources.list.d/duplicate.list" -chmod 0644 "$CASE_DIR/root/etc/apt/sources.list.d/duplicate.list" -if configure_repository >/dev/null 2>&1; then fail "duplicate Cloudflare source accepted"; fi -assert_absent "$CLOUDFLARED_KEYRING" "duplicate source changed key" -assert_absent "$CLOUDFLARED_SOURCE_FILE" "duplicate source wrote managed source" -pass "reject duplicate or extra Cloudflare source before commit" - -new_case -printf 'deb [signed-by=%s trusted=yes] https://pkg.cloudflare.com/cloudflared any main\n' \ - "$CLOUDFLARED_KEYRING" > "$CLOUDFLARED_SOURCE_FILE" -chmod 0644 "$CLOUDFLARED_SOURCE_FILE" -if configure_repository >/dev/null 2>&1; then fail "source injection accepted"; fi -pass "reject unknown source fields and injection content" - -( -new_case -set_old_generation -INSTALL_CALL=0 -FAIL_INSTALL_CALL=1 -repository_install_file() { - INSTALL_CALL=$((INSTALL_CALL + 1)) - (( INSTALL_CALL != FAIL_INSTALL_CALL )) || return 1 - command install -o 0 -g 0 -m "$1" -- "$2" "$3" -} -if configure_repository >/dev/null 2>&1; then fail "key stage write failure succeeded"; fi -assert_old_generation -) -pass "formal key write failure rolls back" - -( -new_case -set_old_generation -INSTALL_CALL=0 -FAIL_INSTALL_CALL=2 -repository_install_file() { - INSTALL_CALL=$((INSTALL_CALL + 1)) - (( INSTALL_CALL != FAIL_INSTALL_CALL )) || return 1 - command install -o 0 -g 0 -m "$1" -- "$2" "$3" -} -if configure_repository >/dev/null 2>&1; then fail "source stage write failure succeeded"; fi -assert_old_generation -) -pass "formal source write failure rolls back" - -( -new_case -set_old_generation -RENAME_CALL=0 -FAIL_RENAME_CALL=2 -repository_rename() { - RENAME_CALL=$((RENAME_CALL + 1)) - (( RENAME_CALL != FAIL_RENAME_CALL )) || return 1 - command mv -fT -- "$1" "$2" -} -if configure_repository >/dev/null 2>&1; then fail "source commit failure succeeded"; fi -assert_old_generation -) -pass "key success/source failure rolls back" - -( -new_case -set_old_generation -RENAME_CALL=0 -FAIL_RENAME_CALL=1 -repository_rename() { - RENAME_CALL=$((RENAME_CALL + 1)) - (( RENAME_CALL != FAIL_RENAME_CALL )) || return 1 - command mv -fT -- "$1" "$2" -} -if configure_repository >/dev/null 2>&1; then fail "key commit failure succeeded"; fi -assert_old_generation -) -pass "source remains old when key commit fails" - -run_failure_case "APT probe failure rollback" 'export FAKE_APT_UPDATE_FAIL=1' 'run_repository_apt_transaction install' -run_failure_case "APT install failure rollback" 'export FAKE_APT_INSTALL_FAIL=1' 'run_repository_apt_transaction install' - -for signal in HUP INT TERM; do - case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - for phase in download validate stage key-commit source-commit apt-probe apt-install; do - new_case - set_old_generation - cat > "$CASE_DIR/signal-runner.sh" <"$CASE_DIR/out" 2>"$CASE_DIR/err" - status=$? - set -e - [[ "$status" == "$expected" ]] || fail "$signal at $phase returned $status, expected $expected" - assert_old_generation - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "$signal at $phase left lock" - pass "$signal=$expected rolls back at $phase" - done -done - -( -new_case -set_old_generation -RENAME_CALL=0 -repository_rename() { - RENAME_CALL=$((RENAME_CALL + 1)) - case "$RENAME_CALL" in 2|3) return 1 ;; esac - command mv -fT -- "$1" "$2" -} -if configure_repository >/dev/null 2>&1; then fail "rollback failure unexpectedly succeeded"; fi -find "$CLOUDFLARED_REPOSITORY_STATE_DIR" -maxdepth 2 -type f -name rollback.log -print -quit | grep -q . || - fail "rollback failure did not preserve evidence" -) -pass "rollback failure returns nonzero and preserves evidence" - -new_case -set_old_generation -configure_repository -current="$CLOUDFLARED_REPOSITORY_STATE_DIR/current" -[[ -f "$current" ]] || fail "generation marker missing" -key_hash=$(sha256sum "$CLOUDFLARED_KEYRING" | awk '{print $1}') -printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflared any main\n' "$CLOUDFLARED_KEYRING" > "$CASE_DIR/independent-source" -assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/independent-source" "source differs from official literal" -source_hash=$(sha256sum "$CASE_DIR/independent-source" | awk '{print $1}') -grep -Fxq "key_sha256=$key_hash" "$current" || fail "key generation hash mismatch" -grep -Fxq "source_sha256=$source_hash" "$current" || fail "source generation hash mismatch" -pass "key/source commit as one recorded generation" - -new_case -set_old_generation -configure_repository -validate_current_repository_manifest || fail "trusted current manifest validation failed" -remove_managed_repository -init_runtime_config -assert_absent "$CLOUDFLARED_SOURCE_FILE" "trusted current did not remove managed source" -[[ -f "$CLOUDFLARED_KEYRING" ]] || fail "managed repository removal deleted keyring" -[[ ! -f "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" ]] || fail "managed repository removal retained current" -find "$CLOUDFLARED_STATE_DIR" -maxdepth 1 -type d -name 'uninstall-*' -print -quit | grep -q . || - fail "managed source backup missing" -[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "managed repository removal left lock" -pass "configure then real repository uninstall removes source and preserves keyring" - -for signal in HUP INT TERM; do - case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - for phase in before-snapshot-create capture-auto_update_script capture-auto_update_timer capture-legacy_marker lock-acquired before-disable-auto-update after-disable-auto-update before-apt-remove after-apt-remove before-source-remove after-source-remove final-apt-update before-lock-release; do - new_case - set_old_generation - configure_repository - cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" - cp "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "$CASE_DIR/expected-current" - set +e - CLOUDFLARED_TEST_INTERNALS=1 SIGNAL_PHASE="$phase" SIGNAL_NAME="$signal" \ - bash -c ' - source "$1" - require_root() { :; } - check_platform() { :; } - uninstall_transaction_hook() { - [[ "$1" != "$SIGNAL_PHASE" ]] || kill -s "$SIGNAL_NAME" "$BASHPID" - } - uninstall_cloudflared --confirmed - ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" >"$CASE_DIR/out" 2>"$CASE_DIR/err" - status=$? - set -e - [[ "$status" == "$expected" ]] || fail "uninstall $signal at $phase returned $status" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall $signal at $phase left lock" - assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" "uninstall $signal at $phase failed source restore" - assert_same "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "$CASE_DIR/expected-current" "uninstall $signal at $phase failed current restore" - acquire_repository_lock || fail "uninstall $signal at $phase lock not reusable" - release_repository_lock || fail "uninstall $signal at $phase lock cleanup failed" - pass "uninstall $signal=$expected cleans $phase" - done -done - -new_case -set_old_generation -configure_repository -uninstall_transaction_hook() { [[ "$1" != after-disable-auto-update ]] || exit 0; } -set +e -( uninstall_cloudflared --confirmed ) >"$CASE_DIR/out" 2>"$CASE_DIR/err" -status=$? -set -e -[[ "$status" == 1 ]] || fail "active uninstall exit 0 returned $status" -[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "active uninstall exit 0 left lock" -grep -R -Fq '活动卸载事务异常退出' "$CLOUDFLARED_REPOSITORY_STATE_DIR" || fail "active uninstall exit 0 lacked evidence" -pass "active uninstall exit 0 returns 1 and cleans transaction" - -for manifest_case in missing duplicate unknown generation key-digest source-digest source-content symlink directory fifo owner gid mode; do - new_case - set_old_generation - configure_repository - current="$CLOUDFLARED_REPOSITORY_STATE_DIR/current" - case "$manifest_case" in - missing) sed -i '/^source_sha256=/d' "$current" ;; - duplicate) cat "$current" >> "$current.copy"; cat "$current.copy" >> "$current"; rm "$current.copy" ;; - unknown) printf 'unknown=value\n' >> "$current" ;; - generation) sed -i 's/^generation=.*/generation=invalid/' "$current" ;; - key-digest) sed -i 's/^key_sha256=.*/key_sha256=0000000000000000000000000000000000000000000000000000000000000000/' "$current" ;; - source-digest) sed -i 's/^source_sha256=.*/source_sha256=0000000000000000000000000000000000000000000000000000000000000000/' "$current" ;; - source-content) printf '# tampered\n' >> "$CLOUDFLARED_SOURCE_FILE" ;; - symlink) rm "$current"; ln -s "$CASE_DIR/external" "$current"; printf external > "$CASE_DIR/external" ;; - directory) rm "$current"; mkdir "$current" ;; - fifo) rm "$current"; mkfifo "$current" ;; - owner) chown 65534:0 "$current" ;; - gid) chown 0:65534 "$current" ;; - mode) chmod 0644 "$current" ;; - esac - if remove_managed_repository >/dev/null 2>&1; then fail "current $manifest_case unexpectedly trusted"; fi - [[ -f "$CLOUDFLARED_KEYRING" ]] || fail "current $manifest_case deleted keyring" - [[ -e "$CLOUDFLARED_SOURCE_FILE" ]] || fail "current $manifest_case deleted source" - [[ ! -e "$CASE_DIR/external" || "$(cat "$CASE_DIR/external")" == external ]] || fail "current symlink target changed" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "current $manifest_case left lock" - pass "current manifest rejects $manifest_case" -done - -new_case -set_old_generation -export FAKE_APT_INSTALL_FAIL=1 -if run_repository_apt_transaction install >/dev/null 2>&1; then fail "old generation rollback test succeeded"; fi -assert_old_generation -pass "old key/source preserved on transaction failure" - -new_case -run_repository_apt_transaction install -grep -Fxq 'apt-get:update' "$FAKE_LOG" || fail "APT probe not called" -grep -Fxq 'apt-get:install -y cloudflared' "$FAKE_LOG" || fail "cloudflared package selection changed" -if grep -Eqi 'remove|purge|autoremove|linux-image|linux-headers' "$FAKE_LOG"; then - fail "repository transaction performed unrelated APT changes" -fi -pass "no kernel deletion or unrelated APT change" - -script_hashes_before=$(sha256sum "$ROOT_DIR/linux_setup.sh" "$ROOT_DIR"/modules/*.sh "$ROOT_DIR"/tools/push.sh \ - "$ROOT_DIR"/tools/setup-motd.sh "$ROOT_DIR"/tools/xanmod-install.sh) -script_hashes_after=$(sha256sum "$ROOT_DIR/linux_setup.sh" "$ROOT_DIR"/modules/*.sh "$ROOT_DIR"/tools/push.sh \ - "$ROOT_DIR"/tools/setup-motd.sh "$ROOT_DIR"/tools/xanmod-install.sh) -[[ "$script_hashes_before" == "$script_hashes_after" ]] || fail "payload or unrelated scripts changed during tests" -pass "payload and unrelated scripts remain unchanged" - -# Keep prior lifecycle coverage with fake systemctl only. -new_case -cat > "$CLOUDFLARED_SERVICE_FILE" < "$CLOUDFLARED_LEGACY_BIN" <<'EOF' -#!/usr/bin/env bash -echo 'cloudflared version 2025.1.0' -EOF -chmod 0755 "$CLOUDFLARED_LEGACY_BIN" -migrate_legacy_binary -assert_absent "$CLOUDFLARED_LEGACY_BIN" "recognized legacy binary was not migrated" -grep -Fq "ExecStart=$CLOUDFLARED_APT_BIN --no-autoupdate --token-file /etc/cloudflared/token" "$CLOUDFLARED_SERVICE_FILE" || - fail "legacy service path was not preserved" -pass "legacy binary and service migration behavior remains" - - -# Each malformed snapshot runs in a separate process: transaction traps and -# intentional corrupt fixtures must not leak into subsequent cases. -for schema_case in version generation duplicate unknown order state mode uid gid digest created payload-name payload-missing payload-extra payload-fifo payload-symlink manifest-fifo ancestor; do - ( - new_case - configure_repository || fail "schema fixture configure" - acquire_repository_lock || fail "schema fixture lock" - begin_uninstall_transaction || fail "schema fixture capture" - manifest="$UNINSTALL_SNAPSHOT_DIR/manifest" - case "$schema_case" in - version) sed -i '1d' "$manifest" ;; - generation) sed -i '2s/.*/generation=20000101T000000Z-1-1/' "$manifest" ;; - duplicate) sed -i '3s/.*/version=1/' "$manifest" ;; - unknown) sed -i '10s/.*/path=\/tmp\/forbidden/' "$manifest" ;; - order) sed -i '3s/auto_update_script/auto_update_timer/' "$manifest" ;; - state) sed -i '4s/.*/state=invalid/' "$manifest" ;; - mode) sed -i '5s/.*/mode=777/' "$manifest" ;; - uid) sed -i '6s/.*/uid=1/' "$manifest" ;; - gid) sed -i '7s/.*/gid=1/' "$manifest" ;; - digest) sed -i '8s/.*/sha256=invalid/' "$manifest" ;; - created) sed -i '0,/^transaction_created=false$/s//transaction_created=invalid/' "$manifest" ;; - payload-name) sed -i 's/^snapshot=target-source$/snapshot=target-current/' "$manifest" ;; - payload-missing) rm -- "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ;; - payload-extra) printf extra > "$UNINSTALL_SNAPSHOT_DIR/files/target-extra"; chmod 600 "$UNINSTALL_SNAPSHOT_DIR/files/target-extra" ;; - payload-fifo) mkfifo "$UNINSTALL_SNAPSHOT_DIR/files/.fifo" ;; - payload-symlink) ln -s target-source "$UNINSTALL_SNAPSHOT_DIR/files/.link" ;; - manifest-fifo) rm -- "$manifest"; mkfifo "$manifest" ;; - ancestor) chmod 0777 "$UNINSTALL_SNAPSHOT_DIR/files" ;; - esac - if uninstall_manifest_valid "$manifest"; then fail "schema accepted $schema_case"; fi - # No rollback is appropriate for this read-only parser test. - UNINSTALL_TRANSACTION_ACTIVE=false - UNINSTALL_SNAPSHOT_BUILDING=false - UNINSTALL_TRANSACTION_STATE=NONE - restore_repository_traps - # Parent alone owns TEST_DIR cleanup; do not run its EXIT trap here. - trap - EXIT - release_repository_lock || fail "schema fixture release" - ) > "$TEST_DIR/schema-$schema_case.log" 2>&1 || fail "schema $schema_case (see $TEST_DIR/schema-$schema_case.log)" - pass "uninstall manifest rejects $schema_case" -done - - -for ownership_case in owned foreign inode mode owner gid symlink directory fifo; do - ( - new_case - uninstall_transaction_hook() { :; } - trap - EXIT - configure_repository || fail "ownership configure" - acquire_repository_lock || fail "ownership lock" - begin_uninstall_transaction || fail "ownership capture" - target="$AUTO_UPDATE_SCRIPT" - printf 'owned-content\n' > "$CASE_DIR/payload" - chmod 0600 "$CASE_DIR/payload" - if [[ "$ownership_case" == foreign ]]; then - printf foreign > "$target" - chmod 0755 "$target" - else - uninstall_create_absent_target auto_update_script "$CASE_DIR/payload" || fail "exclusive creation" - case "$ownership_case" in - inode) mv "$target" "$target.original"; cp "$target.original" "$target" ;; - mode) chmod 0644 "$target" ;; - owner) chown 65534:0 "$target" ;; - gid) chown 0:65534 "$target" ;; - symlink) rm "$target"; ln -s "$CASE_DIR/payload" "$target" ;; - directory) rm "$target"; mkdir "$target" ;; - fifo) rm "$target"; mkfifo "$target" ;; - esac - fi - if [[ "$ownership_case" == owned ]]; then - uninstall_cleanup ownership-test || fail "owned rollback" - assert_absent "$target" "owned file survived" - else - if uninstall_cleanup ownership-test; then fail "foreign identity accepted"; fi - [[ -e "$target" || -L "$target" ]] || fail "foreign object deleted" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" ]] || fail "manifest lost" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ]] || fail "payload lost" - fi - trap - EXIT - ) > "$TEST_DIR/ownership-$ownership_case.log" 2>&1 || { cat "$TEST_DIR/ownership-$ownership_case.log"; fail "ownership $ownership_case"; } - pass "absent ownership $ownership_case" -done - -for journal_case in generation duplicate state unknown reason; do - ( - new_case - trap - EXIT - uninstall_transaction_hook() { :; } - configure_repository || fail "journal configure" - acquire_repository_lock || fail "journal lock" - begin_uninstall_transaction || fail "journal capture" - journal="$UNINSTALL_SNAPSHOT_DIR/journal" - case "$journal_case" in - generation) sed -i '2s/.*/generation=wrong/' "$journal" ;; - duplicate) printf 'state=ACTIVE\n' >> "$journal" ;; - state) sed -i '3s/.*/state=COMMITTED/' "$journal" ;; - unknown) sed -i '1s/.*/unknown=1/' "$journal" ;; - reason) sed -i '4s/.*/path=invalid/' "$journal" ;; - esac - if uninstall_journal_valid "$journal" ACTIVE; then fail "bad journal accepted"; fi - UNINSTALL_TRANSACTION_STATE=FAILED - UNINSTALL_TRANSACTION_ACTIVE=false - UNINSTALL_SNAPSHOT_BUILDING=false - restore_repository_traps - release_repository_lock || fail "journal release" - ) > "$TEST_DIR/journal-$journal_case.log" 2>&1 || fail "journal $journal_case" - pass "journal rejects $journal_case" -done - -for finalization_case in journal archive release restore; do - ( - new_case - trap - EXIT - uninstall_transaction_hook() { :; } - configure_repository || fail "finalization configure" - acquire_repository_lock || fail "finalization lock" - begin_uninstall_transaction || fail "finalization capture" - snapshot_before="$UNINSTALL_SNAPSHOT_DIR" - case "$finalization_case" in - journal|archive) - repository_rename() { - if [[ "$finalization_case" == journal && "$2" == */journal ]] || - [[ "$finalization_case" == archive && "$1" == "$snapshot_before" ]]; then return 1; fi - command mv -fT -- "$1" "$2" - } - ;; - release) - # A nonempty lock is an actual rmdir failure, not a core stub. - printf retained > "$REPOSITORY_LOCK_DIR/injected" - ;; - restore) - repository_install_file() { - [[ "$2" != */target-source ]] || return 1 - command install -o 0 -g 0 -m "$1" -- "$2" "$3" - } - ;; - esac - if uninstall_cleanup "injected-$finalization_case"; then fail "finalization failure accepted"; fi - [[ "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "failed guard state" - [[ "$UNINSTALL_TRANSACTION_ACTIVE" == false ]] || fail "active guard survived" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" ]] || fail "manifest deleted" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ]] || fail "source payload deleted" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-current" ]] || fail "current payload deleted" - cmp "$SOURCE_FILE" "$UNINSTALL_SNAPSHOT_DIR/files/target-source" || fail "source changed" - [[ -f "$KEYRING" ]] || fail "keyring deleted" - if [[ "$finalization_case" == release ]]; then - [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "lock evidence missing" - fi - ) > "$TEST_DIR/finalization-$finalization_case.log" 2>&1 || { - cat "$TEST_DIR/finalization-$finalization_case.log" - fail "finalization $finalization_case" - } - pass "finalization $finalization_case failure preserves complete evidence" -done - -for capture_id in auto_update_script auto_update_timer legacy_marker; do - ( - new_case - trap - EXIT - configure_repository || fail "capture configure" - cp "$SOURCE_FILE" "$CASE_DIR/source-before" - cp "$KEYRING" "$CASE_DIR/key-before" - acquire_repository_lock || fail "capture lock" - uninstall_transaction_hook() { [[ "$1" != "capture-$capture_id" ]]; } - if begin_uninstall_transaction; then fail "capture fault ignored"; fi - [[ "$UNINSTALL_TRANSACTION_STATE" == BUILDING ]] || fail "partial capture activated" - if uninstall_cleanup capture-failed; then fail "partial capture reported recovery"; fi - cmp "$SOURCE_FILE" "$CASE_DIR/source-before" || fail "partial source changed" - cmp "$KEYRING" "$CASE_DIR/key-before" || fail "partial key changed" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/journal" ]] || fail "partial journal missing" - [[ "$UNINSTALL_TRANSACTION_ACTIVE" == false ]] || fail "partial guard active" - ) > "$TEST_DIR/capture-$capture_id.log" 2>&1 || { - cat "$TEST_DIR/capture-$capture_id.log" - fail "capture $capture_id" - } - pass "capture failure $capture_id retains partial evidence without restoration" -done - -for archive_case in intact payload journal; do - ( - new_case - trap - EXIT - uninstall_transaction_hook() { :; } - configure_repository || fail "archive configure" - acquire_repository_lock || fail "archive lock" - begin_uninstall_transaction || fail "archive capture" - uninstall_transaction_hook() { - [[ "$1" == after-evidence-rename ]] || return 0 - case "$archive_case" in - payload) printf corrupt >> "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ;; - journal) printf 'unknown=1\n' >> "$UNINSTALL_SNAPSHOT_DIR/journal" ;; - esac - return 0 - } - rc=0 - uninstall_cleanup archive-verification || rc=$? - if [[ "$archive_case" == intact ]]; then - [[ "$rc" == 0 && "$UNINSTALL_TRANSACTION_STATE" == ROLLED_BACK ]] || fail "valid archive rejected" - else - [[ "$rc" != 0 && "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "corrupt archive accepted" - fi - [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" && -f "$UNINSTALL_SNAPSHOT_DIR/journal" ]] || fail "archive evidence lost" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-current" ]] || fail "archive payload lost" - ) > "$TEST_DIR/archive-$archive_case.log" 2>&1 || { - cat "$TEST_DIR/archive-$archive_case.log" - fail "archive $archive_case" - } - pass "post-rename archive verification $archive_case" -done - -for interrupted_phase in before-snapshot-create after-snapshot-active after-evidence-rename after-evidence-seal before-pending-clear; do - ( - new_case - trap - EXIT - configure_repository || fail "kill configure" - cp "$SOURCE_FILE" "$CASE_DIR/source-before" - ( - trap - EXIT - acquire_repository_lock || exit 1 - uninstall_transaction_hook() { - if [[ "$1" == "$interrupted_phase" ]]; then kill -KILL "$BASHPID"; fi - } - begin_uninstall_transaction || exit 1 - uninstall_cleanup kill-test - ) > "$CASE_DIR/killed.log" 2>&1 & - child=$! - rc=0 - wait "$child" || rc=$? - [[ "$rc" == 137 ]] || fail "SIGKILL did not reach $interrupted_phase" - # Simulate operator removing only the stale lock; pending must still block. - rmdir "$CLOUDFLARED_STATE_DIR.lock" || fail "stale lock remove" - : > "$FAKE_LOG" - for operation in install_cloudflared upgrade_cloudflared uninstall_cloudflared disable_auto_update; do - ( - trap - EXIT - "$operation" --confirmed - ) > "$CASE_DIR/retry-$operation.log" 2>&1 && fail "pending allowed $operation" - done - [[ ! -s "$FAKE_LOG" ]] || fail "pending executed external command" - cmp "$SOURCE_FILE" "$CASE_DIR/source-before" || fail "pending changed source" - compgen -G "$CLOUDFLARED_STATE_DIR/repository/pending-uninstall-*" >/dev/null || fail "pending evidence missing" - ) > "$TEST_DIR/kill-$interrupted_phase.log" 2>&1 || { - cat "$TEST_DIR/kill-$interrupted_phase.log" - fail "SIGKILL $interrupted_phase" - } - pass "SIGKILL $interrupted_phase fails closed in independent entrypoints" -done - -for stage_case in owned foreign; do - ( - new_case - trap - EXIT - configure_repository || fail "stage configure" - acquire_repository_lock || fail "stage lock" - begin_uninstall_transaction || fail "stage capture" - stage=$(mktemp "$CASE_DIR/root/.cloudflared-create.XXXXXX") - uninstall_register_stage "$stage" || fail "stage register" - if [[ "$stage_case" == foreign ]]; then - mv "$stage" "$stage.original" - printf foreign > "$stage" - fi - rc=0 - uninstall_cleanup stage-test || rc=$? - if [[ "$stage_case" == owned ]]; then - [[ "$rc" == 0 && ! -e "$stage" ]] || fail "owned stage not cleaned" - [[ "$(stat -c %a "$UNINSTALL_SNAPSHOT_DIR")" == 500 ]] || fail "archive writable" - [[ "$(stat -c %a "$UNINSTALL_SNAPSHOT_DIR/manifest")" == 400 ]] || fail "manifest writable" - validate_uninstall_archive "$UNINSTALL_SNAPSHOT_DIR" "$UNINSTALL_GENERATION" || fail "sealed archive invalid" - else - [[ "$rc" != 0 && "$(cat "$stage")" == foreign ]] || fail "foreign stage removed" - [[ -f "$UNINSTALL_PENDING" ]] || fail "stage failure lost pending" - fi - ) > "$TEST_DIR/stage-$stage_case.log" 2>&1 || { - cat "$TEST_DIR/stage-$stage_case.log" - fail "stage $stage_case" - } - pass "stage cleanup verifies $stage_case identity" -done - -for terminal_signal in HUP INT TERM; do - case "$terminal_signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - ( - new_case - trap - EXIT - configure_repository || fail "terminal signal configure" - rc=0 - ( - trap - EXIT - acquire_repository_lock || exit 1 - uninstall_transaction_hook() { - if [[ "$1" == after-evidence-seal ]]; then kill -s "$terminal_signal" "$BASHPID"; fi - } - begin_uninstall_transaction || exit 1 - uninstall_cleanup terminal-signal - ) > "$CASE_DIR/signal.log" 2>&1 || rc=$? - [[ "$rc" == "$expected" ]] || fail "terminal signal status $rc" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "terminal signal lock retained" - compgen -G "$CLOUDFLARED_STATE_DIR/repository/pending-uninstall-*" >/dev/null || fail "terminal signal pending missing" - [[ -f "$SOURCE_FILE" ]] || fail "terminal signal source lost" - ) > "$TEST_DIR/terminal-$terminal_signal.log" 2>&1 || { - cat "$TEST_DIR/terminal-$terminal_signal.log" - fail "terminal signal $terminal_signal" - } - pass "terminal $terminal_signal=$expected retains evidence without repeated rollback" -done - -for restore_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do - ( - new_case - trap - EXIT - configure_repository || fail "six target configure" - mkdir -p "$(dirname "$AUTO_UPDATE_SCRIPT")" "$(dirname "$AUTO_UPDATE_SERVICE")" - write_auto_update_files || fail "six target auto update" - printf managed > "$STATE_DIR/repository-managed" - chmod 0600 "$STATE_DIR/repository-managed" - acquire_repository_lock || fail "six target lock" - begin_uninstall_transaction || fail "six target capture" - snapshot=$UNINSTALL_SNAPSHOT_DIR - while IFS= read -r id; do rm -- "$(uninstall_target_path "$id")"; done < <(uninstall_snapshot_targets) - repository_install_file() { - [[ "$2" != "$snapshot/files/target-$restore_id" ]] || return 1 - command install -o 0 -g 0 -m "$1" -- "$2" "$3" - } - if uninstall_cleanup six-target-failure; then fail "restore fault accepted"; fi - while IFS= read -r id; do - [[ -f "$snapshot/files/target-$id" ]] || fail "snapshot payload lost $id" - if [[ "$id" != "$restore_id" ]]; then - cmp "$(uninstall_target_path "$id")" "$snapshot/files/target-$id" || fail "restore skipped $id" - validate_secure_file "$(uninstall_target_path "$id")" "$(uninstall_target_mode "$id")" || fail "restore metadata $id" - fi - done < <(uninstall_snapshot_targets) - [[ -f "$UNINSTALL_PENDING" ]] || fail "restore pending lost" - uninstall_journal_valid "$snapshot/journal" ACTIVE || fail "failure overwrote original journal" - ) > "$TEST_DIR/restore-$restore_id.log" 2>&1 || { - cat "$TEST_DIR/restore-$restore_id.log" - fail "restore failure $restore_id" - } - pass "restore failure $restore_id continues remaining five targets and preserves original evidence" -done - -for target_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do - for bad_metadata in mode owner gid symlink directory fifo; do - ( - new_case - trap - EXIT - configure_repository || fail "metadata configure" - mkdir -p "$(dirname "$AUTO_UPDATE_SCRIPT")" "$(dirname "$AUTO_UPDATE_SERVICE")" - write_auto_update_files || fail "metadata auto update" - printf managed > "$STATE_DIR/repository-managed" - chmod 0600 "$STATE_DIR/repository-managed" - target=$(uninstall_target_path "$target_id") - case "$bad_metadata" in - mode) chmod 0666 "$target" ;; - owner) chown 1:0 "$target" ;; - gid) chown 0:1 "$target" ;; - symlink) mv "$target" "$target.external"; ln -s "$target.external" "$target" ;; - directory) rm "$target"; mkdir "$target" ;; - fifo) rm "$target"; mkfifo "$target" ;; - esac - acquire_repository_lock || fail "metadata lock" - if begin_uninstall_transaction; then fail "bad target captured"; fi - uninstall_cleanup bad-metadata >/dev/null 2>&1 || : - [[ -e "$target" || -L "$target" ]] || fail "untrusted target deleted" - [[ "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "capture failure state lost" - ) > "$TEST_DIR/meta-$target_id-$bad_metadata.log" 2>&1 || { - cat "$TEST_DIR/meta-$target_id-$bad_metadata.log" - fail "metadata $target_id $bad_metadata" - } - pass "capture rejects $target_id $bad_metadata without deleting target" - done -done - -( - unset CLOUDFLARED_TRUST_ANCHOR - init_runtime_config - [[ "$TRUST_ANCHOR" == / ]] || exit 1 - path_is_beneath_anchor / || exit 1 - path_is_beneath_anchor /var/lib || exit 1 - CLOUDFLARED_TRUST_ANCHOR=/var - path_is_beneath_anchor /var || exit 1 - path_is_beneath_anchor /var/lib || exit 1 - if path_is_beneath_anchor /variable; then exit 1; fi - if path_is_beneath_anchor /var/../etc; then exit 1; fi - path_is_beneath_anchor /var/lib/../lib || exit 1 -) || fail "pure anchor boundaries" -pass "pure paths validate default root anchor and normalized non-root boundaries" - -for creation_phase in before-create-marker before-create-manifest-stage before-create-manifest-rename before-create-publish after-create-publish; do - for creation_fault in failure HUP INT TERM; do - ( - new_case - trap - EXIT - configure_repository || fail "creation configure" - printf payload > "$CASE_DIR/payload" - chmod 0600 "$CASE_DIR/payload" - rc=0 - ( - trap - EXIT - acquire_repository_lock || exit 1 - begin_uninstall_transaction || exit 1 - uninstall_transaction_hook() { - [[ "$1" == "$creation_phase" ]] || return 0 - if [[ "$creation_fault" == failure ]]; then return 1; fi - kill -s "$creation_fault" "$BASHPID" - } - if uninstall_create_absent_target auto_update_script "$CASE_DIR/payload"; then exit 90; fi - uninstall_cleanup creation-failure || : - exit 1 - ) > "$CASE_DIR/creation.log" 2>&1 || rc=$? - case "$creation_fault" in failure) expected=1 ;; HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - [[ "$rc" == "$expected" ]] || fail "creation status $rc" - [[ ! -e "$AUTO_UPDATE_SCRIPT" && ! -L "$AUTO_UPDATE_SCRIPT" ]] || fail "owned publication not recovered" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "creation lock leaked" - ) > "$TEST_DIR/create-$creation_phase-$creation_fault.log" 2>&1 || { - cat "$TEST_DIR/create-$creation_phase-$creation_fault.log" - fail "creation $creation_phase $creation_fault" - } - pass "creation $creation_phase $creation_fault preserves ownership across publication" - done -done - -for bookkeeping_fault in marker stage rename; do - ( - new_case - trap - EXIT - configure_repository || fail "bookkeeping configure" - printf payload > "$CASE_DIR/payload"; chmod 0600 "$CASE_DIR/payload" - acquire_repository_lock || fail "bookkeeping lock" - begin_uninstall_transaction || fail "bookkeeping capture" - uninstall_transaction_hook() { - if [[ "$bookkeeping_fault" == marker && "$1" == before-create-marker ]]; then - mkdir "$UNINSTALL_SNAPSHOT_DIR/created-auto_update_script" - fi - return 0 - } - mktemp() { - if [[ "$bookkeeping_fault" == stage && "$1" == */manifest.stage.* ]]; then return 1; fi - command mktemp "$@" - } - repository_rename() { - if [[ "$bookkeeping_fault" == rename && "$1" == */manifest.stage.* ]]; then return 1; fi - command mv -fT -- "$1" "$2" - } - if uninstall_create_absent_target auto_update_script "$CASE_DIR/payload"; then fail "bookkeeping fault ignored"; fi - [[ ! -e "$AUTO_UPDATE_SCRIPT" ]] || fail "published before durable bookkeeping" - uninstall_cleanup bookkeeping-failure || : - ) > "$TEST_DIR/bookkeeping-$bookkeeping_fault.log" 2>&1 || { - cat "$TEST_DIR/bookkeeping-$bookkeeping_fault.log" - fail "bookkeeping $bookkeeping_fault" - } - pass "actual $bookkeeping_fault failure precedes ln publication" -done - -new_case -configure_repository || fail "URL configure" -grep '^curl:' "$FAKE_LOG" | grep -Fq 'https://pkg.cloudflare.com/cloudflare-main.gpg' || fail "key download URL changed" -printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflared any main\n' "$CLOUDFLARED_KEYRING" > "$CASE_DIR/official-source" -assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/official-source" "APT repository URL incorrect" -validate_current_repository_manifest || fail "official current rejected" -remove_managed_repository || fail "official source removal failed" -assert_absent "$CLOUDFLARED_SOURCE_FILE" "official source not removed" -pass "literal key URL and repository source validate current and real removal" - -new_case -printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflare-main.gpg any main\n' "$CLOUDFLARED_KEYRING" > "$CLOUDFLARED_SOURCE_FILE" -chmod 0644 "$CLOUDFLARED_SOURCE_FILE" -cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/wrong-source" -if configure_repository > "$CASE_DIR/rejected.log" 2>&1; then fail "gpg repository accepted"; fi -assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/wrong-source" "unowned wrong source overwritten" -[[ ! -s "$FAKE_LOG" ]] || fail "invalid source triggered external operation" -pass "gpg repository URL rejected without overwrite or download" - -for capture_file in old-key old-source old-current; do - for capture_fault in copy partial chmod HUP INT TERM exit0; do - ( - new_case - trap - EXIT - configure_repository || fail "capture fixture" - for object in "$KEYRING" "$SOURCE_FILE" "$REPOSITORY_STATE_DIR/current"; do - cp -p "$object" "$CASE_DIR/$(basename "$object").before" - done - : > "$FAKE_LOG" - rc=0 - ( - trap - EXIT - repository_copy_file() { - if [[ "$2" == */"$capture_file" ]]; then - case "$capture_fault" in - copy) return 1 ;; - partial) printf partial > "$2"; return 1 ;; - HUP|INT|TERM) kill -s "$capture_fault" "$BASHPID" ;; - exit0) exit 0 ;; - esac - fi - command cp -- "$1" "$2" - } - chmod() { - if [[ "$capture_fault" == chmod && "$2" == */"$capture_file" ]]; then return 1; fi - command chmod "$@" - } - configure_repository - ) > "$CASE_DIR/capture.log" 2>&1 || rc=$? - case "$capture_fault" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; *) expected=1 ;; esac - [[ "$rc" == "$expected" ]] || fail "capture status $rc" - for object in "$KEYRING" "$SOURCE_FILE" "$REPOSITORY_STATE_DIR/current"; do - cmp "$object" "$CASE_DIR/$(basename "$object").before" || fail "capture modified formal bytes" - [[ "$(stat -c '%u:%g:%a' "$object")" == "$(stat -c '%u:%g:%a' "$CASE_DIR/$(basename "$object").before")" ]] || fail "capture modified metadata" - done - [[ ! -s "$FAKE_LOG" ]] || fail "capture downloaded or called APT" - grep -q '未恢复或删除正式文件' "$CASE_DIR/capture.log" || fail "capture diagnostic missing" - acquire_repository_lock || fail "capture lock retained" - release_repository_lock || fail "capture lock release" - ) > "$TEST_DIR/capture-boundary-$capture_file-$capture_fault.log" 2>&1 || { - cat "$TEST_DIR/capture-boundary-$capture_file-$capture_fault.log" - fail "capture boundary $capture_file $capture_fault" - } - pass "repository capture $capture_file $capture_fault preserves formal bytes and metadata" - done -done - -( - new_case - trap - EXIT - set_old_generation - rc=0 - ( - trap - EXIT - begin_repository_transaction || exit 1 - exit 0 - ) > "$CASE_DIR/active-exit.log" 2>&1 || rc=$? - [[ "$rc" == 1 ]] || fail "active exit zero accepted" - assert_old_generation - [[ ! -e "$REPOSITORY_STATE_DIR/current" ]] || fail "absent current restored incorrectly" - acquire_repository_lock || fail "active exit lock retained" - release_repository_lock || fail "active exit release" -) || fail "active exit zero" -pass "complete repository transaction exit zero returns nonzero and restores absent current" - -for build_case in absent archive-failure; do - ( - new_case - trap - EXIT - if [[ "$build_case" == archive-failure ]]; then set_old_generation; fi - trap ':' HUP - before_traps=$(trap -p HUP INT TERM EXIT) - repository_copy_file() { printf partial > "$2"; return 1; } - if [[ "$build_case" == absent ]]; then - # No formal object exists; fail before any download via snapshot wrapper. - chmod() { - [[ "$2" != */transaction-* ]] || return 1 - command chmod "$@" - } - repository_transaction_hook() { exit 0; } - rc=0 - (begin_repository_transaction) > "$CASE_DIR/absent.log" 2>&1 || rc=$? - [[ "$rc" != 0 ]] || fail "absent abnormal exit accepted" - [[ ! -e "$KEYRING" && ! -e "$SOURCE_FILE" && ! -e "$REPOSITORY_STATE_DIR/current" ]] || fail "absent formal object created" - else - repository_rename() { return 1; } - if configure_repository > "$CASE_DIR/build.log" 2>&1; then fail "failed capture accepted"; fi - assert_old_generation - [[ -f "$REPOSITORY_TRANSACTION_DIR/old-key" ]] || fail "partial evidence lost" - [[ -f "$REPOSITORY_TRANSACTION_DIR/rollback.log" ]] || fail "failure diagnostic lost" - grep -q '回滚不完整' "$CASE_DIR/build.log" || fail "cleanup error not reported" - fi - [[ "$(trap -p HUP INT TERM EXIT)" == "$before_traps" ]] || fail "build traps changed" - acquire_repository_lock || fail "build cleanup lock retained" - release_repository_lock || fail "build cleanup lock release" - ) > "$TEST_DIR/build-$build_case.log" 2>&1 || { - cat "$TEST_DIR/build-$build_case.log" - fail "build $build_case" - } - pass "repository building $build_case preserves objects traps and evidence" -done - -for restore_object in key source current; do - for restore_fault in partial wrong-content wrong-mode rename post-content post-mode; do - ( - new_case - trap - EXIT - configure_repository || fail "restore fixture" - begin_repository_transaction || fail "restore transaction" - snapshot=$REPOSITORY_TRANSACTION_DIR - printf historical > "$REPOSITORY_STATE_DIR/current.rollback" - cp "$REPOSITORY_STATE_DIR/current.rollback" "$CASE_DIR/historical" - case "$restore_object" in key) target=$KEYRING ;; source) target=$SOURCE_FILE ;; current) target=$REPOSITORY_STATE_DIR/current ;; esac - cp "$target" "$CASE_DIR/pre-restore" - : > "$CASE_DIR/attempts" - repository_install_file() { - printf '%s\n' "$2" >> "$CASE_DIR/attempts" - if [[ "$2" == "$snapshot/old-$restore_object" ]]; then - case "$restore_fault" in - partial) printf partial > "$3"; return 1 ;; - wrong-content) command install -o 0 -g 0 -m "$1" "$2" "$3"; printf wrong > "$3"; return 0 ;; - wrong-mode) command install -o 0 -g 0 -m 0777 "$2" "$3"; return 0 ;; - esac - fi - command install -o 0 -g 0 -m "$1" "$2" "$3" - } - repository_rename() { - if [[ "$2" == "$target" && "$1" == */.cloudflared-rollback.* ]]; then - [[ "$restore_fault" != rename ]] || return 1 - command mv -fT -- "$1" "$2" || return 1 - case "$restore_fault" in post-content) printf corrupt > "$2" ;; post-mode) chmod 0777 "$2" ;; esac - return 0 - fi - command mv -fT -- "$1" "$2" - } - if rollback_repository_transaction injected > "$CASE_DIR/restore.log" 2>&1; then fail "restore fault accepted"; fi - for item in key source current; do - [[ -f "$REPOSITORY_TRANSACTION_DIR/old-$item" ]] || fail "backup lost $item" - grep -Fxq "$snapshot/old-$item" "$CASE_DIR/attempts" || fail "restore skipped $item" - done - case "$restore_fault" in partial|wrong-content|wrong-mode|rename) cmp "$target" "$CASE_DIR/pre-restore" || fail "invalid stage published" ;; esac - cmp "$REPOSITORY_STATE_DIR/current.rollback" "$CASE_DIR/historical" || fail "historical rollback touched" - if grep -q '旧 key/source 已恢复' "$CASE_DIR/restore.log"; then fail "false recovery success"; fi - grep -q '回滚不完整' "$CASE_DIR/restore.log" || fail "missing failure diagnosis" - acquire_repository_lock || fail "restore lock retained" - release_repository_lock || fail "restore lock release" - ) > "$TEST_DIR/publish-$restore_object-$restore_fault.log" 2>&1 || { - cat "$TEST_DIR/publish-$restore_object-$restore_fault.log" - fail "restore publish $restore_object $restore_fault" - } - pass "repository restore $restore_object $restore_fault rejects corruption and retains all backups" - done -done - -for operation in install_cloudflared upgrade_cloudflared; do - ( - new_case - trap - EXIT - cp "$CASE_DIR/bin/cloudflared" "$APT_BIN" - dpkg-query() { [[ "$*" == *'db:Status-Status'* ]] && { printf installed; return 0; }; return 1; } - timer_enabled=true - systemctl() { - printf 'systemctl:%s\n' "$*" >> "$FAKE_LOG" - if [[ "$*" == *cloudflared-apt-update.timer* && ( "$1" == is-enabled || "$1" == is-active ) ]]; then - [[ "$timer_enabled" == true ]] - else - return 0 - fi - } - write_auto_update_files || fail "timer fixture" - auto_update_file_is_managed "$AUTO_UPDATE_SCRIPT" || fail "main script template rejected" - auto_update_file_is_managed "$AUTO_UPDATE_SERVICE" || fail "main service template rejected" - auto_update_file_is_managed "$AUTO_UPDATE_TIMER" || fail "main timer template rejected" - "$operation" || fail "enabled operation" - [[ "$PRESERVE_AUTO_UPDATE" == true ]] || fail "strategy reset by init" - grep -Fxq 'systemctl:enable --now cloudflared-apt-update.timer' "$FAKE_LOG" || fail "timer not enabled" - timer_enabled=false - : > "$FAKE_LOG" - "$operation" || fail "disabled operation" - [[ "$PRESERVE_AUTO_UPDATE" == false ]] || fail "strategy leaked between operations" - if grep -Fxq 'systemctl:enable --now cloudflared-apt-update.timer' "$FAKE_LOG"; then fail "disabled timer enabled"; fi - timer_enabled=true - export FAKE_APT_UPDATE_FAIL=1 - : > "$FAKE_LOG" - if "$operation"; then fail "repository failure ignored"; fi - if grep -Eq 'systemctl:(disable|stop|enable|daemon-reload)' "$FAKE_LOG"; then fail "repository failure continued lifecycle"; fi - ) > "$TEST_DIR/policy-$operation.log" 2>&1 || { - cat "$TEST_DIR/policy-$operation.log" - fail "policy $operation" - } - pass "policy $operation preserves enabled main timer without cross-operation leakage" -done - -( - new_case - trap - EXIT - write_auto_update_files || fail "template fixture" - printf '# changed\n' >> "$AUTO_UPDATE_TIMER" - if auto_update_file_is_managed "$AUTO_UPDATE_TIMER"; then fail "partial marker trusted"; fi - cp "$AUTO_UPDATE_TIMER" "$CASE_DIR/foreign-timer" - if disable_auto_update --confirmed; then fail "unknown timer deleted"; fi - cmp "$AUTO_UPDATE_TIMER" "$CASE_DIR/foreign-timer" || fail "unknown timer changed" -) || fail "strict main template" -pass "main updater templates reject extra content without deleting unknown files" - -( - new_case - trap - EXIT - begin_repository_transaction || fail "absent transaction" - mv "$SOURCE_FILE" "$CASE_DIR/owned-source" - printf foreign > "$SOURCE_FILE"; chmod 0644 "$SOURCE_FILE" - if rollback_repository_transaction foreign; then fail "foreign absent accepted"; fi - [[ "$(cat "$SOURCE_FILE")" == foreign ]] || fail "foreign absent deleted" - [[ -f "$REPOSITORY_TRANSACTION_DIR/pending" ]] || fail "rollback pending missing" - : > "$FAKE_LOG" - if configure_repository; then fail "pending allowed restart"; fi - [[ ! -s "$FAKE_LOG" ]] || fail "pending performed external work" -) || fail "absent ownership" -pass "repository absent foreign object survives rollback and blocks restart" - -( - new_case - trap - EXIT - acquire_repository_lock || fail "lock fixture" - release_repository_lock || fail "first release" - mkdir "$CLOUDFLARED_STATE_DIR.lock" - release_repository_lock || fail "idempotent release" - [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "released another operation lock" -) || fail "lock identity" -pass "repeated lock release preserves next operation lock" - -( - new_case - trap - EXIT - set_old_generation - mkdir -p "$STATE_DIR" - chmod 0700 "$STATE_DIR" - printf unknown > "$STATE_DIR/repository-managed" - chmod 0600 "$STATE_DIR/repository-managed" - : > "$FAKE_LOG" - if uninstall_cloudflared --confirmed; then fail "bad legacy marker accepted"; fi - [[ ! -s "$FAKE_LOG" ]] || fail "legacy ownership checked after destructive operations" -) || fail "legacy preflight" -pass "legacy uninstall rejects unknown ownership before systemctl or APT" - -for review_fault in timer-enable stage-identity trap-restore; do - ( - new_case - trap - EXIT - case "$review_fault" in - timer-enable) - dpkg-query() { printf installed; } - systemctl() { - printf '%s\n' "$*" >> "$FAKE_LOG" - [[ "$*" != 'enable --now cloudflared-apt-update.timer' ]] - } - if enable_auto_update > "$CASE_DIR/error.log" 2>&1; then fail "timer enable failure hidden"; fi - grep -Fxq 'enable --now cloudflared-apt-update.timer' "$FAKE_LOG" || fail "timer injection not hit" - ;; - stage-identity) - begin_repository_transaction || fail "stage fixture" - stage=$(mktemp "$CASE_DIR/.stage.XXXXXX") - repository_track_stage "$stage" || fail "track stage" - mv "$stage" "$stage.owned" - printf foreign > "$stage" - if rollback_repository_transaction injected; then fail "foreign stage ignored"; fi - [[ "$(cat "$stage")" == foreign ]] || fail "foreign stage deleted" - ;; - trap-restore) - begin_repository_transaction || fail "trap fixture" - restore_one_trap() { printf '%s\n' "$1" >> "$CASE_DIR/trap-attempts"; return 1; } - if rollback_repository_transaction injected; then fail "trap failure hidden"; fi - [[ "$(wc -l < "$CASE_DIR/trap-attempts")" == 4 ]] || fail "trap attempts skipped" - [[ -f "$REPOSITORY_TRANSACTION_DIR/pending" ]] || fail "trap failure pending missing" - ;; - esac - ) > "$TEST_DIR/review-$review_fault.log" 2>&1 || { - cat "$TEST_DIR/review-$review_fault.log" - fail "review $review_fault" - } - pass "review $review_fault injection reached and failure preserved" -done - -for phase in disable-systemctl disable-backup disable-delete; do - for fault in HUP INT TERM zero failure; do - ( - new_case - trap - EXIT - configure_repository || fail "disable fixture" - write_auto_update_files || fail "disable files" - rc=0 - ( - trap - EXIT - updater_transaction_hook() { - [[ "$1" == "$phase" ]] || return 0 - case "$fault" in zero) exit 0 ;; failure) return 1 ;; *) kill -s "$fault" "$BASHPID" ;; esac - } - disable_auto_update --confirmed - ) > "$CASE_DIR/disable.log" 2>&1 || rc=$? - case "$fault" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; *) expected=1 ;; esac - [[ "$rc" == "$expected" ]] || fail "disable exit $rc" - [[ -f "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "premature file deletion" - compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "disable pending missing" - grep -q '自动更新操作未完成' "$CASE_DIR/disable.log" || fail "disable diagnostic missing" - acquire_repository_lock || fail "disable lock retained" - release_repository_lock || fail "disable lock release" - ) > "$TEST_DIR/disable-$phase-$fault.log" 2>&1 || { - cat "$TEST_DIR/disable-$phase-$fault.log" - fail "disable $phase $fault" - } - pass "independent disable $phase $fault preserves pending and exit contract" - done -done - -for phase in enable-validate enable-write enable-reload enable-timer disable-systemctl; do - ( - new_case - trap - EXIT - configure_repository || fail "barrier configure" - write_auto_update_files || fail "barrier files" - mkfifo "$CASE_DIR/ready" "$CASE_DIR/release" - ( - trap - EXIT - dpkg-query() { printf installed; } - updater_transaction_hook() { - [[ "$1" == "$phase" ]] || return 0 - printf ready > "$CASE_DIR/ready" - read -r _ < "$CASE_DIR/release" - } - if [[ "$phase" == disable-* ]]; then disable_auto_update --confirmed; else enable_auto_update; fi - ) > "$CASE_DIR/holder.log" 2>&1 & - holder=$! - read -r ready < "$CASE_DIR/ready" || [[ "$ready" == ready ]] || fail "barrier readiness" - for contender in enable_auto_update disable_auto_update uninstall_cloudflared; do - if ( "$contender" --confirmed ) > "$CASE_DIR/contender.log" 2>&1; then fail "contender bypassed lock"; fi - done - printf 'release\n' > "$CASE_DIR/release" - wait "$holder" || { cat "$CASE_DIR/holder.log"; fail "holder failed"; } - ) > "$TEST_DIR/barrier-$phase.log" 2>&1 || { cat "$TEST_DIR/barrier-$phase.log"; fail "barrier $phase"; } - pass "real updater barrier $phase excludes all competing mutation entrypoints" -done - -for fault in backup delete release; do - ( - new_case - trap - EXIT - configure_repository || fail "file fault fixture" - write_auto_update_files || fail "file fault files" - hit=false - cp() { - if [[ "$fault" == backup && "$*" == *auto-update-* ]]; then hit=true; return 1; fi - command cp "$@" - } - rm() { - if [[ "$fault" == delete && "$*" == *"$AUTO_UPDATE_SERVICE"* ]]; then hit=true; return 1; fi - command rm "$@" - } - rmdir() { - if [[ "$fault" == release ]]; then hit=true; return 1; fi - command rmdir "$@" - } - before=$(trap -p HUP INT TERM EXIT) - if disable_auto_update --confirmed > "$CASE_DIR/error.log" 2>&1; then fail "disable file fault accepted"; fi - [[ "$hit" == true ]] || fail "file fault not hit" - [[ "$(trap -p HUP INT TERM EXIT)" == "$before" ]] || fail "disable traps not restored" - compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "file fault pending missing" - case "$fault" in - backup) [[ -f "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "backup failure deleted files" ;; - delete) [[ ! -e "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "partial deletion group wrong" ;; - release) [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "failed release lost lock" ;; - esac - ) > "$TEST_DIR/disable-file-$fault.log" 2>&1 || { cat "$TEST_DIR/disable-file-$fault.log"; fail "disable file $fault"; } - pass "independent disable actual $fault failure retains group evidence and traps" -done - -# Separate shells load the implementation independently; only file/FIFO state is shared. -( - new_case - trap - EXIT - configure_repository || fail "uninstall barrier configure" - write_auto_update_files || fail "uninstall barrier updater" - mkfifo "$CASE_DIR/uninstall-ready" "$CASE_DIR/uninstall-release" - bash -c ' - source "$1" - check_platform() { :; } - uninstall_transaction_hook() { - [[ "$1" == after-snapshot-active ]] || return 0 - printf "%s\n" "$UNINSTALL_SNAPSHOT_DIR" > "$CASE_DIR/snapshot-path" - printf "ready\n" > "$CASE_DIR/uninstall-ready" - read -r _ < "$CASE_DIR/uninstall-release" - } - uninstall_cloudflared --confirmed - ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" > "$CASE_DIR/holder.log" 2>&1 & - holder=$! - # Timeout bounds a broken readiness handshake; no scheduling sleeps. - timeout 30 bash -c 'read -r ready < "$CASE_DIR/uninstall-ready"; [[ "$ready" == ready ]]' || fail "uninstall never reached active snapshot" - snapshot=$(cat "$CASE_DIR/snapshot-path") - [[ -f "$snapshot/manifest" ]] || fail "snapshot not ready" - lock_before=$(stat -c '%d:%i:%u:%g:%a' "$CLOUDFLARED_STATE_DIR.lock") - find "$REPOSITORY_STATE_DIR" -type f -exec sha256sum {} + | sort > "$CASE_DIR/state-before" - sha256sum "$AUTO_UPDATE_SCRIPT" "$AUTO_UPDATE_SERVICE" "$AUTO_UPDATE_TIMER" > "$CASE_DIR/updater-before" - for contender in enable_auto_update disable_auto_update; do - : > "$CASE_DIR/contender-external.log" - rc=0 - FAKE_LOG="$CASE_DIR/contender-external.log" bash -c ' - source "$1" - check_platform() { :; } - "$2" --confirmed - ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" "$contender" > "$CASE_DIR/contender.log" 2>&1 || rc=$? - [[ "$rc" != 0 ]] || fail "contender entered uninstall lock" - grep -q '无法取得 Cloudflare 仓库事务锁' "$CASE_DIR/contender.log" || fail "competition not reached" - [[ ! -s "$CASE_DIR/contender-external.log" ]] || fail "contender executed external command" - [[ "$(stat -c '%d:%i:%u:%g:%a' "$CLOUDFLARED_STATE_DIR.lock")" == "$lock_before" ]] || fail "holder lock changed" - find "$REPOSITORY_STATE_DIR" -type f -exec sha256sum {} + | sort > "$CASE_DIR/state-after" - cmp "$CASE_DIR/state-before" "$CASE_DIR/state-after" || fail "snapshot or pending changed" - sha256sum -c "$CASE_DIR/updater-before" >/dev/null || fail "contender changed updater" - done - printf 'release\n' > "$CASE_DIR/uninstall-release" - wait "$holder" || { cat "$CASE_DIR/holder.log"; fail "uninstall holder failed"; } - [[ ! -e "$SOURCE_FILE" && ! -e "$AUTO_UPDATE_TIMER" ]] || fail "uninstall did not complete" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall retained lock" -) > "$TEST_DIR/uninstall-holder.log" 2>&1 || { cat "$TEST_DIR/uninstall-holder.log"; fail "uninstall holder barrier"; } -pass "independent uninstall holder excludes enable and disable without changing snapshot pending or lock" - -for dependency_fault in update install; do - ( - new_case - trap - EXIT - write_auto_update_files || fail "dependency updater fixture" - sha256sum "$AUTO_UPDATE_SCRIPT" "$AUTO_UPDATE_SERVICE" "$AUTO_UPDATE_TIMER" > "$CASE_DIR/updater-before" - command() { - if [[ "$*" == '-v flock' ]]; then - printf 'flock-missing\n' >> "$CASE_DIR/dependency-hits" - return 1 - fi - builtin command "$@" - } - dpkg-query() { printf installed; } - apt-get() { - printf 'apt-get:%s\n' "$*" >> "$FAKE_LOG" - if [[ "$1" == update && "$dependency_fault" == update ]]; then return 41; fi - if [[ "$*" == 'install -y util-linux' ]]; then return 42; fi - return 0 - } - trap ':' HUP - before_traps=$(trap -p HUP INT TERM EXIT) - rc=0 - enable_auto_update > "$CASE_DIR/dependency.log" 2>&1 || rc=$? - case "$dependency_fault" in update) expected=41 ;; install) expected=42 ;; esac - [[ "$rc" == "$expected" ]] || fail "dependency status $rc" - grep -Fxq flock-missing "$CASE_DIR/dependency-hits" || fail "flock branch not reached" - grep -Fxq apt-get:update "$FAKE_LOG" || fail "dependency update not reached" - if [[ "$dependency_fault" == update ]]; then - if grep -q 'apt-get:install' "$FAKE_LOG"; then fail "install continued after update failure"; fi - else - grep -Fxq 'apt-get:install -y util-linux' "$FAKE_LOG" || fail "util install not reached" - fi - if grep -Eq 'systemctl:.*(reload|enable)' "$FAKE_LOG"; then fail "dependency failure continued timer work"; fi - sha256sum -c "$CASE_DIR/updater-before" >/dev/null || fail "dependency failure changed updater" - validate_current_repository_manifest || fail "prior repository configuration not recorded" - compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "dependency pending missing" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "dependency lock retained" - [[ "$(trap -p HUP INT TERM EXIT)" == "$before_traps" ]] || fail "dependency traps changed" - grep -q '自动更新操作未完成' "$CASE_DIR/dependency.log" || fail "dependency diagnostic missing" - ) > "$TEST_DIR/dependency-$dependency_fault.log" 2>&1 || { cat "$TEST_DIR/dependency-$dependency_fault.log"; fail "dependency $dependency_fault"; } - pass "missing flock $dependency_fault failure stops updater after recorded repository commit" -done - -entrypoint_output=$(bash -c "$(cat "$ROOT_DIR/tools/cloudflare_tunnel.sh")" cloudflare_tunnel.sh help) -grep -Fq 'cloudflare_tunnel.sh install' <<< "$entrypoint_output" || fail "bash -c entrypoint broken" -pass "bash -c entrypoint remains compatible" - -printf 'All cloudflare wrapper tests passed. PASS=%d\n' "$pass_count" diff --git a/tests/test-motd-01.sh b/tests/test-motd-01.sh new file mode 100644 index 0000000..423ed60 --- /dev/null +++ b/tests/test-motd-01.sh @@ -0,0 +1,555 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-motd-common.sh" +pass "MOTD payloads stay identical"; +assert_eq cf2a4f8dfe2fe26bff5509cdaee56d9e0816fafd0594251b02e892201af5c4d5 "$(sha256sum "$TEST_DIR/t"|awk '{print $1}')" "payload hash" + +( + r="$TEST_DIR/source"; +mkdir -m700 "$r" "$r/lock"; +export_paths "$r"; +b=$(trap -p EXIT HUP INT TERM ERR); +load_motd; +[[ "$b" == "$(trap -p EXIT HUP INT TERM ERR)"&&! -e "$MOTD_STATE_DIR"&&! -e "$MOTD_LOCK_FILE" ]]||fail source +); +pass "source zero side effects" +( + r="$TEST_DIR/basic"; +create_fixture "$r"; +export_paths "$r"; +before=$(target_hash "$r"); +load_motd; +motd_run_locked_operation install; +[[ -f "$MOTD_ETC_ROOT/motd"&&! -s "$MOTD_ETC_ROOT/motd"&&-x "$MOTD_SCRIPT" ]]||fail install; +motd_run_locked_operation restore previous; +assert_eq "$before" "$(target_hash "$r")" "restore previous group"; +motd_run_locked_operation install; +printf changed>"$MOTD_ETC_ROOT/motd"; +motd_run_locked_operation restore initial; +assert_eq "$before" "$(target_hash "$r")" "restore initial group" +) +( + r="$TEST_DIR/types"; +create_fixture "$r"; +export_paths "$r"; +unlink "$MOTD_ETC_ROOT/issue"; +ln -s "$r/link" "$MOTD_ETC_ROOT/issue"; +before=$(target_hash "$r"); +load_motd; +motd_run_locked_operation install; +g=$(motd_read_pointer previous); +motd_validate_snapshot "$MOTD_STATE_DIR/generations/$g" previous "$g"; +assert_eq symlink "${MOTD_SNAPSHOT_STATE[issue]}" "symlink snapshot"; +assert_eq absent "${MOTD_SNAPSHOT_STATE[issue_net]}" "absent snapshot"; +motd_run_locked_operation restore previous; +assert_eq "$before" "$(target_hash "$r")" "symlink restore" +) +( + r="$TEST_DIR/initial"; +create_fixture "$r"; +export_paths "$r"; +before=$(target_hash "$r"); +load_motd; +motd_snapshot_capture_hook(){ [[ "$1" != issue_net ]]; +}; +assert_fail "initial capture atomic" motd_run_locked_operation install; +[[ ! -e "$MOTD_STATE_DIR/initial.current" ]]||fail partial; +assert_eq "$before" "$(target_hash "$r")" "initial failure no target change" +) +( + r="$TEST_DIR/previous"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +old=$(motd_read_pointer previous); +for id in "${MOTD_TARGET_IDS[@]}"; +do motd_snapshot_capture_hook(){ [[ "$1" != "$id" ]]; +}; +assert_fail "previous capture $id" motd_run_locked_operation install; +assert_eq "$old" "$(motd_read_pointer previous)" "previous pointer $id"; +done +) +for failing_id in motd issue_net motd_news; +do +( + r="$TEST_DIR/i-$failing_id"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +printf current>"$MOTD_ETC_ROOT/motd"; +before=$(target_hash "$r"); +motd_target_commit_hook(){ [[ "$2" != "$failing_id" ]]; +}; +assert_fail "install failure $failing_id" motd_run_locked_operation install; +assert_eq "$before" "$(target_hash "$r")" "install rollback $failing_id" +) +( + r="$TEST_DIR/r-$failing_id"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +printf current>"$MOTD_ETC_ROOT/motd"; +before=$(target_hash "$r"); +motd_target_commit_hook(){ [[ "$1" != restore||"$2" != "$failing_id" ]]; +}; +assert_fail "restore failure $failing_id" motd_run_locked_operation restore previous; +assert_eq "$before" "$(target_hash "$r")" "restore rollback $failing_id" +) +done +( + r="$TEST_DIR/manifest"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +g=$(motd_read_pointer previous); +mf="$MOTD_STATE_DIR/generations/$g/manifest"; +cp "$mf" "$r/o"; +for k in missing duplicate unknown digest generation; +do cp "$r/o" "$mf"; +case $k in missing)sed -i '$d' "$mf";; +duplicate)tail -n1 "$mf">>"$mf";; +unknown)echo x=y>>"$mf";; +digest)sed -i 's/[0-9a-f]\{64\}$/0000000000000000000000000000000000000000000000000000000000000000/' "$mf";; +generation)sed -i 's/^generation=.*/generation=00000000000000000000000000000000/' "$mf";; +esac; +before=$(target_hash "$r"); +assert_fail "manifest $k rejected" motd_run_locked_operation restore previous; +assert_eq "$before" "$(target_hash "$r")" "manifest $k no target change"; +done +) +( + r="$TEST_DIR/lock"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_lock_acquired_hook(){ : >"$r/ready"; +sleep 2; +}; +motd_take_lock & h=$!; +while [[ ! -e "$r/ready" ]]; +do sleep .02; +done; +assert_fail "install lock contention" motd_run_locked_operation install; +assert_fail "restore lock contention" motd_run_locked_operation restore previous; +[[ ! -e "$MOTD_STATE_DIR" ]]||fail lockstate; +wait "$h"; +motd_lock_acquired_hook(){ :; +}; +motd_take_lock; +motd_release_lock +); +pass "shared lock nonblocking and reacquirable" +for k in symlink writable fifo; +do( + r="$TEST_DIR/l-$k"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +case $k in symlink)printf x>"$r/x"; +ln -s "$r/x" "$MOTD_LOCK_FILE";; +writable)printf x>"$MOTD_LOCK_FILE"; +chmod 666 "$MOTD_LOCK_FILE";; +fifo)mkfifo "$MOTD_LOCK_FILE";; +esac; +assert_fail "lock rejects $k" motd_take_lock +); +done +for k in symlink writable; +do( + r="$TEST_DIR/s-$k"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +case $k in symlink)mkdir -m700 "$r/s"; +ln -s "$r/s" "$MOTD_STATE_DIR";; +writable)mkdir -m770 "$MOTD_STATE_DIR";; +esac; +motd_take_lock; +assert_fail "state rejects $k" motd_prepare_state_layout; +motd_release_lock +); +done +create_legacy(){ printf legacy>"$MOTD_ETC_ROOT/motd.initial-backup"; +: >"$MOTD_ETC_ROOT/issue.initial-absent"; +printf net>"$MOTD_ETC_ROOT/issue.net.initial-backup"; +: >"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.initial-unknown"; +printf uname>"$MOTD_ETC_ROOT/update-motd.d/10-uname.initial-backup"; +: >"$MOTD_ETC_ROOT/update-motd.d/50-motd-news.initial-absent"; +} +( + r="$TEST_DIR/legacy"; +create_fixture "$r"; +export_paths "$r"; +create_legacy; +load_motd; +motd_run_locked_operation restore initial; +assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "legacy complete restore"; +[[ -e "$MOTD_ETC_ROOT/motd.initial-backup" ]]||fail legacymove +) +for k in partial conflict commit; +do( + r="$TEST_DIR/lg-$k"; +create_fixture "$r"; +export_paths "$r"; +if [[ $k == partial ]]; +then printf x>"$MOTD_ETC_ROOT/motd.initial-backup"; +else create_legacy; +[[ $k == conflict ]]&&: >"$MOTD_ETC_ROOT/motd.initial-absent"; +fi; +load_motd; +[[ $k == commit ]]&&motd_snapshot_commit_hook(){ [[ "$3" != legacy-directory ]]; +}; +assert_fail "legacy $k rejected" motd_run_locked_operation restore initial +); +done +( + r="$TEST_DIR/interop"; +create_fixture "$r"; +export_paths "$r"; +o=$(target_hash "$r"); +bash -c '. "$1/modules/system-customize.sh";ask_yes_no(){ return 0;};motd_preview_hook(){ :;};configure_motd' _ "$ROOT_DIR">/dev/null; +printf x>"$MOTD_ETC_ROOT/motd"; +bash -c '. "$1/tools/setup-motd.sh";motd_run_locked_operation restore previous' _ "$ROOT_DIR"; +assert_eq "$o" "$(target_hash "$r")" "module state tool restore" +) +( + r="$TEST_DIR/exit"; +create_fixture "$r"; +export_paths "$r"; +before=$(target_hash "$r"); +n=0; +bash -c '. "$1";motd_take_lock;motd_install_transaction_traps;motd_prepare_state_layout;motd_validate_target_parents;motd_begin_transaction install;motd_prepare_install_plan;motd_commit_target_plan install false;exit 0' _ "$ROOT_DIR/tools/setup-motd.sh">/dev/null 2>&1||n=$?; +assert_eq 1 "$n" "active exit zero maps one"; +assert_eq "$before" "$(target_hash "$r")" "active exit rollback" +) +( + r="$TEST_DIR/rbf"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +motd_target_commit_hook(){ [[ "$2" != issue_net ]]; +}; +motd_rollback_hook(){ echo "$1">>"$r/a"; +[[ "$1" != motd&&"$1" != custom_welcome ]]; +}; +assert_fail "rollback failure nonzero" motd_run_locked_operation install; +assert_eq 6 "$(wc -l<"$r/a")" "rollback attempts six"; +[[ -e "$MOTD_STATE_DIR/pending" ]]||fail journal +) +cat >"$TEST_DIR/sig.sh"<<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1"; +hit=false +send(){ [[ $hit == false ]]||return; +hit=true; +kill -"$SIGNAL_NAME" "$BASHPID"; +} +motd_snapshot_capture_hook(){ [[ $PHASE == snapshot-building&&$1 == issue ]]&&send; +return 0; +} +motd_transaction_phase_hook(){ [[ $PHASE == snapshot-commit&&$1 == snapshot-commit ]]&&send; +return 0; +} +motd_snapshot_commit_hook(){ [[ $PHASE == snapshot-publish&&$1 == initial&&$3 == pointer ]]&&send; +return 0; +} +motd_target_commit_hook(){ case "$PHASE:$1:$2" in install-first:install:motd|install-middle:install:issue_net|restore-first:restore:motd|restore-middle:restore:issue_net)send;; +esac; +return 0; +} +case $PHASE in restore-*)motd_run_locked_operation restore previous;; +*)motd_run_locked_operation install;; +esac +CHILD +chmod 700 "$TEST_DIR/sig.sh" +for ph in snapshot-building snapshot-commit snapshot-publish install-first install-middle restore-first restore-middle; +do for sig in HUP INT TERM; +do( + r="$TEST_DIR/$ph-$sig"; +create_fixture "$r"; +export_paths "$r"; +if [[ $ph == restore-* ]]; +then bash -c '. "$1";motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh"; +printf current>"$MOTD_ETC_ROOT/motd"; +fi; +before=$(target_hash "$r"); +n=0; +env --default-signal=HUP,INT,TERM PHASE="$ph" SIGNAL_NAME="$sig" bash "$TEST_DIR/sig.sh" "$ROOT_DIR/tools/setup-motd.sh">/dev/null 2>&1||n=$?; +case $sig in HUP)e=129;; +INT)e=130;; +TERM)e=143;; +esac; +assert_eq "$e" "$n" "$sig $ph status"; +assert_eq "$before" "$(target_hash "$r")" "$sig $ph rollback" +); +done; +done +( + r="$TEST_DIR/unknown"; +create_fixture "$r"; +export_paths "$r"; +printf '# linux-setup:managed-motd'>"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; +load_motd; +motd_run_locked_operation install; +g=$(motd_read_pointer initial); +motd_validate_snapshot "$MOTD_STATE_DIR/generations/$g" initial "$g"; +assert_eq unknown "${MOTD_SNAPSHOT_STATE[motd]}" "unknown initial state" +) +( + r="$TEST_DIR/readonly"; +create_fixture "$r"; +export_paths "$r"; +bash "$ROOT_DIR/tools/setup-motd.sh" status>/dev/null; +bash "$ROOT_DIR/tools/setup-motd.sh" help>/dev/null; +[[ ! -e "$MOTD_STATE_DIR"&&! -e "$MOTD_LOCK_FILE" ]]||fail readonly +); +pass "status help no lock" +( + r="$TEST_DIR/publish"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +old=$(motd_read_pointer previous); +printf current>"$MOTD_ETC_ROOT/motd"; +before=$(target_hash "$r"); +motd_snapshot_manifest_hook(){ [[ "$1" != previous ]]; +}; +assert_fail "previous manifest failure" motd_run_locked_operation install; +assert_eq "$old" "$(motd_read_pointer previous)" "manifest preserves previous"; +assert_eq "$before" "$(target_hash "$r")" "manifest failure rolls back"; +motd_snapshot_manifest_hook(){ :; +}; +motd_snapshot_commit_hook(){ [[ "$1" != previous||"$3" != directory ]]; +}; +assert_fail "previous directory commit failure" motd_run_locked_operation install; +assert_eq "$old" "$(motd_read_pointer previous)" "directory failure preserves previous" +) +for k in owner gid; +do( + r="$TEST_DIR/lmeta-$k"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +printf x>"$MOTD_LOCK_FILE"; +chmod 600 "$MOTD_LOCK_FILE"; +stat(){ if [[ "${*: -1}" == "$MOTD_LOCK_FILE"&&"$*" == *"%u:%g:%a"* ]]; +then [[ $k == owner ]]&&printf '99999:%s:600\n' "$MOTD_TRUSTED_GID"||printf '%s:99999:600\n' "$MOTD_TRUSTED_UID"; +else command stat "$@"; +fi; +}; +assert_fail "lock wrong $k" motd_take_lock +); +done +for k in owner gid; +do( + r="$TEST_DIR/smeta-$k"; +create_fixture "$r"; +export_paths "$r"; +mkdir -m700 "$MOTD_STATE_DIR"; +load_motd; +stat(){ if [[ "${*: -1}" == "$MOTD_STATE_DIR"&&"$*" == *"%u:%g:%a"* ]]; +then [[ $k == owner ]]&&printf '99999:%s:700\n' "$MOTD_TRUSTED_GID"||printf '%s:99999:700\n' "$MOTD_TRUSTED_UID"; +else command stat "$@"; +fi; +}; +motd_take_lock; +assert_fail "state wrong $k" motd_prepare_state_layout; +motd_release_lock +); +done +( + r="$TEST_DIR/state-create"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +chown(){ [[ "${*: -1}" == "$MOTD_STATE_DIR" ]]&&return 1; +command chown "$@"; +}; +assert_fail "state create failure" motd_run_locked_operation install; +[[ ! -e "$MOTD_STATE_DIR" ]]||fail state-residue +) +( + r="$TEST_DIR/close"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_take_lock; +motd_lock_close_hook(){ return 1; +}; +assert_fail "lock close failure" motd_release_lock +) +( + r="$TEST_DIR/device"; +create_fixture "$r"; +export_paths "$r"; +if mknod "$MOTD_ETC_ROOT/issue.net" c 1 3 2>/dev/null; +then before=$(target_hash "$r"); +load_motd; +assert_fail "device rejected" motd_run_locked_operation install; +assert_eq "$before" "$(target_hash "$r")" "device no target change"; +else pass "device test skipped"; +fi +) +( + r="$TEST_DIR/legacy-state"; +create_fixture "$r"; +export_paths "$r"; +mkdir -m700 "$MOTD_STATE_DIR"; +printf legacy>"$MOTD_STATE_DIR/motd.previous-backup"; +chmod 600 "$MOTD_STATE_DIR/motd.previous-backup"; +for x in issue issue.net 00-custom-welcome 10-uname 50-motd-news; +do : >"$MOTD_STATE_DIR/$x.previous-absent"; +chmod 600 "$MOTD_STATE_DIR/$x.previous-absent"; +done; +load_motd; +motd_run_locked_operation restore previous; +assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "state legacy restore" +) + +state_fingerprint() { + local root="$1" path + [[ -d "$root" ]] || { printf 'absent\n'; return; } + while IFS= read -r path; do + if [[ -L "$path" ]]; then printf 'L|%s|%s\n' "${path#$root/}" "$(readlink "$path")" + elif [[ -f "$path" ]]; then printf 'F|%s|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" "$(sha256sum "$path" | awk '{print $1}')" + elif [[ -d "$path" ]]; then printf 'D|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" + fi + done < <(find "$root" -mindepth 1 -print | sort) +} + +generation_count() { + local directory="$1" + [[ -d "$directory" ]] || { printf '0\n'; return; } + find "$directory" -mindepth 1 -maxdepth 1 -type d ! -name '.*.stage' | wc -l +} + +assert_no_motd_residue() { + local root="$1" residue + residue=$(find "$root" -name '.linux-setup-motd.*' -o -name '.*.pointer.*' -o -name '.pending.*' -o -name '.*.stage' 2>/dev/null) + [[ -z "$residue" ]] || fail "unexpected MOTD residue: $residue" + if [[ -d "$root/state/transactions" ]]; then [[ -z "$(find "$root/state/transactions" -mindepth 1 -print -quit)" ]] || fail "transaction residue remains"; fi +} + +create_complete_legacy_scope() { + local scope="$1" prefix + cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_ETC_ROOT/motd.${scope}-backup" + cp -a "$MOTD_ETC_ROOT/issue" "$MOTD_ETC_ROOT/issue.${scope}-backup" + : > "$MOTD_ETC_ROOT/issue.net.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/issue.net.${scope}-absent" + cp -a "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.${scope}-backup" + cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_ETC_ROOT/update-motd.d/10-uname.${scope}-backup" + : > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent" +} + +for legacy_phase in random-id stat open create cat chown chmod sha256 manifest mv; do +( + root="$TEST_DIR/legacy-io-$legacy_phase"; create_fixture "$root"; export_paths "$root" + create_complete_legacy_scope initial + before_targets=$(target_hash "$root") + before_legacy=$(find "$MOTD_ETC_ROOT" -name '*.initial-*' -type f -printf '%p|%s|%m\n' | sort) + load_motd + motd_legacy_io_hook() { [[ "$1" != "$legacy_phase" ]]; } + assert_fail "legacy $legacy_phase failure returns nonzero" motd_run_locked_operation restore initial + assert_eq "$before_targets" "$(target_hash "$root")" "legacy $legacy_phase keeps formal targets" + assert_eq "$before_legacy" "$(find "$MOTD_ETC_ROOT" -name '*.initial-*' -type f -printf '%p|%s|%m\n' | sort)" "legacy $legacy_phase keeps legacy files" + [[ ! -e "$MOTD_STATE_DIR/initial.current" ]] || fail "legacy $legacy_phase published initial pointer" + assert_no_motd_residue "$root" +) +done +pass "legacy I/O failure matrix preserves data and state" + +cat > "$TEST_DIR/pointer-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1" +motd_snapshot_commit_hook() { + if [[ "$1" == previous && "$3" == pointer ]]; then kill -"$SIGNAL_NAME" "$BASHPID"; fi +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/pointer-signal-child.sh" + +for signal_name in HUP INT TERM; do +( + root="$TEST_DIR/pointer-signal-$signal_name"; create_fixture "$root"; export_paths "$root" + bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" + old_pointer=$(cat "$MOTD_STATE_DIR/previous.current") + before=$(target_hash "$root") + rc=0 + env --default-signal=HUP,INT,TERM SIGNAL_NAME="$signal_name" bash "$TEST_DIR/pointer-signal-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? + case "$signal_name" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + assert_eq "$expected" "$rc" "$signal_name during previous pointer stage returns conventional status" + assert_eq "$old_pointer" "$(cat "$MOTD_STATE_DIR/previous.current")" "$signal_name preserves previous pointer" + assert_eq "$before" "$(target_hash "$root")" "$signal_name pointer stage restores targets" + assert_no_motd_residue "$root" + bash -c '. "$1"; motd_take_lock; motd_release_lock' _ "$ROOT_DIR/tools/setup-motd.sh" +) +done + +for pending_phase in pending-create pending-write pending-chown pending-chmod pending-publish; do +( + root="$TEST_DIR/pending-failure-$pending_phase"; create_fixture "$root"; export_paths "$root" + load_motd + motd_run_locked_operation install + before=$(target_hash "$root"); before_generations=$(generation_count "$MOTD_STATE_DIR/generations") + motd_transaction_phase_hook() { [[ "$1" != "$pending_phase" ]]; } + assert_fail "pending $pending_phase failure returns nonzero" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "pending $pending_phase keeps targets" + assert_eq "$before_generations" "$(generation_count "$MOTD_STATE_DIR/generations")" "pending $pending_phase leaves no generation orphan" + [[ ! -e "$MOTD_STATE_DIR/pending" && ! -L "$MOTD_STATE_DIR/pending" ]] || fail "pending $pending_phase left pending" + assert_no_motd_residue "$root" + motd_transaction_phase_hook() { :; } + motd_run_locked_operation install +) +done + +for pending_kind in broken-symlink symlink directory fifo mode owner gid; do +( + root="$TEST_DIR/pending-kind-$pending_kind"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + pending="$MOTD_STATE_DIR/pending" + case "$pending_kind" in + broken-symlink) ln -s "$root/missing" "$pending" ;; + symlink) printf x > "$root/pending-target"; chmod 0600 "$root/pending-target"; ln -s "$root/pending-target" "$pending" ;; + directory) mkdir "$pending" ;; + fifo) mkfifo "$pending" ;; + mode) printf x > "$pending"; chmod 0666 "$pending" ;; + owner|gid) printf x > "$pending"; chmod 0600 "$pending" ;; + esac + before=$(target_hash "$root") + if [[ "$pending_kind" == owner || "$pending_kind" == gid ]]; then + stat() { + if [[ "${*: -1}" == "$pending" && "$*" == *"%u:%g:%a"* ]]; then + [[ "$pending_kind" == owner ]] && printf '99999:%s:600\n' "$MOTD_TRUSTED_GID" || printf '%s:99999:600\n' "$MOTD_TRUSTED_UID" + else command stat "$@"; fi + } + fi + assert_fail "pending $pending_kind fails closed" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "pending $pending_kind changes no target" + [[ -e "$pending" || -L "$pending" ]] || fail "pending $pending_kind evidence was removed" +) +done + +cat > "$TEST_DIR/kill-pending-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_transaction_phase_hook() { [[ "$1" == pending-active ]] && kill -KILL "$BASHPID"; return 0; } +motd_run_locked_operation install +CHILD +cat > "$TEST_DIR/kill-previous-pointer-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_snapshot_commit_hook() { [[ "$1" == previous && "$3" == pointer ]] && kill -KILL "$BASHPID"; return 0; } +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/kill-pending-child.sh" "$TEST_DIR/kill-previous-pointer-child.sh" diff --git a/tests/test-motd-02.sh b/tests/test-motd-02.sh new file mode 100644 index 0000000..4656502 --- /dev/null +++ b/tests/test-motd-02.sh @@ -0,0 +1,267 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-motd-common.sh" + +for journal_case in pending-directory journal-transaction generation-scope generation-missing old-new-mix; do +( + root="$TEST_DIR/journal-$journal_case"; create_fixture "$root"; export_paths "$root" + bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" + rc=0 + if [[ "$journal_case" == pending-directory || "$journal_case" == journal-transaction ]]; then + bash "$TEST_DIR/kill-pending-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? + else + bash "$TEST_DIR/kill-previous-pointer-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? + fi + assert_eq 137 "$rc" "$journal_case fixture stops with SIGKILL" + transaction=$(cat "$MOTD_STATE_DIR/pending") + journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" + case "$journal_case" in + pending-directory) printf '%032x\n' 1 > "$MOTD_STATE_DIR/pending" ;; + journal-transaction) sed -i 's/^transaction=.*/transaction=00000000000000000000000000000001/' "$journal" ;; + generation-scope) initial=$(cat "$MOTD_STATE_DIR/initial.current"); sed -i "s/^previous_new=.*/previous_new=$initial/" "$journal" ;; + generation-missing) sed -i 's/^previous_new=.*/previous_new=ffffffffffffffffffffffffffffffff/' "$journal" ;; + old-new-mix) initial=$(cat "$MOTD_STATE_DIR/initial.current"); sed -i "s/^previous_old=.*/previous_old=$initial/" "$journal" ;; + esac + before=$(target_hash "$root") + load_motd + assert_fail "$journal_case pending/journal mismatch rejected" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$journal_case mismatch changes no target" + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$journal_case evidence removed" +) +done + +for tamper_case in content mode inode regular-to-symlink; do +( + root="$TEST_DIR/stage-$tamper_case"; create_fixture "$root"; export_paths "$root" + printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + before=$(target_hash "$root") + load_motd + motd_transaction_phase_hook() { + [[ "$1" == install-plan-ready ]] || return 0 + case "$tamper_case" in + content) printf tampered > "${MOTD_TARGET_STAGE[motd]}" ;; + mode) chmod 0600 "${MOTD_TARGET_STAGE[issue_net]}" ;; + inode) cp "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" "${MOTD_TARGET_STAGE[motd_news]}.replacement"; chmod 0644 "${MOTD_TARGET_STAGE[motd_news]}.replacement"; mv -Tf "${MOTD_TARGET_STAGE[motd_news]}.replacement" "${MOTD_TARGET_STAGE[motd_news]}" ;; + regular-to-symlink) rm -f "${MOTD_TARGET_STAGE[issue_net]}"; ln -s "$root/elsewhere" "${MOTD_TARGET_STAGE[issue_net]}" ;; + esac + } + motd_target_commit_hook() { : > "$root/commit-called"; } + assert_fail "$tamper_case stage tamper rejected before commit" motd_run_locked_operation install + [[ ! -e "$root/commit-called" ]] || fail "$tamper_case reached first formal commit" + assert_eq "$before" "$(target_hash "$root")" "$tamper_case stage tamper changes no target" + assert_no_motd_residue "$root" +) +done + +( + root="$TEST_DIR/stage-symlink-target"; create_fixture "$root"; export_paths "$root" + printf external > "$root/external"; ln -sf "$root/external" "$MOTD_ETC_ROOT/issue" + load_motd; motd_run_locked_operation install + before=$(target_hash "$root") + motd_transaction_phase_hook() { + [[ "$1" == restore-plan-ready ]] || return 0 + rm -f "${MOTD_TARGET_STAGE[issue]}"; ln -s "$root/other" "${MOTD_TARGET_STAGE[issue]}" + } + motd_target_commit_hook() { : > "$root/commit-called"; } + assert_fail "symlink target stage tamper rejected before restore commit" motd_run_locked_operation restore previous + [[ ! -e "$root/commit-called" ]] || fail "symlink target tamper reached formal commit" + assert_eq "$before" "$(target_hash "$root")" "symlink target tamper changes no target" +) + +for native_id in uname motd_news; do +( + root="$TEST_DIR/native-symlink-$native_id"; create_fixture "$root"; export_paths "$root" + case "$native_id" in uname) native_path="$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; motd_news) native_path="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; esac + rm -f "$native_path" + printf 'external-%s\n' "$native_id" > "$root/external-$native_id"; chmod 0755 "$root/external-$native_id" + external_hash=$(sha256sum "$root/external-$native_id"); external_mode=$(stat -c %a "$root/external-$native_id") + ln -s "$root/external-$native_id" "$native_path" + load_motd + motd_run_locked_operation install + [[ ! -e "$native_path" && ! -L "$native_path" ]] || fail "$native_id symlink entry remains enabled" + assert_eq "$external_hash" "$(sha256sum "$root/external-$native_id")" "$native_id external content unchanged" + assert_eq "$external_mode" "$(stat -c %a "$root/external-$native_id")" "$native_id external mode unchanged" + motd_run_locked_operation restore previous + [[ -L "$native_path" && "$(readlink "$native_path")" == "$root/external-$native_id" ]] || fail "$native_id previous symlink not restored" +) +done + +cat > "$TEST_DIR/sigkill-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_transaction_phase_hook() { + [[ "$1" == "$KILL_PHASE" ]] && kill -KILL "$BASHPID" + return 0 +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/sigkill-child.sh" + +for kill_phase in transaction-snapshot-building transaction-final-moved target-committed-motd target-committed-issue_net target-committed-motd_news committed-cleanup committed-old-generation-cleaned committed-pending-removed; do +( + root="$TEST_DIR/sigkill-$kill_phase"; create_fixture "$root"; export_paths "$root" + printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" + before=$(target_hash "$root") + rc=0 + KILL_PHASE="$kill_phase" bash "$TEST_DIR/sigkill-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? + assert_eq 137 "$rc" "SIGKILL at $kill_phase terminates process" + after_kill=$(target_hash "$root") + load_motd + motd_take_lock; motd_prepare_state_layout; motd_reconcile_state; motd_release_lock + if [[ "$kill_phase" == committed-* ]]; then + assert_eq "$after_kill" "$(target_hash "$root")" "committed cleanup SIGKILL does not roll back" + else + assert_eq "$before" "$(target_hash "$root")" "SIGKILL at $kill_phase rolls back to pre-transaction state" + fi + assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "SIGKILL at $kill_phase leaves bounded generations" + assert_no_motd_residue "$root" + motd_take_lock; motd_release_lock +) +done + +( + root="$TEST_DIR/generation-bounded"; create_fixture "$root"; export_paths "$root" + load_motd + for _ in {1..6}; do motd_run_locked_operation install; done + assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "repeated install keeps two generations" +) + +( + root="$TEST_DIR/generation-commit-failure"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + before_count=$(generation_count "$MOTD_STATE_DIR/generations") + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + assert_fail "target commit failure returns nonzero for generation cleanup" motd_run_locked_operation install + assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "target commit failure leaves no orphan generation" +) + +( + root="$TEST_DIR/interop-module-tool"; create_fixture "$root"; export_paths "$root" + original=$(target_hash "$root") + bash -c '. "$1/modules/system-customize.sh"; ask_yes_no(){ return 0; }; motd_preview_hook(){ :; }; configure_motd' _ "$ROOT_DIR" >/dev/null + printf changed > "$MOTD_ETC_ROOT/motd" + bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation restore previous' _ "$ROOT_DIR" + assert_eq "$original" "$(target_hash "$root")" "module install to tool previous restore" + printf changed > "$MOTD_ETC_ROOT/motd" + bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation restore initial' _ "$ROOT_DIR" + assert_eq "$original" "$(target_hash "$root")" "module install to tool initial restore" +) + +( + root="$TEST_DIR/interop-tool-module"; create_fixture "$root"; export_paths "$root" + original=$(target_hash "$root") + bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation install' _ "$ROOT_DIR" + printf between > "$MOTD_ETC_ROOT/motd"; between=$(target_hash "$root") + bash -c '. "$1/modules/system-customize.sh"; ask_yes_no(){ return 0; }; motd_preview_hook(){ :; }; configure_motd' _ "$ROOT_DIR" >/dev/null + bash -c '. "$1/modules/system-customize.sh"; motd_run_locked_operation restore previous' _ "$ROOT_DIR" + assert_eq "$between" "$(target_hash "$root")" "tool install to module previous restore" + printf changed > "$MOTD_ETC_ROOT/motd" + bash -c '. "$1/modules/system-customize.sh"; motd_run_locked_operation restore initial' _ "$ROOT_DIR" + assert_eq "$original" "$(target_hash "$root")" "tool install to module initial restore" +) + + +( + root="$TEST_DIR/cleanup-old-generation-failure"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + old_previous=$(cat "$MOTD_STATE_DIR/previous.current"); blocked="$MOTD_STATE_DIR/generations/$old_previous" + motd_cleanup_path_hook() { [[ "$1" != "$blocked" ]]; } + assert_fail "old previous generation delete failure returns nonzero" motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "old generation delete failure lost pending journal" + transaction=$(cat "$MOTD_STATE_DIR/pending"); grep -Fq 'phase=committed' "$MOTD_TRANSACTION_PARENT/$transaction/journal" || fail "old generation failure journal not committed" + [[ -d "$blocked" ]] || fail "blocked old generation unexpectedly removed" + motd_cleanup_path_hook() { :; } + motd_run_locked_operation install + [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "old generation cleanup retry left pending" +) + +( + root="$TEST_DIR/cleanup-rollback-generation-failure"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + current_initial=$(cat "$MOTD_STATE_DIR/initial.current"); current_previous=$(cat "$MOTD_STATE_DIR/previous.current") + motd_cleanup_path_hook() { + local name=${1##*/} + if [[ "$1" == "$MOTD_STATE_DIR/generations/"* && "$name" != "$current_initial" && "$name" != "$current_previous" ]]; then return 1; fi + return 0 + } + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + assert_fail "rollback generation delete failure returns nonzero" motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback generation failure lost pending" + transaction=$(cat "$MOTD_STATE_DIR/pending"); grep -Fq 'phase=rolledback' "$MOTD_TRANSACTION_PARENT/$transaction/journal" || fail "rollback generation failure journal not rolledback" + motd_cleanup_path_hook() { :; }; motd_target_commit_hook() { :; } + motd_run_locked_operation install + [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "rolledback cleanup retry left pending" + assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "rolledback cleanup retry restores bounded generations" +) + + +for hook_case in content-motd mode-issue_net inode-motd_news type-issue_net; do +( + root="$TEST_DIR/hook-target-$hook_case"; create_fixture "$root"; export_paths "$root" + printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + before=$(target_hash "$root") + load_motd + motd_target_commit_hook() { + local id="$2" stage replacement + case "$hook_case:$id" in + content-motd:motd) printf changed > "${MOTD_TARGET_STAGE[motd]}" ;; + mode-issue_net:issue_net) chmod 0600 "${MOTD_TARGET_STAGE[issue_net]}" ;; + inode-motd_news:motd_news) + stage=${MOTD_TARGET_STAGE[motd_news]}; replacement="${stage}.replacement" + cp "$stage" "$replacement"; chmod "${MOTD_STAGE_MODE[motd_news]}" "$replacement"; mv -Tf "$replacement" "$stage" + ;; + type-issue_net:issue_net) rm -f "${MOTD_TARGET_STAGE[issue_net]}"; ln -s "$root/other" "${MOTD_TARGET_STAGE[issue_net]}" ;; + esac + return 0 + } + assert_fail "$hook_case target hook tamper rejected" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$hook_case leaves formal targets unchanged" + assert_no_motd_residue "$root" +) +done + +( + root="$TEST_DIR/hook-rollback-stage"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + before=$(target_hash "$root") + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + motd_rollback_hook() { [[ "$1" != motd ]] || printf tampered > "${MOTD_TARGET_STAGE[motd]}"; return 0; } + assert_fail "rollback hook stage tamper rejected" motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback hook tamper did not preserve pending journal" + motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } + motd_run_locked_operation install + [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback hook retry left pending" + assert_eq "$before" "$(target_hash "$root")" "rollback hook retry restores original group before next install" +) + +for pointer_case in old-generation other-generation extra-line inode; do +( + root="$TEST_DIR/pointer-hook-$pointer_case"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + before=$(target_hash "$root"); old_pointer=$(cat "$MOTD_STATE_DIR/previous.current"); before_count=$(generation_count "$MOTD_STATE_DIR/generations") + motd_snapshot_commit_hook() { + local scope="$1" generation="$2" kind="$3" stage other source + [[ "$scope" == previous && "$kind" == pointer ]] || return 0 + stage=${MOTD_POINTER_STAGE_PATHS[${#MOTD_POINTER_STAGE_PATHS[@]}-1]} + case "$pointer_case" in + old-generation) printf '%s\n' "$old_pointer" > "$stage" ;; + other-generation) + other=11111111111111111111111111111111; source="$MOTD_STATE_DIR/generations/$generation" + cp -a "$source" "$MOTD_STATE_DIR/generations/$other" + sed -i "s/^generation=.*/generation=$other/" "$MOTD_STATE_DIR/generations/$other/manifest" + MOTD_NEW_GENERATIONS+=("$other") + printf '%s\n' "$other" > "$stage" + ;; + extra-line) printf 'extra\n' >> "$stage" ;; + inode) cp "$stage" "${stage}.replacement"; chmod 0600 "${stage}.replacement"; mv -Tf "${stage}.replacement" "$stage" ;; + esac + } + assert_fail "pointer hook $pointer_case rejected" motd_run_locked_operation install + assert_eq "$old_pointer" "$(cat "$MOTD_STATE_DIR/previous.current")" "pointer hook $pointer_case preserves pointer" + assert_eq "$before" "$(target_hash "$root")" "pointer hook $pointer_case preserves targets" + assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "pointer hook $pointer_case preserves generation count" + assert_no_motd_residue "$root" +) +done diff --git a/tests/test-motd-03.sh b/tests/test-motd-03.sh new file mode 100644 index 0000000..e6b7ece --- /dev/null +++ b/tests/test-motd-03.sh @@ -0,0 +1,233 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-motd-common.sh" + +for pending_hook_case in id extra-line inode; do +( + root="$TEST_DIR/pending-hook-$pending_hook_case"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + before=$(target_hash "$root") + motd_transaction_phase_hook() { + local phase="$1" stage + [[ "$phase" == pending-publish ]] || return 0 + stage=${MOTD_PENDING_STAGE_PATHS[${#MOTD_PENDING_STAGE_PATHS[@]}-1]} + case "$pending_hook_case" in + id) printf '%032x\n' 1 > "$stage" ;; + extra-line) printf 'extra\n' >> "$stage" ;; + inode) cp "$stage" "${stage}.replacement"; chmod 0600 "${stage}.replacement"; mv -Tf "${stage}.replacement" "$stage" ;; + esac + } + motd_target_commit_hook() { : > "$root/commit-called"; } + assert_fail "pending publish hook $pending_hook_case rejected" motd_run_locked_operation install + [[ ! -e "$root/commit-called" ]] || fail "pending hook $pending_hook_case reached formal target commit" + assert_eq "$before" "$(target_hash "$root")" "pending hook $pending_hook_case preserves targets" + [[ ! -e "$MOTD_STATE_DIR/pending" && ! -L "$MOTD_STATE_DIR/pending" ]] || fail "pending hook $pending_hook_case left pending" + assert_no_motd_residue "$root" +) +done + +cat > "$TEST_DIR/building-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1" +send_building_signal() { kill -"$SIGNAL_NAME" "$BASHPID"; } +motd_transaction_phase_hook() { + case "$FAIL_PHASE:$1" in + transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) send_building_signal ;; + esac + return 0 +} +motd_snapshot_capture_hook() { + [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0 + case "$FAIL_PHASE:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) send_building_signal ;; esac + return 0 +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/building-signal-child.sh" + +for building_phase in transaction-snapshot-building capture-motd capture-issue_net capture-motd_news snapshot-commit transaction-final-move; do +( + root="$TEST_DIR/building-fail-$building_phase"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + before=$(target_hash "$root"); before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")"; before_count=$(generation_count "$MOTD_STATE_DIR/generations") + motd_transaction_phase_hook() { case "$building_phase:$1" in transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) return 1 ;; esac; return 0; } + motd_snapshot_capture_hook() { [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0; case "$building_phase:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) return 1 ;; esac; return 0; } + assert_fail "$building_phase ordinary failure returns nonzero" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$building_phase ordinary failure preserves targets" + assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$building_phase ordinary failure preserves pointers" + assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "$building_phase ordinary failure preserves generations" + assert_no_motd_residue "$root" + motd_take_lock; motd_release_lock +) + for signal_name in HUP INT TERM; do + ( + root="$TEST_DIR/building-$building_phase-$signal_name"; create_fixture "$root"; export_paths "$root" + bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" + before=$(target_hash "$root"); before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")"; before_count=$(generation_count "$MOTD_STATE_DIR/generations") + rc=0 + env --default-signal=HUP,INT,TERM FAIL_PHASE="$building_phase" SIGNAL_NAME="$signal_name" bash "$TEST_DIR/building-signal-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? + case "$signal_name" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + assert_eq "$expected" "$rc" "$building_phase $signal_name status" + assert_eq "$before" "$(target_hash "$root")" "$building_phase $signal_name preserves targets" + assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$building_phase $signal_name preserves pointers" + assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "$building_phase $signal_name preserves generations" + assert_no_motd_residue "$root" + bash -c '. "$1"; motd_take_lock; motd_release_lock' _ "$ROOT_DIR/tools/setup-motd.sh" + ) + done +done + +for previous_impl in tool module; do +( + root="$TEST_DIR/previous-only-$previous_impl"; create_fixture "$root"; export_paths "$root" + create_complete_legacy_scope previous + case "$previous_impl" in tool) script="$ROOT_DIR/tools/setup-motd.sh" ;; module) script="$ROOT_DIR/modules/system-customize.sh" ;; esac + # shellcheck source=/dev/null + . "$script" + motd_run_locked_operation restore previous + [[ ! -e "$MOTD_STATE_DIR/initial.current" && ! -L "$MOTD_STATE_DIR/initial.current" ]] || fail "$previous_impl previous-only created initial pointer" + [[ -e "$MOTD_STATE_DIR/previous.current" ]] || fail "$previous_impl previous-only missing previous pointer" + motd_run_locked_operation restore previous + [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$previous_impl repeated previous restore left transaction" + motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/initial.current" && -e "$MOTD_STATE_DIR/previous.current" ]] || fail "$previous_impl install did not complete initial/previous pointers" + assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$previous_impl previous-only lifecycle bounded" +) +done + +for time_direction in tool-module module-tool; do +( + root="$TEST_DIR/timestamps-$time_direction"; create_fixture "$root"; export_paths "$root" + printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + for native in 10-uname 50-motd-news; do touch -a -d @1000000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1000200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done + expected_uname=$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname"); expected_news=$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news") + case "$time_direction" in + tool-module) install_script="$ROOT_DIR/tools/setup-motd.sh"; restore_script="$ROOT_DIR/modules/system-customize.sh" ;; + module-tool) install_script="$ROOT_DIR/modules/system-customize.sh"; restore_script="$ROOT_DIR/tools/setup-motd.sh" ;; + esac + # shellcheck source=/dev/null + . "$install_script"; motd_run_locked_operation install + assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction install preserves uname timestamps" + assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction install preserves news timestamps" + for native in 10-uname 50-motd-news; do touch -a -d @1100000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1100200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done + bash -c '. "$1"; motd_run_locked_operation restore previous' _ "$restore_script" + assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction previous restores uname timestamps" + assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction previous restores news timestamps" + for native in 10-uname 50-motd-news; do touch -a -d @1200000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1200200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done + bash -c '. "$1"; motd_run_locked_operation restore initial' _ "$restore_script" + assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction initial restores uname timestamps" + assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction initial restores news timestamps" +) +done + +for hidden_location in transactions generations root; do + for hidden_kind in file symlink fifo dir; do + ( + root="$TEST_DIR/hidden-$hidden_location-$hidden_kind"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + case "$hidden_location" in transactions) parent="$MOTD_TRANSACTION_PARENT" ;; generations) parent="$MOTD_STATE_DIR/generations" ;; root) parent="$MOTD_STATE_DIR" ;; esac + object="$parent/.evil-$hidden_kind" + case "$hidden_kind" in file) printf evil > "$object" ;; symlink) ln -s "$root/missing" "$object" ;; fifo) mkfifo "$object" ;; dir) mkdir "$object" ;; esac + before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + if [[ "$hidden_kind" == file || "$hidden_kind" == fifo ]]; then operation=(install); else operation=(restore previous); fi + assert_fail "$hidden_location hidden $hidden_kind rejected" motd_run_locked_operation "${operation[@]}" + assert_eq "$before" "$(target_hash "$root")" "$hidden_location hidden $hidden_kind preserves targets" + assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$hidden_location hidden $hidden_kind preserves pointers" + [[ -e "$object" || -L "$object" ]] || fail "$hidden_location hidden $hidden_kind evidence removed" + motd_take_lock; motd_release_lock + ) + done +done + +( + root="$TEST_DIR/hidden-transaction-short"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + printf evil > "$MOTD_TRANSACTION_PARENT/.123" + before=$(target_hash "$root") + assert_fail "transactions .123 rejected" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "transactions .123 preserves targets" + [[ -e "$MOTD_TRANSACTION_PARENT/.123" ]] || fail "transactions .123 evidence removed" + motd_take_lock; motd_release_lock +) + +( + transaction_block="$TEST_DIR/transaction-block" + sed -n '/# === MOTD group transaction ===/,/^configure_motd()/p' "$ROOT_DIR/tools/setup-motd.sh" > "$transaction_block" + if grep -Eq "stat .*%F|regular file|regular empty file|symbolic link" "$transaction_block"; then fail "MOTD transaction depends on localized stat type text"; fi + pass "MOTD transaction avoids localized stat type text" +) + +( + root="$TEST_DIR/locale-types"; create_fixture "$root"; export_paths "$root" + printf outside > "$root/outside"; rm -f "$MOTD_ETC_ROOT/issue"; ln -s "$root/outside" "$MOTD_ETC_ROOT/issue" + create_complete_legacy_scope initial + load_motd + stat() { + if [[ "$*" == *"%F"* ]]; then printf '本地化文件类型\n'; else command stat "$@"; fi + } + LANG=zh_CN.UTF-8 motd_run_locked_operation restore initial + [[ -L "$MOTD_ETC_ROOT/issue" ]] || fail "localized stat wrapper broke symlink restore" + assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "localized stat wrapper preserves regular legacy import" + LANG=zh_CN.UTF-8 bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/modules/system-customize.sh" +) + +for missing_command in bash chown flock ln od readlink sha256sum touch wc; do +( + root="$TEST_DIR/module-missing-$missing_command"; create_fixture "$root"; export_paths "$root" + before=$(target_hash "$root"); rc=0; mkdir "$root/bin" + for available_command in apt-get apt-cache awk bash basename cat chmod chown cp curl df dirname dpkg flock grep hostname id install ln mkdir mktemp mv od readlink rm sed sha256sum sleep sort stat touch tr uname uptime wc; do + [[ "$available_command" == "$missing_command" ]] && continue + available_path=$(command -v "$available_command" || true) + if [[ -n "$available_path" ]]; then ln -s "$available_path" "$root/bin/$available_command" + else printf '#!/bin/sh\nexit 0\n' > "$root/bin/$available_command"; chmod 0755 "$root/bin/$available_command"; fi + done + PATH="$root/bin" /bin/bash -c ' + . "$1" + require_root() { :; } + main motd + ' _ "$ROOT_DIR/modules/system-customize.sh" > "$root/output" 2>&1 || rc=$? + assert_eq 1 "$rc" "module missing $missing_command fails before action" + grep -Fq "缺少必要命令: $missing_command" "$root/output" || fail "module missing $missing_command not reported" + pass "module preflight reports missing $missing_command" + assert_eq "$before" "$(target_hash "$root")" "module missing $missing_command preserves MOTD" + [[ ! -e "$MOTD_STATE_DIR" && ! -e "$MOTD_LOCK_FILE" ]] || fail "module missing $missing_command created lock/state" +) +done + +grep -Fq 'error "缺少必要命令: $required_command"' "$ROOT_DIR/modules/system-customize.sh" || fail "module missing-command error output branch absent" +pass "module dependency preflight retains missing-command output" + + +for noop_impl in tool module; do + for noop_created_type in regular symlink fifo; do + ( + root="$TEST_DIR/noop-absent-$noop_impl-$noop_created_type"; create_fixture "$root"; export_paths "$root" + path="$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; before=$(target_hash "$root") + if [[ "$noop_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_target_commit_hook() { if [[ "$2" == motd_news ]]; then case "$noop_created_type" in regular) printf created > "$3"; chmod 0755 "$3" ;; symlink) ln -s "$root/external" "$3" ;; fifo) mkfifo "$3" ;; esac; fi; return 0; } + assert_fail "$noop_impl absent noop rejects $noop_created_type creation" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$noop_impl absent noop $noop_created_type rolls back group" + [[ ! -e "$path" && ! -L "$path" ]] || fail "$noop_impl absent noop $noop_created_type left created object" + assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock + ) + done +done + +for noop_impl in tool module; do + for noop_change in regular-content regular-mode regular-inode regular-symlink symlink-target absent-regular; do + ( + root="$TEST_DIR/noop-unknown-$noop_impl-$noop_change"; create_fixture "$root"; export_paths "$root" + printf '# linux-setup:managed-motd\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" + if [[ "$noop_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_run_locked_operation install + case "$noop_change" in regular-content|regular-symlink) target_id=motd ;; regular-mode) target_id=issue ;; regular-inode) target_id=custom_welcome ;; symlink-target) target_id="uname"; rm -f "${MOTD_PATH_BY_ID[$target_id]}"; ln -s "$root/link-one" "${MOTD_PATH_BY_ID[$target_id]}" ;; absent-regular) target_id=motd_news ;; esac + before=$(target_hash "$root") + motd_target_commit_hook() { local id="$2" path="$3" replacement; [[ "$id" == "$target_id" ]] || return 0; case "$noop_change" in regular-content) printf changed > "$path" ;; regular-mode) chmod 0600 "$path" ;; regular-inode) replacement="${path}.replacement"; cp "$path" "$replacement"; chmod 0755 "$replacement"; mv -Tf "$replacement" "$path" ;; regular-symlink) rm -f "$path"; ln -s "$root/other" "$path" ;; symlink-target) rm -f "$path"; ln -s "$root/link-two" "$path" ;; absent-regular) printf created > "$path"; chmod 0755 "$path" ;; esac; return 0; } + assert_fail "$noop_impl unknown noop rejects $noop_change" motd_run_locked_operation restore initial + assert_eq "$before" "$(target_hash "$root")" "$noop_impl unknown noop $noop_change rolls back group" + assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock + ) + done +done diff --git a/tests/test-motd-04.sh b/tests/test-motd-04.sh new file mode 100644 index 0000000..8f0116a --- /dev/null +++ b/tests/test-motd-04.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-motd-common.sh" + +for group_impl in tool module; do + for group_change in motd-content issue-mode custom-inode; do + ( + root="$TEST_DIR/final-group-$group_impl-$group_change"; create_fixture "$root"; export_paths "$root"; before=$(target_hash "$root") + if [[ "$group_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_transaction_phase_hook() { local phase="$1" path replacement; [[ "$phase" == target-committed-motd_news ]] || return 0; case "$group_change" in motd-content) printf changed > "$MOTD_ETC_ROOT/motd" ;; issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; custom-inode) path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; replacement="${path}.replacement"; cp "$path" "$replacement"; chmod 0755 "$replacement"; mv -Tf "$replacement" "$path" ;; esac; } + assert_fail "$group_impl final group rejects $group_change" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$group_impl final group $group_change rolls back all targets" + [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "$group_impl final group $group_change left pending" + assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock + ) + done +done + +for illegal_legacy_name in motd.previous-unknown issue.previous-unknown motd.invalid-backup evil.initial-backup; do +( + root="$TEST_DIR/legacy-name-${illegal_legacy_name//[^a-zA-Z0-9]/-}"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install + object="$MOTD_STATE_DIR/$illegal_legacy_name"; printf invalid > "$object"; chmod 0600 "$object"; before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + assert_fail "illegal legacy name $illegal_legacy_name rejected" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "illegal legacy name $illegal_legacy_name preserves targets" + assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "illegal legacy name $illegal_legacy_name preserves pointers" + [[ -e "$object" || -L "$object" ]] || fail "illegal legacy name $illegal_legacy_name evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "illegal legacy name $illegal_legacy_name created pending"; motd_take_lock; motd_release_lock +) +done + +for legacy_backup_kind in directory fifo socket writable wrong-owner wrong-gid device; do +( + root="$TEST_DIR/legacy-backup-$legacy_backup_kind"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install; object="$MOTD_STATE_DIR/motd.initial-backup" + case "$legacy_backup_kind" in + directory) mkdir "$object" ;; fifo) mkfifo "$object" ;; + socket) if command -v python3 >/dev/null 2>&1; then python3 - "$object" <<'PY' +import socket,sys +s=socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.close() +PY + else pass "socket legacy test skipped without python3"; exit 0; fi ;; + writable) printf bad > "$object"; chmod 0666 "$object" ;; + wrong-owner) if (( EUID == 0 )); then printf bad > "$object"; chmod 0600 "$object"; chown "$((MOTD_TRUSTED_UID+1)):$MOTD_TRUSTED_GID" "$object"; else pass "wrong-owner legacy test skipped without privilege"; exit 0; fi ;; + wrong-gid) if (( EUID == 0 )); then printf bad > "$object"; chmod 0600 "$object"; chown "$MOTD_TRUSTED_UID:$((MOTD_TRUSTED_GID+1))" "$object"; else pass "wrong-gid legacy test skipped without privilege"; exit 0; fi ;; + device) if (( EUID == 0 )) && mknod "$object" c 1 3 2>/dev/null; then :; else pass "device legacy test skipped without privilege"; exit 0; fi ;; + esac + before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + assert_fail "legacy backup $legacy_backup_kind rejected" motd_run_locked_operation restore initial + assert_eq "$before" "$(target_hash "$root")" "legacy backup $legacy_backup_kind preserves targets" + assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "legacy backup $legacy_backup_kind preserves pointers" + [[ -e "$object" || -L "$object" ]] || fail "legacy backup $legacy_backup_kind evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "legacy backup $legacy_backup_kind created pending"; motd_take_lock; motd_release_lock +) +done + +for legacy_marker_name in motd.initial-absent issue.initial-unknown motd.previous-absent; do + for legacy_marker_kind in symlink directory fifo writable; do + ( + root="$TEST_DIR/legacy-marker-${legacy_marker_name//[^a-zA-Z0-9]/-}-$legacy_marker_kind"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install; object="$MOTD_STATE_DIR/$legacy_marker_name" + case "$legacy_marker_kind" in symlink) ln -s "$root/missing" "$object" ;; directory) mkdir "$object" ;; fifo) mkfifo "$object" ;; writable) printf bad > "$object"; chmod 0666 "$object" ;; esac + before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + assert_fail "legacy marker $legacy_marker_name $legacy_marker_kind rejected" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "legacy marker $legacy_marker_name $legacy_marker_kind preserves targets" + assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "legacy marker $legacy_marker_name $legacy_marker_kind preserves pointers" + [[ -e "$object" || -L "$object" ]] || fail "legacy marker $legacy_marker_name $legacy_marker_kind evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "legacy marker $legacy_marker_name $legacy_marker_kind created pending"; motd_take_lock; motd_release_lock + ) + done +done + +( + root="$TEST_DIR/legal-state-root-legacy"; create_fixture "$root"; export_paths "$root"; mkdir -p "$MOTD_STATE_DIR"; chmod 0700 "$MOTD_STATE_DIR" + cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_STATE_DIR/motd.initial-backup"; printf target > "$root/symlink-target"; ln -s "$root/symlink-target" "$MOTD_STATE_DIR/issue.initial-backup" + : > "$MOTD_STATE_DIR/issue.net.initial-absent"; chmod 0600 "$MOTD_STATE_DIR/issue.net.initial-absent"; : > "$MOTD_STATE_DIR/00-custom-welcome.initial-unknown"; chmod 0600 "$MOTD_STATE_DIR/00-custom-welcome.initial-unknown" + cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_STATE_DIR/10-uname.initial-backup"; : > "$MOTD_STATE_DIR/50-motd-news.initial-absent"; chmod 0600 "$MOTD_STATE_DIR/50-motd-news.initial-absent" + touch -a -d @1000000000 "$MOTD_STATE_DIR/motd.initial-backup"; touch -m -d @1000200000 "$MOTD_STATE_DIR/motd.initial-backup"; legacy_times=$(stat -c %X:%Y "$MOTD_STATE_DIR/motd.initial-backup") + load_motd; motd_run_locked_operation restore initial + [[ -L "$MOTD_ETC_ROOT/issue" && "$(readlink "$MOTD_ETC_ROOT/issue")" == "$root/symlink-target" ]] || fail "legal state root symlink backup not imported" + assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "legal state root regular backup imported"; [[ -e "$MOTD_STATE_DIR/initial.current" ]] || fail "legal state root import did not publish initial pointer" + assert_eq "$legacy_times" "$(stat -c %X:%Y "$MOTD_STATE_DIR/motd.initial-backup")" "legacy root metadata validation/import preserves timestamps"; pass "legal state root regular, symlink, and markers import" +) + + +for rollback_impl in tool module; do + for rollback_damage in motd-content issue-mode custom-inode absent-regular absent-symlink absent-fifo symlink-target; do + ( + root="$TEST_DIR/rollback-final-$rollback_impl-$rollback_damage"; create_fixture "$root"; export_paths "$root" + if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_run_locked_operation install + case "$rollback_damage" in + motd-content) printf 'pre-run-motd\n' > "$MOTD_ETC_ROOT/motd" ;; + issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; + custom-inode) printf 'pre-run-custom\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0700 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" ;; + absent-regular|absent-symlink|absent-fifo) rm -f "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; + symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/original-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; + esac + before=$(target_hash "$root") + before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + motd_rollback_hook() { + local id="$1" path target + [[ "$id" == motd_news ]] || return 0 + case "$rollback_damage" in + motd-content) printf 'rollback-corrupt\n' > "$MOTD_ETC_ROOT/motd" ;; + issue-mode) chmod 0777 "$MOTD_ETC_ROOT/issue" ;; + custom-inode) + path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" + cp "$path" "${path}.replacement"; chmod 0700 "${path}.replacement"; mv -Tf "${path}.replacement" "$path" + ;; + absent-regular|absent-symlink|absent-fifo) + ROLLBACK_RECREATE_TARGET="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + rm() { + local status + command rm "$@"; status=$?; unset -f rm + case "$rollback_damage" in + absent-regular) printf recreated > "$ROLLBACK_RECREATE_TARGET"; chmod 0755 "$ROLLBACK_RECREATE_TARGET" ;; + absent-symlink) ln -s "$root/recreated-link" "$ROLLBACK_RECREATE_TARGET" ;; + absent-fifo) mkfifo "$ROLLBACK_RECREATE_TARGET" ;; + esac + return "$status" + } + ;; + symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/changed-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; + esac + return 0 + } + assert_fail "$rollback_impl rollback final group rejects $rollback_damage" motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$rollback_impl rollback $rollback_damage lost pending" + transaction=$(cat "$MOTD_STATE_DIR/pending") + journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" + grep -Fxq 'phase=active' "$journal" || fail "$rollback_impl rollback $rollback_damage journal not active" + [[ -d "$MOTD_TRANSACTION_PARENT/$transaction/rollback" ]] || fail "$rollback_impl rollback $rollback_damage lost rollback snapshot" + pass "$rollback_impl rollback $rollback_damage preserves active recovery evidence" + motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } + motd_take_lock; motd_reconcile_state; motd_release_lock + assert_eq "$before" "$(target_hash "$root")" "$rollback_impl rollback $rollback_damage retry restores targets" + assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$rollback_impl rollback $rollback_damage retry restores pointers" + [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl rollback $rollback_damage retry left journal" + assert_no_motd_residue "$root"; assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$rollback_impl rollback $rollback_damage retry keeps bounded generations" + motd_take_lock; motd_release_lock + ) + done + +done + +for rollback_impl in tool module; do +( + root="$TEST_DIR/rollback-final-success-$rollback_impl"; create_fixture "$root"; export_paths "$root" + if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_run_locked_operation install; printf 'normal-pre-run\n' > "$MOTD_ETC_ROOT/motd"; before=$(target_hash "$root") + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + assert_fail "$rollback_impl normal rollback still returns original operation failure" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$rollback_impl normal rollback final group succeeds" + [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl normal rollback left journal" + assert_no_motd_residue "$root" +) +done + +printf 'All MOTD rollback final validation tests passed.\n' diff --git a/tests/test-motd.sh b/tests/test-motd.sh deleted file mode 100644 index cccdae2..0000000 --- a/tests/test-motd.sh +++ /dev/null @@ -1,1259 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."&&pwd); -readonly ROOT_DIR -TEST_DIR=$(mktemp -d); -readonly TEST_DIR -fail(){ echo "FAIL: $*" >&2; -exit 1; -}; -pass(){ echo "PASS: $*"; -}; -assert_eq(){ [[ "$1" == "$2" ]]||fail "$3: $1 != $2"; -pass "$3"; -}; -assert_fail(){ local l="$1"; -shift; -if "$@"; -then fail "$l"; -fi; -pass "$l"; -} -extract_payload(){ awk '/install -m 0755 \/dev\/stdin .*<<.SCRIPT./{c=1;next}c&&$0=="SCRIPT"{exit}c{print}' "$1"; -} -load_motd() { - . "$ROOT_DIR/tools/setup-motd.sh" -} - -create_fixture(){ local r="$1" e - e="$r/e"; -e+="tc"; -mkdir -m700 "$r" "$e" "$e/update-motd.d" "$r/lock"; -printf original>"$e/motd"; -chmod 640 "$e/motd"; -printf issue>"$e/issue"; -chmod 644 "$e/issue"; -printf old>"$e/update-motd.d/00-custom-welcome"; -chmod 700 "$e/update-motd.d/00-custom-welcome"; -printf uname>"$e/update-motd.d/10-uname"; -chmod 754 "$e/update-motd.d/10-uname"; -} -export_paths(){ local r="$1" e - e="$r/e"; -e+="tc"; -export MOTD_TEST_MODE=1 MOTD_ETC_ROOT="$e" MOTD_STATE_DIR="$r/state" MOTD_LOCK_FILE="$r/lock/motd.lock" MOTD_TRANSACTION_PARENT="$r/state/transactions"; -} -target_hash(){ local r="$1" e p - e="$r/e"; -e+="tc"; -for p in "$e/motd" "$e/issue" "$e/issue.net" "$e/update-motd.d/00-custom-welcome" "$e/update-motd.d/10-uname" "$e/update-motd.d/50-motd-news"; -do if [[ -L "$p" ]]; -then echo "L|$p|$(readlink "$p")"; -elif [[ -f "$p" ]]; -then echo "F|$p|$(stat -c %u:%g:%a "$p")|$(sha256sum "$p"|awk '{print $1}')"; -elif [[ ! -e "$p" ]]; -then echo "A|$p"; -else echo "O|$p|$(stat -c %F "$p")"; -fi; -done; -} -extract_payload "$ROOT_DIR/tools/setup-motd.sh">"$TEST_DIR/t"; -extract_payload "$ROOT_DIR/modules/system-customize.sh">"$TEST_DIR/m"; -cmp -s "$TEST_DIR/t" "$TEST_DIR/m"||fail payload; -pass "MOTD payloads stay identical"; -assert_eq cf2a4f8dfe2fe26bff5509cdaee56d9e0816fafd0594251b02e892201af5c4d5 "$(sha256sum "$TEST_DIR/t"|awk '{print $1}')" "payload hash" -( - r="$TEST_DIR/source"; -mkdir -m700 "$r" "$r/lock"; -export_paths "$r"; -b=$(trap -p EXIT HUP INT TERM ERR); -load_motd; -[[ "$b" == "$(trap -p EXIT HUP INT TERM ERR)"&&! -e "$MOTD_STATE_DIR"&&! -e "$MOTD_LOCK_FILE" ]]||fail source -); -pass "source zero side effects" -( - r="$TEST_DIR/basic"; -create_fixture "$r"; -export_paths "$r"; -before=$(target_hash "$r"); -load_motd; -motd_run_locked_operation install; -[[ -f "$MOTD_ETC_ROOT/motd"&&! -s "$MOTD_ETC_ROOT/motd"&&-x "$MOTD_SCRIPT" ]]||fail install; -motd_run_locked_operation restore previous; -assert_eq "$before" "$(target_hash "$r")" "restore previous group"; -motd_run_locked_operation install; -printf changed>"$MOTD_ETC_ROOT/motd"; -motd_run_locked_operation restore initial; -assert_eq "$before" "$(target_hash "$r")" "restore initial group" -) -( - r="$TEST_DIR/types"; -create_fixture "$r"; -export_paths "$r"; -unlink "$MOTD_ETC_ROOT/issue"; -ln -s "$r/link" "$MOTD_ETC_ROOT/issue"; -before=$(target_hash "$r"); -load_motd; -motd_run_locked_operation install; -g=$(motd_read_pointer previous); -motd_validate_snapshot "$MOTD_STATE_DIR/generations/$g" previous "$g"; -assert_eq symlink "${MOTD_SNAPSHOT_STATE[issue]}" "symlink snapshot"; -assert_eq absent "${MOTD_SNAPSHOT_STATE[issue_net]}" "absent snapshot"; -motd_run_locked_operation restore previous; -assert_eq "$before" "$(target_hash "$r")" "symlink restore" -) -( - r="$TEST_DIR/initial"; -create_fixture "$r"; -export_paths "$r"; -before=$(target_hash "$r"); -load_motd; -motd_snapshot_capture_hook(){ [[ "$1" != issue_net ]]; -}; -assert_fail "initial capture atomic" motd_run_locked_operation install; -[[ ! -e "$MOTD_STATE_DIR/initial.current" ]]||fail partial; -assert_eq "$before" "$(target_hash "$r")" "initial failure no target change" -) -( - r="$TEST_DIR/previous"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -old=$(motd_read_pointer previous); -for id in "${MOTD_TARGET_IDS[@]}"; -do motd_snapshot_capture_hook(){ [[ "$1" != "$id" ]]; -}; -assert_fail "previous capture $id" motd_run_locked_operation install; -assert_eq "$old" "$(motd_read_pointer previous)" "previous pointer $id"; -done -) -for failing_id in motd issue_net motd_news; -do -( - r="$TEST_DIR/i-$failing_id"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -printf current>"$MOTD_ETC_ROOT/motd"; -before=$(target_hash "$r"); -motd_target_commit_hook(){ [[ "$2" != "$failing_id" ]]; -}; -assert_fail "install failure $failing_id" motd_run_locked_operation install; -assert_eq "$before" "$(target_hash "$r")" "install rollback $failing_id" -) -( - r="$TEST_DIR/r-$failing_id"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -printf current>"$MOTD_ETC_ROOT/motd"; -before=$(target_hash "$r"); -motd_target_commit_hook(){ [[ "$1" != restore||"$2" != "$failing_id" ]]; -}; -assert_fail "restore failure $failing_id" motd_run_locked_operation restore previous; -assert_eq "$before" "$(target_hash "$r")" "restore rollback $failing_id" -) -done -( - r="$TEST_DIR/manifest"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -g=$(motd_read_pointer previous); -mf="$MOTD_STATE_DIR/generations/$g/manifest"; -cp "$mf" "$r/o"; -for k in missing duplicate unknown digest generation; -do cp "$r/o" "$mf"; -case $k in missing)sed -i '$d' "$mf";; -duplicate)tail -n1 "$mf">>"$mf";; -unknown)echo x=y>>"$mf";; -digest)sed -i 's/[0-9a-f]\{64\}$/0000000000000000000000000000000000000000000000000000000000000000/' "$mf";; -generation)sed -i 's/^generation=.*/generation=00000000000000000000000000000000/' "$mf";; -esac; -before=$(target_hash "$r"); -assert_fail "manifest $k rejected" motd_run_locked_operation restore previous; -assert_eq "$before" "$(target_hash "$r")" "manifest $k no target change"; -done -) -( - r="$TEST_DIR/lock"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_lock_acquired_hook(){ : >"$r/ready"; -sleep 2; -}; -motd_take_lock & h=$!; -while [[ ! -e "$r/ready" ]]; -do sleep .02; -done; -assert_fail "install lock contention" motd_run_locked_operation install; -assert_fail "restore lock contention" motd_run_locked_operation restore previous; -[[ ! -e "$MOTD_STATE_DIR" ]]||fail lockstate; -wait "$h"; -motd_lock_acquired_hook(){ :; -}; -motd_take_lock; -motd_release_lock -); -pass "shared lock nonblocking and reacquirable" -for k in symlink writable fifo; -do( - r="$TEST_DIR/l-$k"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -case $k in symlink)printf x>"$r/x"; -ln -s "$r/x" "$MOTD_LOCK_FILE";; -writable)printf x>"$MOTD_LOCK_FILE"; -chmod 666 "$MOTD_LOCK_FILE";; -fifo)mkfifo "$MOTD_LOCK_FILE";; -esac; -assert_fail "lock rejects $k" motd_take_lock -); -done -for k in symlink writable; -do( - r="$TEST_DIR/s-$k"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -case $k in symlink)mkdir -m700 "$r/s"; -ln -s "$r/s" "$MOTD_STATE_DIR";; -writable)mkdir -m770 "$MOTD_STATE_DIR";; -esac; -motd_take_lock; -assert_fail "state rejects $k" motd_prepare_state_layout; -motd_release_lock -); -done -create_legacy(){ printf legacy>"$MOTD_ETC_ROOT/motd.initial-backup"; -: >"$MOTD_ETC_ROOT/issue.initial-absent"; -printf net>"$MOTD_ETC_ROOT/issue.net.initial-backup"; -: >"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.initial-unknown"; -printf uname>"$MOTD_ETC_ROOT/update-motd.d/10-uname.initial-backup"; -: >"$MOTD_ETC_ROOT/update-motd.d/50-motd-news.initial-absent"; -} -( - r="$TEST_DIR/legacy"; -create_fixture "$r"; -export_paths "$r"; -create_legacy; -load_motd; -motd_run_locked_operation restore initial; -assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "legacy complete restore"; -[[ -e "$MOTD_ETC_ROOT/motd.initial-backup" ]]||fail legacymove -) -for k in partial conflict commit; -do( - r="$TEST_DIR/lg-$k"; -create_fixture "$r"; -export_paths "$r"; -if [[ $k == partial ]]; -then printf x>"$MOTD_ETC_ROOT/motd.initial-backup"; -else create_legacy; -[[ $k == conflict ]]&&: >"$MOTD_ETC_ROOT/motd.initial-absent"; -fi; -load_motd; -[[ $k == commit ]]&&motd_snapshot_commit_hook(){ [[ "$3" != legacy-directory ]]; -}; -assert_fail "legacy $k rejected" motd_run_locked_operation restore initial -); -done -( - r="$TEST_DIR/interop"; -create_fixture "$r"; -export_paths "$r"; -o=$(target_hash "$r"); -bash -c '. "$1/modules/system-customize.sh";ask_yes_no(){ return 0;};motd_preview_hook(){ :;};configure_motd' _ "$ROOT_DIR">/dev/null; -printf x>"$MOTD_ETC_ROOT/motd"; -bash -c '. "$1/tools/setup-motd.sh";motd_run_locked_operation restore previous' _ "$ROOT_DIR"; -assert_eq "$o" "$(target_hash "$r")" "module state tool restore" -) -( - r="$TEST_DIR/exit"; -create_fixture "$r"; -export_paths "$r"; -before=$(target_hash "$r"); -n=0; -bash -c '. "$1";motd_take_lock;motd_install_transaction_traps;motd_prepare_state_layout;motd_validate_target_parents;motd_begin_transaction install;motd_prepare_install_plan;motd_commit_target_plan install false;exit 0' _ "$ROOT_DIR/tools/setup-motd.sh">/dev/null 2>&1||n=$?; -assert_eq 1 "$n" "active exit zero maps one"; -assert_eq "$before" "$(target_hash "$r")" "active exit rollback" -) -( - r="$TEST_DIR/rbf"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -motd_target_commit_hook(){ [[ "$2" != issue_net ]]; -}; -motd_rollback_hook(){ echo "$1">>"$r/a"; -[[ "$1" != motd&&"$1" != custom_welcome ]]; -}; -assert_fail "rollback failure nonzero" motd_run_locked_operation install; -assert_eq 6 "$(wc -l<"$r/a")" "rollback attempts six"; -[[ -e "$MOTD_STATE_DIR/pending" ]]||fail journal -) -cat >"$TEST_DIR/sig.sh"<<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1"; -hit=false -send(){ [[ $hit == false ]]||return; -hit=true; -kill -"$SIGNAL_NAME" "$BASHPID"; -} -motd_snapshot_capture_hook(){ [[ $PHASE == snapshot-building&&$1 == issue ]]&&send; -return 0; -} -motd_transaction_phase_hook(){ [[ $PHASE == snapshot-commit&&$1 == snapshot-commit ]]&&send; -return 0; -} -motd_snapshot_commit_hook(){ [[ $PHASE == snapshot-publish&&$1 == initial&&$3 == pointer ]]&&send; -return 0; -} -motd_target_commit_hook(){ case "$PHASE:$1:$2" in install-first:install:motd|install-middle:install:issue_net|restore-first:restore:motd|restore-middle:restore:issue_net)send;; -esac; -return 0; -} -case $PHASE in restore-*)motd_run_locked_operation restore previous;; -*)motd_run_locked_operation install;; -esac -CHILD -chmod 700 "$TEST_DIR/sig.sh" -for ph in snapshot-building snapshot-commit snapshot-publish install-first install-middle restore-first restore-middle; -do for sig in HUP INT TERM; -do( - r="$TEST_DIR/$ph-$sig"; -create_fixture "$r"; -export_paths "$r"; -if [[ $ph == restore-* ]]; -then bash -c '. "$1";motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh"; -printf current>"$MOTD_ETC_ROOT/motd"; -fi; -before=$(target_hash "$r"); -n=0; -env --default-signal=HUP,INT,TERM PHASE="$ph" SIGNAL_NAME="$sig" bash "$TEST_DIR/sig.sh" "$ROOT_DIR/tools/setup-motd.sh">/dev/null 2>&1||n=$?; -case $sig in HUP)e=129;; -INT)e=130;; -TERM)e=143;; -esac; -assert_eq "$e" "$n" "$sig $ph status"; -assert_eq "$before" "$(target_hash "$r")" "$sig $ph rollback" -); -done; -done -( - r="$TEST_DIR/unknown"; -create_fixture "$r"; -export_paths "$r"; -printf '# linux-setup:managed-motd'>"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; -load_motd; -motd_run_locked_operation install; -g=$(motd_read_pointer initial); -motd_validate_snapshot "$MOTD_STATE_DIR/generations/$g" initial "$g"; -assert_eq unknown "${MOTD_SNAPSHOT_STATE[motd]}" "unknown initial state" -) -( - r="$TEST_DIR/readonly"; -create_fixture "$r"; -export_paths "$r"; -bash "$ROOT_DIR/tools/setup-motd.sh" status>/dev/null; -bash "$ROOT_DIR/tools/setup-motd.sh" help>/dev/null; -[[ ! -e "$MOTD_STATE_DIR"&&! -e "$MOTD_LOCK_FILE" ]]||fail readonly -); -pass "status help no lock" -( - r="$TEST_DIR/publish"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -old=$(motd_read_pointer previous); -printf current>"$MOTD_ETC_ROOT/motd"; -before=$(target_hash "$r"); -motd_snapshot_manifest_hook(){ [[ "$1" != previous ]]; -}; -assert_fail "previous manifest failure" motd_run_locked_operation install; -assert_eq "$old" "$(motd_read_pointer previous)" "manifest preserves previous"; -assert_eq "$before" "$(target_hash "$r")" "manifest failure rolls back"; -motd_snapshot_manifest_hook(){ :; -}; -motd_snapshot_commit_hook(){ [[ "$1" != previous||"$3" != directory ]]; -}; -assert_fail "previous directory commit failure" motd_run_locked_operation install; -assert_eq "$old" "$(motd_read_pointer previous)" "directory failure preserves previous" -) -for k in owner gid; -do( - r="$TEST_DIR/lmeta-$k"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -printf x>"$MOTD_LOCK_FILE"; -chmod 600 "$MOTD_LOCK_FILE"; -stat(){ if [[ "${*: -1}" == "$MOTD_LOCK_FILE"&&"$*" == *"%u:%g:%a"* ]]; -then [[ $k == owner ]]&&printf '99999:%s:600\n' "$MOTD_TRUSTED_GID"||printf '%s:99999:600\n' "$MOTD_TRUSTED_UID"; -else command stat "$@"; -fi; -}; -assert_fail "lock wrong $k" motd_take_lock -); -done -for k in owner gid; -do( - r="$TEST_DIR/smeta-$k"; -create_fixture "$r"; -export_paths "$r"; -mkdir -m700 "$MOTD_STATE_DIR"; -load_motd; -stat(){ if [[ "${*: -1}" == "$MOTD_STATE_DIR"&&"$*" == *"%u:%g:%a"* ]]; -then [[ $k == owner ]]&&printf '99999:%s:700\n' "$MOTD_TRUSTED_GID"||printf '%s:99999:700\n' "$MOTD_TRUSTED_UID"; -else command stat "$@"; -fi; -}; -motd_take_lock; -assert_fail "state wrong $k" motd_prepare_state_layout; -motd_release_lock -); -done -( - r="$TEST_DIR/state-create"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -chown(){ [[ "${*: -1}" == "$MOTD_STATE_DIR" ]]&&return 1; -command chown "$@"; -}; -assert_fail "state create failure" motd_run_locked_operation install; -[[ ! -e "$MOTD_STATE_DIR" ]]||fail state-residue -) -( - r="$TEST_DIR/close"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_take_lock; -motd_lock_close_hook(){ return 1; -}; -assert_fail "lock close failure" motd_release_lock -) -( - r="$TEST_DIR/device"; -create_fixture "$r"; -export_paths "$r"; -if mknod "$MOTD_ETC_ROOT/issue.net" c 1 3 2>/dev/null; -then before=$(target_hash "$r"); -load_motd; -assert_fail "device rejected" motd_run_locked_operation install; -assert_eq "$before" "$(target_hash "$r")" "device no target change"; -else pass "device test skipped"; -fi -) -( - r="$TEST_DIR/legacy-state"; -create_fixture "$r"; -export_paths "$r"; -mkdir -m700 "$MOTD_STATE_DIR"; -printf legacy>"$MOTD_STATE_DIR/motd.previous-backup"; -chmod 600 "$MOTD_STATE_DIR/motd.previous-backup"; -for x in issue issue.net 00-custom-welcome 10-uname 50-motd-news; -do : >"$MOTD_STATE_DIR/$x.previous-absent"; -chmod 600 "$MOTD_STATE_DIR/$x.previous-absent"; -done; -load_motd; -motd_run_locked_operation restore previous; -assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "state legacy restore" -) - -state_fingerprint() { - local root="$1" path - [[ -d "$root" ]] || { printf 'absent\n'; return; } - while IFS= read -r path; do - if [[ -L "$path" ]]; then printf 'L|%s|%s\n' "${path#$root/}" "$(readlink "$path")" - elif [[ -f "$path" ]]; then printf 'F|%s|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" "$(sha256sum "$path" | awk '{print $1}')" - elif [[ -d "$path" ]]; then printf 'D|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" - fi - done < <(find "$root" -mindepth 1 -print | sort) -} - -generation_count() { - local directory="$1" - [[ -d "$directory" ]] || { printf '0\n'; return; } - find "$directory" -mindepth 1 -maxdepth 1 -type d ! -name '.*.stage' | wc -l -} - -assert_no_motd_residue() { - local root="$1" residue - residue=$(find "$root" -name '.linux-setup-motd.*' -o -name '.*.pointer.*' -o -name '.pending.*' -o -name '.*.stage' 2>/dev/null) - [[ -z "$residue" ]] || fail "unexpected MOTD residue: $residue" - if [[ -d "$root/state/transactions" ]]; then [[ -z "$(find "$root/state/transactions" -mindepth 1 -print -quit)" ]] || fail "transaction residue remains"; fi -} - -create_complete_legacy_scope() { - local scope="$1" prefix - cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_ETC_ROOT/motd.${scope}-backup" - cp -a "$MOTD_ETC_ROOT/issue" "$MOTD_ETC_ROOT/issue.${scope}-backup" - : > "$MOTD_ETC_ROOT/issue.net.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/issue.net.${scope}-absent" - cp -a "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.${scope}-backup" - cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_ETC_ROOT/update-motd.d/10-uname.${scope}-backup" - : > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent" -} - -for legacy_phase in random-id stat open create cat chown chmod sha256 manifest mv; do -( - root="$TEST_DIR/legacy-io-$legacy_phase"; create_fixture "$root"; export_paths "$root" - create_complete_legacy_scope initial - before_targets=$(target_hash "$root") - before_legacy=$(find "$MOTD_ETC_ROOT" -name '*.initial-*' -type f -printf '%p|%s|%m\n' | sort) - load_motd - motd_legacy_io_hook() { [[ "$1" != "$legacy_phase" ]]; } - assert_fail "legacy $legacy_phase failure returns nonzero" motd_run_locked_operation restore initial - assert_eq "$before_targets" "$(target_hash "$root")" "legacy $legacy_phase keeps formal targets" - assert_eq "$before_legacy" "$(find "$MOTD_ETC_ROOT" -name '*.initial-*' -type f -printf '%p|%s|%m\n' | sort)" "legacy $legacy_phase keeps legacy files" - [[ ! -e "$MOTD_STATE_DIR/initial.current" ]] || fail "legacy $legacy_phase published initial pointer" - assert_no_motd_residue "$root" -) -done -pass "legacy I/O failure matrix preserves data and state" - -cat > "$TEST_DIR/pointer-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1" -motd_snapshot_commit_hook() { - if [[ "$1" == previous && "$3" == pointer ]]; then kill -"$SIGNAL_NAME" "$BASHPID"; fi -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/pointer-signal-child.sh" - -for signal_name in HUP INT TERM; do -( - root="$TEST_DIR/pointer-signal-$signal_name"; create_fixture "$root"; export_paths "$root" - bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" - old_pointer=$(cat "$MOTD_STATE_DIR/previous.current") - before=$(target_hash "$root") - rc=0 - env --default-signal=HUP,INT,TERM SIGNAL_NAME="$signal_name" bash "$TEST_DIR/pointer-signal-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? - case "$signal_name" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - assert_eq "$expected" "$rc" "$signal_name during previous pointer stage returns conventional status" - assert_eq "$old_pointer" "$(cat "$MOTD_STATE_DIR/previous.current")" "$signal_name preserves previous pointer" - assert_eq "$before" "$(target_hash "$root")" "$signal_name pointer stage restores targets" - assert_no_motd_residue "$root" - bash -c '. "$1"; motd_take_lock; motd_release_lock' _ "$ROOT_DIR/tools/setup-motd.sh" -) -done - -for pending_phase in pending-create pending-write pending-chown pending-chmod pending-publish; do -( - root="$TEST_DIR/pending-failure-$pending_phase"; create_fixture "$root"; export_paths "$root" - load_motd - motd_run_locked_operation install - before=$(target_hash "$root"); before_generations=$(generation_count "$MOTD_STATE_DIR/generations") - motd_transaction_phase_hook() { [[ "$1" != "$pending_phase" ]]; } - assert_fail "pending $pending_phase failure returns nonzero" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "pending $pending_phase keeps targets" - assert_eq "$before_generations" "$(generation_count "$MOTD_STATE_DIR/generations")" "pending $pending_phase leaves no generation orphan" - [[ ! -e "$MOTD_STATE_DIR/pending" && ! -L "$MOTD_STATE_DIR/pending" ]] || fail "pending $pending_phase left pending" - assert_no_motd_residue "$root" - motd_transaction_phase_hook() { :; } - motd_run_locked_operation install -) -done - -for pending_kind in broken-symlink symlink directory fifo mode owner gid; do -( - root="$TEST_DIR/pending-kind-$pending_kind"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - pending="$MOTD_STATE_DIR/pending" - case "$pending_kind" in - broken-symlink) ln -s "$root/missing" "$pending" ;; - symlink) printf x > "$root/pending-target"; chmod 0600 "$root/pending-target"; ln -s "$root/pending-target" "$pending" ;; - directory) mkdir "$pending" ;; - fifo) mkfifo "$pending" ;; - mode) printf x > "$pending"; chmod 0666 "$pending" ;; - owner|gid) printf x > "$pending"; chmod 0600 "$pending" ;; - esac - before=$(target_hash "$root") - if [[ "$pending_kind" == owner || "$pending_kind" == gid ]]; then - stat() { - if [[ "${*: -1}" == "$pending" && "$*" == *"%u:%g:%a"* ]]; then - [[ "$pending_kind" == owner ]] && printf '99999:%s:600\n' "$MOTD_TRUSTED_GID" || printf '%s:99999:600\n' "$MOTD_TRUSTED_UID" - else command stat "$@"; fi - } - fi - assert_fail "pending $pending_kind fails closed" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "pending $pending_kind changes no target" - [[ -e "$pending" || -L "$pending" ]] || fail "pending $pending_kind evidence was removed" -) -done - -cat > "$TEST_DIR/kill-pending-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_transaction_phase_hook() { [[ "$1" == pending-active ]] && kill -KILL "$BASHPID"; return 0; } -motd_run_locked_operation install -CHILD -cat > "$TEST_DIR/kill-previous-pointer-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_snapshot_commit_hook() { [[ "$1" == previous && "$3" == pointer ]] && kill -KILL "$BASHPID"; return 0; } -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/kill-pending-child.sh" "$TEST_DIR/kill-previous-pointer-child.sh" - -for journal_case in pending-directory journal-transaction generation-scope generation-missing old-new-mix; do -( - root="$TEST_DIR/journal-$journal_case"; create_fixture "$root"; export_paths "$root" - bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" - rc=0 - if [[ "$journal_case" == pending-directory || "$journal_case" == journal-transaction ]]; then - bash "$TEST_DIR/kill-pending-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? - else - bash "$TEST_DIR/kill-previous-pointer-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? - fi - assert_eq 137 "$rc" "$journal_case fixture stops with SIGKILL" - transaction=$(cat "$MOTD_STATE_DIR/pending") - journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" - case "$journal_case" in - pending-directory) printf '%032x\n' 1 > "$MOTD_STATE_DIR/pending" ;; - journal-transaction) sed -i 's/^transaction=.*/transaction=00000000000000000000000000000001/' "$journal" ;; - generation-scope) initial=$(cat "$MOTD_STATE_DIR/initial.current"); sed -i "s/^previous_new=.*/previous_new=$initial/" "$journal" ;; - generation-missing) sed -i 's/^previous_new=.*/previous_new=ffffffffffffffffffffffffffffffff/' "$journal" ;; - old-new-mix) initial=$(cat "$MOTD_STATE_DIR/initial.current"); sed -i "s/^previous_old=.*/previous_old=$initial/" "$journal" ;; - esac - before=$(target_hash "$root") - load_motd - assert_fail "$journal_case pending/journal mismatch rejected" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$journal_case mismatch changes no target" - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$journal_case evidence removed" -) -done - -for tamper_case in content mode inode regular-to-symlink; do -( - root="$TEST_DIR/stage-$tamper_case"; create_fixture "$root"; export_paths "$root" - printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - before=$(target_hash "$root") - load_motd - motd_transaction_phase_hook() { - [[ "$1" == install-plan-ready ]] || return 0 - case "$tamper_case" in - content) printf tampered > "${MOTD_TARGET_STAGE[motd]}" ;; - mode) chmod 0600 "${MOTD_TARGET_STAGE[issue_net]}" ;; - inode) cp "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" "${MOTD_TARGET_STAGE[motd_news]}.replacement"; chmod 0644 "${MOTD_TARGET_STAGE[motd_news]}.replacement"; mv -Tf "${MOTD_TARGET_STAGE[motd_news]}.replacement" "${MOTD_TARGET_STAGE[motd_news]}" ;; - regular-to-symlink) rm -f "${MOTD_TARGET_STAGE[issue_net]}"; ln -s "$root/elsewhere" "${MOTD_TARGET_STAGE[issue_net]}" ;; - esac - } - motd_target_commit_hook() { : > "$root/commit-called"; } - assert_fail "$tamper_case stage tamper rejected before commit" motd_run_locked_operation install - [[ ! -e "$root/commit-called" ]] || fail "$tamper_case reached first formal commit" - assert_eq "$before" "$(target_hash "$root")" "$tamper_case stage tamper changes no target" - assert_no_motd_residue "$root" -) -done - -( - root="$TEST_DIR/stage-symlink-target"; create_fixture "$root"; export_paths "$root" - printf external > "$root/external"; ln -sf "$root/external" "$MOTD_ETC_ROOT/issue" - load_motd; motd_run_locked_operation install - before=$(target_hash "$root") - motd_transaction_phase_hook() { - [[ "$1" == restore-plan-ready ]] || return 0 - rm -f "${MOTD_TARGET_STAGE[issue]}"; ln -s "$root/other" "${MOTD_TARGET_STAGE[issue]}" - } - motd_target_commit_hook() { : > "$root/commit-called"; } - assert_fail "symlink target stage tamper rejected before restore commit" motd_run_locked_operation restore previous - [[ ! -e "$root/commit-called" ]] || fail "symlink target tamper reached formal commit" - assert_eq "$before" "$(target_hash "$root")" "symlink target tamper changes no target" -) - -for native_id in uname motd_news; do -( - root="$TEST_DIR/native-symlink-$native_id"; create_fixture "$root"; export_paths "$root" - case "$native_id" in uname) native_path="$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; motd_news) native_path="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; esac - rm -f "$native_path" - printf 'external-%s\n' "$native_id" > "$root/external-$native_id"; chmod 0755 "$root/external-$native_id" - external_hash=$(sha256sum "$root/external-$native_id"); external_mode=$(stat -c %a "$root/external-$native_id") - ln -s "$root/external-$native_id" "$native_path" - load_motd - motd_run_locked_operation install - [[ ! -e "$native_path" && ! -L "$native_path" ]] || fail "$native_id symlink entry remains enabled" - assert_eq "$external_hash" "$(sha256sum "$root/external-$native_id")" "$native_id external content unchanged" - assert_eq "$external_mode" "$(stat -c %a "$root/external-$native_id")" "$native_id external mode unchanged" - motd_run_locked_operation restore previous - [[ -L "$native_path" && "$(readlink "$native_path")" == "$root/external-$native_id" ]] || fail "$native_id previous symlink not restored" -) -done - -cat > "$TEST_DIR/sigkill-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_transaction_phase_hook() { - [[ "$1" == "$KILL_PHASE" ]] && kill -KILL "$BASHPID" - return 0 -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/sigkill-child.sh" - -for kill_phase in transaction-snapshot-building transaction-final-moved target-committed-motd target-committed-issue_net target-committed-motd_news committed-cleanup committed-old-generation-cleaned committed-pending-removed; do -( - root="$TEST_DIR/sigkill-$kill_phase"; create_fixture "$root"; export_paths "$root" - printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" - before=$(target_hash "$root") - rc=0 - KILL_PHASE="$kill_phase" bash "$TEST_DIR/sigkill-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? - assert_eq 137 "$rc" "SIGKILL at $kill_phase terminates process" - after_kill=$(target_hash "$root") - load_motd - motd_take_lock; motd_prepare_state_layout; motd_reconcile_state; motd_release_lock - if [[ "$kill_phase" == committed-* ]]; then - assert_eq "$after_kill" "$(target_hash "$root")" "committed cleanup SIGKILL does not roll back" - else - assert_eq "$before" "$(target_hash "$root")" "SIGKILL at $kill_phase rolls back to pre-transaction state" - fi - assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "SIGKILL at $kill_phase leaves bounded generations" - assert_no_motd_residue "$root" - motd_take_lock; motd_release_lock -) -done - -( - root="$TEST_DIR/generation-bounded"; create_fixture "$root"; export_paths "$root" - load_motd - for _ in {1..6}; do motd_run_locked_operation install; done - assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "repeated install keeps two generations" -) - -( - root="$TEST_DIR/generation-commit-failure"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - before_count=$(generation_count "$MOTD_STATE_DIR/generations") - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - assert_fail "target commit failure returns nonzero for generation cleanup" motd_run_locked_operation install - assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "target commit failure leaves no orphan generation" -) - -( - root="$TEST_DIR/interop-module-tool"; create_fixture "$root"; export_paths "$root" - original=$(target_hash "$root") - bash -c '. "$1/modules/system-customize.sh"; ask_yes_no(){ return 0; }; motd_preview_hook(){ :; }; configure_motd' _ "$ROOT_DIR" >/dev/null - printf changed > "$MOTD_ETC_ROOT/motd" - bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation restore previous' _ "$ROOT_DIR" - assert_eq "$original" "$(target_hash "$root")" "module install to tool previous restore" - printf changed > "$MOTD_ETC_ROOT/motd" - bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation restore initial' _ "$ROOT_DIR" - assert_eq "$original" "$(target_hash "$root")" "module install to tool initial restore" -) - -( - root="$TEST_DIR/interop-tool-module"; create_fixture "$root"; export_paths "$root" - original=$(target_hash "$root") - bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation install' _ "$ROOT_DIR" - printf between > "$MOTD_ETC_ROOT/motd"; between=$(target_hash "$root") - bash -c '. "$1/modules/system-customize.sh"; ask_yes_no(){ return 0; }; motd_preview_hook(){ :; }; configure_motd' _ "$ROOT_DIR" >/dev/null - bash -c '. "$1/modules/system-customize.sh"; motd_run_locked_operation restore previous' _ "$ROOT_DIR" - assert_eq "$between" "$(target_hash "$root")" "tool install to module previous restore" - printf changed > "$MOTD_ETC_ROOT/motd" - bash -c '. "$1/modules/system-customize.sh"; motd_run_locked_operation restore initial' _ "$ROOT_DIR" - assert_eq "$original" "$(target_hash "$root")" "tool install to module initial restore" -) - - -( - root="$TEST_DIR/cleanup-old-generation-failure"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - old_previous=$(cat "$MOTD_STATE_DIR/previous.current"); blocked="$MOTD_STATE_DIR/generations/$old_previous" - motd_cleanup_path_hook() { [[ "$1" != "$blocked" ]]; } - assert_fail "old previous generation delete failure returns nonzero" motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "old generation delete failure lost pending journal" - transaction=$(cat "$MOTD_STATE_DIR/pending"); grep -Fq 'phase=committed' "$MOTD_TRANSACTION_PARENT/$transaction/journal" || fail "old generation failure journal not committed" - [[ -d "$blocked" ]] || fail "blocked old generation unexpectedly removed" - motd_cleanup_path_hook() { :; } - motd_run_locked_operation install - [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "old generation cleanup retry left pending" -) - -( - root="$TEST_DIR/cleanup-rollback-generation-failure"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - current_initial=$(cat "$MOTD_STATE_DIR/initial.current"); current_previous=$(cat "$MOTD_STATE_DIR/previous.current") - motd_cleanup_path_hook() { - local name=${1##*/} - if [[ "$1" == "$MOTD_STATE_DIR/generations/"* && "$name" != "$current_initial" && "$name" != "$current_previous" ]]; then return 1; fi - return 0 - } - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - assert_fail "rollback generation delete failure returns nonzero" motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback generation failure lost pending" - transaction=$(cat "$MOTD_STATE_DIR/pending"); grep -Fq 'phase=rolledback' "$MOTD_TRANSACTION_PARENT/$transaction/journal" || fail "rollback generation failure journal not rolledback" - motd_cleanup_path_hook() { :; }; motd_target_commit_hook() { :; } - motd_run_locked_operation install - [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "rolledback cleanup retry left pending" - assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "rolledback cleanup retry restores bounded generations" -) - - -for hook_case in content-motd mode-issue_net inode-motd_news type-issue_net; do -( - root="$TEST_DIR/hook-target-$hook_case"; create_fixture "$root"; export_paths "$root" - printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - before=$(target_hash "$root") - load_motd - motd_target_commit_hook() { - local id="$2" stage replacement - case "$hook_case:$id" in - content-motd:motd) printf changed > "${MOTD_TARGET_STAGE[motd]}" ;; - mode-issue_net:issue_net) chmod 0600 "${MOTD_TARGET_STAGE[issue_net]}" ;; - inode-motd_news:motd_news) - stage=${MOTD_TARGET_STAGE[motd_news]}; replacement="${stage}.replacement" - cp "$stage" "$replacement"; chmod "${MOTD_STAGE_MODE[motd_news]}" "$replacement"; mv -Tf "$replacement" "$stage" - ;; - type-issue_net:issue_net) rm -f "${MOTD_TARGET_STAGE[issue_net]}"; ln -s "$root/other" "${MOTD_TARGET_STAGE[issue_net]}" ;; - esac - return 0 - } - assert_fail "$hook_case target hook tamper rejected" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$hook_case leaves formal targets unchanged" - assert_no_motd_residue "$root" -) -done - -( - root="$TEST_DIR/hook-rollback-stage"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - before=$(target_hash "$root") - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - motd_rollback_hook() { [[ "$1" != motd ]] || printf tampered > "${MOTD_TARGET_STAGE[motd]}"; return 0; } - assert_fail "rollback hook stage tamper rejected" motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback hook tamper did not preserve pending journal" - motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } - motd_run_locked_operation install - [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback hook retry left pending" - assert_eq "$before" "$(target_hash "$root")" "rollback hook retry restores original group before next install" -) - -for pointer_case in old-generation other-generation extra-line inode; do -( - root="$TEST_DIR/pointer-hook-$pointer_case"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - before=$(target_hash "$root"); old_pointer=$(cat "$MOTD_STATE_DIR/previous.current"); before_count=$(generation_count "$MOTD_STATE_DIR/generations") - motd_snapshot_commit_hook() { - local scope="$1" generation="$2" kind="$3" stage other source - [[ "$scope" == previous && "$kind" == pointer ]] || return 0 - stage=${MOTD_POINTER_STAGE_PATHS[${#MOTD_POINTER_STAGE_PATHS[@]}-1]} - case "$pointer_case" in - old-generation) printf '%s\n' "$old_pointer" > "$stage" ;; - other-generation) - other=11111111111111111111111111111111; source="$MOTD_STATE_DIR/generations/$generation" - cp -a "$source" "$MOTD_STATE_DIR/generations/$other" - sed -i "s/^generation=.*/generation=$other/" "$MOTD_STATE_DIR/generations/$other/manifest" - MOTD_NEW_GENERATIONS+=("$other") - printf '%s\n' "$other" > "$stage" - ;; - extra-line) printf 'extra\n' >> "$stage" ;; - inode) cp "$stage" "${stage}.replacement"; chmod 0600 "${stage}.replacement"; mv -Tf "${stage}.replacement" "$stage" ;; - esac - } - assert_fail "pointer hook $pointer_case rejected" motd_run_locked_operation install - assert_eq "$old_pointer" "$(cat "$MOTD_STATE_DIR/previous.current")" "pointer hook $pointer_case preserves pointer" - assert_eq "$before" "$(target_hash "$root")" "pointer hook $pointer_case preserves targets" - assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "pointer hook $pointer_case preserves generation count" - assert_no_motd_residue "$root" -) -done - -for pending_hook_case in id extra-line inode; do -( - root="$TEST_DIR/pending-hook-$pending_hook_case"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - before=$(target_hash "$root") - motd_transaction_phase_hook() { - local phase="$1" stage - [[ "$phase" == pending-publish ]] || return 0 - stage=${MOTD_PENDING_STAGE_PATHS[${#MOTD_PENDING_STAGE_PATHS[@]}-1]} - case "$pending_hook_case" in - id) printf '%032x\n' 1 > "$stage" ;; - extra-line) printf 'extra\n' >> "$stage" ;; - inode) cp "$stage" "${stage}.replacement"; chmod 0600 "${stage}.replacement"; mv -Tf "${stage}.replacement" "$stage" ;; - esac - } - motd_target_commit_hook() { : > "$root/commit-called"; } - assert_fail "pending publish hook $pending_hook_case rejected" motd_run_locked_operation install - [[ ! -e "$root/commit-called" ]] || fail "pending hook $pending_hook_case reached formal target commit" - assert_eq "$before" "$(target_hash "$root")" "pending hook $pending_hook_case preserves targets" - [[ ! -e "$MOTD_STATE_DIR/pending" && ! -L "$MOTD_STATE_DIR/pending" ]] || fail "pending hook $pending_hook_case left pending" - assert_no_motd_residue "$root" -) -done - -cat > "$TEST_DIR/building-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1" -send_building_signal() { kill -"$SIGNAL_NAME" "$BASHPID"; } -motd_transaction_phase_hook() { - case "$FAIL_PHASE:$1" in - transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) send_building_signal ;; - esac - return 0 -} -motd_snapshot_capture_hook() { - [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0 - case "$FAIL_PHASE:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) send_building_signal ;; esac - return 0 -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/building-signal-child.sh" - -for building_phase in transaction-snapshot-building capture-motd capture-issue_net capture-motd_news snapshot-commit transaction-final-move; do -( - root="$TEST_DIR/building-fail-$building_phase"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - before=$(target_hash "$root"); before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")"; before_count=$(generation_count "$MOTD_STATE_DIR/generations") - motd_transaction_phase_hook() { case "$building_phase:$1" in transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) return 1 ;; esac; return 0; } - motd_snapshot_capture_hook() { [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0; case "$building_phase:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) return 1 ;; esac; return 0; } - assert_fail "$building_phase ordinary failure returns nonzero" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$building_phase ordinary failure preserves targets" - assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$building_phase ordinary failure preserves pointers" - assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "$building_phase ordinary failure preserves generations" - assert_no_motd_residue "$root" - motd_take_lock; motd_release_lock -) - for signal_name in HUP INT TERM; do - ( - root="$TEST_DIR/building-$building_phase-$signal_name"; create_fixture "$root"; export_paths "$root" - bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" - before=$(target_hash "$root"); before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")"; before_count=$(generation_count "$MOTD_STATE_DIR/generations") - rc=0 - env --default-signal=HUP,INT,TERM FAIL_PHASE="$building_phase" SIGNAL_NAME="$signal_name" bash "$TEST_DIR/building-signal-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? - case "$signal_name" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - assert_eq "$expected" "$rc" "$building_phase $signal_name status" - assert_eq "$before" "$(target_hash "$root")" "$building_phase $signal_name preserves targets" - assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$building_phase $signal_name preserves pointers" - assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "$building_phase $signal_name preserves generations" - assert_no_motd_residue "$root" - bash -c '. "$1"; motd_take_lock; motd_release_lock' _ "$ROOT_DIR/tools/setup-motd.sh" - ) - done -done - -for previous_impl in tool module; do -( - root="$TEST_DIR/previous-only-$previous_impl"; create_fixture "$root"; export_paths "$root" - create_complete_legacy_scope previous - case "$previous_impl" in tool) script="$ROOT_DIR/tools/setup-motd.sh" ;; module) script="$ROOT_DIR/modules/system-customize.sh" ;; esac - # shellcheck source=/dev/null - . "$script" - motd_run_locked_operation restore previous - [[ ! -e "$MOTD_STATE_DIR/initial.current" && ! -L "$MOTD_STATE_DIR/initial.current" ]] || fail "$previous_impl previous-only created initial pointer" - [[ -e "$MOTD_STATE_DIR/previous.current" ]] || fail "$previous_impl previous-only missing previous pointer" - motd_run_locked_operation restore previous - [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$previous_impl repeated previous restore left transaction" - motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/initial.current" && -e "$MOTD_STATE_DIR/previous.current" ]] || fail "$previous_impl install did not complete initial/previous pointers" - assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$previous_impl previous-only lifecycle bounded" -) -done - -for time_direction in tool-module module-tool; do -( - root="$TEST_DIR/timestamps-$time_direction"; create_fixture "$root"; export_paths "$root" - printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - for native in 10-uname 50-motd-news; do touch -a -d @1000000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1000200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done - expected_uname=$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname"); expected_news=$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news") - case "$time_direction" in - tool-module) install_script="$ROOT_DIR/tools/setup-motd.sh"; restore_script="$ROOT_DIR/modules/system-customize.sh" ;; - module-tool) install_script="$ROOT_DIR/modules/system-customize.sh"; restore_script="$ROOT_DIR/tools/setup-motd.sh" ;; - esac - # shellcheck source=/dev/null - . "$install_script"; motd_run_locked_operation install - assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction install preserves uname timestamps" - assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction install preserves news timestamps" - for native in 10-uname 50-motd-news; do touch -a -d @1100000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1100200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done - bash -c '. "$1"; motd_run_locked_operation restore previous' _ "$restore_script" - assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction previous restores uname timestamps" - assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction previous restores news timestamps" - for native in 10-uname 50-motd-news; do touch -a -d @1200000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1200200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done - bash -c '. "$1"; motd_run_locked_operation restore initial' _ "$restore_script" - assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction initial restores uname timestamps" - assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction initial restores news timestamps" -) -done - -for hidden_location in transactions generations root; do - for hidden_kind in file symlink fifo dir; do - ( - root="$TEST_DIR/hidden-$hidden_location-$hidden_kind"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - case "$hidden_location" in transactions) parent="$MOTD_TRANSACTION_PARENT" ;; generations) parent="$MOTD_STATE_DIR/generations" ;; root) parent="$MOTD_STATE_DIR" ;; esac - object="$parent/.evil-$hidden_kind" - case "$hidden_kind" in file) printf evil > "$object" ;; symlink) ln -s "$root/missing" "$object" ;; fifo) mkfifo "$object" ;; dir) mkdir "$object" ;; esac - before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - if [[ "$hidden_kind" == file || "$hidden_kind" == fifo ]]; then operation=(install); else operation=(restore previous); fi - assert_fail "$hidden_location hidden $hidden_kind rejected" motd_run_locked_operation "${operation[@]}" - assert_eq "$before" "$(target_hash "$root")" "$hidden_location hidden $hidden_kind preserves targets" - assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$hidden_location hidden $hidden_kind preserves pointers" - [[ -e "$object" || -L "$object" ]] || fail "$hidden_location hidden $hidden_kind evidence removed" - motd_take_lock; motd_release_lock - ) - done -done - -( - root="$TEST_DIR/hidden-transaction-short"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - printf evil > "$MOTD_TRANSACTION_PARENT/.123" - before=$(target_hash "$root") - assert_fail "transactions .123 rejected" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "transactions .123 preserves targets" - [[ -e "$MOTD_TRANSACTION_PARENT/.123" ]] || fail "transactions .123 evidence removed" - motd_take_lock; motd_release_lock -) - -( - transaction_block="$TEST_DIR/transaction-block" - sed -n '/# === MOTD group transaction ===/,/^configure_motd()/p' "$ROOT_DIR/tools/setup-motd.sh" > "$transaction_block" - if grep -Eq "stat .*%F|regular file|regular empty file|symbolic link" "$transaction_block"; then fail "MOTD transaction depends on localized stat type text"; fi - pass "MOTD transaction avoids localized stat type text" -) - -( - root="$TEST_DIR/locale-types"; create_fixture "$root"; export_paths "$root" - printf outside > "$root/outside"; rm -f "$MOTD_ETC_ROOT/issue"; ln -s "$root/outside" "$MOTD_ETC_ROOT/issue" - create_complete_legacy_scope initial - load_motd - stat() { - if [[ "$*" == *"%F"* ]]; then printf '本地化文件类型\n'; else command stat "$@"; fi - } - LANG=zh_CN.UTF-8 motd_run_locked_operation restore initial - [[ -L "$MOTD_ETC_ROOT/issue" ]] || fail "localized stat wrapper broke symlink restore" - assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "localized stat wrapper preserves regular legacy import" - LANG=zh_CN.UTF-8 bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/modules/system-customize.sh" -) - -for missing_command in bash chown flock ln od readlink sha256sum touch wc; do -( - root="$TEST_DIR/module-missing-$missing_command"; create_fixture "$root"; export_paths "$root" - before=$(target_hash "$root"); rc=0; mkdir "$root/bin" - for available_command in apt-get apt-cache awk bash basename cat chmod chown cp curl df dirname dpkg flock grep hostname id install ln mkdir mktemp mv od readlink rm sed sha256sum sleep sort stat touch tr uname uptime wc; do - [[ "$available_command" == "$missing_command" ]] && continue - available_path=$(command -v "$available_command" || true) - if [[ -n "$available_path" ]]; then ln -s "$available_path" "$root/bin/$available_command" - else printf '#!/bin/sh\nexit 0\n' > "$root/bin/$available_command"; chmod 0755 "$root/bin/$available_command"; fi - done - PATH="$root/bin" /bin/bash -c ' - . "$1" - require_root() { :; } - main motd - ' _ "$ROOT_DIR/modules/system-customize.sh" > "$root/output" 2>&1 || rc=$? - assert_eq 1 "$rc" "module missing $missing_command fails before action" - grep -Fq "缺少必要命令: $missing_command" "$root/output" || fail "module missing $missing_command not reported" - pass "module preflight reports missing $missing_command" - assert_eq "$before" "$(target_hash "$root")" "module missing $missing_command preserves MOTD" - [[ ! -e "$MOTD_STATE_DIR" && ! -e "$MOTD_LOCK_FILE" ]] || fail "module missing $missing_command created lock/state" -) -done - -grep -Fq 'error "缺少必要命令: $required_command"' "$ROOT_DIR/modules/system-customize.sh" || fail "module missing-command error output branch absent" -pass "module dependency preflight retains missing-command output" - - -for noop_impl in tool module; do - for noop_created_type in regular symlink fifo; do - ( - root="$TEST_DIR/noop-absent-$noop_impl-$noop_created_type"; create_fixture "$root"; export_paths "$root" - path="$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; before=$(target_hash "$root") - if [[ "$noop_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_target_commit_hook() { if [[ "$2" == motd_news ]]; then case "$noop_created_type" in regular) printf created > "$3"; chmod 0755 "$3" ;; symlink) ln -s "$root/external" "$3" ;; fifo) mkfifo "$3" ;; esac; fi; return 0; } - assert_fail "$noop_impl absent noop rejects $noop_created_type creation" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$noop_impl absent noop $noop_created_type rolls back group" - [[ ! -e "$path" && ! -L "$path" ]] || fail "$noop_impl absent noop $noop_created_type left created object" - assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock - ) - done -done - -for noop_impl in tool module; do - for noop_change in regular-content regular-mode regular-inode regular-symlink symlink-target absent-regular; do - ( - root="$TEST_DIR/noop-unknown-$noop_impl-$noop_change"; create_fixture "$root"; export_paths "$root" - printf '# linux-setup:managed-motd\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" - if [[ "$noop_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_run_locked_operation install - case "$noop_change" in regular-content|regular-symlink) target_id=motd ;; regular-mode) target_id=issue ;; regular-inode) target_id=custom_welcome ;; symlink-target) target_id="uname"; rm -f "${MOTD_PATH_BY_ID[$target_id]}"; ln -s "$root/link-one" "${MOTD_PATH_BY_ID[$target_id]}" ;; absent-regular) target_id=motd_news ;; esac - before=$(target_hash "$root") - motd_target_commit_hook() { local id="$2" path="$3" replacement; [[ "$id" == "$target_id" ]] || return 0; case "$noop_change" in regular-content) printf changed > "$path" ;; regular-mode) chmod 0600 "$path" ;; regular-inode) replacement="${path}.replacement"; cp "$path" "$replacement"; chmod 0755 "$replacement"; mv -Tf "$replacement" "$path" ;; regular-symlink) rm -f "$path"; ln -s "$root/other" "$path" ;; symlink-target) rm -f "$path"; ln -s "$root/link-two" "$path" ;; absent-regular) printf created > "$path"; chmod 0755 "$path" ;; esac; return 0; } - assert_fail "$noop_impl unknown noop rejects $noop_change" motd_run_locked_operation restore initial - assert_eq "$before" "$(target_hash "$root")" "$noop_impl unknown noop $noop_change rolls back group" - assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock - ) - done -done - -for group_impl in tool module; do - for group_change in motd-content issue-mode custom-inode; do - ( - root="$TEST_DIR/final-group-$group_impl-$group_change"; create_fixture "$root"; export_paths "$root"; before=$(target_hash "$root") - if [[ "$group_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_transaction_phase_hook() { local phase="$1" path replacement; [[ "$phase" == target-committed-motd_news ]] || return 0; case "$group_change" in motd-content) printf changed > "$MOTD_ETC_ROOT/motd" ;; issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; custom-inode) path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; replacement="${path}.replacement"; cp "$path" "$replacement"; chmod 0755 "$replacement"; mv -Tf "$replacement" "$path" ;; esac; } - assert_fail "$group_impl final group rejects $group_change" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$group_impl final group $group_change rolls back all targets" - [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "$group_impl final group $group_change left pending" - assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock - ) - done -done - -for illegal_legacy_name in motd.previous-unknown issue.previous-unknown motd.invalid-backup evil.initial-backup; do -( - root="$TEST_DIR/legacy-name-${illegal_legacy_name//[^a-zA-Z0-9]/-}"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install - object="$MOTD_STATE_DIR/$illegal_legacy_name"; printf invalid > "$object"; chmod 0600 "$object"; before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - assert_fail "illegal legacy name $illegal_legacy_name rejected" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "illegal legacy name $illegal_legacy_name preserves targets" - assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "illegal legacy name $illegal_legacy_name preserves pointers" - [[ -e "$object" || -L "$object" ]] || fail "illegal legacy name $illegal_legacy_name evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "illegal legacy name $illegal_legacy_name created pending"; motd_take_lock; motd_release_lock -) -done - -for legacy_backup_kind in directory fifo socket writable wrong-owner wrong-gid device; do -( - root="$TEST_DIR/legacy-backup-$legacy_backup_kind"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install; object="$MOTD_STATE_DIR/motd.initial-backup" - case "$legacy_backup_kind" in - directory) mkdir "$object" ;; fifo) mkfifo "$object" ;; - socket) if command -v python3 >/dev/null 2>&1; then python3 - "$object" <<'PY' -import socket,sys -s=socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.close() -PY - else pass "socket legacy test skipped without python3"; exit 0; fi ;; - writable) printf bad > "$object"; chmod 0666 "$object" ;; - wrong-owner) if (( EUID == 0 )); then printf bad > "$object"; chmod 0600 "$object"; chown "$((MOTD_TRUSTED_UID+1)):$MOTD_TRUSTED_GID" "$object"; else pass "wrong-owner legacy test skipped without privilege"; exit 0; fi ;; - wrong-gid) if (( EUID == 0 )); then printf bad > "$object"; chmod 0600 "$object"; chown "$MOTD_TRUSTED_UID:$((MOTD_TRUSTED_GID+1))" "$object"; else pass "wrong-gid legacy test skipped without privilege"; exit 0; fi ;; - device) if (( EUID == 0 )) && mknod "$object" c 1 3 2>/dev/null; then :; else pass "device legacy test skipped without privilege"; exit 0; fi ;; - esac - before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - assert_fail "legacy backup $legacy_backup_kind rejected" motd_run_locked_operation restore initial - assert_eq "$before" "$(target_hash "$root")" "legacy backup $legacy_backup_kind preserves targets" - assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "legacy backup $legacy_backup_kind preserves pointers" - [[ -e "$object" || -L "$object" ]] || fail "legacy backup $legacy_backup_kind evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "legacy backup $legacy_backup_kind created pending"; motd_take_lock; motd_release_lock -) -done - -for legacy_marker_name in motd.initial-absent issue.initial-unknown motd.previous-absent; do - for legacy_marker_kind in symlink directory fifo writable; do - ( - root="$TEST_DIR/legacy-marker-${legacy_marker_name//[^a-zA-Z0-9]/-}-$legacy_marker_kind"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install; object="$MOTD_STATE_DIR/$legacy_marker_name" - case "$legacy_marker_kind" in symlink) ln -s "$root/missing" "$object" ;; directory) mkdir "$object" ;; fifo) mkfifo "$object" ;; writable) printf bad > "$object"; chmod 0666 "$object" ;; esac - before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - assert_fail "legacy marker $legacy_marker_name $legacy_marker_kind rejected" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "legacy marker $legacy_marker_name $legacy_marker_kind preserves targets" - assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "legacy marker $legacy_marker_name $legacy_marker_kind preserves pointers" - [[ -e "$object" || -L "$object" ]] || fail "legacy marker $legacy_marker_name $legacy_marker_kind evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "legacy marker $legacy_marker_name $legacy_marker_kind created pending"; motd_take_lock; motd_release_lock - ) - done -done - -( - root="$TEST_DIR/legal-state-root-legacy"; create_fixture "$root"; export_paths "$root"; mkdir -p "$MOTD_STATE_DIR"; chmod 0700 "$MOTD_STATE_DIR" - cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_STATE_DIR/motd.initial-backup"; printf target > "$root/symlink-target"; ln -s "$root/symlink-target" "$MOTD_STATE_DIR/issue.initial-backup" - : > "$MOTD_STATE_DIR/issue.net.initial-absent"; chmod 0600 "$MOTD_STATE_DIR/issue.net.initial-absent"; : > "$MOTD_STATE_DIR/00-custom-welcome.initial-unknown"; chmod 0600 "$MOTD_STATE_DIR/00-custom-welcome.initial-unknown" - cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_STATE_DIR/10-uname.initial-backup"; : > "$MOTD_STATE_DIR/50-motd-news.initial-absent"; chmod 0600 "$MOTD_STATE_DIR/50-motd-news.initial-absent" - touch -a -d @1000000000 "$MOTD_STATE_DIR/motd.initial-backup"; touch -m -d @1000200000 "$MOTD_STATE_DIR/motd.initial-backup"; legacy_times=$(stat -c %X:%Y "$MOTD_STATE_DIR/motd.initial-backup") - load_motd; motd_run_locked_operation restore initial - [[ -L "$MOTD_ETC_ROOT/issue" && "$(readlink "$MOTD_ETC_ROOT/issue")" == "$root/symlink-target" ]] || fail "legal state root symlink backup not imported" - assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "legal state root regular backup imported"; [[ -e "$MOTD_STATE_DIR/initial.current" ]] || fail "legal state root import did not publish initial pointer" - assert_eq "$legacy_times" "$(stat -c %X:%Y "$MOTD_STATE_DIR/motd.initial-backup")" "legacy root metadata validation/import preserves timestamps"; pass "legal state root regular, symlink, and markers import" -) - - -for rollback_impl in tool module; do - for rollback_damage in motd-content issue-mode custom-inode absent-regular absent-symlink absent-fifo symlink-target; do - ( - root="$TEST_DIR/rollback-final-$rollback_impl-$rollback_damage"; create_fixture "$root"; export_paths "$root" - if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_run_locked_operation install - case "$rollback_damage" in - motd-content) printf 'pre-run-motd\n' > "$MOTD_ETC_ROOT/motd" ;; - issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; - custom-inode) printf 'pre-run-custom\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0700 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" ;; - absent-regular|absent-symlink|absent-fifo) rm -f "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; - symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/original-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; - esac - before=$(target_hash "$root") - before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - motd_rollback_hook() { - local id="$1" path target - [[ "$id" == motd_news ]] || return 0 - case "$rollback_damage" in - motd-content) printf 'rollback-corrupt\n' > "$MOTD_ETC_ROOT/motd" ;; - issue-mode) chmod 0777 "$MOTD_ETC_ROOT/issue" ;; - custom-inode) - path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" - cp "$path" "${path}.replacement"; chmod 0700 "${path}.replacement"; mv -Tf "${path}.replacement" "$path" - ;; - absent-regular|absent-symlink|absent-fifo) - ROLLBACK_RECREATE_TARGET="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - rm() { - local status - command rm "$@"; status=$?; unset -f rm - case "$rollback_damage" in - absent-regular) printf recreated > "$ROLLBACK_RECREATE_TARGET"; chmod 0755 "$ROLLBACK_RECREATE_TARGET" ;; - absent-symlink) ln -s "$root/recreated-link" "$ROLLBACK_RECREATE_TARGET" ;; - absent-fifo) mkfifo "$ROLLBACK_RECREATE_TARGET" ;; - esac - return "$status" - } - ;; - symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/changed-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; - esac - return 0 - } - assert_fail "$rollback_impl rollback final group rejects $rollback_damage" motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$rollback_impl rollback $rollback_damage lost pending" - transaction=$(cat "$MOTD_STATE_DIR/pending") - journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" - grep -Fxq 'phase=active' "$journal" || fail "$rollback_impl rollback $rollback_damage journal not active" - [[ -d "$MOTD_TRANSACTION_PARENT/$transaction/rollback" ]] || fail "$rollback_impl rollback $rollback_damage lost rollback snapshot" - pass "$rollback_impl rollback $rollback_damage preserves active recovery evidence" - motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } - motd_take_lock; motd_reconcile_state; motd_release_lock - assert_eq "$before" "$(target_hash "$root")" "$rollback_impl rollback $rollback_damage retry restores targets" - assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$rollback_impl rollback $rollback_damage retry restores pointers" - [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl rollback $rollback_damage retry left journal" - assert_no_motd_residue "$root"; assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$rollback_impl rollback $rollback_damage retry keeps bounded generations" - motd_take_lock; motd_release_lock - ) - done - -done - -for rollback_impl in tool module; do -( - root="$TEST_DIR/rollback-final-success-$rollback_impl"; create_fixture "$root"; export_paths "$root" - if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_run_locked_operation install; printf 'normal-pre-run\n' > "$MOTD_ETC_ROOT/motd"; before=$(target_hash "$root") - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - assert_fail "$rollback_impl normal rollback still returns original operation failure" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$rollback_impl normal rollback final group succeeds" - [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl normal rollback left journal" - assert_no_motd_residue "$root" -) -done - -printf 'All MOTD rollback final validation tests passed.\n' diff --git a/tests/test-push-worker-registration-01.sh b/tests/test-push-worker-registration-01.sh new file mode 100644 index 0000000..e1a8df0 --- /dev/null +++ b/tests/test-push-worker-registration-01.sh @@ -0,0 +1,576 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-push-worker-registration-common.sh" +assert_eq source-safe "$source_result" "source keeps zero runtime side effects" + +( + # Inert direct-child job table: no SSH or production services. + sleep 10 & snapshot_child=$! + cleanup_snapshot_child() { kill -TERM "$snapshot_child" 2>/dev/null || true; wait "$snapshot_child" 2>/dev/null || true; } + trap cleanup_snapshot_child EXIT + job_is_active "$snapshot_child" || fail "snapshot misses active direct child" + if job_is_active 0; then fail "snapshot accepts absent PID"; fi + kill -TERM "$snapshot_child" + wait "$snapshot_child" 2>/dev/null || true + if job_is_active "$snapshot_child"; then fail "snapshot retains reaped child"; fi + trap - EXIT + pass "job snapshot preserves direct-child matching and reap semantics" +) +trap 'rm -rf "$TEST_DIR"' EXIT + +# Deterministic coordinator contract: a live worker can republish after cleanup. +( + root="$TEST_DIR/republish-tracking"; mkdir -m 0700 "$root" + state_file="$root/worker.state"; : > "$state_file" + ACTIVE_WORKERS=([424242]=12345) + ACTIVE_WORKER_STATE_FILES=([424242]="$state_file") + ACTIVE_WORKER_STATE_STARTS=([424242]=12345) + worker_session_state_maybe_cleanup_failed() { return 0; } + cleanup_calls=0 + cleanup_worker_session_state_file() { cleanup_calls=$((cleanup_calls + 1)); rm -f -- "$1"; } + cleanup_active_failed_worker_sessions + [[ ${ACTIVE_WORKER_STATE_FILES[424242]:-} == "$state_file" && ${ACTIVE_WORKER_STATE_STARTS[424242]:-} == 12345 ]] || fail "active cleanup lost live publisher tracking" + : > "$state_file" + ACTIVE_WORKERS=() + job_is_active() { return 1; } + cleanup_published_worker_sessions + assert_eq 2 "$cleanup_calls" "post-reap fallback handles republished state" + [[ ! -e $state_file && ${#ACTIVE_WORKER_STATE_FILES[@]} == 0 && ${#ACTIVE_WORKER_STATE_STARTS[@]} == 0 ]] || fail "republished state remains" + pass "live publisher tracking survives successful active cleanup" +) + +setup_fixture() { + local root="$1" + CURRENT_FIXTURE_ROOT=$root + mkdir -m 0700 "$root" "$root/runtime" "$root/capture" + TMPDIR="$root/runtime"; initialize_runtime + MAX_PARALLEL=2 + MAX_RETRIES=1 + BATCH_WORKER_FAILED=false + BATCH_WORKER_ERROR=false +} +assert_no_registration_residue() { + local root="$1" + [[ -z "$(find "$root/runtime" \( -name 'worker-registration.*' -o -name '.worker-registration.*' -o -name 'worker-session.*.state' \) -print -quit)" ]] || fail "registration/session residue remains" + (( ${#REGISTERING_WORKERS[@]} == 0 && ${#ACTIVE_WORKERS[@]} == 0 )) || fail "worker arrays remain" + (( ${#WORKER_REGISTRATION_READY_FILES[@]} == 0 && ${#WORKER_REGISTRATION_RELEASE_FILES[@]} == 0 && ${#WORKER_REGISTRATION_STARTS[@]} == 0 && ${#WORKER_REGISTRATION_STAGE_FILES[@]} == 0 )) || fail "registration arrays remain" +} + +run_delayed_parent_reap_case() ( + local depth="$1" root leader leader_start leader_pgid leader_sid + root="$TEST_DIR/delayed-reap-$depth" + local state_file elapsed_start elapsed_end rc=0 + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + setup_fixture "$root" + mkfifo "$root/reap-clock" + cat > "$root/parent.sh" <<'CHILD' +#!/usr/bin/env bash +set -u +root="$1" depth="$2" +trap '' HUP INT TERM + : > "$root/parent-open" +if (( depth == 0 )); then + sleep 300 & child=$! + printf '%s\n' "$child" > "$root/leaf.pid" +else + bash "$root/parent.sh" "$root" "$((depth - 1))" & child=$! +fi +kill -STOP "$BASHPID" +wait "$child" 2>/dev/null || true +exit 0 +CHILD + chmod 0700 "$root/parent.sh" + setsid bash "$root/parent.sh" "$root" "$depth" & leader=$! + leader_start=$(wait_test_process_start "$leader") + for _ in {1..100}; do [[ -e "$root/parent-open" ]] && break; sleep .01; done + read_process_record "$leader"; leader_pgid=$PROC_PGID; leader_sid=$PROC_SID + [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "delayed $depth leader identity invalid" + WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start + WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid + publish_worker_session_state active; state_file=$WORKER_SESSION_STATE_FILE + leaf=$(<"$root/leaf.pid"); leaf_start=$(wait_test_process_start "$leaf") + kill -KILL "$leaf" 2>/dev/null || true + old_reap_status=1 + for _ in {1..30}; do + session_process_identity_matches "$leaf" "$leaf_start" "$leader_sid" || { old_reap_status=0; break; } + sleep 0.01 + done + assert_eq 1 "$old_reap_status" "delayed $depth old 300ms window remains pending" + ( sleep 1; kill -CONT -"$leader" 2>/dev/null || true ) & releaser=$! + cleanup_clock; elapsed_start=$CLEANUP_NOW + terminate_worker_transfer || rc=$? + wait "$releaser" 2>/dev/null || true + cleanup_clock; elapsed_end=$CLEANUP_NOW + assert_eq 0 "$rc" "delayed $depth parent reap succeeds" + (( elapsed_end - elapsed_start >= 100 )) || fail "delayed $depth reap did not wait for parent" + wait_test_process_identity_gone "$leader" "$leader_start" || fail "delayed $depth left leader" + collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states + assert_eq 0 "${#session_pids[@]}" "delayed $depth managed SID clears" + [[ ! -e "$state_file" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "delayed $depth state cleanup incomplete" + cleanup_runtime + exec 9>&- + pass "delayed parent reap depth $depth preserves leaf-first order" +) +run_delayed_parent_reap_case 1 +run_delayed_parent_reap_case 4 + +( + root="$TEST_DIR/reap-deadline"; setup_fixture "$root" + cat > "$root/parent.sh" <<'CHILD' +#!/usr/bin/env bash +set -u +root="$1" +trap '' HUP INT TERM +sleep 300 & child=$! +: > "$root/ready" +while [[ ! -e "$root/release" ]]; do :; done +wait "$child" 2>/dev/null || true +exit 0 +CHILD + chmod 0700 "$root/parent.sh" + setsid bash "$root/parent.sh" "$root" & leader=$! + for _ in {1..100}; do [[ -e "$root/ready" ]] && break; sleep .01; done + leader_start=$(wait_test_process_start "$leader"); read_process_record "$leader" + WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start; WORKER_TRANSFER_PGID=$PROC_PGID; WORKER_TRANSFER_SID=$PROC_SID + publish_worker_session_state active; state_file=$WORKER_SESSION_STATE_FILE + MANAGED_SESSION_CLEANUP_TICKS=300 + rc=0; terminate_worker_transfer || rc=$? + (( rc != 0 )) || fail "deadline failure unexpectedly succeeded" + [[ -d "$TEMP_DIR" && -f "$state_file" && "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || fail "deadline failure discarded evidence" + [[ "$(stat -c %a "$TEMP_DIR")" == 700 && "$(stat -c %a "$state_file")" == 600 ]] || fail "deadline evidence trust changed" + : > "$root/release"; wait "$leader" 2>/dev/null || true + remove_worker_session_state + WORKER_TRANSFER_PID=""; WORKER_TRANSFER_START=""; WORKER_TRANSFER_PGID=""; WORKER_TRANSFER_SID=""; WORKER_TRANSFER_CLEANUP_FAILED=false + clear_managed_session_cleanup_tracking + cleanup_runtime + [[ ! -e "$TEMP_DIR" ]] || fail "deadline teardown left runtime" + exec 9>&- + pass "deadline failure is bounded and preserves then safely clears evidence" +) +if [[ "${PUSH_REAP_ONLY:-false}" == true ]]; then + exit 0 +fi +test_watchdog_process() { + local seconds="$1" marker="$2" target_pid="$3" timer="" + trap '[[ -z "$timer" ]] || kill "$timer" 2>/dev/null || true; [[ -z "$timer" ]] || wait "$timer" 2>/dev/null || true; exit 0' HUP INT TERM + sleep "$seconds" & timer=$! + wait "$timer" 2>/dev/null || exit 0 + printf fired > "$marker" + kill -KILL "$target_pid" 2>/dev/null || true +} + +run_with_watchdog() { + local name="$1" seconds="$2"; shift 2 + local marker="$TEST_DIR/watchdog-$name" pid watchdog rc=0 + "$@" & pid=$! + test_watchdog_process "$seconds" "$marker" "$pid" & watchdog=$! + wait "$pid" || rc=$? + kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$marker" ]] || fail "$name exceeded watchdog deadline" + return "$rc" +} + +root_cause_regression_case() ( + root="$TEST_DIR/root-cause"; setup_fixture "$root" + GET_START_CALLS=0 + get_process_start_time() { ((GET_START_CALLS += 1)); return 1; } + push_to_server() { + local server="$1" lock_fd + exec {lock_fd}>>"$root/capture/calls.lock"; flock -x "$lock_fd" + printf '%s\n' "$server" >> "$root/capture/calls"; : > "$root/capture/ready.$server" + flock -u "$lock_fd"; exec {lock_fd}>&- + while [[ ! -e "$root/capture/release" ]]; do sleep 0.02; done + return "$MANAGED_CLEANUP_FAILURE_STATUS" + } + ( + coordinator_status=0; deadline=$((SECONDS + 15)) + while (( SECONDS < deadline )); do + if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release"; break; fi + sleep 0.02 + done + if [[ ! -e "$root/capture/release" ]]; then : > "$root/capture/coordinator-timeout"; : > "$root/capture/release"; coordinator_status=1; fi + while [[ ! -e "$root/capture/coordinator-stop" ]]; do sleep 0.02; done + exit "$coordinator_status" + ) & coordinator=$! + rc=0; run_server_batch source destination one.example two.example three.example four.example || rc=$? + : > "$root/capture/coordinator-stop"; coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? + assert_eq 0 "$coordinator_rc" "registration coordinator sees both first-slot workers" + assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "cleanup failure remains lifecycle status" + assert_eq 2 "$(wc -l < "$root/capture/calls")" "both first-slot workers enter fake transfer" + grep -Fxq one.example "$root/capture/calls" || fail "one.example did not enter fake transfer" + grep -Fxq two.example "$root/capture/calls" || fail "two.example did not enter fake transfer" + grep -Eq 'three\.example|four\.example' "$root/capture/calls" && fail "registration failure scheduled later servers" + assert_eq 0 "$GET_START_CALLS" "parent no longer depends on short get_process_start_time window" + terminate_active_workers || true; cleanup_runtime; assert_no_registration_residue "$root" + pass "deterministic registration regression avoids worker-one/worker-two deadlock" +) +run_with_watchdog root-cause 20 root_cause_regression_case + +run_ready_failure_case() ( + kind="$1"; root="$TEST_DIR/ready-$kind"; setup_fixture "$root" + WORKER_REGISTRATION_TIMEOUT_TICKS=10 + push_to_server() { : > "$root/capture/transfer"; return 0; } + case "$kind" in + missing) worker_registration_phase_hook() { [[ "$1" != before-ready ]] || while :; do :; done; } ;; + malformed) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf malformed > "$3"; } ;; + symlink) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || { rm -f "$3"; ln -s "$root/other" "$3"; }; } ;; + pid-mismatch) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || sed -i 's/^worker_pid=.*/worker_pid=999999/' "$3"; } ;; + wrong-mode) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || chmod 0644 "$3"; } ;; + duplicate-field) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf 'worker_pid=1\n' >> "$3"; } ;; + unknown-field) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf 'unexpected=value\n' >> "$3"; } ;; + illegal-number) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || sed -i 's/^worker_start=.*/worker_start=0/' "$3"; } ;; + release-failure) publish_worker_registration_release() { return 1; } ;; + esac + rc=0; launch_worker one.example source destination 1 1 || rc=$? + (( rc != 0 )) || fail "$kind registration failure unexpectedly succeeded" + [[ ! -e "$root/capture/transfer" ]] || fail "$kind reached transfer before trusted handoff" + cleanup_runtime || true; assert_no_registration_residue "$root" + [[ -z "$(jobs -pr)" ]] || fail "$kind left worker job" + pass "$kind registration failure is bounded and side-effect free" +) +for registration_failure in missing malformed symlink pid-mismatch wrong-mode duplicate-field unknown-field illegal-number release-failure; do run_ready_failure_case "$registration_failure"; done + +( + root="$TEST_DIR/release-replaced"; setup_fixture "$root" + push_to_server() { : > "$root/capture/transfer"; return 0; } + worker_registration_phase_hook() { [[ "$1" != ready-published ]] || while [[ ! -e "$root/capture/release-mutated" ]]; do sleep 0.02; done; } + parent_worker_registration_hook() { [[ "$1" != release-published ]] || { chmod 0644 "$4"; : > "$root/capture/release-mutated"; }; } + launch_worker one.example source destination 1 1 + rc=0; wait_for_all_workers || rc=$? + assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "worker rejects replaced release before transfer" + [[ ! -e "$root/capture/transfer" ]] || fail "replaced release reached transfer" + cleanup_runtime || true; assert_no_registration_residue "$root" + pass "release replacement is bounded and leaves no residue" +) +( + root="$TEST_DIR/parent-disappeared"; setup_fixture "$root" + push_to_server() { : > "$root/capture/transfer"; return 0; } + sleep 300 & fake_parent=$!; read_process_record "$fake_parent"; fake_parent_start=$PROC_START + registered_worker_entry "$fake_parent" "$fake_parent_start" one.example source destination 1 1 & worker=$! + for _ in {1..200}; do [[ -e "$TEMP_DIR/worker-registration.$worker.ready" ]] && break; sleep 0.02; done + [[ -e "$TEMP_DIR/worker-registration.$worker.ready" ]] || fail "worker did not publish ready before parent disappearance" + kill -TERM "$fake_parent" 2>/dev/null || true; wait "$fake_parent" 2>/dev/null || true + rc=0; wait "$worker" || rc=$?; (( rc != 0 )) || fail "worker accepted disappeared parent" + [[ ! -e "$root/capture/transfer" ]] || fail "worker transferred after parent disappeared" + cleanup_worker_registration_for_pid "$worker" || true; cleanup_runtime; assert_no_registration_residue "$root" + pass "worker exits before transfer when parent identity disappears" +) + +( + sleep 60 & proc_fallback_worker=$! + proc_fallback_start=$(wait_test_process_start "$proc_fallback_worker") + eval "$(declare -f read_process_record | sed '1s/read_process_record/original_read_process_record/')" + read_process_record() { [[ "$1" != "$proc_fallback_worker" ]] && original_read_process_record "$@"; } + proc_fallback_status=0 + read_process_record_for_session_scan "$proc_fallback_worker" || proc_fallback_status=$? + assert_eq 0 "$proc_fallback_status" "proc owner fallback recovers transient status-read failure" + [[ "$PROC_START" == "$proc_fallback_start" && "$PROC_UID" == "$EUID" ]] || fail "proc owner fallback identity mismatch" + kill -TERM "$proc_fallback_worker"; wait "$proc_fallback_worker" 2>/dev/null || true + pass "proc owner fallback binds UID to stable PID start" +) + +prune_identity_failure_case() ( + mode="$1"; root="$TEST_DIR/prune-$mode"; setup_fixture "$root" + sleep 300 & worker=$!; read_process_record "$worker"; start=$PROC_START + ACTIVE_WORKERS[$worker]=$start; ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state"; ACTIVE_WORKER_STATE_STARTS[$worker]=$start + eval "$(declare -f process_identity_matches | sed '1s/process_identity_matches/original_process_identity_matches/')" + eval "$(declare -f read_process_record_for_session_scan | sed '1s/read_process_record_for_session_scan/original_read_process_record_for_session_scan/')" + failures=1; scan_failures=1; [[ "$mode" == repeated ]] && { failures=100; scan_failures=100; } + process_identity_matches() { if [[ "$1" == "$worker" && "$failures" -gt 0 ]]; then ((failures -= 1)); return 1; fi; original_process_identity_matches "$@"; } + read_process_record_for_session_scan() { + if [[ "$1" == "$worker" && "$scan_failures" -gt 0 && -r "/proc/$worker/stat" ]]; then ((scan_failures -= 1)); return 1; fi + original_read_process_record_for_session_scan "$@" + } + rc=0; prune_active_workers || rc=$?; (( rc != 0 )) || fail "$mode identity failure did not stop prune" + [[ "$BATCH_WORKER_FAILED" == true ]] || fail "$mode identity failure did not set lifecycle barrier" + process_is_running "$worker" || fail "$mode prune shortened active worker grace" + [[ -n "${ACTIVE_WORKERS[$worker]:-}" ]] || fail "$mode prune discarded active worker evidence" + rc=0; wait_for_worker_slot || rc=$?; assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "$mode identity failure propagates lifecycle status" + terminate_active_workers || true + process_is_running "$worker" && fail "$mode unified active cleanup left worker" + cleanup_runtime || true; assert_no_registration_residue "$root" + pass "$mode active identity failure defers termination to unified cleanup" +) +prune_identity_failure_case once +prune_identity_failure_case repeated +( + root="$TEST_DIR/prune-cleanup-failure"; setup_fixture "$root" + sleep 300 & worker=$!; read_process_record "$worker"; start=$PROC_START + ACTIVE_WORKERS[$worker]=$start + ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state" + ACTIVE_WORKER_STATE_STARTS[$worker]=$start + eval "$(declare -f process_identity_matches | sed '1s/process_identity_matches/original_process_identity_matches/')" + eval "$(declare -f read_process_record_for_session_scan | sed '1s/read_process_record_for_session_scan/original_read_process_record_for_session_scan/')" + process_identity_matches() { [[ "$1" != "$worker" ]] && original_process_identity_matches "$@"; } + read_process_record_for_session_scan() { [[ "$1" != "$worker" || ! -r "/proc/$worker/stat" ]] && original_read_process_record_for_session_scan "$@"; } + rc=0; wait_for_worker_slot || rc=$? + assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "prune trust failure propagates 125" + [[ -n "${ACTIVE_WORKERS[$worker]:-}" ]] || fail "prune trust failure discarded active worker evidence" + eval "$(declare -f original_process_identity_matches | sed '1s/original_process_identity_matches/process_identity_matches/')" + eval "$(declare -f original_read_process_record_for_session_scan | sed '1s/original_read_process_record_for_session_scan/read_process_record_for_session_scan/')" + eval "$(declare -f cleanup_all_worker_registration_residue | sed '1s/cleanup_all_worker_registration_residue/original_cleanup_all_worker_registration_residue/')" + cleanup_all_worker_registration_residue() { return 1; } + rc=0; stop_batch_after_lifecycle_failure || rc=$? + assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "active cleanup failure remains lifecycle status 125" + eval "$(declare -f original_cleanup_all_worker_registration_residue | sed '1s/original_cleanup_all_worker_registration_residue/cleanup_all_worker_registration_residue/')" + cleanup_runtime || true; assert_no_registration_residue "$root" + pass "active cleanup failure preserves lifecycle status" +) +( + root="$TEST_DIR/prune-exited"; setup_fixture "$root" + sleep 0.1 & worker=$!; read_process_record "$worker"; start=$PROC_START + ACTIVE_WORKERS[$worker]=$start + ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state" + ACTIVE_WORKER_STATE_STARTS[$worker]=$start + sleep 0.2; prune_active_workers + assert_eq 0 "${#ACTIVE_WORKERS[@]}" "exited worker is normally reaped" + [[ "$BATCH_WORKER_FAILED" == false ]] || fail "normal exited worker set lifecycle barrier" + cleanup_runtime; assert_no_registration_residue "$root" + pass "normal exited worker accounting remains intact" +) + +cat > "$TEST_DIR/registration-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" root="$2" phase="$3" +mkdir -p -m 0700 "$root/runtime" "$root/capture" +export TMPDIR="$root/runtime" +source "$script" +initialize_runtime; install_runtime_traps +WORKER_REGISTRATION_TIMEOUT_TICKS=200 +push_to_server() { printf 'push pid=%s root=%s\n' "$BASHPID" "$root" >> "$root/capture/transfer-calls"; : > "$root/capture/transfer"; return 0; } +worker_registration_phase_hook() { local current="$1"; [[ "$phase" == "$current" ]] || return 0; : > "$root/capture/phase"; while :; do :; done; } +parent_worker_registration_hook() { local current="$1"; [[ "$phase" == "$current" ]] || return 0; : > "$root/capture/phase"; while :; do :; done; } +launch_worker one.example source destination 1 1 +wait_for_all_workers +if [[ "$phase" == normal ]]; then [[ -e "$root/capture/transfer" ]] || { printf 'normal transfer missing\n' >&2; jobs -l >&2 || true; exit 91; }; fi +cleanup_runtime +CHILD +chmod 0700 "$TEST_DIR/registration-signal-child.sh" +run_registration_signal_case() { + local phase="$1" signal_name="$2" expected="$3" root retry_root + local pid watchdog unrelated rc=0 + root="$TEST_DIR/signal-$phase-$signal_name"; CURRENT_FIXTURE_ROOT=$root + sleep 60 & unrelated=$! + env --default-signal=HUP,INT,TERM bash "$TEST_DIR/registration-signal-child.sh" "$SCRIPT" "$root" "$phase" > "$root.log" 2>&1 & pid=$! + for _ in {1..300}; do [[ -e "$root/capture/phase" ]] && break; sleep 0.05; done + [[ -e "$root/capture/phase" ]] || { cat "$root.log"; kill "$pid" "$unrelated" 2>/dev/null || true; fail "$phase/$signal_name did not reach registration phase"; } + test_watchdog_process 30 "$root.watchdog" "$pid" & watchdog=$! + kill "-$signal_name" "$pid"; wait "$pid" || rc=$? + kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root.watchdog" ]] || fail "$phase/$signal_name exceeded watchdog" + assert_eq "$expected" "$rc" "$phase registration $signal_name status" + [[ ! -e "$root/capture/transfer" ]] || fail "$phase/$signal_name reached transfer" + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$phase/$signal_name left runtime state" + kill -0 "$unrelated" 2>/dev/null || fail "$phase/$signal_name killed unrelated process" + kill "$unrelated" 2>/dev/null || true; wait "$unrelated" 2>/dev/null || true + retry_root="${root}-retry" + env bash "$TEST_DIR/registration-signal-child.sh" "$SCRIPT" "$retry_root" normal > "$retry_root.log" 2>&1 + if [[ ! -e "$retry_root/capture/transfer" ]]; then cat "$retry_root.log" >&2 2>/dev/null || true; fail "$phase/$signal_name retry did not transfer"; fi + [[ -z "$(find "$retry_root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$phase/$signal_name retry left runtime" + pass "$phase/$signal_name leaves registration clean and retryable" +} +for registration_phase in before-ready accepted-before-release release-validated; do + run_registration_signal_case "$registration_phase" HUP 129 + run_registration_signal_case "$registration_phase" INT 130 + run_registration_signal_case "$registration_phase" TERM 143 +done + +( + sleep 60 & reused_pid=$! + replacement_start=$(get_process_start_time "$reused_pid") + expected_start=$((replacement_start + 1)) + eval "$(declare -f collect_owned_session_records | sed '1s/collect_owned_session_records/original_collect_owned_session_records/')" + collect_owned_session_records() { fail "PID reuse path attempted to inspect or terminate replacement session"; } + terminate_managed_session "$reused_pid" "$expected_start" "$reused_pid" "$reused_pid" + test_process_identity_exists "$reused_pid" "$replacement_start" || fail "PID reuse path touched replacement process" + kill -TERM "$reused_pid"; wait "$reused_pid" 2>/dev/null || true + pass "managed leader PID reuse is treated as original session gone" +) + +cat > "$TEST_DIR/failed-reap-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" root="$2" mode="$3" +mkdir -p -m 0700 "$root/runtime" "$root/capture" +export TMPDIR="$root/runtime" +source "$script" +initialize_runtime +setsid bash -c 'trap "" HUP INT TERM; while :; do :; done' & leader=$! +for _ in {1..100}; do + leader_start=$(get_process_start_time "$leader" 2>/dev/null || true) + leader_pgid=$(get_process_group_id "$leader" 2>/dev/null || true) + leader_sid=$(get_process_session_id "$leader" 2>/dev/null || true) + [[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] && break + sleep 0.01 +done +[[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || exit 90 +printf '%s\n%s\n%s\n%s\n' "$leader" "$leader_start" "$leader_pgid" "$leader_sid" > "$root/capture/leader" +WORKER_TRANSFER_PID=$leader +WORKER_TRANSFER_START=$leader_start +WORKER_TRANSFER_PGID=$leader_pgid +WORKER_TRANSFER_SID=$leader_sid +publish_worker_session_state active +state_file=$WORKER_SESSION_STATE_FILE +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +terminate_managed_session() { return 1; } +start_ns=$(date +%s%N) +rc=0 +terminate_worker_transfer || rc=$? +elapsed_ns=$(( $(date +%s%N) - start_ns )) +[[ "$rc" != 0 && "$elapsed_ns" -lt 2000000000 ]] || exit 91 +[[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && + "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && + "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || exit 92 +read_worker_session_state "$state_file" +[[ "$SESSION_STATE" == cleanup_failed ]] || exit 93 +next_rc=0 +run_managed_command "$TEMP_DIR/forbidden-next-command" true || next_rc=$? +[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-next-command" ]] || exit 94 +printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\nstate_file=%s\n' "$rc" "$elapsed_ns" "$next_rc" "$state_file" > "$root/capture/result" +: > "$root/capture/returned" +if [[ "$mode" == hold ]]; then + while [[ ! -e "$root/capture/release-worker" ]]; do sleep 0.01; done + for _ in {1..400}; do + reap_status=0 + reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" || reap_status=$? + case "$reap_status" in + 0|"$MANAGED_PROCESS_REUSED_STATUS") break ;; + *) sleep 0.01 ;; + esac + done + (( reap_status == 0 || reap_status == MANAGED_PROCESS_REUSED_STATUS )) || exit 95 + : > "$root/capture/reaped" + for _ in {1..400}; do + [[ ! -e "$state_file" && ! -L "$state_file" ]] && break + sleep 0.01 + done + [[ ! -e "$state_file" && ! -L "$state_file" ]] || exit 96 + terminate_managed_session() { original_terminate_managed_session "$@"; } +else + for _ in {1..400}; do [[ -e "$root/capture/release-recovery" ]] && break; sleep 0.01; done + [[ -e "$root/capture/release-recovery" ]] || exit 97 + terminate_managed_session() { original_terminate_managed_session "$@"; } +fi +cleanup_runtime +CHILD +chmod 0700 "$TEST_DIR/failed-reap-child.sh" + +run_failed_reap_case() { + local run="$1" root="$TEST_DIR/failed-reap-$1" child watchdog rc=0 + bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" recover > "$root.log" 2>&1 & child=$! + test_watchdog_process 3 "$root.watchdog-fired" "$child" & watchdog=$! + for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done + [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "failed reap run $run did not return before watchdog"; } + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root.watchdog-fired" ]] || fail "failed reap run $run fired watchdog" + : > "$root/capture/release-recovery" + wait "$child" || rc=$? + [[ "$rc" == 0 ]] || cat "$root.log" >&2 2>/dev/null || true + assert_eq 0 "$rc" "failed managed-session reap run $run returns bounded" + [[ -z "$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null)" ]] || fail "failed reap run $run left worker-session state" +} +for failed_reap_run in $(seq 1 20); do run_failed_reap_case "$failed_reap_run"; done +pass "failed managed-session reap remains bounded for 20 deterministic runs" + +( + trap - EXIT HUP INT TERM + root="$TEST_DIR/failed-reap-parent"; mkdir -p "$root/capture"; chmod 0700 "$root" "$root/capture" + bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" hold > "$root.log" 2>&1 & worker=$! + worker_start=$(wait_test_process_start "$worker") + for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done + [[ -e "$root/capture/returned" ]] || fail "parent fallback worker did not return bounded" + mapfile -t leader_fields < "$root/capture/leader" + leader=${leader_fields[0]}; leader_start=${leader_fields[1]}; leader_sid=${leader_fields[3]} + state_file=$(sed -n 's/^state_file=//p' "$root/capture/result") + TEMP_DIR=$(dirname "$state_file") + ACTIVE_WORKERS=(); ACTIVE_WORKER_STATE_FILES=(); ACTIVE_WORKER_STATE_STARTS=() + BATCH_WORKER_FAILED=false + BATCH_WORKER_ERROR=false + MAX_PARALLEL=1 + ACTIVE_WORKERS[$worker]=$worker_start + ACTIVE_WORKER_STATE_FILES[$worker]=$state_file + ACTIVE_WORKER_STATE_STARTS[$worker]=$worker_start + : > "$root/capture/release-worker" + wait_for_all_workers + assert_eq 0 "${#ACTIVE_WORKERS[@]}" "scheduler parent fallback reaps active worker" + assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "scheduler parent fallback removes cleanup_failed state" + wait_test_process_identity_gone "$leader" "$leader_start" || fail "parent fallback left same-identity leader" + session_pids=(); session_starts=(); session_ppids=(); session_pgids=(); session_states=() + collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states + assert_eq 0 "${#session_pids[@]}" "parent fallback leaves no managed SID member" + [[ ! -e "$state_file" && -e "$root/capture/reaped" ]] || fail "parent fallback evidence incomplete" + pass "parent fallback cleans a live failed session without PPID 1 zombie" +) + +( + trap - EXIT HUP INT TERM + sleep 60 & replacement=$! + replacement_start=$(wait_test_process_start "$replacement") + wait_called=false + wait() { wait_called=true; return 1; } + reap_status=0 + reap_direct_managed_leader "$replacement" "$((replacement_start + 1))" "$replacement" "$replacement" || reap_status=$? + assert_eq "$MANAGED_PROCESS_REUSED_STATUS" "$reap_status" "direct leader PID reuse returns safe status" + [[ "$wait_called" == false ]] || fail "direct leader PID reuse waited on replacement process" + test_process_identity_exists "$replacement" "$replacement_start" || fail "direct leader PID reuse touched replacement process" + kill -TERM "$replacement"; builtin wait "$replacement" 2>/dev/null || true + pass "direct leader PID reuse sends no signal and performs no wait" +) + +( + trap - EXIT HUP INT TERM + leader=424242; leader_start=777777; wait_called=false; after_wait=false + collect_owned_session_records() { + local -n pids_ref="$2" starts_ref="$3" ppids_ref="$4" pgids_ref="$5" states_ref="$6" + pids_ref=(); starts_ref=(); ppids_ref=(); pgids_ref=(); states_ref=() + [[ "$after_wait" == false ]] || return 0 + pids_ref=("$leader"); starts_ref[$leader]=$leader_start; ppids_ref[$leader]=$BASHPID + pgids_ref[$leader]=$leader; states_ref[$leader]=Z + } + read_process_record_for_session_scan() { + [[ "$after_wait" == false ]] || return 2 + PROC_UID=$EUID; PROC_PPID=$BASHPID; PROC_START=$leader_start + PROC_STATE=Z; PROC_PGID=$leader; PROC_SID=$leader + } + wait() { wait_called=true; after_wait=true; return 0; } + reap_direct_managed_leader "$leader" "$leader_start" "$leader" "$leader" + [[ "$wait_called" == true ]] || fail "same-identity leader zombie was not wait/reaped" + pass "same-identity direct leader zombie is immediately wait/reaped" +) + +cat > "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" root="$2" +mkdir -p -m 0700 "$root/runtime" "$root/capture" +export TMPDIR="$root/runtime" +source "$script" +initialize_runtime +TOTAL_TIMEOUT=1 +TIMEOUT_KILL_AFTER=1 +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +failures=1 +terminate_managed_session() { + if (( failures > 0 )); then ((failures -= 1)); return 1; fi + original_terminate_managed_session "$@" +} +start_ns=$(date +%s%N) +rc=0 +run_managed_command "$TEMP_DIR/hard-output" timeout 1 bash -c \ + 'trap "" HUP INT TERM; bash -c '\''trap "" HUP INT TERM; while :; do kill -STOP "$$"; done'\'' & wait "$!"' || rc=$? +elapsed_ns=$(( $(date +%s%N) - start_ns )) +[[ "$rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && "$elapsed_ns" -lt 5000000000 ]] || exit 90 +[[ "$WORKER_TRANSFER_CLEANUP_FAILED" == true && -n "$WORKER_TRANSFER_PID" && -f "$WORKER_SESSION_STATE_FILE" ]] || exit 91 +read_worker_session_state "$WORKER_SESSION_STATE_FILE" +[[ "$SESSION_STATE" == cleanup_failed ]] || exit 92 +next_rc=0 +run_managed_command "$TEMP_DIR/forbidden-hard-next" true || next_rc=$? +[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-hard-next" ]] || exit 93 +printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\n' "$rc" "$elapsed_ns" "$next_rc" > "$root/capture/result" +: > "$root/capture/returned" +terminate_worker_transfer +cleanup_runtime +CHILD +chmod 0700 "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" \ No newline at end of file diff --git a/tests/test-push-worker-registration-02.sh b/tests/test-push-worker-registration-02.sh new file mode 100644 index 0000000..d0fe671 --- /dev/null +++ b/tests/test-push-worker-registration-02.sh @@ -0,0 +1,358 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-push-worker-registration-common.sh" + +( + trap - EXIT HUP INT TERM + root="$TEST_DIR/hard-timeout-cleanup-failure" + bash "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" "$SCRIPT" "$root" > "$root.log" 2>&1 & child=$! + test_watchdog_process 5 "$root.watchdog-fired" "$child" & watchdog=$! + for _ in {1..120}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done + [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "hard timeout cleanup failure did not return bounded"; } + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root.watchdog-fired" ]] || fail "hard timeout cleanup failure fired watchdog" + rc=0; wait "$child" || rc=$? + assert_eq 0 "$rc" "hard timeout cleanup failure recovers fixture" + grep -Fxq "rc=$MANAGED_CLEANUP_FAILURE_STATUS" "$root/capture/result" || fail "hard timeout cleanup failure did not propagate 125" + grep -Fxq "next_rc=$MANAGED_CLEANUP_FAILURE_STATUS" "$root/capture/result" || fail "hard timeout cleanup failure allowed next command" + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "hard timeout cleanup failure left runtime" + pass "hard timeout cleanup failure is bounded and blocks subsequent command" +) + +write_test_worker_session_manifest() { + local file="$1" state="$2" worker_pid="$3" worker_start="$4" leader_pid="$5" leader_start="$6" pgid="$7" sid="$8" + printf 'version=1\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\n' \ + "$state" "$worker_pid" "$worker_start" "$leader_pid" "$leader_start" "$pgid" "$sid" > "$file" + chmod 0600 "$file" +} + + +assert_retained_runtime_trusted() { + local path="$1" expected_dev="$2" expected_inode="$3" metadata="" dev inode owner gid mode trusted_gid + [[ -d "$path" && ! -L "$path" ]] || fail "retained runtime is not a regular directory: $path" + metadata=$(stat -Lc '%d:%i:%u:%g:%a' -- "$path") || fail "cannot stat retained runtime: $path" + IFS=: read -r dev inode owner gid mode <<< "$metadata" + trusted_gid=$(current_gid) + [[ "$dev" == "$expected_dev" && "$inode" == "$expected_inode" && + "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 700 ]] || + fail "retained runtime trust mismatch: $metadata" +} + +assert_retained_state_trusted() { + local file="$1" metadata="" owner gid mode trusted_gid + [[ -f "$file" && ! -L "$file" ]] || fail "retained worker state is not a regular file: $file" + metadata=$(stat -Lc '%u:%g:%a' -- "$file") || fail "cannot stat retained worker state: $file" + IFS=: read -r owner gid mode <<< "$metadata" + trusted_gid=$(current_gid) + [[ "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 600 ]] || + fail "retained worker state trust mismatch: $metadata" +} + +run_preserved_cleanup_retry_case() ( + local run="$1" root runtime runtime_dev runtime_inode credential leader leader_start leader_pgid leader_sid state_file + local owner watchdog rc=0 metadata="" state_status=0 + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + root="$TEST_DIR/runtime-evidence-retry-$run"; setup_fixture "$root" + runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE + credential="$runtime/private-key" + printf 'fixture-credential-do-not-print\n' > "$credential"; chmod 0600 "$credential" + RUNTIME_KEY_FILE=$credential + + setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! + leader_start=$(wait_test_process_start "$leader") + for _ in {1..100}; do + read_process_record "$leader" || true + [[ "$PROC_START" == "$leader_start" && "$PROC_STATE" == T ]] && break + sleep 0.01 + done + read_process_record "$leader" + leader_pgid=$PROC_PGID; leader_sid=$PROC_SID + [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "retry run $run leader identity invalid" + WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start + WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid + publish_worker_session_state active + state_file=$WORKER_SESSION_STATE_FILE + + eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" + terminate_managed_session() { return 1; } + owner=$BASHPID + test_watchdog_process 3 "$root/watchdog-fired" "$owner" & watchdog=$! + cleanup_runtime > "$root/capture/first-cleanup.log" 2>&1 || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + + (( rc != 0 )) || fail "retry run $run first cleanup unexpectedly succeeded" + [[ ! -e "$root/watchdog-fired" ]] || fail "retry run $run watchdog fired" + [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" ]] || + fail "retry run $run lost runtime identity" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + assert_retained_state_trusted "$state_file" + metadata=$(stat -Lc '%u:%g:%a' -- "$credential") + [[ "$metadata" == "$EUID:$(current_gid):600" ]] || fail "retry run $run credential permissions changed: $metadata" + grep -Fq 'fixture-credential-do-not-print' "$root/capture/first-cleanup.log" && fail "retry run $run printed credential content" + state_status=0; read_worker_session_state "$state_file" || state_status=$? + assert_eq 0 "$state_status" "retry run $run retained state parses" + [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && + "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || + fail "retry run $run retained state identity mismatch" + [[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && + "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && + "$WORKER_TRANSFER_CLEANUP_FAILED" == true && "$WORKER_SESSION_STATE_FILE" == "$state_file" && + "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run lost cleanup globals" + test_process_identity_exists "$leader" "$leader_start" || fail "retry run $run lost live leader tracking" + read_process_record "$leader" + [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "retry run $run orphaned or zombified leader" + + eval "$(declare -f original_terminate_managed_session | sed '1s/original_terminate_managed_session/terminate_managed_session/')" + cleanup_runtime + wait_test_process_identity_gone "$leader" "$leader_start" || fail "retry run $run left leader identity" + collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states + assert_eq 0 "${#session_pids[@]}" "retry run $run clears managed SID" + [[ ! -e "$state_file" && ! -L "$state_file" && ! -e "$runtime" && ! -L "$runtime" ]] || + fail "retry run $run left state or runtime" + runtime_lifecycle_globals_resolved || fail "retry run $run left lifecycle globals" + [[ "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run runtime state not reset" + pass "failed cleanup evidence retry run $run preserves then clears runtime" +) + +runtime_evidence_stress=${PUSH_RUNTIME_EVIDENCE_STRESS:-20} +for runtime_evidence_run in $(seq 1 "$runtime_evidence_stress"); do + run_preserved_cleanup_retry_case "$runtime_evidence_run" +done +pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" + +( + local_cleanup_root="$TEST_DIR/active-cleanup-evidence"; setup_fixture "$local_cleanup_root" + runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE + sleep 300 & worker=$!; worker_start=$(wait_test_process_start "$worker") + sleep 300 & replacement=$!; replacement_start=$(wait_test_process_start "$replacement") + state_file="$runtime/worker-session.$worker.state" + write_test_worker_session_manifest "$state_file" cleanup_failed "$worker" "$worker_start" \ + "$replacement" "$((replacement_start + 1))" "$replacement" "$replacement" + ACTIVE_WORKERS[$worker]=$worker_start + ACTIVE_WORKER_STATE_FILES[$worker]=$state_file + ACTIVE_WORKER_STATE_STARTS[$worker]=$worker_start + eval "$(declare -f terminate_active_workers | sed '1s/terminate_active_workers/original_terminate_active_workers/')" + terminate_active_workers() { return 1; } + rc=0; cleanup_runtime > "$local_cleanup_root/capture/first-cleanup.log" 2>&1 || rc=$? + (( rc != 0 )) || fail "active cleanup evidence first call unexpectedly succeeded" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + assert_retained_state_trusted "$state_file" + [[ -n "${ACTIVE_WORKERS[$worker]:-}" && "${ACTIVE_WORKER_STATE_FILES[$worker]:-}" == "$state_file" && + "${ACTIVE_WORKER_STATE_STARTS[$worker]:-}" == "$worker_start" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || + fail "active cleanup evidence maps were discarded" + test_process_identity_exists "$worker" "$worker_start" || fail "active cleanup evidence worker disappeared" + eval "$(declare -f original_terminate_active_workers | sed '1s/original_terminate_active_workers/terminate_active_workers/')" + cleanup_runtime + wait_test_process_identity_gone "$worker" "$worker_start" || fail "active cleanup retry left worker" + [[ ! -e "$state_file" && ! -e "$runtime" && ${#ACTIVE_WORKERS[@]} == 0 && + ${#ACTIVE_WORKER_STATE_FILES[@]} == 0 && ${#ACTIVE_WORKER_STATE_STARTS[@]} == 0 ]] || + fail "active cleanup retry left runtime, state, or maps" + test_process_identity_exists "$replacement" "$replacement_start" || fail "active cleanup retry touched PID-reuse replacement" + kill -TERM "$replacement"; wait "$replacement" 2>/dev/null || true + pass "active cleanup failure preserves evidence and retry clears it" +) + +run_runtime_trust_retention_case() ( + local kind="$1" root runtime runtime_dev runtime_inode original_path replacement_path rc=0 + root="$TEST_DIR/runtime-trust-$kind"; setup_fixture "$root" + runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE + case "$kind" in + symlink) + original_path="$runtime.original"; replacement_path="$runtime.replacement" + mv "$runtime" "$original_path"; mkdir -m 0700 "$replacement_path"; : > "$replacement_path/marker" + ln -s "$replacement_path" "$runtime" + ;; + wrong-mode) chmod 0755 "$runtime" ;; + wrong-owner|wrong-gid) + eval "$(declare -f stat | sed '1s/stat/original_stat/')" 2>/dev/null || true + stat() { + local last="${!#}" metadata dev inode owner gid mode + if [[ "$last" == "$runtime" && "$1" == -Lc && "$2" == '%d:%i:%u:%g:%a' ]]; then + metadata=$(command stat -Lc '%d:%i:%u:%g:%a' -- "$runtime") || return 1 + IFS=: read -r dev inode owner gid mode <<< "$metadata" + [[ "$kind" == wrong-owner ]] && owner=$((owner + 1)) || gid=$((gid + 1)) + printf '%s:%s:%s:%s:%s\n' "$dev" "$inode" "$owner" "$gid" "$mode" + else + command stat "$@" + fi + } + ;; + *) fail "unknown runtime trust case: $kind" ;; + esac + cleanup_runtime > "$root/capture/cleanup.log" 2>&1 || rc=$? + (( rc != 0 )) || fail "$kind untrusted runtime unexpectedly deleted" + [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" && + "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "$kind untrusted runtime lost path identity" + case "$kind" in + symlink) + [[ -L "$runtime" && -f "$replacement_path/marker" && -d "$original_path" ]] || fail "symlink replacement was modified" + rm -f "$runtime"; rmdir "$replacement_path" 2>/dev/null || { rm -f "$replacement_path/marker"; rmdir "$replacement_path"; } + mv "$original_path" "$runtime" + ;; + wrong-mode) chmod 0700 "$runtime" ;; + wrong-owner|wrong-gid) + unset -f stat + ;; + esac + cleanup_runtime + [[ ! -e "$runtime" && ! -L "$runtime" ]] || fail "$kind trusted retry left runtime" + pass "$kind runtime trust failure preserves evidence and replacement" +) +for runtime_trust_kind in symlink wrong-owner wrong-gid wrong-mode; do + run_runtime_trust_retention_case "$runtime_trust_kind" +done + +for final_residue_kind in worker-session registration-ready registration-stage; do + ( + root="$TEST_DIR/runtime-final-proof-$final_residue_kind"; setup_fixture "$root" + runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE + case "$final_residue_kind" in + worker-session) residue="$runtime/worker-session.99999.state" ;; + registration-ready) residue="$runtime/worker-registration.99999.ready" ;; + registration-stage) residue="$runtime/.worker-registration.99999.ready.token.stage" ;; + esac + eval "$(declare -f terminate_active_workers | sed '1s/terminate_active_workers/original_terminate_active_workers/')" + terminate_active_workers() { + original_terminate_active_workers || return 1 + printf 'retained-lifecycle-evidence\n' > "$residue"; chmod 0600 "$residue" + } + rc=0; cleanup_runtime > "$root/capture/cleanup.log" 2>&1 || rc=$? + (( rc != 0 )) || fail "$final_residue_kind final proof unexpectedly deleted runtime" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + [[ -f "$residue" && ! -L "$residue" && "$TEMP_DIR" == "$runtime" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || + fail "$final_residue_kind final proof discarded evidence" + eval "$(declare -f original_terminate_active_workers | sed '1s/original_terminate_active_workers/terminate_active_workers/')" + rm -f -- "$residue" + cleanup_runtime + [[ ! -e "$runtime" ]] || fail "$final_residue_kind final proof retry left runtime" + pass "$final_residue_kind blocks runtime deletion until evidence is cleared" + ) +done + +( + root="$TEST_DIR/runtime-final-proof-success"; setup_fixture "$root"; runtime=$TEMP_DIR + cleanup_runtime + [[ ! -e "$runtime" && -z "$TEMP_DIR" && "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || + fail "clean runtime final proof did not clear state" + pass "clean runtime final proof preserves normal success path" +) + + +cat > "$TEST_DIR/runtime-retention-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" root="$2" leader="$3" leader_start="$4" leader_pgid="$5" leader_sid="$6" +mkdir -p "$root/runtime" "$root/capture" +chmod 0700 "$root" "$root/runtime" "$root/capture" +TMPDIR="$root/runtime" +# shellcheck source=../tools/push.sh +source "$script" +initialize_runtime +credential="$TEMP_DIR/private-key" +printf 'signal-fixture-credential-do-not-print\n' > "$credential" +chmod 0600 "$credential" +RUNTIME_KEY_FILE=$credential +WORKER_TRANSFER_PID=$leader +WORKER_TRANSFER_START=$leader_start +WORKER_TRANSFER_PGID=$leader_pgid +WORKER_TRANSFER_SID=$leader_sid +publish_worker_session_state active +state_file=$WORKER_SESSION_STATE_FILE +terminate_managed_session() { return 1; } +install_runtime_traps +worker_start=$(get_process_start_time "$BASHPID") +stage="$root/capture/ready.stage" +{ + printf 'target_pid=%s\n' "$BASHPID" + printf 'target_start=%s\n' "$worker_start" + printf 'runtime=%s\n' "$TEMP_DIR" + printf 'runtime_dev=%s\n' "$TEMP_DIR_DEV" + printf 'runtime_inode=%s\n' "$TEMP_DIR_INODE" + printf 'state_file=%s\n' "$state_file" + printf 'credential=%s\n' "$credential" +} > "$stage" +chmod 0600 "$stage" +mv -fT "$stage" "$root/capture/ready" +while :; do :; done +CHILD +chmod 0700 "$TEST_DIR/runtime-retention-signal-child.sh" + +run_runtime_retention_signal_case() ( + local signal_name="$1" expected="$2" root leader="" leader_start="" leader_pgid="" leader_sid="" target="" target_pid="" target_start="" + local runtime runtime_dev runtime_inode state_file credential watchdog="" rc=0 line key value state_status=0 reap_status=0 + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + cleanup_runtime_retention_signal_fixture() { + if [[ -n "$target" ]]; then kill -KILL "$target" 2>/dev/null || true; wait "$target" 2>/dev/null || true; fi + if [[ -n "$leader" ]]; then kill -CONT "$leader" 2>/dev/null || true; kill -KILL "$leader" 2>/dev/null || true; wait "$leader" 2>/dev/null || true; fi + if [[ -n "$watchdog" ]]; then kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true; fi + } + trap cleanup_runtime_retention_signal_fixture EXIT + root="$TEST_DIR/runtime-retention-signal-$signal_name"; CURRENT_FIXTURE_ROOT=$root + mkdir -m 0700 "$root" + setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! + leader_start=$(wait_test_process_start "$leader") + for _ in {1..100}; do read_process_record "$leader" || true; [[ "$PROC_STATE" == T ]] && break; sleep 0.01; done + read_process_record "$leader"; leader_pgid=$PROC_PGID; leader_sid=$PROC_SID + [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "$signal_name retention leader identity invalid" + + ( + exec env --default-signal=HUP,INT,TERM bash "$TEST_DIR/runtime-retention-signal-child.sh" \ + "$SCRIPT" "$root" "$leader" "$leader_start" "$leader_pgid" "$leader_sid" + ) > "$root/output.log" 2>&1 & target=$! + for _ in {1..300}; do [[ -f "$root/capture/ready" ]] && break; sleep 0.02; done + [[ -f "$root/capture/ready" ]] || { cat "$root/output.log" >&2 || true; fail "$signal_name retention child not ready"; } + while IFS= read -r line; do + key=${line%%=*}; value=${line#*=} + case "$key" in + target_pid) target_pid=$value ;; + target_start) target_start=$value ;; + runtime) runtime=$value ;; + runtime_dev) runtime_dev=$value ;; + runtime_inode) runtime_inode=$value ;; + state_file) state_file=$value ;; + credential) credential=$value ;; + esac + done < "$root/capture/ready" + [[ "$target_pid" == "$target" && "$target_start" =~ ^[1-9][0-9]*$ && + "$runtime_dev" =~ ^[0-9]+$ && "$runtime_inode" =~ ^[1-9][0-9]*$ ]] || + fail "$signal_name retention ready metadata malformed" + job_is_active "$target" || fail "$signal_name retention target job missing" + test_watchdog_process 5 "$root/watchdog-fired" "$target" & watchdog=$! + kill "-$signal_name" "$target" + wait "$target" || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root/watchdog-fired" ]] || fail "$signal_name retention watchdog fired" + assert_eq "$expected" "$rc" "$signal_name failed cleanup preserves signal status" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + assert_retained_state_trusted "$state_file" + [[ "$(stat -Lc '%u:%g:%a' -- "$credential")" == "$EUID:$(current_gid):600" ]] || + fail "$signal_name retained credential permissions changed" + grep -Fq "$runtime" "$root/output.log" || fail "$signal_name failure output omitted runtime path" + grep -Fq 'signal-fixture-credential-do-not-print' "$root/output.log" && fail "$signal_name failure output leaked credential" + TEMP_DIR=$runtime + state_status=0; read_worker_session_state "$state_file" || state_status=$? + assert_eq 0 "$state_status" "$signal_name retained cleanup_failed state parses" + [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && + "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || + fail "$signal_name retained cleanup_failed identity mismatch" + test_process_identity_exists "$leader" "$leader_start" || fail "$signal_name failed cleanup lost leader tracking" + read_process_record "$leader" + [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "$signal_name teardown leader lost safe parent" + + terminate_managed_session "$leader" "$leader_start" "$leader_pgid" "$leader_sid" true + reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" "$BASHPID" || reap_status=$? + assert_eq 0 "$reap_status" "$signal_name teardown safely reaps leader" + collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states + assert_eq 0 "${#session_pids[@]}" "$signal_name teardown clears managed SID" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + rm -rf -- "$runtime" + TEMP_DIR="" + leader=""; target=""; watchdog="" + trap - EXIT + pass "$signal_name cleanup failure retains trusted runtime and state" +) +run_runtime_retention_signal_case HUP 129 +run_runtime_retention_signal_case INT 130 +run_runtime_retention_signal_case TERM 143 diff --git a/tests/test-push-worker-registration-03.sh b/tests/test-push-worker-registration-03.sh new file mode 100644 index 0000000..086aeda --- /dev/null +++ b/tests/test-push-worker-registration-03.sh @@ -0,0 +1,491 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-push-worker-registration-common.sh" + +( + root="$TEST_DIR/session-dual-publish"; setup_fixture "$root" + : > "$root/capture/terminate-calls" + terminate_managed_session() { printf '%s:%s:%s:%s:%s\n' "$@" >> "$root/capture/terminate-calls"; return 0; } + worker_session_state_publish_hook() { + local phase="$1" state="$2" + if [[ "$phase" == stage-ready && "$state" == cleanup_failed ]]; then + : > "$root/capture/stage.$BASHPID" + while [[ ! -e "$root/capture/release" ]]; do sleep 0.01; done + elif [[ "$phase" == after-rename && "$state" == cleanup_failed ]]; then + : > "$root/capture/renamed.$BASHPID" + fi + } + session_publisher() { + WORKER_TRANSFER_PID=$BASHPID; WORKER_TRANSFER_START=11 + WORKER_TRANSFER_PGID=$BASHPID; WORKER_TRANSFER_SID=$BASHPID + publish_worker_session_state active + publish_worker_session_state cleanup_failed + } + session_publisher & first=$!; session_publisher & second=$! + first_start=$(get_process_start_time "$first"); second_start=$(get_process_start_time "$second") + ACTIVE_WORKERS[$first]=$first_start; ACTIVE_WORKERS[$second]=$second_start + ACTIVE_WORKER_STATE_FILES[$first]="$TEMP_DIR/worker-session.$first.state" + ACTIVE_WORKER_STATE_FILES[$second]="$TEMP_DIR/worker-session.$second.state" + ACTIVE_WORKER_STATE_STARTS[$first]=$first_start; ACTIVE_WORKER_STATE_STARTS[$second]=$second_start + for _ in {1..300}; do [[ -e "$root/capture/stage.$first" && -e "$root/capture/stage.$second" ]] && break; sleep 0.01; done + [[ -e "$root/capture/stage.$first" && -e "$root/capture/stage.$second" ]] || fail "dual publishers did not reach pre-rename hook" + cleanup_published_worker_sessions + assert_eq 2 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback waits while both workers can still publish" + : > "$root/capture/release" + wait "$first"; wait "$second" + [[ -e "$root/capture/renamed.$first" && -e "$root/capture/renamed.$second" ]] || fail "dual publishers did not complete rename" + prune_active_workers + cleanup_published_worker_sessions + assert_eq 2 "$(wc -l < "$root/capture/terminate-calls")" "fallback processes both stable cleanup_failed manifests" + assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "dual fallback clears both state maps" + assert_eq 0 "${#ACTIVE_WORKER_STATE_STARTS[@]}" "dual fallback clears both state identities" + runtime_has_published_worker_state && fail "dual fallback left worker session state" + cleanup_runtime; assert_no_registration_residue "$root" + pass "dual worker state transitions stabilize after wait/reap" +) + +run_transient_state_case() ( + kind="$1"; root="$TEST_DIR/session-transient-$kind"; setup_fixture "$root" + worker=11111; start=22222; file="$TEMP_DIR/worker-session.$worker.state" + write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 + ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start + : > "$root/capture/terminate-calls" + terminate_managed_session() { printf 'call\n' >> "$root/capture/terminate-calls"; return 0; } + failures=2 + case "$kind" in + inode) + worker_session_state_read_hook() { + if [[ "$1" == after-open && "$failures" -gt 0 ]]; then + next="$file.next.$failures" + write_test_worker_session_manifest "$next" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 + mv -fT "$next" "$file"; ((failures -= 1)) + fi + } + ;; + open) + eval "$(declare -f open_worker_session_state_fd | sed '1s/open_worker_session_state_fd/original_open_worker_session_state_fd/')" + open_worker_session_state_fd() { + if (( failures > 0 )); then ((failures -= 1)); return "$WORKER_SESSION_STATE_TRANSIENT_STATUS"; fi + original_open_worker_session_state_fd "$@" + } + ;; + stat) + eval "$(declare -f worker_session_state_path_metadata | sed '1s/worker_session_state_path_metadata/original_worker_session_state_path_metadata/')" + worker_session_state_path_metadata() { + if (( failures > 0 )); then ((failures -= 1)); return "$WORKER_SESSION_STATE_TRANSIENT_STATUS"; fi + original_worker_session_state_path_metadata "$@" + } + ;; + esac + cleanup_published_worker_sessions + assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "$kind transient failure clears state map after stabilization" + assert_eq 1 "$(wc -l < "$root/capture/terminate-calls")" "$kind transient failure performs one trusted session cleanup" + [[ ! -e "$file" ]] || fail "$kind transient failure left state file" + cleanup_runtime; pass "$kind transient worker state failure is bounded and recoverable" +) +run_transient_state_case inode +run_transient_state_case open +run_transient_state_case stat + +( + root="$TEST_DIR/session-not-reaped"; setup_fixture "$root" + sleep 300 & worker=$!; start=$(get_process_start_time "$worker") + file="$TEMP_DIR/worker-session.$worker.state" + write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 + ACTIVE_WORKERS[$worker]=$start; ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start + : > "$root/capture/terminate-calls" + terminate_managed_session() { printf call >> "$root/capture/terminate-calls"; return 0; } + cleanup_published_worker_sessions + assert_eq 1 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback retains state while worker is not reaped" + assert_eq 0 "$(wc -c < "$root/capture/terminate-calls")" "fallback performs no session cleanup before reap" + kill -TERM "$worker"; wait "$worker" 2>/dev/null || true; prune_active_workers + cleanup_published_worker_sessions + assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback succeeds after worker reap" + cleanup_runtime; pass "worker reap gates final fallback" +) + +run_permanent_state_case() ( + kind="$1"; root="$TEST_DIR/session-permanent-$kind"; setup_fixture "$root" + worker=11111; start=22222; file="$TEMP_DIR/worker-session.$worker.state" + write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 + ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start + : > "$root/capture/terminate-calls" + terminate_managed_session() { printf call >> "$root/capture/terminate-calls"; return 0; } + case "$kind" in + symlink) mv "$file" "$file.real"; ln -s "$file.real" "$file" ;; + wrong-mode) chmod 0644 "$file" ;; + wrong-owner|wrong-gid) + stat() { + last=${!#} + if [[ "$last" == "$file" ]]; then + raw=$(command stat -Lc '%d:%i:%u:%g:%a:%s' -- "$file") + IFS=: read -r dev ino owner gid mode size <<< "$raw" + [[ "$kind" == wrong-owner ]] && owner=$((EUID + 1)) || gid=$((gid + 1)) + printf '%s:%s:%s:%s:%s:%s\n' "$dev" "$ino" "$owner" "$gid" "$mode" "$size" + else command stat "$@"; fi + } + ;; + malformed) printf malformed > "$file" ;; + duplicate) printf 'worker_pid=11111\n' >> "$file" ;; + unknown) printf 'unexpected=value\n' >> "$file" ;; + missing) sed -i '/^sid=/d' "$file" ;; + pid-mismatch) sed -i 's/^worker_pid=.*/worker_pid=99999/' "$file" ;; + stable-tamper) + worker_session_state_read_hook() { [[ "$1" != after-open ]] || printf tampered > "$file"; } + ;; + esac + rc=0; cleanup_published_worker_sessions || rc=$? + (( rc != 0 )) || fail "$kind permanent state unexpectedly succeeded" + assert_eq 1 "${#ACTIVE_WORKER_STATE_FILES[@]}" "$kind permanent state preserves map evidence" + assert_eq 0 "$(wc -c < "$root/capture/terminate-calls")" "$kind permanent state executes no session cleanup" + [[ -e "$file" || -L "$file" ]] || fail "$kind permanent state deleted evidence" + unset 'ACTIVE_WORKER_STATE_FILES[$worker]' 'ACTIVE_WORKER_STATE_STARTS[$worker]' + command rm -f -- "$file" "$file.real" 2>/dev/null || true + cleanup_runtime; pass "$kind permanent worker state remains rejected" +) +for permanent_state in symlink wrong-mode wrong-owner wrong-gid malformed duplicate unknown missing pid-mismatch stable-tamper; do + run_permanent_state_case "$permanent_state" +done + +run_parallel_fallback_iteration() ( + run="$1"; root="$TEST_DIR/session-parallel-$run"; setup_fixture "$root" + MAX_PARALLEL=2 + push_to_server() { + local server="$1" lock_fd + exec {lock_fd}>>"$root/capture/calls.lock"; flock -x "$lock_fd" + printf '%s\n' "$server" >> "$root/capture/calls"; : > "$root/capture/ready.$server" + flock -u "$lock_fd"; exec {lock_fd}>&- + while [[ ! -e "$root/capture/release" ]]; do sleep 0.01; done + WORKER_TRANSFER_PID=$BASHPID + WORKER_TRANSFER_START=11 + WORKER_TRANSFER_PGID=$BASHPID + WORKER_TRANSFER_SID=$BASHPID + publish_worker_session_state cleanup_failed + return "$MANAGED_CLEANUP_FAILURE_STATUS" + } + terminate_managed_session() { return 0; } + ( + deadline=$((SECONDS + 15)); status=0 + while (( SECONDS < deadline )); do + if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release"; break; fi + sleep 0.01 + done + [[ -e "$root/capture/release" ]] || { : > "$root/capture/coordinator-timeout"; : > "$root/capture/release"; status=1; } + while [[ ! -e "$root/capture/stop" ]]; do sleep 0.01; done + exit "$status" + ) & coordinator=$! + rc=0; run_server_batch source destination one.example two.example three.example four.example || rc=$? + : > "$root/capture/stop"; coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? + assert_eq 0 "$coordinator_rc" "parallel fallback run $run coordinator releases both workers" + assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "parallel fallback run $run returns lifecycle status" + assert_eq 2 "$(wc -l < "$root/capture/calls")" "parallel fallback run $run starts only first two workers" + assert_eq 0 "${#ACTIVE_WORKERS[@]}" "parallel fallback run $run reaps active workers" + assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "parallel fallback run $run clears state maps" + assert_eq 0 "${#ACTIVE_WORKER_STATE_STARTS[@]}" "parallel fallback run $run clears state identities" + [[ ! -e "$root/capture/ready.three.example" && ! -e "$root/capture/ready.four.example" ]] || fail "parallel fallback run $run launched later worker" + runtime_has_published_worker_state && fail "parallel fallback run $run left session state" + reset_batch_lifecycle_barrier; cleanup_runtime; assert_no_registration_residue "$root" +) +parallel_fallback_runs=${PUSH_WORKER_REGISTRATION_PARALLEL_STRESS:-1} +for parallel_run in $(seq 1 "$parallel_fallback_runs"); do run_parallel_fallback_iteration "$parallel_run"; done +pass "parallel fallback completed $parallel_fallback_runs deterministic run(s)" + +write_active_grace_fixture() { + local root="$1" + mkdir -m 0700 "$root" "$root/bin" "$root/runtime" "$root/ssh" "$root/pids" + printf source > "$root/source" + cat > "$root/config.conf" < "$root/bin/timeout" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/timeout.pid" +[[ ${1:-} == --kill-after=* ]] && shift +shift +"$@" & +wait "$!" +EOF + cat > "$root/bin/sshpass" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/sshpass.pid" +[[ ${1:-} == -e ]] && shift +"$@" & +wait "$!" +EOF + cat > "$root/bin/rsync" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +set -m +printf '%s\n' "$$" > "$PID_DIR/rsync.pid" +ssh fake-target & +wait "$!" +EOF + cat > "$root/bin/ssh" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/ssh.pid" +if [[ "${STATE_PUBLISH_MODE:-active}" == cleanup ]]; then + while [[ ! -e "$STATE_CLEANUP_RELEASE" ]]; do sleep 0.01; done + exit 0 +fi +sleep 300 & +printf '%s\n' "$!" > "$PID_DIR/leaf.pid" +wait "$!" +EOF + chmod 0700 "$root/bin"/* +} +cat > "$TEST_DIR/active-grace-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" +shift +source "$script" +worker_after_session_cleanup_hook() { + : > "$ACTIVE_GRACE_STARTED" + sleep 3.2 + : > "$ACTIVE_GRACE_FINISHED" +} +main "$@" +CHILD +chmod 0700 "$TEST_DIR/active-grace-child.sh" +run_active_grace_signal_case() { + local signal_name="$1" expected="$2" root + local main_pid watchdog unrelated rc=0 state_file file pid + root="$TEST_DIR/active-grace-$signal_name" + CURRENT_FIXTURE_ROOT=$root + write_active_grace_fixture "$root" + sleep 60 & unrelated=$! + ( + cd "$root" + exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ + ACTIVE_GRACE_STARTED="$root/cleanup-started" ACTIVE_GRACE_FINISHED="$root/cleanup-finished" \ + bash "$TEST_DIR/active-grace-child.sh" "$SCRIPT" "$root/source" /remote/path + ) > "$root/output.log" 2>&1 & + main_pid=$! + for _ in {1..400}; do + state_file=$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null || true) + [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] && break + sleep 0.05 + done + if [[ -z "${state_file:-}" || ! -f "$root/pids/leaf.pid" ]]; then + cat "$root/output.log" >&2 || true + kill "$main_pid" "$unrelated" 2>/dev/null || true + fail "$signal_name active grace fixture did not enter managed session" + fi + test_watchdog_process 20 "$root/watchdog-timeout" "$main_pid" & watchdog=$! + kill "-$signal_name" "$main_pid" + wait "$main_pid" || rc=$? + kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root/watchdog-timeout" ]] || fail "$signal_name active grace exceeded watchdog" + assert_eq "$expected" "$rc" "$signal_name active worker preserves main signal status" + [[ -e "$root/cleanup-started" && -e "$root/cleanup-finished" ]] || { cat "$root/output.log" >&2 || true; fail "$signal_name active worker was killed before cleanup hook completed"; } + for file in "$root/pids"/*.pid; do + [[ -f "$file" ]] || continue + pid=$(<"$file") + process_is_running "$pid" && fail "$signal_name active grace left $(basename "$file") pid=$pid" + done + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$signal_name active grace left runtime state" + kill -0 "$unrelated" 2>/dev/null || fail "$signal_name active grace killed unrelated process" + kill "$unrelated" 2>/dev/null || true; wait "$unrelated" 2>/dev/null || true + pass "$signal_name active worker receives full managed-session cleanup grace" +} +run_active_grace_signal_case HUP 129 +run_active_grace_signal_case INT 130 +run_active_grace_signal_case TERM 143 + +cat > "$TEST_DIR/live-cleanup-failure-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" +shift +source "$script" +main_owner=$BASHPID +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +terminate_managed_session() { + if [[ "$BASHPID" != "$main_owner" ]]; then return 1; fi + original_terminate_managed_session "$@" +} +main "$@" +CHILD +chmod 0700 "$TEST_DIR/live-cleanup-failure-child.sh" + +run_live_cleanup_failure_signal_case() ( + trap - EXIT HUP INT TERM + local signal_name="$1" expected="$2" root main_pid main_start state_file="" saved_temp + local worker_pid="" worker_start="" leader_pid="" leader_start="" managed_sid="" watchdog="" unrelated="" unrelated_start="" rc=0 + cleanup_live_failure_fixture() { + [[ -z "$worker_pid" || -z "$worker_start" ]] || { test_process_identity_exists "$worker_pid" "$worker_start" && kill -KILL "$worker_pid" 2>/dev/null || true; } + [[ -z "$main_pid" || -z "$main_start" ]] || { test_process_identity_exists "$main_pid" "$main_start" && kill -KILL "$main_pid" 2>/dev/null || true; } + [[ -z "$unrelated" || -z "$unrelated_start" ]] || { test_process_identity_exists "$unrelated" "$unrelated_start" && kill -TERM "$unrelated" 2>/dev/null || true; } + [[ -z "$main_pid" ]] || wait "$main_pid" 2>/dev/null || true + [[ -z "$watchdog" ]] || { kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true; } + [[ -z "$unrelated" ]] || wait "$unrelated" 2>/dev/null || true + } + trap cleanup_live_failure_fixture EXIT + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + root="$TEST_DIR/live-cleanup-failure-$signal_name"; CURRENT_FIXTURE_ROOT=$root + write_active_grace_fixture "$root" + sleep 60 & unrelated=$!; unrelated_start=$(wait_test_process_start "$unrelated") + ( + cd "$root" + exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ + bash "$TEST_DIR/live-cleanup-failure-child.sh" "$SCRIPT" "$root/source" /remote/path + ) > "$root/output.log" 2>&1 & + main_pid=$!; main_start=$(wait_test_process_start "$main_pid") + for _ in {1..500}; do + state_file=$(find "$root/runtime" -type f -name 'worker-session.*.state' -print -quit 2>/dev/null || true) + [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] && break + sleep 0.05 + done + [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] || fail "$signal_name cleanup-failure fixture did not reach live session" + saved_temp=$TEMP_DIR; TEMP_DIR=$(dirname "$state_file") + read_worker_session_state "$state_file" || fail "$signal_name cleanup-failure state unreadable" + worker_pid=$SESSION_WORKER_PID; worker_start=$SESSION_WORKER_START + leader_pid=$SESSION_LEADER_PID; leader_start=$SESSION_LEADER_START; managed_sid=$SESSION_SID + TEMP_DIR=$saved_temp + wait_test_process_identity_present "$worker_pid" "$worker_start" || fail "$signal_name cleanup-failure worker identity missing" + wait_test_process_identity_present "$leader_pid" "$leader_start" || fail "$signal_name cleanup-failure leader identity missing" + test_watchdog_process 30 "$root/watchdog-fired" "$main_pid" & watchdog=$! + kill "-$signal_name" "$main_pid" + wait "$main_pid" || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root/watchdog-fired" ]] || fail "$signal_name cleanup failure fired watchdog" + assert_eq "$expected" "$rc" "$signal_name cleanup failure preserves main signal status" + wait_test_process_identity_gone "$worker_pid" "$worker_start" || fail "$signal_name cleanup failure left worker identity" + wait_test_process_identity_gone "$leader_pid" "$leader_start" || fail "$signal_name cleanup failure left leader identity" + collect_owned_session_records "$managed_sid" session_pids session_starts session_ppids session_pgids session_states || fail "$signal_name cleanup failure cannot verify managed SID" + assert_eq 0 "${#session_pids[@]}" "$signal_name cleanup failure leaves no managed SID member or zombie" + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$signal_name cleanup failure left runtime or state" + test_process_identity_exists "$unrelated" "$unrelated_start" || fail "$signal_name cleanup failure killed unrelated process" + kill -TERM "$unrelated"; wait "$unrelated" 2>/dev/null || true + main_pid=""; main_start=""; watchdog=""; unrelated=""; unrelated_start="" + trap - EXIT + pass "$signal_name cleanup failure uses parent fallback without watchdog or zombie" +) +run_live_cleanup_failure_signal_case HUP 129 +run_live_cleanup_failure_signal_case INT 130 +run_live_cleanup_failure_signal_case TERM 143 + +cat > "$TEST_DIR/state-publication-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" +shift +source "$script" +main_owner=$BASHPID +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +terminate_managed_session() { + if [[ "$STATE_PUBLISH_MODE" == cleanup && "$BASHPID" != "$main_owner" ]]; then return 1; fi + original_terminate_managed_session "$@" +} +worker_session_state_publish_hook() { + local phase="$1" state="$2" state_file="$3" hook_stage="$4" marker_stage worker_start + if [[ "$STATE_PUBLISH_MODE" == cleanup && "$state" == active && "$phase" == state-file-assigned ]]; then + : > "$STATE_CLEANUP_RELEASE" + fi + [[ "$phase" == "$STATE_PUBLISH_PHASE" && "$state" == "$STATE_PUBLISH_STATE" ]] || return 0 + [[ ! -e "$STATE_PUBLISH_MARKER" ]] || return 0 + worker_start="" + for _ in {1..20}; do + if read_process_record "$BASHPID" && [[ "$PROC_UID" == "$EUID" && "$PROC_STATE" != Z ]]; then worker_start=$PROC_START; break; fi + sleep 0.01 + done + [[ -n "$worker_start" ]] || return 1 + marker_stage="$STATE_PUBLISH_MARKER.$BASHPID.stage" + (umask 077; set -o noclobber; : > "$marker_stage") || return 1 + printf 'phase=%s\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\nstate_file=%s\nhook_stage=%s\n' \ + "$phase" "$state" "$BASHPID" "$worker_start" "$WORKER_TRANSFER_PID" "$WORKER_TRANSFER_START" \ + "$WORKER_TRANSFER_PGID" "$WORKER_TRANSFER_SID" "$state_file" "$hook_stage" > "$marker_stage" + chmod 0600 "$marker_stage"; mv -fT "$marker_stage" "$STATE_PUBLISH_MARKER" + while :; do :; done +} +main "$@" +CHILD +chmod 0700 "$TEST_DIR/state-publication-child.sh" + +run_state_publication_signal_case() ( + local state="$1" phase="$2" signal_name="$3" expected="$4" mode root main_pid="" main_start="" worker_pid="" worker_start="" leader_pid="" leader_start="" managed_sid="" + local watchdog="" watchdog_start="" unrelated="" unrelated_start="" rc=0 line key value marker state_file hook_stage + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + cleanup_publication_fixture() { + [[ -z "$worker_pid" || -z "$worker_start" ]] || { test_process_identity_exists "$worker_pid" "$worker_start" && kill -KILL "$worker_pid" 2>/dev/null || true; } + [[ -z "$main_pid" || -z "$main_start" ]] || { test_process_identity_exists "$main_pid" "$main_start" && kill -KILL "$main_pid" 2>/dev/null || true; } + [[ -z "$watchdog" || -z "$watchdog_start" ]] || { test_process_identity_exists "$watchdog" "$watchdog_start" && kill -TERM "$watchdog" 2>/dev/null || true; } + [[ -z "$unrelated" || -z "$unrelated_start" ]] || { test_process_identity_exists "$unrelated" "$unrelated_start" && kill -TERM "$unrelated" 2>/dev/null || true; } + [[ -z "$main_pid" ]] || wait "$main_pid" 2>/dev/null || true + [[ -z "$watchdog" ]] || wait "$watchdog" 2>/dev/null || true + [[ -z "$unrelated" ]] || wait "$unrelated" 2>/dev/null || true + } + trap cleanup_publication_fixture EXIT + mode=active; [[ "$state" == cleanup_failed ]] && mode=cleanup + root="$TEST_DIR/state-publish-$state-$phase-$signal_name${5:-}"; CURRENT_FIXTURE_ROOT=$root + write_active_grace_fixture "$root" + mkdir -m 0700 "$root/capture" + marker="$root/capture/publish-marker" + sleep 60 & unrelated=$!; unrelated_start=$(wait_test_process_start "$unrelated") + ( + cd "$root" + exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ + STATE_PUBLISH_MODE="$mode" STATE_PUBLISH_STATE="$state" STATE_PUBLISH_PHASE="$phase" STATE_PUBLISH_MARKER="$marker" \ + STATE_CLEANUP_RELEASE="$root/capture/cleanup-release" bash "$TEST_DIR/state-publication-child.sh" "$SCRIPT" "$root/source" /remote/path + ) > "$root/output.log" 2>&1 & + main_pid=$!; main_start=$(wait_test_process_start "$main_pid") + for _ in {1..600}; do [[ -f "$marker" ]] && break; sleep 0.05; done + [[ -f "$marker" && ! -L "$marker" && "$(stat -Lc %a "$marker")" == 600 ]] || { cat "$root/output.log" >&2 || true; fail "$state/$phase/$signal_name did not reach publish hook"; } + while IFS= read -r line; do key=${line%%=*}; value=${line#*=}; case "$key" in worker_pid) worker_pid=$value;;worker_start) worker_start=$value;;leader_pid) leader_pid=$value;;leader_start) leader_start=$value;;sid) managed_sid=$value;;state_file) state_file=$value;;hook_stage) hook_stage=$value;;esac; done < "$marker" + [[ "$worker_pid" =~ ^[1-9][0-9]*$ && "$worker_start" =~ ^[1-9][0-9]*$ && "$managed_sid" =~ ^[1-9][0-9]*$ ]] || fail "$state/$phase/$signal_name marker identity malformed" + wait_test_process_identity_present "$worker_pid" "$worker_start" || { cat "$marker" >&2; fail "$state/$phase/$signal_name worker identity missing at hook"; } + test_watchdog_process 30 "$root/watchdog-timeout" "$main_pid" & watchdog=$!; watchdog_start=$(wait_test_process_start "$watchdog") + printf "DIAG: publication state=%s phase=%s signal=%s main=%s/%s worker=%s/%s sid=%s\n" \ + "$state" "$phase" "$signal_name" "$main_pid" "$main_start" "$worker_pid" "$worker_start" "$managed_sid" + kill "-$signal_name" "$main_pid"; wait "$main_pid" || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root/watchdog-timeout" ]] || fail "$state/$phase/$signal_name watchdog fired" + assert_eq "$expected" "$rc" "$state/$phase preserves $signal_name status" + wait_test_process_identity_gone "$worker_pid" "$worker_start" || fail "$state/$phase/$signal_name left worker identity" + if [[ "$leader_pid" =~ ^[1-9][0-9]*$ && "$leader_start" =~ ^[1-9][0-9]*$ ]]; then + if ! wait_test_process_identity_gone "$leader_pid" "$leader_start"; then + read_process_record "$leader_pid" || true + printf 'leader remains pid=%s expected_start=%s actual_start=%s state=%s ppid=%s pgid=%s sid=%s\n' \ + "$leader_pid" "$leader_start" "${PROC_START:-unknown}" "${PROC_STATE:-unknown}" "${PROC_PPID:-unknown}" "${PROC_PGID:-unknown}" "${PROC_SID:-unknown}" >&2 + cat "/proc/$leader_pid/stat" >&2 2>/dev/null || true + fail "$state/$phase/$signal_name left leader identity" + fi + fi + collect_owned_session_records "$managed_sid" session_pids session_starts session_ppids session_pgids session_states || fail "$state/$phase/$signal_name cannot verify managed SID" + (( ${#session_pids[@]} == 0 )) || fail "$state/$phase/$signal_name left managed SID member: ${session_pids[*]}" + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$state/$phase/$signal_name left runtime/session/registration state" + [[ ! -e "$state_file" && ! -e "$hook_stage" && ! -L "$state_file" && ! -L "$hook_stage" ]] || fail "$state/$phase/$signal_name left publication path" + test_process_identity_exists "$unrelated" "$unrelated_start" || fail "$state/$phase/$signal_name killed unrelated process" + kill -TERM "$unrelated"; wait "$unrelated" 2>/dev/null || true + unrelated=""; unrelated_start=""; main_pid=""; main_start=""; watchdog=""; watchdog_start="" + trap - EXIT + pass "$state/$phase/$signal_name publication window cleans process tree and runtime" +) +publication_states=${PUSH_STATE_PUBLICATION_STATES:-"active cleanup_failed"} +publication_phases=${PUSH_STATE_PUBLICATION_PHASES:-"before-stage-create stage-ready after-rename state-file-assigned"} +publication_signals=${PUSH_STATE_PUBLICATION_SIGNALS:-"HUP INT TERM"} \ No newline at end of file diff --git a/tests/test-push-worker-registration-04.sh b/tests/test-push-worker-registration-04.sh new file mode 100644 index 0000000..019fb99 --- /dev/null +++ b/tests/test-push-worker-registration-04.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +set -euo pipefail +source "$(dirname "${BASH_SOURCE[0]}")/lib/test-push-worker-registration-common.sh" + +for publication_state in $publication_states; do + for publication_phase in $publication_phases; do + for publication_signal in $publication_signals; do + case "$publication_signal" in HUP) publication_status=129 ;; INT) publication_status=130 ;; TERM) publication_status=143 ;; *) fail "unknown publication signal" ;; esac + run_state_publication_signal_case "$publication_state" "$publication_phase" "$publication_signal" "$publication_status" + done + done +done + +( + # Three bounded extra probes; preserve original matrix and stop at first failure. + for diagnostic_round in 1 2 3; do + printf "DIAG: extra HUP probe=%s/3\n" "$diagnostic_round" + run_state_publication_signal_case cleanup_failed after-rename HUP 129 "-probe-$diagnostic_round" + done + root="$TEST_DIR/normal-parallel"; setup_fixture "$root" + : > "$root/capture/current"; : > "$root/capture/max" + push_to_server() { + local server="$1" lock_fd current max group + case "$server" in one.example|two.example) group=first ;; *) group=second ;; esac + exec {lock_fd}>>"$root/capture/lock"; flock -x "$lock_fd" + current=$(<"$root/capture/current"); ((current += 1)); printf '%s' "$current" > "$root/capture/current" + max=$(<"$root/capture/max"); (( current > max )) && printf '%s' "$current" > "$root/capture/max" + printf '%s\n' "$server" >> "$root/capture/transfer-calls"; : > "$root/capture/ready.$server" + flock -u "$lock_fd"; exec {lock_fd}>&- + while [[ ! -e "$root/capture/release.$group" ]]; do sleep 0.02; done + exec {lock_fd}>>"$root/capture/lock"; flock -x "$lock_fd" + current=$(<"$root/capture/current"); ((current -= 1)); printf '%s' "$current" > "$root/capture/current" + flock -u "$lock_fd"; exec {lock_fd}>&- + record_result success "$server" + } + ( + deadline=$((SECONDS + 15)) + while (( SECONDS < deadline )); do + if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release.first"; break; fi + sleep 0.02 + done + [[ -e "$root/capture/release.first" ]] || { : > "$root/capture/coordinator-timeout"; : > "$root/capture/release.first"; : > "$root/capture/release.second"; exit 1; } + while (( SECONDS < deadline + 15 )); do + if [[ -e "$root/capture/ready.three.example" && -e "$root/capture/ready.four.example" ]]; then : > "$root/capture/release.second"; exit 0; fi + sleep 0.02 + done + : > "$root/capture/coordinator-timeout"; : > "$root/capture/release.second"; exit 1 + ) & coordinator=$! + run_server_batch source destination one.example two.example three.example four.example + coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? + assert_eq 0 "$coordinator_rc" "normal parallel coordinator releases both worker pairs" + assert_eq 2 "$(<"$root/capture/max")" "normal registration preserves MAX_PARALLEL=2" + assert_eq 4 "$(wc -l < "$root/capture/transfer-calls")" "normal registration completes four fake servers" + assert_no_registration_residue "$root"; cleanup_runtime; assert_no_registration_residue "$root" + pass "normal registration/session lifecycle cleans all state" +) +printf 'All push worker registration tests passed.\n' diff --git a/tests/test-push-worker-registration.sh b/tests/test-push-worker-registration.sh deleted file mode 100755 index 3ef0fcf..0000000 --- a/tests/test-push-worker-registration.sh +++ /dev/null @@ -1,1553 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -# Safe runtime identity only: no environment dump, credentials, or xtrace. -printf "DIAG: bash=%s kernel=%s\n" "$BASH_VERSION" "$(uname -r)" -dpkg-query -W bash libc6 2>/dev/null || true - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) -readonly ROOT_DIR -readonly SCRIPT="$ROOT_DIR/tools/push.sh" -TEST_DIR=$(mktemp -d) -readonly TEST_DIR -CURRENT_FIXTURE_ROOT="" -trap 'rm -rf "$TEST_DIR"' EXIT - -failure_diagnostics() { - local root=${CURRENT_FIXTURE_ROOT:-} file pid - [[ -n "$root" && -d "$root" ]] || return 0 - printf '%s\n' '=== worker registration diagnostics ===' >&2 - printf '%s\n' '--- coordinator/status ---' >&2 - for file in "$root"/capture/coordinator* "$root"/capture/watchdog* "$root"/capture/timeout*; do - [[ -e "$file" || -L "$file" ]] || continue - printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true - done - printf '%s\n' '--- registration/release/stages ---' >&2 - find "$root" \( -name 'worker-registration.*' -o -name '.worker-registration.*' \) -print 2>/dev/null | sort >&2 || true - for file in "$root"/runtime/push-runtime.*/worker-registration.*; do - [[ -e "$file" || -L "$file" ]] || continue - printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true - done - printf '%s\n' '--- worker/session states ---' >&2 - for file in "$root"/runtime/push-runtime.*/worker-session.*.state; do - [[ -e "$file" || -L "$file" ]] || continue - printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true - done - printf '%s\n' '--- jobs ---' >&2; jobs -l >&2 || true - printf '%s\n' '--- runtime tree ---' >&2 - find "$root/runtime" -mindepth 1 -maxdepth 4 -printf '%y %m %p\n' 2>/dev/null | sort >&2 || true - printf '%s\n' '--- process output ---' >&2 - [[ -f "$root/output.log" ]] && cat "$root/output.log" >&2 2>/dev/null || true - printf '%s\n' '--- fake command calls ---' >&2 - for file in "$root"/capture/calls "$root"/capture/transfer-calls; do - [[ -e "$file" ]] && { printf '[%s]\n' "$file" >&2; cat "$file" >&2 || true; } - done - printf '%s\n' '--- recorded proc identities ---' >&2 - for file in "$root"/capture/*.pid; do - [[ -f "$file" && ! -L "$file" ]] || continue - pid=$(<"$file") - [[ "$pid" =~ ^[1-9][0-9]*$ && -r "/proc/$pid/stat" ]] && printf 'pid=%s stat=%s\n' "$pid" "$(<"/proc/$pid/stat")" >&2 - done -} -fail() { failure_diagnostics; printf 'FAIL: %s\n' "$*" >&2; exit 1; } -pass() { printf 'PASS: %s\n' "$*"; } -assert_eq() { [[ "$1" == "$2" ]] || fail "$3: expected '$1', got '$2'"; pass "$3"; } -process_is_running() { local pid="$1" state; [[ -r "/proc/$pid/stat" ]] || return 1; state=$(awk '{print $3}' "/proc/$pid/stat" 2>/dev/null) || return 1; [[ "$state" != Z ]]; } -test_process_identity_exists() { - local pid="$1" start="$2" - read_process_record "$pid" || return 1 - [[ "$PROC_UID" == "$EUID" && "$PROC_START" == "$start" ]] -} -wait_test_process_identity_gone() { - local pid="$1" start="$2" _ - for _ in {1..100}; do test_process_identity_exists "$pid" "$start" || return 0; sleep 0.05; done - return 1 -} -wait_test_process_identity_present() { - local pid="$1" start="$2" _ - for _ in {1..30}; do test_process_identity_exists "$pid" "$start" && return 0; [[ -r "/proc/$pid/stat" ]] || return 1; sleep 0.01; done - return 1 -} -wait_test_process_start() { - local pid="$1" value="" _ - for _ in {1..100}; do value=$(get_process_start_time "$pid" 2>/dev/null || true); [[ -n "$value" ]] && { printf '%s\n' "$value"; return 0; }; sleep 0.005; done - return 1 -} - -source_root="$TEST_DIR/source"; mkdir -m 0700 "$source_root" -source_result=$(env TMPDIR="$source_root" SCRIPT="$SCRIPT" bash -c ' -set -euo pipefail -before=$(find "$TMPDIR" -mindepth 1 -print -quit) -source "$SCRIPT" -[[ -z "$TEMP_DIR" && -z "$SUCCESS_FILE" && -z "$FAILED_FILE" ]] -[[ "$before" == "$(find "$TMPDIR" -mindepth 1 -print -quit)" ]] -printf source-safe -') -assert_eq source-safe "$source_result" "source keeps zero runtime side effects" -# shellcheck source=../tools/push.sh -source "$SCRIPT" -( - # Inert direct-child job table: no SSH or production services. - sleep 10 & snapshot_child=$! - cleanup_snapshot_child() { kill -TERM "$snapshot_child" 2>/dev/null || true; wait "$snapshot_child" 2>/dev/null || true; } - trap cleanup_snapshot_child EXIT - job_is_active "$snapshot_child" || fail "snapshot misses active direct child" - if job_is_active 0; then fail "snapshot accepts absent PID"; fi - kill -TERM "$snapshot_child" - wait "$snapshot_child" 2>/dev/null || true - if job_is_active "$snapshot_child"; then fail "snapshot retains reaped child"; fi - trap - EXIT - pass "job snapshot preserves direct-child matching and reap semantics" -) -trap 'rm -rf "$TEST_DIR"' EXIT - -# Deterministic coordinator contract: a live worker can republish after cleanup. -( - root="$TEST_DIR/republish-tracking"; mkdir -m 0700 "$root" - state_file="$root/worker.state"; : > "$state_file" - ACTIVE_WORKERS=([424242]=12345) - ACTIVE_WORKER_STATE_FILES=([424242]="$state_file") - ACTIVE_WORKER_STATE_STARTS=([424242]=12345) - worker_session_state_maybe_cleanup_failed() { return 0; } - cleanup_calls=0 - cleanup_worker_session_state_file() { cleanup_calls=$((cleanup_calls + 1)); rm -f -- "$1"; } - cleanup_active_failed_worker_sessions - [[ ${ACTIVE_WORKER_STATE_FILES[424242]:-} == "$state_file" && ${ACTIVE_WORKER_STATE_STARTS[424242]:-} == 12345 ]] || fail "active cleanup lost live publisher tracking" - : > "$state_file" - ACTIVE_WORKERS=() - job_is_active() { return 1; } - cleanup_published_worker_sessions - assert_eq 2 "$cleanup_calls" "post-reap fallback handles republished state" - [[ ! -e $state_file && ${#ACTIVE_WORKER_STATE_FILES[@]} == 0 && ${#ACTIVE_WORKER_STATE_STARTS[@]} == 0 ]] || fail "republished state remains" - pass "live publisher tracking survives successful active cleanup" -) - -setup_fixture() { - local root="$1" - CURRENT_FIXTURE_ROOT=$root - mkdir -m 0700 "$root" "$root/runtime" "$root/capture" - TMPDIR="$root/runtime"; initialize_runtime - MAX_PARALLEL=2 - MAX_RETRIES=1 - BATCH_WORKER_FAILED=false - BATCH_WORKER_ERROR=false -} -assert_no_registration_residue() { - local root="$1" - [[ -z "$(find "$root/runtime" \( -name 'worker-registration.*' -o -name '.worker-registration.*' -o -name 'worker-session.*.state' \) -print -quit)" ]] || fail "registration/session residue remains" - (( ${#REGISTERING_WORKERS[@]} == 0 && ${#ACTIVE_WORKERS[@]} == 0 )) || fail "worker arrays remain" - (( ${#WORKER_REGISTRATION_READY_FILES[@]} == 0 && ${#WORKER_REGISTRATION_RELEASE_FILES[@]} == 0 && ${#WORKER_REGISTRATION_STARTS[@]} == 0 && ${#WORKER_REGISTRATION_STAGE_FILES[@]} == 0 )) || fail "registration arrays remain" -} - -run_delayed_parent_reap_case() ( - local depth="$1" root leader leader_start leader_pgid leader_sid - root="$TEST_DIR/delayed-reap-$depth" - local state_file elapsed_start elapsed_end rc=0 - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - setup_fixture "$root" - mkfifo "$root/reap-clock" - cat > "$root/parent.sh" <<'CHILD' -#!/usr/bin/env bash -set -u -root="$1" depth="$2" -trap '' HUP INT TERM - : > "$root/parent-open" -if (( depth == 0 )); then - sleep 300 & child=$! - printf '%s\n' "$child" > "$root/leaf.pid" -else - bash "$root/parent.sh" "$root" "$((depth - 1))" & child=$! -fi -kill -STOP "$BASHPID" -wait "$child" 2>/dev/null || true -exit 0 -CHILD - chmod 0700 "$root/parent.sh" - setsid bash "$root/parent.sh" "$root" "$depth" & leader=$! - leader_start=$(wait_test_process_start "$leader") - for _ in {1..100}; do [[ -e "$root/parent-open" ]] && break; sleep .01; done - read_process_record "$leader"; leader_pgid=$PROC_PGID; leader_sid=$PROC_SID - [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "delayed $depth leader identity invalid" - WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start - WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid - publish_worker_session_state active; state_file=$WORKER_SESSION_STATE_FILE - leaf=$(<"$root/leaf.pid"); leaf_start=$(wait_test_process_start "$leaf") - kill -KILL "$leaf" 2>/dev/null || true - old_reap_status=1 - for _ in {1..30}; do - session_process_identity_matches "$leaf" "$leaf_start" "$leader_sid" || { old_reap_status=0; break; } - sleep 0.01 - done - assert_eq 1 "$old_reap_status" "delayed $depth old 300ms window remains pending" - ( sleep 1; kill -CONT -"$leader" 2>/dev/null || true ) & releaser=$! - cleanup_clock; elapsed_start=$CLEANUP_NOW - terminate_worker_transfer || rc=$? - wait "$releaser" 2>/dev/null || true - cleanup_clock; elapsed_end=$CLEANUP_NOW - assert_eq 0 "$rc" "delayed $depth parent reap succeeds" - (( elapsed_end - elapsed_start >= 100 )) || fail "delayed $depth reap did not wait for parent" - wait_test_process_identity_gone "$leader" "$leader_start" || fail "delayed $depth left leader" - collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states - assert_eq 0 "${#session_pids[@]}" "delayed $depth managed SID clears" - [[ ! -e "$state_file" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "delayed $depth state cleanup incomplete" - cleanup_runtime - exec 9>&- - pass "delayed parent reap depth $depth preserves leaf-first order" -) -run_delayed_parent_reap_case 1 -run_delayed_parent_reap_case 4 - -( - root="$TEST_DIR/reap-deadline"; setup_fixture "$root" - cat > "$root/parent.sh" <<'CHILD' -#!/usr/bin/env bash -set -u -root="$1" -trap '' HUP INT TERM -sleep 300 & child=$! -: > "$root/ready" -while [[ ! -e "$root/release" ]]; do :; done -wait "$child" 2>/dev/null || true -exit 0 -CHILD - chmod 0700 "$root/parent.sh" - setsid bash "$root/parent.sh" "$root" & leader=$! - for _ in {1..100}; do [[ -e "$root/ready" ]] && break; sleep .01; done - leader_start=$(wait_test_process_start "$leader"); read_process_record "$leader" - WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start; WORKER_TRANSFER_PGID=$PROC_PGID; WORKER_TRANSFER_SID=$PROC_SID - publish_worker_session_state active; state_file=$WORKER_SESSION_STATE_FILE - MANAGED_SESSION_CLEANUP_TICKS=300 - rc=0; terminate_worker_transfer || rc=$? - (( rc != 0 )) || fail "deadline failure unexpectedly succeeded" - [[ -d "$TEMP_DIR" && -f "$state_file" && "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || fail "deadline failure discarded evidence" - [[ "$(stat -c %a "$TEMP_DIR")" == 700 && "$(stat -c %a "$state_file")" == 600 ]] || fail "deadline evidence trust changed" - : > "$root/release"; wait "$leader" 2>/dev/null || true - remove_worker_session_state - WORKER_TRANSFER_PID=""; WORKER_TRANSFER_START=""; WORKER_TRANSFER_PGID=""; WORKER_TRANSFER_SID=""; WORKER_TRANSFER_CLEANUP_FAILED=false - clear_managed_session_cleanup_tracking - cleanup_runtime - [[ ! -e "$TEMP_DIR" ]] || fail "deadline teardown left runtime" - exec 9>&- - pass "deadline failure is bounded and preserves then safely clears evidence" -) -if [[ "${PUSH_REAP_ONLY:-false}" == true ]]; then - exit 0 -fi -test_watchdog_process() { - local seconds="$1" marker="$2" target_pid="$3" timer="" - trap '[[ -z "$timer" ]] || kill "$timer" 2>/dev/null || true; [[ -z "$timer" ]] || wait "$timer" 2>/dev/null || true; exit 0' HUP INT TERM - sleep "$seconds" & timer=$! - wait "$timer" 2>/dev/null || exit 0 - printf fired > "$marker" - kill -KILL "$target_pid" 2>/dev/null || true -} - -run_with_watchdog() { - local name="$1" seconds="$2"; shift 2 - local marker="$TEST_DIR/watchdog-$name" pid watchdog rc=0 - "$@" & pid=$! - test_watchdog_process "$seconds" "$marker" "$pid" & watchdog=$! - wait "$pid" || rc=$? - kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$marker" ]] || fail "$name exceeded watchdog deadline" - return "$rc" -} - -root_cause_regression_case() ( - root="$TEST_DIR/root-cause"; setup_fixture "$root" - GET_START_CALLS=0 - get_process_start_time() { ((GET_START_CALLS += 1)); return 1; } - push_to_server() { - local server="$1" lock_fd - exec {lock_fd}>>"$root/capture/calls.lock"; flock -x "$lock_fd" - printf '%s\n' "$server" >> "$root/capture/calls"; : > "$root/capture/ready.$server" - flock -u "$lock_fd"; exec {lock_fd}>&- - while [[ ! -e "$root/capture/release" ]]; do sleep 0.02; done - return "$MANAGED_CLEANUP_FAILURE_STATUS" - } - ( - coordinator_status=0; deadline=$((SECONDS + 15)) - while (( SECONDS < deadline )); do - if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release"; break; fi - sleep 0.02 - done - if [[ ! -e "$root/capture/release" ]]; then : > "$root/capture/coordinator-timeout"; : > "$root/capture/release"; coordinator_status=1; fi - while [[ ! -e "$root/capture/coordinator-stop" ]]; do sleep 0.02; done - exit "$coordinator_status" - ) & coordinator=$! - rc=0; run_server_batch source destination one.example two.example three.example four.example || rc=$? - : > "$root/capture/coordinator-stop"; coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? - assert_eq 0 "$coordinator_rc" "registration coordinator sees both first-slot workers" - assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "cleanup failure remains lifecycle status" - assert_eq 2 "$(wc -l < "$root/capture/calls")" "both first-slot workers enter fake transfer" - grep -Fxq one.example "$root/capture/calls" || fail "one.example did not enter fake transfer" - grep -Fxq two.example "$root/capture/calls" || fail "two.example did not enter fake transfer" - grep -Eq 'three\.example|four\.example' "$root/capture/calls" && fail "registration failure scheduled later servers" - assert_eq 0 "$GET_START_CALLS" "parent no longer depends on short get_process_start_time window" - terminate_active_workers || true; cleanup_runtime; assert_no_registration_residue "$root" - pass "deterministic registration regression avoids worker-one/worker-two deadlock" -) -run_with_watchdog root-cause 20 root_cause_regression_case - -run_ready_failure_case() ( - kind="$1"; root="$TEST_DIR/ready-$kind"; setup_fixture "$root" - WORKER_REGISTRATION_TIMEOUT_TICKS=10 - push_to_server() { : > "$root/capture/transfer"; return 0; } - case "$kind" in - missing) worker_registration_phase_hook() { [[ "$1" != before-ready ]] || while :; do :; done; } ;; - malformed) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf malformed > "$3"; } ;; - symlink) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || { rm -f "$3"; ln -s "$root/other" "$3"; }; } ;; - pid-mismatch) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || sed -i 's/^worker_pid=.*/worker_pid=999999/' "$3"; } ;; - wrong-mode) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || chmod 0644 "$3"; } ;; - duplicate-field) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf 'worker_pid=1\n' >> "$3"; } ;; - unknown-field) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf 'unexpected=value\n' >> "$3"; } ;; - illegal-number) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || sed -i 's/^worker_start=.*/worker_start=0/' "$3"; } ;; - release-failure) publish_worker_registration_release() { return 1; } ;; - esac - rc=0; launch_worker one.example source destination 1 1 || rc=$? - (( rc != 0 )) || fail "$kind registration failure unexpectedly succeeded" - [[ ! -e "$root/capture/transfer" ]] || fail "$kind reached transfer before trusted handoff" - cleanup_runtime || true; assert_no_registration_residue "$root" - [[ -z "$(jobs -pr)" ]] || fail "$kind left worker job" - pass "$kind registration failure is bounded and side-effect free" -) -for registration_failure in missing malformed symlink pid-mismatch wrong-mode duplicate-field unknown-field illegal-number release-failure; do run_ready_failure_case "$registration_failure"; done - -( - root="$TEST_DIR/release-replaced"; setup_fixture "$root" - push_to_server() { : > "$root/capture/transfer"; return 0; } - worker_registration_phase_hook() { [[ "$1" != ready-published ]] || while [[ ! -e "$root/capture/release-mutated" ]]; do sleep 0.02; done; } - parent_worker_registration_hook() { [[ "$1" != release-published ]] || { chmod 0644 "$4"; : > "$root/capture/release-mutated"; }; } - launch_worker one.example source destination 1 1 - rc=0; wait_for_all_workers || rc=$? - assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "worker rejects replaced release before transfer" - [[ ! -e "$root/capture/transfer" ]] || fail "replaced release reached transfer" - cleanup_runtime || true; assert_no_registration_residue "$root" - pass "release replacement is bounded and leaves no residue" -) -( - root="$TEST_DIR/parent-disappeared"; setup_fixture "$root" - push_to_server() { : > "$root/capture/transfer"; return 0; } - sleep 300 & fake_parent=$!; read_process_record "$fake_parent"; fake_parent_start=$PROC_START - registered_worker_entry "$fake_parent" "$fake_parent_start" one.example source destination 1 1 & worker=$! - for _ in {1..200}; do [[ -e "$TEMP_DIR/worker-registration.$worker.ready" ]] && break; sleep 0.02; done - [[ -e "$TEMP_DIR/worker-registration.$worker.ready" ]] || fail "worker did not publish ready before parent disappearance" - kill -TERM "$fake_parent" 2>/dev/null || true; wait "$fake_parent" 2>/dev/null || true - rc=0; wait "$worker" || rc=$?; (( rc != 0 )) || fail "worker accepted disappeared parent" - [[ ! -e "$root/capture/transfer" ]] || fail "worker transferred after parent disappeared" - cleanup_worker_registration_for_pid "$worker" || true; cleanup_runtime; assert_no_registration_residue "$root" - pass "worker exits before transfer when parent identity disappears" -) - -( - sleep 60 & proc_fallback_worker=$! - proc_fallback_start=$(wait_test_process_start "$proc_fallback_worker") - eval "$(declare -f read_process_record | sed '1s/read_process_record/original_read_process_record/')" - read_process_record() { [[ "$1" != "$proc_fallback_worker" ]] && original_read_process_record "$@"; } - proc_fallback_status=0 - read_process_record_for_session_scan "$proc_fallback_worker" || proc_fallback_status=$? - assert_eq 0 "$proc_fallback_status" "proc owner fallback recovers transient status-read failure" - [[ "$PROC_START" == "$proc_fallback_start" && "$PROC_UID" == "$EUID" ]] || fail "proc owner fallback identity mismatch" - kill -TERM "$proc_fallback_worker"; wait "$proc_fallback_worker" 2>/dev/null || true - pass "proc owner fallback binds UID to stable PID start" -) - -prune_identity_failure_case() ( - mode="$1"; root="$TEST_DIR/prune-$mode"; setup_fixture "$root" - sleep 300 & worker=$!; read_process_record "$worker"; start=$PROC_START - ACTIVE_WORKERS[$worker]=$start; ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state"; ACTIVE_WORKER_STATE_STARTS[$worker]=$start - eval "$(declare -f process_identity_matches | sed '1s/process_identity_matches/original_process_identity_matches/')" - eval "$(declare -f read_process_record_for_session_scan | sed '1s/read_process_record_for_session_scan/original_read_process_record_for_session_scan/')" - failures=1; scan_failures=1; [[ "$mode" == repeated ]] && { failures=100; scan_failures=100; } - process_identity_matches() { if [[ "$1" == "$worker" && "$failures" -gt 0 ]]; then ((failures -= 1)); return 1; fi; original_process_identity_matches "$@"; } - read_process_record_for_session_scan() { - if [[ "$1" == "$worker" && "$scan_failures" -gt 0 && -r "/proc/$worker/stat" ]]; then ((scan_failures -= 1)); return 1; fi - original_read_process_record_for_session_scan "$@" - } - rc=0; prune_active_workers || rc=$?; (( rc != 0 )) || fail "$mode identity failure did not stop prune" - [[ "$BATCH_WORKER_FAILED" == true ]] || fail "$mode identity failure did not set lifecycle barrier" - process_is_running "$worker" || fail "$mode prune shortened active worker grace" - [[ -n "${ACTIVE_WORKERS[$worker]:-}" ]] || fail "$mode prune discarded active worker evidence" - rc=0; wait_for_worker_slot || rc=$?; assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "$mode identity failure propagates lifecycle status" - terminate_active_workers || true - process_is_running "$worker" && fail "$mode unified active cleanup left worker" - cleanup_runtime || true; assert_no_registration_residue "$root" - pass "$mode active identity failure defers termination to unified cleanup" -) -prune_identity_failure_case once -prune_identity_failure_case repeated -( - root="$TEST_DIR/prune-cleanup-failure"; setup_fixture "$root" - sleep 300 & worker=$!; read_process_record "$worker"; start=$PROC_START - ACTIVE_WORKERS[$worker]=$start - ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state" - ACTIVE_WORKER_STATE_STARTS[$worker]=$start - eval "$(declare -f process_identity_matches | sed '1s/process_identity_matches/original_process_identity_matches/')" - eval "$(declare -f read_process_record_for_session_scan | sed '1s/read_process_record_for_session_scan/original_read_process_record_for_session_scan/')" - process_identity_matches() { [[ "$1" != "$worker" ]] && original_process_identity_matches "$@"; } - read_process_record_for_session_scan() { [[ "$1" != "$worker" || ! -r "/proc/$worker/stat" ]] && original_read_process_record_for_session_scan "$@"; } - rc=0; wait_for_worker_slot || rc=$? - assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "prune trust failure propagates 125" - [[ -n "${ACTIVE_WORKERS[$worker]:-}" ]] || fail "prune trust failure discarded active worker evidence" - eval "$(declare -f original_process_identity_matches | sed '1s/original_process_identity_matches/process_identity_matches/')" - eval "$(declare -f original_read_process_record_for_session_scan | sed '1s/original_read_process_record_for_session_scan/read_process_record_for_session_scan/')" - eval "$(declare -f cleanup_all_worker_registration_residue | sed '1s/cleanup_all_worker_registration_residue/original_cleanup_all_worker_registration_residue/')" - cleanup_all_worker_registration_residue() { return 1; } - rc=0; stop_batch_after_lifecycle_failure || rc=$? - assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "active cleanup failure remains lifecycle status 125" - eval "$(declare -f original_cleanup_all_worker_registration_residue | sed '1s/original_cleanup_all_worker_registration_residue/cleanup_all_worker_registration_residue/')" - cleanup_runtime || true; assert_no_registration_residue "$root" - pass "active cleanup failure preserves lifecycle status" -) -( - root="$TEST_DIR/prune-exited"; setup_fixture "$root" - sleep 0.1 & worker=$!; read_process_record "$worker"; start=$PROC_START - ACTIVE_WORKERS[$worker]=$start - ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state" - ACTIVE_WORKER_STATE_STARTS[$worker]=$start - sleep 0.2; prune_active_workers - assert_eq 0 "${#ACTIVE_WORKERS[@]}" "exited worker is normally reaped" - [[ "$BATCH_WORKER_FAILED" == false ]] || fail "normal exited worker set lifecycle barrier" - cleanup_runtime; assert_no_registration_residue "$root" - pass "normal exited worker accounting remains intact" -) - -cat > "$TEST_DIR/registration-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" root="$2" phase="$3" -mkdir -p -m 0700 "$root/runtime" "$root/capture" -export TMPDIR="$root/runtime" -source "$script" -initialize_runtime; install_runtime_traps -WORKER_REGISTRATION_TIMEOUT_TICKS=200 -push_to_server() { printf 'push pid=%s root=%s\n' "$BASHPID" "$root" >> "$root/capture/transfer-calls"; : > "$root/capture/transfer"; return 0; } -worker_registration_phase_hook() { local current="$1"; [[ "$phase" == "$current" ]] || return 0; : > "$root/capture/phase"; while :; do :; done; } -parent_worker_registration_hook() { local current="$1"; [[ "$phase" == "$current" ]] || return 0; : > "$root/capture/phase"; while :; do :; done; } -launch_worker one.example source destination 1 1 -wait_for_all_workers -if [[ "$phase" == normal ]]; then [[ -e "$root/capture/transfer" ]] || { printf 'normal transfer missing\n' >&2; jobs -l >&2 || true; exit 91; }; fi -cleanup_runtime -CHILD -chmod 0700 "$TEST_DIR/registration-signal-child.sh" -run_registration_signal_case() { - local phase="$1" signal_name="$2" expected="$3" root retry_root - local pid watchdog unrelated rc=0 - root="$TEST_DIR/signal-$phase-$signal_name"; CURRENT_FIXTURE_ROOT=$root - sleep 60 & unrelated=$! - env --default-signal=HUP,INT,TERM bash "$TEST_DIR/registration-signal-child.sh" "$SCRIPT" "$root" "$phase" > "$root.log" 2>&1 & pid=$! - for _ in {1..300}; do [[ -e "$root/capture/phase" ]] && break; sleep 0.05; done - [[ -e "$root/capture/phase" ]] || { cat "$root.log"; kill "$pid" "$unrelated" 2>/dev/null || true; fail "$phase/$signal_name did not reach registration phase"; } - test_watchdog_process 30 "$root.watchdog" "$pid" & watchdog=$! - kill "-$signal_name" "$pid"; wait "$pid" || rc=$? - kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root.watchdog" ]] || fail "$phase/$signal_name exceeded watchdog" - assert_eq "$expected" "$rc" "$phase registration $signal_name status" - [[ ! -e "$root/capture/transfer" ]] || fail "$phase/$signal_name reached transfer" - [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$phase/$signal_name left runtime state" - kill -0 "$unrelated" 2>/dev/null || fail "$phase/$signal_name killed unrelated process" - kill "$unrelated" 2>/dev/null || true; wait "$unrelated" 2>/dev/null || true - retry_root="${root}-retry" - env bash "$TEST_DIR/registration-signal-child.sh" "$SCRIPT" "$retry_root" normal > "$retry_root.log" 2>&1 - if [[ ! -e "$retry_root/capture/transfer" ]]; then cat "$retry_root.log" >&2 2>/dev/null || true; fail "$phase/$signal_name retry did not transfer"; fi - [[ -z "$(find "$retry_root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$phase/$signal_name retry left runtime" - pass "$phase/$signal_name leaves registration clean and retryable" -} -for registration_phase in before-ready accepted-before-release release-validated; do - run_registration_signal_case "$registration_phase" HUP 129 - run_registration_signal_case "$registration_phase" INT 130 - run_registration_signal_case "$registration_phase" TERM 143 -done - -( - sleep 60 & reused_pid=$! - replacement_start=$(get_process_start_time "$reused_pid") - expected_start=$((replacement_start + 1)) - eval "$(declare -f collect_owned_session_records | sed '1s/collect_owned_session_records/original_collect_owned_session_records/')" - collect_owned_session_records() { fail "PID reuse path attempted to inspect or terminate replacement session"; } - terminate_managed_session "$reused_pid" "$expected_start" "$reused_pid" "$reused_pid" - test_process_identity_exists "$reused_pid" "$replacement_start" || fail "PID reuse path touched replacement process" - kill -TERM "$reused_pid"; wait "$reused_pid" 2>/dev/null || true - pass "managed leader PID reuse is treated as original session gone" -) - -cat > "$TEST_DIR/failed-reap-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" root="$2" mode="$3" -mkdir -p -m 0700 "$root/runtime" "$root/capture" -export TMPDIR="$root/runtime" -source "$script" -initialize_runtime -setsid bash -c 'trap "" HUP INT TERM; while :; do :; done' & leader=$! -for _ in {1..100}; do - leader_start=$(get_process_start_time "$leader" 2>/dev/null || true) - leader_pgid=$(get_process_group_id "$leader" 2>/dev/null || true) - leader_sid=$(get_process_session_id "$leader" 2>/dev/null || true) - [[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] && break - sleep 0.01 -done -[[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || exit 90 -printf '%s\n%s\n%s\n%s\n' "$leader" "$leader_start" "$leader_pgid" "$leader_sid" > "$root/capture/leader" -WORKER_TRANSFER_PID=$leader -WORKER_TRANSFER_START=$leader_start -WORKER_TRANSFER_PGID=$leader_pgid -WORKER_TRANSFER_SID=$leader_sid -publish_worker_session_state active -state_file=$WORKER_SESSION_STATE_FILE -eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" -terminate_managed_session() { return 1; } -start_ns=$(date +%s%N) -rc=0 -terminate_worker_transfer || rc=$? -elapsed_ns=$(( $(date +%s%N) - start_ns )) -[[ "$rc" != 0 && "$elapsed_ns" -lt 2000000000 ]] || exit 91 -[[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && - "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && - "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || exit 92 -read_worker_session_state "$state_file" -[[ "$SESSION_STATE" == cleanup_failed ]] || exit 93 -next_rc=0 -run_managed_command "$TEMP_DIR/forbidden-next-command" true || next_rc=$? -[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-next-command" ]] || exit 94 -printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\nstate_file=%s\n' "$rc" "$elapsed_ns" "$next_rc" "$state_file" > "$root/capture/result" -: > "$root/capture/returned" -if [[ "$mode" == hold ]]; then - while [[ ! -e "$root/capture/release-worker" ]]; do sleep 0.01; done - for _ in {1..400}; do - reap_status=0 - reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" || reap_status=$? - case "$reap_status" in - 0|"$MANAGED_PROCESS_REUSED_STATUS") break ;; - *) sleep 0.01 ;; - esac - done - (( reap_status == 0 || reap_status == MANAGED_PROCESS_REUSED_STATUS )) || exit 95 - : > "$root/capture/reaped" - for _ in {1..400}; do - [[ ! -e "$state_file" && ! -L "$state_file" ]] && break - sleep 0.01 - done - [[ ! -e "$state_file" && ! -L "$state_file" ]] || exit 96 - terminate_managed_session() { original_terminate_managed_session "$@"; } -else - for _ in {1..400}; do [[ -e "$root/capture/release-recovery" ]] && break; sleep 0.01; done - [[ -e "$root/capture/release-recovery" ]] || exit 97 - terminate_managed_session() { original_terminate_managed_session "$@"; } -fi -cleanup_runtime -CHILD -chmod 0700 "$TEST_DIR/failed-reap-child.sh" - -run_failed_reap_case() { - local run="$1" root="$TEST_DIR/failed-reap-$1" child watchdog rc=0 - bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" recover > "$root.log" 2>&1 & child=$! - test_watchdog_process 3 "$root.watchdog-fired" "$child" & watchdog=$! - for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done - [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "failed reap run $run did not return before watchdog"; } - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root.watchdog-fired" ]] || fail "failed reap run $run fired watchdog" - : > "$root/capture/release-recovery" - wait "$child" || rc=$? - [[ "$rc" == 0 ]] || cat "$root.log" >&2 2>/dev/null || true - assert_eq 0 "$rc" "failed managed-session reap run $run returns bounded" - [[ -z "$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null)" ]] || fail "failed reap run $run left worker-session state" -} -for failed_reap_run in $(seq 1 20); do run_failed_reap_case "$failed_reap_run"; done -pass "failed managed-session reap remains bounded for 20 deterministic runs" - -( - trap - EXIT HUP INT TERM - root="$TEST_DIR/failed-reap-parent"; mkdir -p "$root/capture"; chmod 0700 "$root" "$root/capture" - bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" hold > "$root.log" 2>&1 & worker=$! - worker_start=$(wait_test_process_start "$worker") - for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done - [[ -e "$root/capture/returned" ]] || fail "parent fallback worker did not return bounded" - mapfile -t leader_fields < "$root/capture/leader" - leader=${leader_fields[0]}; leader_start=${leader_fields[1]}; leader_sid=${leader_fields[3]} - state_file=$(sed -n 's/^state_file=//p' "$root/capture/result") - TEMP_DIR=$(dirname "$state_file") - ACTIVE_WORKERS=(); ACTIVE_WORKER_STATE_FILES=(); ACTIVE_WORKER_STATE_STARTS=() - BATCH_WORKER_FAILED=false - BATCH_WORKER_ERROR=false - MAX_PARALLEL=1 - ACTIVE_WORKERS[$worker]=$worker_start - ACTIVE_WORKER_STATE_FILES[$worker]=$state_file - ACTIVE_WORKER_STATE_STARTS[$worker]=$worker_start - : > "$root/capture/release-worker" - wait_for_all_workers - assert_eq 0 "${#ACTIVE_WORKERS[@]}" "scheduler parent fallback reaps active worker" - assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "scheduler parent fallback removes cleanup_failed state" - wait_test_process_identity_gone "$leader" "$leader_start" || fail "parent fallback left same-identity leader" - session_pids=(); session_starts=(); session_ppids=(); session_pgids=(); session_states=() - collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states - assert_eq 0 "${#session_pids[@]}" "parent fallback leaves no managed SID member" - [[ ! -e "$state_file" && -e "$root/capture/reaped" ]] || fail "parent fallback evidence incomplete" - pass "parent fallback cleans a live failed session without PPID 1 zombie" -) - -( - trap - EXIT HUP INT TERM - sleep 60 & replacement=$! - replacement_start=$(wait_test_process_start "$replacement") - wait_called=false - wait() { wait_called=true; return 1; } - reap_status=0 - reap_direct_managed_leader "$replacement" "$((replacement_start + 1))" "$replacement" "$replacement" || reap_status=$? - assert_eq "$MANAGED_PROCESS_REUSED_STATUS" "$reap_status" "direct leader PID reuse returns safe status" - [[ "$wait_called" == false ]] || fail "direct leader PID reuse waited on replacement process" - test_process_identity_exists "$replacement" "$replacement_start" || fail "direct leader PID reuse touched replacement process" - kill -TERM "$replacement"; builtin wait "$replacement" 2>/dev/null || true - pass "direct leader PID reuse sends no signal and performs no wait" -) - -( - trap - EXIT HUP INT TERM - leader=424242; leader_start=777777; wait_called=false; after_wait=false - collect_owned_session_records() { - local -n pids_ref="$2" starts_ref="$3" ppids_ref="$4" pgids_ref="$5" states_ref="$6" - pids_ref=(); starts_ref=(); ppids_ref=(); pgids_ref=(); states_ref=() - [[ "$after_wait" == false ]] || return 0 - pids_ref=("$leader"); starts_ref[$leader]=$leader_start; ppids_ref[$leader]=$BASHPID - pgids_ref[$leader]=$leader; states_ref[$leader]=Z - } - read_process_record_for_session_scan() { - [[ "$after_wait" == false ]] || return 2 - PROC_UID=$EUID; PROC_PPID=$BASHPID; PROC_START=$leader_start - PROC_STATE=Z; PROC_PGID=$leader; PROC_SID=$leader - } - wait() { wait_called=true; after_wait=true; return 0; } - reap_direct_managed_leader "$leader" "$leader_start" "$leader" "$leader" - [[ "$wait_called" == true ]] || fail "same-identity leader zombie was not wait/reaped" - pass "same-identity direct leader zombie is immediately wait/reaped" -) - -cat > "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" root="$2" -mkdir -p -m 0700 "$root/runtime" "$root/capture" -export TMPDIR="$root/runtime" -source "$script" -initialize_runtime -TOTAL_TIMEOUT=1 -TIMEOUT_KILL_AFTER=1 -eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" -failures=1 -terminate_managed_session() { - if (( failures > 0 )); then ((failures -= 1)); return 1; fi - original_terminate_managed_session "$@" -} -start_ns=$(date +%s%N) -rc=0 -run_managed_command "$TEMP_DIR/hard-output" timeout 1 bash -c \ - 'trap "" HUP INT TERM; bash -c '\''trap "" HUP INT TERM; while :; do kill -STOP "$$"; done'\'' & wait "$!"' || rc=$? -elapsed_ns=$(( $(date +%s%N) - start_ns )) -[[ "$rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && "$elapsed_ns" -lt 5000000000 ]] || exit 90 -[[ "$WORKER_TRANSFER_CLEANUP_FAILED" == true && -n "$WORKER_TRANSFER_PID" && -f "$WORKER_SESSION_STATE_FILE" ]] || exit 91 -read_worker_session_state "$WORKER_SESSION_STATE_FILE" -[[ "$SESSION_STATE" == cleanup_failed ]] || exit 92 -next_rc=0 -run_managed_command "$TEMP_DIR/forbidden-hard-next" true || next_rc=$? -[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-hard-next" ]] || exit 93 -printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\n' "$rc" "$elapsed_ns" "$next_rc" > "$root/capture/result" -: > "$root/capture/returned" -terminate_worker_transfer -cleanup_runtime -CHILD -chmod 0700 "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" -( - trap - EXIT HUP INT TERM - root="$TEST_DIR/hard-timeout-cleanup-failure" - bash "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" "$SCRIPT" "$root" > "$root.log" 2>&1 & child=$! - test_watchdog_process 5 "$root.watchdog-fired" "$child" & watchdog=$! - for _ in {1..120}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done - [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "hard timeout cleanup failure did not return bounded"; } - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root.watchdog-fired" ]] || fail "hard timeout cleanup failure fired watchdog" - rc=0; wait "$child" || rc=$? - assert_eq 0 "$rc" "hard timeout cleanup failure recovers fixture" - grep -Fxq "rc=$MANAGED_CLEANUP_FAILURE_STATUS" "$root/capture/result" || fail "hard timeout cleanup failure did not propagate 125" - grep -Fxq "next_rc=$MANAGED_CLEANUP_FAILURE_STATUS" "$root/capture/result" || fail "hard timeout cleanup failure allowed next command" - [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "hard timeout cleanup failure left runtime" - pass "hard timeout cleanup failure is bounded and blocks subsequent command" -) - -write_test_worker_session_manifest() { - local file="$1" state="$2" worker_pid="$3" worker_start="$4" leader_pid="$5" leader_start="$6" pgid="$7" sid="$8" - printf 'version=1\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\n' \ - "$state" "$worker_pid" "$worker_start" "$leader_pid" "$leader_start" "$pgid" "$sid" > "$file" - chmod 0600 "$file" -} - - -assert_retained_runtime_trusted() { - local path="$1" expected_dev="$2" expected_inode="$3" metadata="" dev inode owner gid mode trusted_gid - [[ -d "$path" && ! -L "$path" ]] || fail "retained runtime is not a regular directory: $path" - metadata=$(stat -Lc '%d:%i:%u:%g:%a' -- "$path") || fail "cannot stat retained runtime: $path" - IFS=: read -r dev inode owner gid mode <<< "$metadata" - trusted_gid=$(current_gid) - [[ "$dev" == "$expected_dev" && "$inode" == "$expected_inode" && - "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 700 ]] || - fail "retained runtime trust mismatch: $metadata" -} - -assert_retained_state_trusted() { - local file="$1" metadata="" owner gid mode trusted_gid - [[ -f "$file" && ! -L "$file" ]] || fail "retained worker state is not a regular file: $file" - metadata=$(stat -Lc '%u:%g:%a' -- "$file") || fail "cannot stat retained worker state: $file" - IFS=: read -r owner gid mode <<< "$metadata" - trusted_gid=$(current_gid) - [[ "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 600 ]] || - fail "retained worker state trust mismatch: $metadata" -} - -run_preserved_cleanup_retry_case() ( - local run="$1" root runtime runtime_dev runtime_inode credential leader leader_start leader_pgid leader_sid state_file - local owner watchdog rc=0 metadata="" state_status=0 - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - root="$TEST_DIR/runtime-evidence-retry-$run"; setup_fixture "$root" - runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE - credential="$runtime/private-key" - printf 'fixture-credential-do-not-print\n' > "$credential"; chmod 0600 "$credential" - RUNTIME_KEY_FILE=$credential - - setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! - leader_start=$(wait_test_process_start "$leader") - for _ in {1..100}; do - read_process_record "$leader" || true - [[ "$PROC_START" == "$leader_start" && "$PROC_STATE" == T ]] && break - sleep 0.01 - done - read_process_record "$leader" - leader_pgid=$PROC_PGID; leader_sid=$PROC_SID - [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "retry run $run leader identity invalid" - WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start - WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid - publish_worker_session_state active - state_file=$WORKER_SESSION_STATE_FILE - - eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" - terminate_managed_session() { return 1; } - owner=$BASHPID - test_watchdog_process 3 "$root/watchdog-fired" "$owner" & watchdog=$! - cleanup_runtime > "$root/capture/first-cleanup.log" 2>&1 || rc=$? - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - - (( rc != 0 )) || fail "retry run $run first cleanup unexpectedly succeeded" - [[ ! -e "$root/watchdog-fired" ]] || fail "retry run $run watchdog fired" - [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" ]] || - fail "retry run $run lost runtime identity" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - assert_retained_state_trusted "$state_file" - metadata=$(stat -Lc '%u:%g:%a' -- "$credential") - [[ "$metadata" == "$EUID:$(current_gid):600" ]] || fail "retry run $run credential permissions changed: $metadata" - grep -Fq 'fixture-credential-do-not-print' "$root/capture/first-cleanup.log" && fail "retry run $run printed credential content" - state_status=0; read_worker_session_state "$state_file" || state_status=$? - assert_eq 0 "$state_status" "retry run $run retained state parses" - [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && - "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || - fail "retry run $run retained state identity mismatch" - [[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && - "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && - "$WORKER_TRANSFER_CLEANUP_FAILED" == true && "$WORKER_SESSION_STATE_FILE" == "$state_file" && - "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run lost cleanup globals" - test_process_identity_exists "$leader" "$leader_start" || fail "retry run $run lost live leader tracking" - read_process_record "$leader" - [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "retry run $run orphaned or zombified leader" - - eval "$(declare -f original_terminate_managed_session | sed '1s/original_terminate_managed_session/terminate_managed_session/')" - cleanup_runtime - wait_test_process_identity_gone "$leader" "$leader_start" || fail "retry run $run left leader identity" - collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states - assert_eq 0 "${#session_pids[@]}" "retry run $run clears managed SID" - [[ ! -e "$state_file" && ! -L "$state_file" && ! -e "$runtime" && ! -L "$runtime" ]] || - fail "retry run $run left state or runtime" - runtime_lifecycle_globals_resolved || fail "retry run $run left lifecycle globals" - [[ "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run runtime state not reset" - pass "failed cleanup evidence retry run $run preserves then clears runtime" -) - -runtime_evidence_stress=${PUSH_RUNTIME_EVIDENCE_STRESS:-20} -for runtime_evidence_run in $(seq 1 "$runtime_evidence_stress"); do - run_preserved_cleanup_retry_case "$runtime_evidence_run" -done -pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" - -( - local_cleanup_root="$TEST_DIR/active-cleanup-evidence"; setup_fixture "$local_cleanup_root" - runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE - sleep 300 & worker=$!; worker_start=$(wait_test_process_start "$worker") - sleep 300 & replacement=$!; replacement_start=$(wait_test_process_start "$replacement") - state_file="$runtime/worker-session.$worker.state" - write_test_worker_session_manifest "$state_file" cleanup_failed "$worker" "$worker_start" \ - "$replacement" "$((replacement_start + 1))" "$replacement" "$replacement" - ACTIVE_WORKERS[$worker]=$worker_start - ACTIVE_WORKER_STATE_FILES[$worker]=$state_file - ACTIVE_WORKER_STATE_STARTS[$worker]=$worker_start - eval "$(declare -f terminate_active_workers | sed '1s/terminate_active_workers/original_terminate_active_workers/')" - terminate_active_workers() { return 1; } - rc=0; cleanup_runtime > "$local_cleanup_root/capture/first-cleanup.log" 2>&1 || rc=$? - (( rc != 0 )) || fail "active cleanup evidence first call unexpectedly succeeded" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - assert_retained_state_trusted "$state_file" - [[ -n "${ACTIVE_WORKERS[$worker]:-}" && "${ACTIVE_WORKER_STATE_FILES[$worker]:-}" == "$state_file" && - "${ACTIVE_WORKER_STATE_STARTS[$worker]:-}" == "$worker_start" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || - fail "active cleanup evidence maps were discarded" - test_process_identity_exists "$worker" "$worker_start" || fail "active cleanup evidence worker disappeared" - eval "$(declare -f original_terminate_active_workers | sed '1s/original_terminate_active_workers/terminate_active_workers/')" - cleanup_runtime - wait_test_process_identity_gone "$worker" "$worker_start" || fail "active cleanup retry left worker" - [[ ! -e "$state_file" && ! -e "$runtime" && ${#ACTIVE_WORKERS[@]} == 0 && - ${#ACTIVE_WORKER_STATE_FILES[@]} == 0 && ${#ACTIVE_WORKER_STATE_STARTS[@]} == 0 ]] || - fail "active cleanup retry left runtime, state, or maps" - test_process_identity_exists "$replacement" "$replacement_start" || fail "active cleanup retry touched PID-reuse replacement" - kill -TERM "$replacement"; wait "$replacement" 2>/dev/null || true - pass "active cleanup failure preserves evidence and retry clears it" -) - -run_runtime_trust_retention_case() ( - local kind="$1" root runtime runtime_dev runtime_inode original_path replacement_path rc=0 - root="$TEST_DIR/runtime-trust-$kind"; setup_fixture "$root" - runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE - case "$kind" in - symlink) - original_path="$runtime.original"; replacement_path="$runtime.replacement" - mv "$runtime" "$original_path"; mkdir -m 0700 "$replacement_path"; : > "$replacement_path/marker" - ln -s "$replacement_path" "$runtime" - ;; - wrong-mode) chmod 0755 "$runtime" ;; - wrong-owner|wrong-gid) - eval "$(declare -f stat | sed '1s/stat/original_stat/')" 2>/dev/null || true - stat() { - local last="${!#}" metadata dev inode owner gid mode - if [[ "$last" == "$runtime" && "$1" == -Lc && "$2" == '%d:%i:%u:%g:%a' ]]; then - metadata=$(command stat -Lc '%d:%i:%u:%g:%a' -- "$runtime") || return 1 - IFS=: read -r dev inode owner gid mode <<< "$metadata" - [[ "$kind" == wrong-owner ]] && owner=$((owner + 1)) || gid=$((gid + 1)) - printf '%s:%s:%s:%s:%s\n' "$dev" "$inode" "$owner" "$gid" "$mode" - else - command stat "$@" - fi - } - ;; - *) fail "unknown runtime trust case: $kind" ;; - esac - cleanup_runtime > "$root/capture/cleanup.log" 2>&1 || rc=$? - (( rc != 0 )) || fail "$kind untrusted runtime unexpectedly deleted" - [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" && - "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "$kind untrusted runtime lost path identity" - case "$kind" in - symlink) - [[ -L "$runtime" && -f "$replacement_path/marker" && -d "$original_path" ]] || fail "symlink replacement was modified" - rm -f "$runtime"; rmdir "$replacement_path" 2>/dev/null || { rm -f "$replacement_path/marker"; rmdir "$replacement_path"; } - mv "$original_path" "$runtime" - ;; - wrong-mode) chmod 0700 "$runtime" ;; - wrong-owner|wrong-gid) - unset -f stat - ;; - esac - cleanup_runtime - [[ ! -e "$runtime" && ! -L "$runtime" ]] || fail "$kind trusted retry left runtime" - pass "$kind runtime trust failure preserves evidence and replacement" -) -for runtime_trust_kind in symlink wrong-owner wrong-gid wrong-mode; do - run_runtime_trust_retention_case "$runtime_trust_kind" -done - -for final_residue_kind in worker-session registration-ready registration-stage; do - ( - root="$TEST_DIR/runtime-final-proof-$final_residue_kind"; setup_fixture "$root" - runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE - case "$final_residue_kind" in - worker-session) residue="$runtime/worker-session.99999.state" ;; - registration-ready) residue="$runtime/worker-registration.99999.ready" ;; - registration-stage) residue="$runtime/.worker-registration.99999.ready.token.stage" ;; - esac - eval "$(declare -f terminate_active_workers | sed '1s/terminate_active_workers/original_terminate_active_workers/')" - terminate_active_workers() { - original_terminate_active_workers || return 1 - printf 'retained-lifecycle-evidence\n' > "$residue"; chmod 0600 "$residue" - } - rc=0; cleanup_runtime > "$root/capture/cleanup.log" 2>&1 || rc=$? - (( rc != 0 )) || fail "$final_residue_kind final proof unexpectedly deleted runtime" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - [[ -f "$residue" && ! -L "$residue" && "$TEMP_DIR" == "$runtime" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || - fail "$final_residue_kind final proof discarded evidence" - eval "$(declare -f original_terminate_active_workers | sed '1s/original_terminate_active_workers/terminate_active_workers/')" - rm -f -- "$residue" - cleanup_runtime - [[ ! -e "$runtime" ]] || fail "$final_residue_kind final proof retry left runtime" - pass "$final_residue_kind blocks runtime deletion until evidence is cleared" - ) -done - -( - root="$TEST_DIR/runtime-final-proof-success"; setup_fixture "$root"; runtime=$TEMP_DIR - cleanup_runtime - [[ ! -e "$runtime" && -z "$TEMP_DIR" && "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || - fail "clean runtime final proof did not clear state" - pass "clean runtime final proof preserves normal success path" -) - - -cat > "$TEST_DIR/runtime-retention-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" root="$2" leader="$3" leader_start="$4" leader_pgid="$5" leader_sid="$6" -mkdir -p "$root/runtime" "$root/capture" -chmod 0700 "$root" "$root/runtime" "$root/capture" -TMPDIR="$root/runtime" -# shellcheck source=../tools/push.sh -source "$script" -initialize_runtime -credential="$TEMP_DIR/private-key" -printf 'signal-fixture-credential-do-not-print\n' > "$credential" -chmod 0600 "$credential" -RUNTIME_KEY_FILE=$credential -WORKER_TRANSFER_PID=$leader -WORKER_TRANSFER_START=$leader_start -WORKER_TRANSFER_PGID=$leader_pgid -WORKER_TRANSFER_SID=$leader_sid -publish_worker_session_state active -state_file=$WORKER_SESSION_STATE_FILE -terminate_managed_session() { return 1; } -install_runtime_traps -worker_start=$(get_process_start_time "$BASHPID") -stage="$root/capture/ready.stage" -{ - printf 'target_pid=%s\n' "$BASHPID" - printf 'target_start=%s\n' "$worker_start" - printf 'runtime=%s\n' "$TEMP_DIR" - printf 'runtime_dev=%s\n' "$TEMP_DIR_DEV" - printf 'runtime_inode=%s\n' "$TEMP_DIR_INODE" - printf 'state_file=%s\n' "$state_file" - printf 'credential=%s\n' "$credential" -} > "$stage" -chmod 0600 "$stage" -mv -fT "$stage" "$root/capture/ready" -while :; do :; done -CHILD -chmod 0700 "$TEST_DIR/runtime-retention-signal-child.sh" - -run_runtime_retention_signal_case() ( - local signal_name="$1" expected="$2" root leader="" leader_start="" leader_pgid="" leader_sid="" target="" target_pid="" target_start="" - local runtime runtime_dev runtime_inode state_file credential watchdog="" rc=0 line key value state_status=0 reap_status=0 - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - cleanup_runtime_retention_signal_fixture() { - if [[ -n "$target" ]]; then kill -KILL "$target" 2>/dev/null || true; wait "$target" 2>/dev/null || true; fi - if [[ -n "$leader" ]]; then kill -CONT "$leader" 2>/dev/null || true; kill -KILL "$leader" 2>/dev/null || true; wait "$leader" 2>/dev/null || true; fi - if [[ -n "$watchdog" ]]; then kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true; fi - } - trap cleanup_runtime_retention_signal_fixture EXIT - root="$TEST_DIR/runtime-retention-signal-$signal_name"; CURRENT_FIXTURE_ROOT=$root - mkdir -m 0700 "$root" - setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! - leader_start=$(wait_test_process_start "$leader") - for _ in {1..100}; do read_process_record "$leader" || true; [[ "$PROC_STATE" == T ]] && break; sleep 0.01; done - read_process_record "$leader"; leader_pgid=$PROC_PGID; leader_sid=$PROC_SID - [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "$signal_name retention leader identity invalid" - - ( - exec env --default-signal=HUP,INT,TERM bash "$TEST_DIR/runtime-retention-signal-child.sh" \ - "$SCRIPT" "$root" "$leader" "$leader_start" "$leader_pgid" "$leader_sid" - ) > "$root/output.log" 2>&1 & target=$! - for _ in {1..300}; do [[ -f "$root/capture/ready" ]] && break; sleep 0.02; done - [[ -f "$root/capture/ready" ]] || { cat "$root/output.log" >&2 || true; fail "$signal_name retention child not ready"; } - while IFS= read -r line; do - key=${line%%=*}; value=${line#*=} - case "$key" in - target_pid) target_pid=$value ;; - target_start) target_start=$value ;; - runtime) runtime=$value ;; - runtime_dev) runtime_dev=$value ;; - runtime_inode) runtime_inode=$value ;; - state_file) state_file=$value ;; - credential) credential=$value ;; - esac - done < "$root/capture/ready" - [[ "$target_pid" == "$target" && "$target_start" =~ ^[1-9][0-9]*$ && - "$runtime_dev" =~ ^[0-9]+$ && "$runtime_inode" =~ ^[1-9][0-9]*$ ]] || - fail "$signal_name retention ready metadata malformed" - job_is_active "$target" || fail "$signal_name retention target job missing" - test_watchdog_process 5 "$root/watchdog-fired" "$target" & watchdog=$! - kill "-$signal_name" "$target" - wait "$target" || rc=$? - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root/watchdog-fired" ]] || fail "$signal_name retention watchdog fired" - assert_eq "$expected" "$rc" "$signal_name failed cleanup preserves signal status" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - assert_retained_state_trusted "$state_file" - [[ "$(stat -Lc '%u:%g:%a' -- "$credential")" == "$EUID:$(current_gid):600" ]] || - fail "$signal_name retained credential permissions changed" - grep -Fq "$runtime" "$root/output.log" || fail "$signal_name failure output omitted runtime path" - grep -Fq 'signal-fixture-credential-do-not-print' "$root/output.log" && fail "$signal_name failure output leaked credential" - TEMP_DIR=$runtime - state_status=0; read_worker_session_state "$state_file" || state_status=$? - assert_eq 0 "$state_status" "$signal_name retained cleanup_failed state parses" - [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && - "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || - fail "$signal_name retained cleanup_failed identity mismatch" - test_process_identity_exists "$leader" "$leader_start" || fail "$signal_name failed cleanup lost leader tracking" - read_process_record "$leader" - [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "$signal_name teardown leader lost safe parent" - - terminate_managed_session "$leader" "$leader_start" "$leader_pgid" "$leader_sid" true - reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" "$BASHPID" || reap_status=$? - assert_eq 0 "$reap_status" "$signal_name teardown safely reaps leader" - collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states - assert_eq 0 "${#session_pids[@]}" "$signal_name teardown clears managed SID" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - rm -rf -- "$runtime" - TEMP_DIR="" - leader=""; target=""; watchdog="" - trap - EXIT - pass "$signal_name cleanup failure retains trusted runtime and state" -) -run_runtime_retention_signal_case HUP 129 -run_runtime_retention_signal_case INT 130 -run_runtime_retention_signal_case TERM 143 - -( - root="$TEST_DIR/session-dual-publish"; setup_fixture "$root" - : > "$root/capture/terminate-calls" - terminate_managed_session() { printf '%s:%s:%s:%s:%s\n' "$@" >> "$root/capture/terminate-calls"; return 0; } - worker_session_state_publish_hook() { - local phase="$1" state="$2" - if [[ "$phase" == stage-ready && "$state" == cleanup_failed ]]; then - : > "$root/capture/stage.$BASHPID" - while [[ ! -e "$root/capture/release" ]]; do sleep 0.01; done - elif [[ "$phase" == after-rename && "$state" == cleanup_failed ]]; then - : > "$root/capture/renamed.$BASHPID" - fi - } - session_publisher() { - WORKER_TRANSFER_PID=$BASHPID; WORKER_TRANSFER_START=11 - WORKER_TRANSFER_PGID=$BASHPID; WORKER_TRANSFER_SID=$BASHPID - publish_worker_session_state active - publish_worker_session_state cleanup_failed - } - session_publisher & first=$!; session_publisher & second=$! - first_start=$(get_process_start_time "$first"); second_start=$(get_process_start_time "$second") - ACTIVE_WORKERS[$first]=$first_start; ACTIVE_WORKERS[$second]=$second_start - ACTIVE_WORKER_STATE_FILES[$first]="$TEMP_DIR/worker-session.$first.state" - ACTIVE_WORKER_STATE_FILES[$second]="$TEMP_DIR/worker-session.$second.state" - ACTIVE_WORKER_STATE_STARTS[$first]=$first_start; ACTIVE_WORKER_STATE_STARTS[$second]=$second_start - for _ in {1..300}; do [[ -e "$root/capture/stage.$first" && -e "$root/capture/stage.$second" ]] && break; sleep 0.01; done - [[ -e "$root/capture/stage.$first" && -e "$root/capture/stage.$second" ]] || fail "dual publishers did not reach pre-rename hook" - cleanup_published_worker_sessions - assert_eq 2 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback waits while both workers can still publish" - : > "$root/capture/release" - wait "$first"; wait "$second" - [[ -e "$root/capture/renamed.$first" && -e "$root/capture/renamed.$second" ]] || fail "dual publishers did not complete rename" - prune_active_workers - cleanup_published_worker_sessions - assert_eq 2 "$(wc -l < "$root/capture/terminate-calls")" "fallback processes both stable cleanup_failed manifests" - assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "dual fallback clears both state maps" - assert_eq 0 "${#ACTIVE_WORKER_STATE_STARTS[@]}" "dual fallback clears both state identities" - runtime_has_published_worker_state && fail "dual fallback left worker session state" - cleanup_runtime; assert_no_registration_residue "$root" - pass "dual worker state transitions stabilize after wait/reap" -) - -run_transient_state_case() ( - kind="$1"; root="$TEST_DIR/session-transient-$kind"; setup_fixture "$root" - worker=11111; start=22222; file="$TEMP_DIR/worker-session.$worker.state" - write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 - ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start - : > "$root/capture/terminate-calls" - terminate_managed_session() { printf 'call\n' >> "$root/capture/terminate-calls"; return 0; } - failures=2 - case "$kind" in - inode) - worker_session_state_read_hook() { - if [[ "$1" == after-open && "$failures" -gt 0 ]]; then - next="$file.next.$failures" - write_test_worker_session_manifest "$next" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 - mv -fT "$next" "$file"; ((failures -= 1)) - fi - } - ;; - open) - eval "$(declare -f open_worker_session_state_fd | sed '1s/open_worker_session_state_fd/original_open_worker_session_state_fd/')" - open_worker_session_state_fd() { - if (( failures > 0 )); then ((failures -= 1)); return "$WORKER_SESSION_STATE_TRANSIENT_STATUS"; fi - original_open_worker_session_state_fd "$@" - } - ;; - stat) - eval "$(declare -f worker_session_state_path_metadata | sed '1s/worker_session_state_path_metadata/original_worker_session_state_path_metadata/')" - worker_session_state_path_metadata() { - if (( failures > 0 )); then ((failures -= 1)); return "$WORKER_SESSION_STATE_TRANSIENT_STATUS"; fi - original_worker_session_state_path_metadata "$@" - } - ;; - esac - cleanup_published_worker_sessions - assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "$kind transient failure clears state map after stabilization" - assert_eq 1 "$(wc -l < "$root/capture/terminate-calls")" "$kind transient failure performs one trusted session cleanup" - [[ ! -e "$file" ]] || fail "$kind transient failure left state file" - cleanup_runtime; pass "$kind transient worker state failure is bounded and recoverable" -) -run_transient_state_case inode -run_transient_state_case open -run_transient_state_case stat - -( - root="$TEST_DIR/session-not-reaped"; setup_fixture "$root" - sleep 300 & worker=$!; start=$(get_process_start_time "$worker") - file="$TEMP_DIR/worker-session.$worker.state" - write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 - ACTIVE_WORKERS[$worker]=$start; ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start - : > "$root/capture/terminate-calls" - terminate_managed_session() { printf call >> "$root/capture/terminate-calls"; return 0; } - cleanup_published_worker_sessions - assert_eq 1 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback retains state while worker is not reaped" - assert_eq 0 "$(wc -c < "$root/capture/terminate-calls")" "fallback performs no session cleanup before reap" - kill -TERM "$worker"; wait "$worker" 2>/dev/null || true; prune_active_workers - cleanup_published_worker_sessions - assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback succeeds after worker reap" - cleanup_runtime; pass "worker reap gates final fallback" -) - -run_permanent_state_case() ( - kind="$1"; root="$TEST_DIR/session-permanent-$kind"; setup_fixture "$root" - worker=11111; start=22222; file="$TEMP_DIR/worker-session.$worker.state" - write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 - ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start - : > "$root/capture/terminate-calls" - terminate_managed_session() { printf call >> "$root/capture/terminate-calls"; return 0; } - case "$kind" in - symlink) mv "$file" "$file.real"; ln -s "$file.real" "$file" ;; - wrong-mode) chmod 0644 "$file" ;; - wrong-owner|wrong-gid) - stat() { - last=${!#} - if [[ "$last" == "$file" ]]; then - raw=$(command stat -Lc '%d:%i:%u:%g:%a:%s' -- "$file") - IFS=: read -r dev ino owner gid mode size <<< "$raw" - [[ "$kind" == wrong-owner ]] && owner=$((EUID + 1)) || gid=$((gid + 1)) - printf '%s:%s:%s:%s:%s:%s\n' "$dev" "$ino" "$owner" "$gid" "$mode" "$size" - else command stat "$@"; fi - } - ;; - malformed) printf malformed > "$file" ;; - duplicate) printf 'worker_pid=11111\n' >> "$file" ;; - unknown) printf 'unexpected=value\n' >> "$file" ;; - missing) sed -i '/^sid=/d' "$file" ;; - pid-mismatch) sed -i 's/^worker_pid=.*/worker_pid=99999/' "$file" ;; - stable-tamper) - worker_session_state_read_hook() { [[ "$1" != after-open ]] || printf tampered > "$file"; } - ;; - esac - rc=0; cleanup_published_worker_sessions || rc=$? - (( rc != 0 )) || fail "$kind permanent state unexpectedly succeeded" - assert_eq 1 "${#ACTIVE_WORKER_STATE_FILES[@]}" "$kind permanent state preserves map evidence" - assert_eq 0 "$(wc -c < "$root/capture/terminate-calls")" "$kind permanent state executes no session cleanup" - [[ -e "$file" || -L "$file" ]] || fail "$kind permanent state deleted evidence" - unset 'ACTIVE_WORKER_STATE_FILES[$worker]' 'ACTIVE_WORKER_STATE_STARTS[$worker]' - command rm -f -- "$file" "$file.real" 2>/dev/null || true - cleanup_runtime; pass "$kind permanent worker state remains rejected" -) -for permanent_state in symlink wrong-mode wrong-owner wrong-gid malformed duplicate unknown missing pid-mismatch stable-tamper; do - run_permanent_state_case "$permanent_state" -done - -run_parallel_fallback_iteration() ( - run="$1"; root="$TEST_DIR/session-parallel-$run"; setup_fixture "$root" - MAX_PARALLEL=2 - push_to_server() { - local server="$1" lock_fd - exec {lock_fd}>>"$root/capture/calls.lock"; flock -x "$lock_fd" - printf '%s\n' "$server" >> "$root/capture/calls"; : > "$root/capture/ready.$server" - flock -u "$lock_fd"; exec {lock_fd}>&- - while [[ ! -e "$root/capture/release" ]]; do sleep 0.01; done - WORKER_TRANSFER_PID=$BASHPID - WORKER_TRANSFER_START=11 - WORKER_TRANSFER_PGID=$BASHPID - WORKER_TRANSFER_SID=$BASHPID - publish_worker_session_state cleanup_failed - return "$MANAGED_CLEANUP_FAILURE_STATUS" - } - terminate_managed_session() { return 0; } - ( - deadline=$((SECONDS + 15)); status=0 - while (( SECONDS < deadline )); do - if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release"; break; fi - sleep 0.01 - done - [[ -e "$root/capture/release" ]] || { : > "$root/capture/coordinator-timeout"; : > "$root/capture/release"; status=1; } - while [[ ! -e "$root/capture/stop" ]]; do sleep 0.01; done - exit "$status" - ) & coordinator=$! - rc=0; run_server_batch source destination one.example two.example three.example four.example || rc=$? - : > "$root/capture/stop"; coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? - assert_eq 0 "$coordinator_rc" "parallel fallback run $run coordinator releases both workers" - assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "parallel fallback run $run returns lifecycle status" - assert_eq 2 "$(wc -l < "$root/capture/calls")" "parallel fallback run $run starts only first two workers" - assert_eq 0 "${#ACTIVE_WORKERS[@]}" "parallel fallback run $run reaps active workers" - assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "parallel fallback run $run clears state maps" - assert_eq 0 "${#ACTIVE_WORKER_STATE_STARTS[@]}" "parallel fallback run $run clears state identities" - [[ ! -e "$root/capture/ready.three.example" && ! -e "$root/capture/ready.four.example" ]] || fail "parallel fallback run $run launched later worker" - runtime_has_published_worker_state && fail "parallel fallback run $run left session state" - reset_batch_lifecycle_barrier; cleanup_runtime; assert_no_registration_residue "$root" -) -parallel_fallback_runs=${PUSH_WORKER_REGISTRATION_PARALLEL_STRESS:-1} -for parallel_run in $(seq 1 "$parallel_fallback_runs"); do run_parallel_fallback_iteration "$parallel_run"; done -pass "parallel fallback completed $parallel_fallback_runs deterministic run(s)" - -write_active_grace_fixture() { - local root="$1" - mkdir -m 0700 "$root" "$root/bin" "$root/runtime" "$root/ssh" "$root/pids" - printf source > "$root/source" - cat > "$root/config.conf" < "$root/bin/timeout" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/timeout.pid" -[[ ${1:-} == --kill-after=* ]] && shift -shift -"$@" & -wait "$!" -EOF - cat > "$root/bin/sshpass" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/sshpass.pid" -[[ ${1:-} == -e ]] && shift -"$@" & -wait "$!" -EOF - cat > "$root/bin/rsync" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -set -m -printf '%s\n' "$$" > "$PID_DIR/rsync.pid" -ssh fake-target & -wait "$!" -EOF - cat > "$root/bin/ssh" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/ssh.pid" -if [[ "${STATE_PUBLISH_MODE:-active}" == cleanup ]]; then - while [[ ! -e "$STATE_CLEANUP_RELEASE" ]]; do sleep 0.01; done - exit 0 -fi -sleep 300 & -printf '%s\n' "$!" > "$PID_DIR/leaf.pid" -wait "$!" -EOF - chmod 0700 "$root/bin"/* -} -cat > "$TEST_DIR/active-grace-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" -shift -source "$script" -worker_after_session_cleanup_hook() { - : > "$ACTIVE_GRACE_STARTED" - sleep 3.2 - : > "$ACTIVE_GRACE_FINISHED" -} -main "$@" -CHILD -chmod 0700 "$TEST_DIR/active-grace-child.sh" -run_active_grace_signal_case() { - local signal_name="$1" expected="$2" root - local main_pid watchdog unrelated rc=0 state_file file pid - root="$TEST_DIR/active-grace-$signal_name" - CURRENT_FIXTURE_ROOT=$root - write_active_grace_fixture "$root" - sleep 60 & unrelated=$! - ( - cd "$root" - exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ - ACTIVE_GRACE_STARTED="$root/cleanup-started" ACTIVE_GRACE_FINISHED="$root/cleanup-finished" \ - bash "$TEST_DIR/active-grace-child.sh" "$SCRIPT" "$root/source" /remote/path - ) > "$root/output.log" 2>&1 & - main_pid=$! - for _ in {1..400}; do - state_file=$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null || true) - [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] && break - sleep 0.05 - done - if [[ -z "${state_file:-}" || ! -f "$root/pids/leaf.pid" ]]; then - cat "$root/output.log" >&2 || true - kill "$main_pid" "$unrelated" 2>/dev/null || true - fail "$signal_name active grace fixture did not enter managed session" - fi - test_watchdog_process 20 "$root/watchdog-timeout" "$main_pid" & watchdog=$! - kill "-$signal_name" "$main_pid" - wait "$main_pid" || rc=$? - kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root/watchdog-timeout" ]] || fail "$signal_name active grace exceeded watchdog" - assert_eq "$expected" "$rc" "$signal_name active worker preserves main signal status" - [[ -e "$root/cleanup-started" && -e "$root/cleanup-finished" ]] || { cat "$root/output.log" >&2 || true; fail "$signal_name active worker was killed before cleanup hook completed"; } - for file in "$root/pids"/*.pid; do - [[ -f "$file" ]] || continue - pid=$(<"$file") - process_is_running "$pid" && fail "$signal_name active grace left $(basename "$file") pid=$pid" - done - [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$signal_name active grace left runtime state" - kill -0 "$unrelated" 2>/dev/null || fail "$signal_name active grace killed unrelated process" - kill "$unrelated" 2>/dev/null || true; wait "$unrelated" 2>/dev/null || true - pass "$signal_name active worker receives full managed-session cleanup grace" -} -run_active_grace_signal_case HUP 129 -run_active_grace_signal_case INT 130 -run_active_grace_signal_case TERM 143 - -cat > "$TEST_DIR/live-cleanup-failure-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" -shift -source "$script" -main_owner=$BASHPID -eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" -terminate_managed_session() { - if [[ "$BASHPID" != "$main_owner" ]]; then return 1; fi - original_terminate_managed_session "$@" -} -main "$@" -CHILD -chmod 0700 "$TEST_DIR/live-cleanup-failure-child.sh" - -run_live_cleanup_failure_signal_case() ( - trap - EXIT HUP INT TERM - local signal_name="$1" expected="$2" root main_pid main_start state_file="" saved_temp - local worker_pid="" worker_start="" leader_pid="" leader_start="" managed_sid="" watchdog="" unrelated="" unrelated_start="" rc=0 - cleanup_live_failure_fixture() { - [[ -z "$worker_pid" || -z "$worker_start" ]] || { test_process_identity_exists "$worker_pid" "$worker_start" && kill -KILL "$worker_pid" 2>/dev/null || true; } - [[ -z "$main_pid" || -z "$main_start" ]] || { test_process_identity_exists "$main_pid" "$main_start" && kill -KILL "$main_pid" 2>/dev/null || true; } - [[ -z "$unrelated" || -z "$unrelated_start" ]] || { test_process_identity_exists "$unrelated" "$unrelated_start" && kill -TERM "$unrelated" 2>/dev/null || true; } - [[ -z "$main_pid" ]] || wait "$main_pid" 2>/dev/null || true - [[ -z "$watchdog" ]] || { kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true; } - [[ -z "$unrelated" ]] || wait "$unrelated" 2>/dev/null || true - } - trap cleanup_live_failure_fixture EXIT - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - root="$TEST_DIR/live-cleanup-failure-$signal_name"; CURRENT_FIXTURE_ROOT=$root - write_active_grace_fixture "$root" - sleep 60 & unrelated=$!; unrelated_start=$(wait_test_process_start "$unrelated") - ( - cd "$root" - exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ - bash "$TEST_DIR/live-cleanup-failure-child.sh" "$SCRIPT" "$root/source" /remote/path - ) > "$root/output.log" 2>&1 & - main_pid=$!; main_start=$(wait_test_process_start "$main_pid") - for _ in {1..500}; do - state_file=$(find "$root/runtime" -type f -name 'worker-session.*.state' -print -quit 2>/dev/null || true) - [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] && break - sleep 0.05 - done - [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] || fail "$signal_name cleanup-failure fixture did not reach live session" - saved_temp=$TEMP_DIR; TEMP_DIR=$(dirname "$state_file") - read_worker_session_state "$state_file" || fail "$signal_name cleanup-failure state unreadable" - worker_pid=$SESSION_WORKER_PID; worker_start=$SESSION_WORKER_START - leader_pid=$SESSION_LEADER_PID; leader_start=$SESSION_LEADER_START; managed_sid=$SESSION_SID - TEMP_DIR=$saved_temp - wait_test_process_identity_present "$worker_pid" "$worker_start" || fail "$signal_name cleanup-failure worker identity missing" - wait_test_process_identity_present "$leader_pid" "$leader_start" || fail "$signal_name cleanup-failure leader identity missing" - test_watchdog_process 30 "$root/watchdog-fired" "$main_pid" & watchdog=$! - kill "-$signal_name" "$main_pid" - wait "$main_pid" || rc=$? - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root/watchdog-fired" ]] || fail "$signal_name cleanup failure fired watchdog" - assert_eq "$expected" "$rc" "$signal_name cleanup failure preserves main signal status" - wait_test_process_identity_gone "$worker_pid" "$worker_start" || fail "$signal_name cleanup failure left worker identity" - wait_test_process_identity_gone "$leader_pid" "$leader_start" || fail "$signal_name cleanup failure left leader identity" - collect_owned_session_records "$managed_sid" session_pids session_starts session_ppids session_pgids session_states || fail "$signal_name cleanup failure cannot verify managed SID" - assert_eq 0 "${#session_pids[@]}" "$signal_name cleanup failure leaves no managed SID member or zombie" - [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$signal_name cleanup failure left runtime or state" - test_process_identity_exists "$unrelated" "$unrelated_start" || fail "$signal_name cleanup failure killed unrelated process" - kill -TERM "$unrelated"; wait "$unrelated" 2>/dev/null || true - main_pid=""; main_start=""; watchdog=""; unrelated=""; unrelated_start="" - trap - EXIT - pass "$signal_name cleanup failure uses parent fallback without watchdog or zombie" -) -run_live_cleanup_failure_signal_case HUP 129 -run_live_cleanup_failure_signal_case INT 130 -run_live_cleanup_failure_signal_case TERM 143 - -cat > "$TEST_DIR/state-publication-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" -shift -source "$script" -main_owner=$BASHPID -eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" -terminate_managed_session() { - if [[ "$STATE_PUBLISH_MODE" == cleanup && "$BASHPID" != "$main_owner" ]]; then return 1; fi - original_terminate_managed_session "$@" -} -worker_session_state_publish_hook() { - local phase="$1" state="$2" state_file="$3" hook_stage="$4" marker_stage worker_start - if [[ "$STATE_PUBLISH_MODE" == cleanup && "$state" == active && "$phase" == state-file-assigned ]]; then - : > "$STATE_CLEANUP_RELEASE" - fi - [[ "$phase" == "$STATE_PUBLISH_PHASE" && "$state" == "$STATE_PUBLISH_STATE" ]] || return 0 - [[ ! -e "$STATE_PUBLISH_MARKER" ]] || return 0 - worker_start="" - for _ in {1..20}; do - if read_process_record "$BASHPID" && [[ "$PROC_UID" == "$EUID" && "$PROC_STATE" != Z ]]; then worker_start=$PROC_START; break; fi - sleep 0.01 - done - [[ -n "$worker_start" ]] || return 1 - marker_stage="$STATE_PUBLISH_MARKER.$BASHPID.stage" - (umask 077; set -o noclobber; : > "$marker_stage") || return 1 - printf 'phase=%s\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\nstate_file=%s\nhook_stage=%s\n' \ - "$phase" "$state" "$BASHPID" "$worker_start" "$WORKER_TRANSFER_PID" "$WORKER_TRANSFER_START" \ - "$WORKER_TRANSFER_PGID" "$WORKER_TRANSFER_SID" "$state_file" "$hook_stage" > "$marker_stage" - chmod 0600 "$marker_stage"; mv -fT "$marker_stage" "$STATE_PUBLISH_MARKER" - while :; do :; done -} -main "$@" -CHILD -chmod 0700 "$TEST_DIR/state-publication-child.sh" - -run_state_publication_signal_case() ( - local state="$1" phase="$2" signal_name="$3" expected="$4" mode root main_pid="" main_start="" worker_pid="" worker_start="" leader_pid="" leader_start="" managed_sid="" - local watchdog="" watchdog_start="" unrelated="" unrelated_start="" rc=0 line key value marker state_file hook_stage - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - cleanup_publication_fixture() { - [[ -z "$worker_pid" || -z "$worker_start" ]] || { test_process_identity_exists "$worker_pid" "$worker_start" && kill -KILL "$worker_pid" 2>/dev/null || true; } - [[ -z "$main_pid" || -z "$main_start" ]] || { test_process_identity_exists "$main_pid" "$main_start" && kill -KILL "$main_pid" 2>/dev/null || true; } - [[ -z "$watchdog" || -z "$watchdog_start" ]] || { test_process_identity_exists "$watchdog" "$watchdog_start" && kill -TERM "$watchdog" 2>/dev/null || true; } - [[ -z "$unrelated" || -z "$unrelated_start" ]] || { test_process_identity_exists "$unrelated" "$unrelated_start" && kill -TERM "$unrelated" 2>/dev/null || true; } - [[ -z "$main_pid" ]] || wait "$main_pid" 2>/dev/null || true - [[ -z "$watchdog" ]] || wait "$watchdog" 2>/dev/null || true - [[ -z "$unrelated" ]] || wait "$unrelated" 2>/dev/null || true - } - trap cleanup_publication_fixture EXIT - mode=active; [[ "$state" == cleanup_failed ]] && mode=cleanup - root="$TEST_DIR/state-publish-$state-$phase-$signal_name${5:-}"; CURRENT_FIXTURE_ROOT=$root - write_active_grace_fixture "$root" - mkdir -m 0700 "$root/capture" - marker="$root/capture/publish-marker" - sleep 60 & unrelated=$!; unrelated_start=$(wait_test_process_start "$unrelated") - ( - cd "$root" - exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ - STATE_PUBLISH_MODE="$mode" STATE_PUBLISH_STATE="$state" STATE_PUBLISH_PHASE="$phase" STATE_PUBLISH_MARKER="$marker" \ - STATE_CLEANUP_RELEASE="$root/capture/cleanup-release" bash "$TEST_DIR/state-publication-child.sh" "$SCRIPT" "$root/source" /remote/path - ) > "$root/output.log" 2>&1 & - main_pid=$!; main_start=$(wait_test_process_start "$main_pid") - for _ in {1..600}; do [[ -f "$marker" ]] && break; sleep 0.05; done - [[ -f "$marker" && ! -L "$marker" && "$(stat -Lc %a "$marker")" == 600 ]] || { cat "$root/output.log" >&2 || true; fail "$state/$phase/$signal_name did not reach publish hook"; } - while IFS= read -r line; do key=${line%%=*}; value=${line#*=}; case "$key" in worker_pid) worker_pid=$value;;worker_start) worker_start=$value;;leader_pid) leader_pid=$value;;leader_start) leader_start=$value;;sid) managed_sid=$value;;state_file) state_file=$value;;hook_stage) hook_stage=$value;;esac; done < "$marker" - [[ "$worker_pid" =~ ^[1-9][0-9]*$ && "$worker_start" =~ ^[1-9][0-9]*$ && "$managed_sid" =~ ^[1-9][0-9]*$ ]] || fail "$state/$phase/$signal_name marker identity malformed" - wait_test_process_identity_present "$worker_pid" "$worker_start" || { cat "$marker" >&2; fail "$state/$phase/$signal_name worker identity missing at hook"; } - test_watchdog_process 30 "$root/watchdog-timeout" "$main_pid" & watchdog=$!; watchdog_start=$(wait_test_process_start "$watchdog") - printf "DIAG: publication state=%s phase=%s signal=%s main=%s/%s worker=%s/%s sid=%s\n" \ - "$state" "$phase" "$signal_name" "$main_pid" "$main_start" "$worker_pid" "$worker_start" "$managed_sid" - kill "-$signal_name" "$main_pid"; wait "$main_pid" || rc=$? - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root/watchdog-timeout" ]] || fail "$state/$phase/$signal_name watchdog fired" - assert_eq "$expected" "$rc" "$state/$phase preserves $signal_name status" - wait_test_process_identity_gone "$worker_pid" "$worker_start" || fail "$state/$phase/$signal_name left worker identity" - if [[ "$leader_pid" =~ ^[1-9][0-9]*$ && "$leader_start" =~ ^[1-9][0-9]*$ ]]; then - if ! wait_test_process_identity_gone "$leader_pid" "$leader_start"; then - read_process_record "$leader_pid" || true - printf 'leader remains pid=%s expected_start=%s actual_start=%s state=%s ppid=%s pgid=%s sid=%s\n' \ - "$leader_pid" "$leader_start" "${PROC_START:-unknown}" "${PROC_STATE:-unknown}" "${PROC_PPID:-unknown}" "${PROC_PGID:-unknown}" "${PROC_SID:-unknown}" >&2 - cat "/proc/$leader_pid/stat" >&2 2>/dev/null || true - fail "$state/$phase/$signal_name left leader identity" - fi - fi - collect_owned_session_records "$managed_sid" session_pids session_starts session_ppids session_pgids session_states || fail "$state/$phase/$signal_name cannot verify managed SID" - (( ${#session_pids[@]} == 0 )) || fail "$state/$phase/$signal_name left managed SID member: ${session_pids[*]}" - [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$state/$phase/$signal_name left runtime/session/registration state" - [[ ! -e "$state_file" && ! -e "$hook_stage" && ! -L "$state_file" && ! -L "$hook_stage" ]] || fail "$state/$phase/$signal_name left publication path" - test_process_identity_exists "$unrelated" "$unrelated_start" || fail "$state/$phase/$signal_name killed unrelated process" - kill -TERM "$unrelated"; wait "$unrelated" 2>/dev/null || true - unrelated=""; unrelated_start=""; main_pid=""; main_start=""; watchdog=""; watchdog_start="" - trap - EXIT - pass "$state/$phase/$signal_name publication window cleans process tree and runtime" -) -publication_states=${PUSH_STATE_PUBLICATION_STATES:-"active cleanup_failed"} -publication_phases=${PUSH_STATE_PUBLICATION_PHASES:-"before-stage-create stage-ready after-rename state-file-assigned"} -publication_signals=${PUSH_STATE_PUBLICATION_SIGNALS:-"HUP INT TERM"} -for publication_state in $publication_states; do - for publication_phase in $publication_phases; do - for publication_signal in $publication_signals; do - case "$publication_signal" in HUP) publication_status=129 ;; INT) publication_status=130 ;; TERM) publication_status=143 ;; *) fail "unknown publication signal" ;; esac - run_state_publication_signal_case "$publication_state" "$publication_phase" "$publication_signal" "$publication_status" - done - done -done - -( - # Three bounded extra probes; preserve original matrix and stop at first failure. - for diagnostic_round in 1 2 3; do - printf "DIAG: extra HUP probe=%s/3\n" "$diagnostic_round" - run_state_publication_signal_case cleanup_failed after-rename HUP 129 "-probe-$diagnostic_round" - done - root="$TEST_DIR/normal-parallel"; setup_fixture "$root" - : > "$root/capture/current"; : > "$root/capture/max" - push_to_server() { - local server="$1" lock_fd current max group - case "$server" in one.example|two.example) group=first ;; *) group=second ;; esac - exec {lock_fd}>>"$root/capture/lock"; flock -x "$lock_fd" - current=$(<"$root/capture/current"); ((current += 1)); printf '%s' "$current" > "$root/capture/current" - max=$(<"$root/capture/max"); (( current > max )) && printf '%s' "$current" > "$root/capture/max" - printf '%s\n' "$server" >> "$root/capture/transfer-calls"; : > "$root/capture/ready.$server" - flock -u "$lock_fd"; exec {lock_fd}>&- - while [[ ! -e "$root/capture/release.$group" ]]; do sleep 0.02; done - exec {lock_fd}>>"$root/capture/lock"; flock -x "$lock_fd" - current=$(<"$root/capture/current"); ((current -= 1)); printf '%s' "$current" > "$root/capture/current" - flock -u "$lock_fd"; exec {lock_fd}>&- - record_result success "$server" - } - ( - deadline=$((SECONDS + 15)) - while (( SECONDS < deadline )); do - if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release.first"; break; fi - sleep 0.02 - done - [[ -e "$root/capture/release.first" ]] || { : > "$root/capture/coordinator-timeout"; : > "$root/capture/release.first"; : > "$root/capture/release.second"; exit 1; } - while (( SECONDS < deadline + 15 )); do - if [[ -e "$root/capture/ready.three.example" && -e "$root/capture/ready.four.example" ]]; then : > "$root/capture/release.second"; exit 0; fi - sleep 0.02 - done - : > "$root/capture/coordinator-timeout"; : > "$root/capture/release.second"; exit 1 - ) & coordinator=$! - run_server_batch source destination one.example two.example three.example four.example - coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? - assert_eq 0 "$coordinator_rc" "normal parallel coordinator releases both worker pairs" - assert_eq 2 "$(<"$root/capture/max")" "normal registration preserves MAX_PARALLEL=2" - assert_eq 4 "$(wc -l < "$root/capture/transfer-calls")" "normal registration completes four fake servers" - assert_no_registration_residue "$root"; cleanup_runtime; assert_no_registration_residue "$root" - pass "normal registration/session lifecycle cleans all state" -) -printf 'All push worker registration tests passed.\n' From c9104848ef32b0bd681fde7f5c0f8154adff15a3 Mon Sep 17 00:00:00 2001 From: LUNO Date: Sat, 24 Oct 2026 12:54:25 +0800 Subject: [PATCH 02/12] ci: run the motd suite as four parallel segments motd is the CI critical path (335 s of a 349 s run) and dominates its shard, so sharding by suite cannot shorten it. Give its top-level cases segment guards: each case is wrapped in 'if seg_skip; then :; else ... fi', and the guard runs everything unless TEST_SEGMENT is set. Only units whose text parses on its own are wrapped, so a guard never lands inside a subshell, loop, case or heredoc; function definitions, fixture writers and loose setup stay unguarded and therefore run in every process, which keeps shared fixtures available to every segment. TEST_SEGMENT is unset by default, so bash tests/test-motd.sh is unchanged. tests/run.sh learns TEST_JOBS (bounded parallelism over a shard's suites, serial and byte-identical output by default) and TEST_SEGMENTS (one process per segment for the suites listed as segmented). CI uses 4 for both. Also map modules/system-optimize.sh to its suite instead of falling back to the full run, fetch one commit for the lint job, and run Debian's whole-tree bash -n once instead of per shard. This reverts the per-file split of the three heavy suites: their parts depended on fixtures created by other parts, which the full run caught. --- .github/workflows/shell-tests.yml | 4 +- README.md | 2 +- tests/lib/test-cloudflare-tunnel-common.sh | 181 -- tests/lib/test-motd-common.sh | 62 - .../test-push-worker-registration-common.sh | 86 - tests/run.sh | 85 +- tests/test-cloudflare-tunnel-01.sh | 349 ---- tests/test-cloudflare-tunnel-02.sh | 380 ---- tests/test-cloudflare-tunnel-03.sh | 272 --- tests/test-cloudflare-tunnel-04.sh | 375 ---- tests/test-cloudflare-tunnel.sh | 1544 ++++++++++++++++ tests/test-motd-01.sh | 555 ------ tests/test-motd-02.sh | 267 --- tests/test-motd-03.sh | 233 --- tests/test-motd-04.sh | 155 -- tests/test-motd.sh | 1337 ++++++++++++++ tests/test-push-worker-registration-01.sh | 576 ------ tests/test-push-worker-registration-02.sh | 358 ---- tests/test-push-worker-registration-03.sh | 491 ------ tests/test-push-worker-registration-04.sh | 57 - tests/test-push-worker-registration.sh | 1553 +++++++++++++++++ 21 files changed, 4493 insertions(+), 4429 deletions(-) delete mode 100644 tests/lib/test-cloudflare-tunnel-common.sh delete mode 100644 tests/lib/test-motd-common.sh delete mode 100644 tests/lib/test-push-worker-registration-common.sh delete mode 100644 tests/test-cloudflare-tunnel-01.sh delete mode 100644 tests/test-cloudflare-tunnel-02.sh delete mode 100644 tests/test-cloudflare-tunnel-03.sh delete mode 100644 tests/test-cloudflare-tunnel-04.sh create mode 100644 tests/test-cloudflare-tunnel.sh delete mode 100644 tests/test-motd-01.sh delete mode 100644 tests/test-motd-02.sh delete mode 100644 tests/test-motd-03.sh delete mode 100644 tests/test-motd-04.sh create mode 100644 tests/test-motd.sh delete mode 100644 tests/test-push-worker-registration-01.sh delete mode 100644 tests/test-push-worker-registration-02.sh delete mode 100644 tests/test-push-worker-registration-03.sh delete mode 100644 tests/test-push-worker-registration-04.sh create mode 100755 tests/test-push-worker-registration.sh diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index 173fae9..2f1b589 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -48,7 +48,7 @@ jobs: # Check both groups even when production lint fails. status=0 git ls-files -z -- "*.sh" ":!:tests/**" | xargs -0 -r shellcheck -x --severity=warning -- || status=1 - git ls-files -z -- "tests/*.sh" "tests/lib/*.sh" | xargs -0 -r shellcheck -x --severity=error -- || status=1 + git ls-files -z -- "tests/*.sh" | xargs -0 -r shellcheck -x --severity=error -- || status=1 exit "$status" ubuntu-tests: @@ -86,6 +86,7 @@ jobs: TEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }} TEST_SHARD: ${{ matrix.shard }} TEST_JOBS: 4 + TEST_SEGMENTS: 4 run: bash tests/run.sh "$TEST_SHARD" debian-tests: @@ -128,6 +129,7 @@ jobs: TEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }} TEST_SHARD: ${{ matrix.shard }} TEST_JOBS: 4 + TEST_SEGMENTS: 4 run: bash tests/run.sh "$TEST_SHARD" # Preserve existing required-check names; skipped/cancelled dependencies fail closed. diff --git a/README.md b/README.md index ac278f9..8e44f0f 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ 生产宿主机（包括 Netcup）仅执行差异检查及改动脚本的 `bash -n`，不运行全量 ShellCheck 或完整测试集； 分析和测试交由 CI：PR 按完整差异及依赖映射选择套件，README/许可证修改无需业务测试； 公共设施或未知依赖改动回退全量，main 推送仍执行全量。两个必需 Job 始终运行，输出选择和跳过记录。 -motd、Cloudflare Tunnel 与 push worker 注册三个套件按用例族拆成 `-NN.sh` 分片，共用 `tests/lib/-common.sh` 的定义；同一分片内的文件由 `tests/run.sh` 以 `TEST_JOBS` 限定的并发执行（CI 取 4，本机默认串行）。 +motd 套件内的用例带有 CI 分段守卫：`TEST_SEGMENTS` 个进程各执行互不重叠的一份（CI 取 4，本机默认 1，即原样串行）；同一分片内的套件由 `tests/run.sh` 以 `TEST_JOBS` 限定的并发执行。 Ubuntu 使用固定 ShellCheck v0.10.0：所有受 Git 跟踪、位于 tests/ 之外的 .sh 文件执行 warning 门槛，tests/ 与 tests/lib/ 内的 .sh 文件执行 error 门槛；新目录中的生产 .sh 文件自动纳入，不维护生产文件白名单。Zsh 配置 p10k-config.zsh 不交给 Bash ShellCheck。两组 lint 都会执行并累计失败，Debian 不重复 lint。 例外仅允许局部明确的 SC 规则编号和邻近理由；禁止文件级或全局 warning 抑制。测试 warning 不阻断不等于被 disable，禁止用抑制隐藏行为缺陷。CI 工作流改动继续触发全量测试回退；warning 门槛失败时后续 Ubuntu 行为套件不会执行，不能以 Debian 通过代替双环境验收。 语法检查仍覆盖全部 .sh 脚本；文档修改无需本地全套测试，但仍须通过必需 CI。 diff --git a/tests/lib/test-cloudflare-tunnel-common.sh b/tests/lib/test-cloudflare-tunnel-common.sh deleted file mode 100644 index cea083f..0000000 --- a/tests/lib/test-cloudflare-tunnel-common.sh +++ /dev/null @@ -1,181 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if (( EUID != 0 )); then - exec sudo --preserve-env=PATH bash "$0" "$@" -fi - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -readonly ROOT_DIR -TEST_DIR=$(mktemp -d -p "$ROOT_DIR") -readonly TEST_DIR -trap 'rm -rf "$TEST_DIR"' EXIT - -pass_count=0 -fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } -pass() { pass_count=$((pass_count + 1)); printf 'PASS: %s\n' "$*"; } -assert_same() { cmp -s -- "$1" "$2" || fail "$3"; } -assert_absent() { [[ ! -e "$1" && ! -L "$1" ]] || fail "$2"; } - -make_fake_commands() { - local bin="$1" - mkdir -p "$bin" - cat > "$bin/curl" <<'FAKE' -#!/usr/bin/env bash -set -eu -printf 'curl:%s\n' "$*" >> "$FAKE_LOG" -[[ "${FAKE_CURL_FAIL:-}" != 1 ]] || exit 22 -output="" -while (( $# )); do - if [[ "$1" == -o ]]; then output="$2"; shift 2; continue; fi - shift -done -[[ -n "$output" ]] -printf 'fake-cloudflare-key\n' > "$output" -FAKE - cat > "$bin/gpg" <<'FAKE' -#!/usr/bin/env bash -set -eu -printf 'gpg:%s\n' "$*" >> "$FAKE_LOG" -[[ "${FAKE_GPG_FAIL:-}" != 1 ]] || exit 2 -case "${FAKE_GPG_CASE:-good}" in - good) - cat <<'EOF' -pub:-:4096:1:8A682D308D4E5E73:1761229688:::-:::scESC::::::23::0: -fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: -uid:-::::1761229688::HASH::CloudFlare Software Packaging 2025 ::::::::::0: -sub:-:4096:1:029E1444B7D9F50F:1761229688::::::e::::::23: -fpr:::::::::06C89DB3B80A8F4349697C76029E1444B7D9F50F: -EOF - ;; - fingerprint) - cat <<'EOF' -pub:-:4096:1:BAD:0:::-:::scESC: -fpr:::::::::0000000000000000000000000000000000000000: -uid:-::::0::HASH::CloudFlare Software Packaging 2025 ::: -EOF - ;; - uid) - cat <<'EOF' -pub:-:4096:1:8A682D308D4E5E73:0:::-:::scESC: -fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: -uid:-::::0::HASH::Attacker ::: -EOF - ;; - multi) - cat <<'EOF' -pub:-:4096:1:8A682D308D4E5E73:0:::-:::scESC: -fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: -uid:-::::0::HASH::CloudFlare Software Packaging 2025 ::: -pub:-:4096:1:BAD:0:::-:::scESC: -fpr:::::::::0000000000000000000000000000000000000000: -uid:-::::0::HASH::Other ::: -EOF - ;; -esac -FAKE - cat > "$bin/apt-get" <<'FAKE' -#!/usr/bin/env bash -set -eu -printf 'apt-get:%s\n' "$*" >> "$FAKE_LOG" -case "${1:-}" in - update) [[ "${FAKE_APT_UPDATE_FAIL:-}" != 1 ]] ;; - install) [[ "${FAKE_APT_INSTALL_FAIL:-}" != 1 ]] ;; - *) exit 0 ;; -esac -FAKE - cat > "$bin/systemctl" <<'FAKE' -#!/usr/bin/env bash -printf 'systemctl:%s\n' "$*" >> "$FAKE_LOG" -exit 0 -FAKE - cat > "$bin/cloudflared" <<'FAKE' -#!/usr/bin/env bash -printf 'cloudflared:%s\n' "$*" >> "$FAKE_LOG" -exit 0 -FAKE - chmod 0755 "$bin"/* -} - -new_case() { - CASE_DIR="$TEST_DIR/case" - rm -rf "$CASE_DIR" - mkdir -p "$CASE_DIR" - export CASE_DIR FAKE_LOG="$CASE_DIR/fake.log" - : > "$FAKE_LOG" - mkdir -p "$CASE_DIR/root/usr/share/keyrings" "$CASE_DIR/root/etc/apt/sources.list.d" \ - "$CASE_DIR/root/var/lib" - chmod 0755 "$CASE_DIR/root" "$CASE_DIR/root/usr" "$CASE_DIR/root/usr/share" \ - "$CASE_DIR/root/usr/share/keyrings" "$CASE_DIR/root/etc" "$CASE_DIR/root/etc/apt" \ - "$CASE_DIR/root/etc/apt/sources.list.d" "$CASE_DIR/root/var" "$CASE_DIR/root/var/lib" - make_fake_commands "$CASE_DIR/bin" - export PATH="$CASE_DIR/bin:$ORIGINAL_PATH" - export CLOUDFLARED_TRUST_ANCHOR="$CASE_DIR" - export CLOUDFLARED_KEYRING="$CASE_DIR/root/usr/share/keyrings/cloudflare-main.gpg" - export CLOUDFLARED_SOURCE_FILE="$CASE_DIR/root/etc/apt/sources.list.d/cloudflared.list" - export CLOUDFLARED_APT_SOURCE_ROOT="$CASE_DIR/root/etc/apt" - export CLOUDFLARED_STATE_DIR="$CASE_DIR/root/var/lib/cloudflared-wrapper" - export CLOUDFLARED_REPOSITORY_STATE_DIR="$CLOUDFLARED_STATE_DIR/repository" - export CLOUDFLARED_LEGACY_BIN="$CASE_DIR/legacy-cloudflared" - export CLOUDFLARED_APT_BIN="$CASE_DIR/usr-bin-cloudflared" - export CLOUDFLARED_LEGACY_UPDATER="$CASE_DIR/cloudflared-update" - export CLOUDFLARED_LEGACY_SERVICE="$CASE_DIR/cloudflared-updater.service" - export CLOUDFLARED_LEGACY_TIMER="$CASE_DIR/cloudflared-updater.timer" - export CLOUDFLARED_AUTO_UPDATE_SCRIPT="$CASE_DIR/cloudflared-apt-update" - export CLOUDFLARED_AUTO_UPDATE_SERVICE="$CASE_DIR/cloudflared-apt-update.service" - export CLOUDFLARED_AUTO_UPDATE_TIMER="$CASE_DIR/cloudflared-apt-update.timer" - export CLOUDFLARED_SERVICE_FILE="$CASE_DIR/cloudflared.service" - export CLOUDFLARED_BINARY_UPDATE_SERVICE="$CASE_DIR/cloudflared-update.service" - export CLOUDFLARED_BINARY_UPDATE_TIMER="$CASE_DIR/cloudflared-update.timer" - export CLOUDFLARED_TEST_INTERNALS=1 - unset FAKE_CURL_FAIL FAKE_GPG_FAIL FAKE_GPG_CASE FAKE_APT_UPDATE_FAIL FAKE_APT_INSTALL_FAIL - unset FAIL_INSTALL_CALL FAIL_RENAME_CALL INSTALL_CALL RENAME_CALL SIGNAL_PHASE - if [[ "${SCRIPT_SOURCED:-}" != 1 ]]; then - # shellcheck source=../tools/cloudflare_tunnel.sh - source "$ROOT_DIR/tools/cloudflare_tunnel.sh" - SCRIPT_SOURCED=1 - fi - init_runtime_config -} - -set_old_generation() { - printf 'old-key\n' > "$CLOUDFLARED_KEYRING" - repository_legacy_source_content > "$CLOUDFLARED_SOURCE_FILE" - chmod 0644 "$CLOUDFLARED_KEYRING" "$CLOUDFLARED_SOURCE_FILE" - cp "$CLOUDFLARED_KEYRING" "$CASE_DIR/expected-key" - cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" -} - -assert_old_generation() { - assert_same "$CLOUDFLARED_KEYRING" "$CASE_DIR/expected-key" "old key was not restored" - assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" "old source was not restored" -} - -run_failure_case() { - local name="$1" setup="$2" action="${3:-configure_repository}" - new_case - set_old_generation - eval "$setup" - set +e - eval "$action" >"$CASE_DIR/out" 2>"$CASE_DIR/err" - status=$? - set -e - if (( status == 0 )); then - fail "$name unexpectedly succeeded" - fi - assert_old_generation - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "$name left lock" - find "$CLOUDFLARED_REPOSITORY_STATE_DIR" -maxdepth 1 -type d -name 'failure-*' -print -quit | grep -q . || - fail "$name did not preserve failure evidence" - pass "$name" -} - -ORIGINAL_PATH=$PATH - -new_case -before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) -repository_source_content > "$CASE_DIR/source-output" -after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) -[[ "$before" == "$after" ]] || fail "source rendering had side effects" -[[ "$(cat "$CASE_DIR/source-output")" == "deb [signed-by=$CLOUDFLARED_KEYRING] https://pkg.cloudflare.com/cloudflared any main" ]] || - fail "source is not exact official definition" diff --git a/tests/lib/test-motd-common.sh b/tests/lib/test-motd-common.sh deleted file mode 100644 index 8e681ba..0000000 --- a/tests/lib/test-motd-common.sh +++ /dev/null @@ -1,62 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.."&&pwd); -readonly ROOT_DIR -TEST_DIR=$(mktemp -d); -readonly TEST_DIR -fail(){ echo "FAIL: $*" >&2; -exit 1; -}; -pass(){ echo "PASS: $*"; -}; -assert_eq(){ [[ "$1" == "$2" ]]||fail "$3: $1 != $2"; -pass "$3"; -}; -assert_fail(){ local l="$1"; -shift; -if "$@"; -then fail "$l"; -fi; -pass "$l"; -} -extract_payload(){ awk '/install -m 0755 \/dev\/stdin .*<<.SCRIPT./{c=1;next}c&&$0=="SCRIPT"{exit}c{print}' "$1"; -} -load_motd() { - . "$ROOT_DIR/tools/setup-motd.sh" -} - -create_fixture(){ local r="$1" e - e="$r/e"; -e+="tc"; -mkdir -m700 "$r" "$e" "$e/update-motd.d" "$r/lock"; -printf original>"$e/motd"; -chmod 640 "$e/motd"; -printf issue>"$e/issue"; -chmod 644 "$e/issue"; -printf old>"$e/update-motd.d/00-custom-welcome"; -chmod 700 "$e/update-motd.d/00-custom-welcome"; -printf uname>"$e/update-motd.d/10-uname"; -chmod 754 "$e/update-motd.d/10-uname"; -} -export_paths(){ local r="$1" e - e="$r/e"; -e+="tc"; -export MOTD_TEST_MODE=1 MOTD_ETC_ROOT="$e" MOTD_STATE_DIR="$r/state" MOTD_LOCK_FILE="$r/lock/motd.lock" MOTD_TRANSACTION_PARENT="$r/state/transactions"; -} -target_hash(){ local r="$1" e p - e="$r/e"; -e+="tc"; -for p in "$e/motd" "$e/issue" "$e/issue.net" "$e/update-motd.d/00-custom-welcome" "$e/update-motd.d/10-uname" "$e/update-motd.d/50-motd-news"; -do if [[ -L "$p" ]]; -then echo "L|$p|$(readlink "$p")"; -elif [[ -f "$p" ]]; -then echo "F|$p|$(stat -c %u:%g:%a "$p")|$(sha256sum "$p"|awk '{print $1}')"; -elif [[ ! -e "$p" ]]; -then echo "A|$p"; -else echo "O|$p|$(stat -c %F "$p")"; -fi; -done; -} -extract_payload "$ROOT_DIR/tools/setup-motd.sh">"$TEST_DIR/t"; -extract_payload "$ROOT_DIR/modules/system-customize.sh">"$TEST_DIR/m"; -cmp -s "$TEST_DIR/t" "$TEST_DIR/m"||fail payload; diff --git a/tests/lib/test-push-worker-registration-common.sh b/tests/lib/test-push-worker-registration-common.sh deleted file mode 100644 index 05cfeab..0000000 --- a/tests/lib/test-push-worker-registration-common.sh +++ /dev/null @@ -1,86 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -# Safe runtime identity only: no environment dump, credentials, or xtrace. -printf "DIAG: bash=%s kernel=%s\n" "$BASH_VERSION" "$(uname -r)" -dpkg-query -W bash libc6 2>/dev/null || true - -ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd) -readonly ROOT_DIR -readonly SCRIPT="$ROOT_DIR/tools/push.sh" -TEST_DIR=$(mktemp -d) -readonly TEST_DIR -CURRENT_FIXTURE_ROOT="" -trap 'rm -rf "$TEST_DIR"' EXIT - -failure_diagnostics() { - local root=${CURRENT_FIXTURE_ROOT:-} file pid - [[ -n "$root" && -d "$root" ]] || return 0 - printf '%s\n' '=== worker registration diagnostics ===' >&2 - printf '%s\n' '--- coordinator/status ---' >&2 - for file in "$root"/capture/coordinator* "$root"/capture/watchdog* "$root"/capture/timeout*; do - [[ -e "$file" || -L "$file" ]] || continue - printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true - done - printf '%s\n' '--- registration/release/stages ---' >&2 - find "$root" \( -name 'worker-registration.*' -o -name '.worker-registration.*' \) -print 2>/dev/null | sort >&2 || true - for file in "$root"/runtime/push-runtime.*/worker-registration.*; do - [[ -e "$file" || -L "$file" ]] || continue - printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true - done - printf '%s\n' '--- worker/session states ---' >&2 - for file in "$root"/runtime/push-runtime.*/worker-session.*.state; do - [[ -e "$file" || -L "$file" ]] || continue - printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true - done - printf '%s\n' '--- jobs ---' >&2; jobs -l >&2 || true - printf '%s\n' '--- runtime tree ---' >&2 - find "$root/runtime" -mindepth 1 -maxdepth 4 -printf '%y %m %p\n' 2>/dev/null | sort >&2 || true - printf '%s\n' '--- process output ---' >&2 - [[ -f "$root/output.log" ]] && cat "$root/output.log" >&2 2>/dev/null || true - printf '%s\n' '--- fake command calls ---' >&2 - for file in "$root"/capture/calls "$root"/capture/transfer-calls; do - [[ -e "$file" ]] && { printf '[%s]\n' "$file" >&2; cat "$file" >&2 || true; } - done - printf '%s\n' '--- recorded proc identities ---' >&2 - for file in "$root"/capture/*.pid; do - [[ -f "$file" && ! -L "$file" ]] || continue - pid=$(<"$file") - [[ "$pid" =~ ^[1-9][0-9]*$ && -r "/proc/$pid/stat" ]] && printf 'pid=%s stat=%s\n' "$pid" "$(<"/proc/$pid/stat")" >&2 - done -} -fail() { failure_diagnostics; printf 'FAIL: %s\n' "$*" >&2; exit 1; } -pass() { printf 'PASS: %s\n' "$*"; } -assert_eq() { [[ "$1" == "$2" ]] || fail "$3: expected '$1', got '$2'"; pass "$3"; } -process_is_running() { local pid="$1" state; [[ -r "/proc/$pid/stat" ]] || return 1; state=$(awk '{print $3}' "/proc/$pid/stat" 2>/dev/null) || return 1; [[ "$state" != Z ]]; } -test_process_identity_exists() { - local pid="$1" start="$2" - read_process_record "$pid" || return 1 - [[ "$PROC_UID" == "$EUID" && "$PROC_START" == "$start" ]] -} -wait_test_process_identity_gone() { - local pid="$1" start="$2" _ - for _ in {1..100}; do test_process_identity_exists "$pid" "$start" || return 0; sleep 0.05; done - return 1 -} -wait_test_process_identity_present() { - local pid="$1" start="$2" _ - for _ in {1..30}; do test_process_identity_exists "$pid" "$start" && return 0; [[ -r "/proc/$pid/stat" ]] || return 1; sleep 0.01; done - return 1 -} -wait_test_process_start() { - local pid="$1" value="" _ - for _ in {1..100}; do value=$(get_process_start_time "$pid" 2>/dev/null || true); [[ -n "$value" ]] && { printf '%s\n' "$value"; return 0; }; sleep 0.005; done - return 1 -} - -source_root="$TEST_DIR/source"; mkdir -m 0700 "$source_root" -source_result=$(env TMPDIR="$source_root" SCRIPT="$SCRIPT" bash -c ' -set -euo pipefail -before=$(find "$TMPDIR" -mindepth 1 -print -quit) -source "$SCRIPT" -[[ -z "$TEMP_DIR" && -z "$SUCCESS_FILE" && -z "$FAILED_FILE" ]] -[[ "$before" == "$(find "$TMPDIR" -mindepth 1 -print -quit)" ]] -printf source-safe -') -# shellcheck source=../tools/push.sh -source "$SCRIPT" diff --git a/tests/run.sh b/tests/run.sh index 48f7830..15d741b 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -14,14 +14,6 @@ case "$shard" in *) printf 'Invalid test shard: %s\n' "$shard" >&2; exit 2 ;; esac -# Suites split across several files are named -NN.sh; every part belongs -# to the family's shard and every part is selected by a change to the family. -family_of() { - local name=${1##*/} - name=${name%.sh} - printf '%s.sh\n' "${name%-[0-9][0-9]}" -} - declare -A selected=() full=true if [[ -n "${TEST_BASE_SHA:-}" && -n "${TEST_HEAD_SHA:-}" ]]; then @@ -40,8 +32,8 @@ if [[ -n "${TEST_BASE_SHA:-}" && -n "${TEST_HEAD_SHA:-}" ]]; then modules/system-optimize.sh) selected[test-system-optimize.sh]=1 ;; linux_setup.sh) selected[test-linux-setup.sh]=1; selected[test-setup-exec.sh]=1 ;; modules/zsh-setup.sh|p10k-config.zsh) selected[test-linux-setup.sh]=1 ;; - tests/test-*.sh|tests/lib/*.sh) - if [[ -f "$ROOT_DIR/$path" ]]; then selected["$(family_of "$path")"]=1; else full=true; fi ;; + tests/test-*.sh) + if [[ -f "$ROOT_DIR/$path" ]]; then selected["${path##*/}"]=1; else full=true; fi ;; README.md|LICENSE) ;; *) full=true ;; esac @@ -52,19 +44,14 @@ if [[ -n "${TEST_BASE_SHA:-}" && -n "${TEST_HEAD_SHA:-}" ]]; then fi printf 'Test selection: full=%s\n' "$full" -# Bounded parallelism: 1 keeps the original serial output byte for byte. -jobs=${TEST_JOBS:-1} -[[ "$jobs" =~ ^[1-9][0-9]*$ ]] || { printf 'Invalid TEST_JOBS: %s\n' "$jobs" >&2; exit 2; } - plan=() for test_file in "$ROOT_DIR"/tests/test-*.sh; do test_name=$(basename "$test_file") - family=$(family_of "$test_name") - if [[ "$full" != true && -z "${selected[${family}]:-}" ]]; then + if [[ "$full" != true && -z "${selected[${test_name}]:-}" ]]; then printf 'SKIP: %s (unaffected by PR diff)\n' "$test_name" continue fi - case "$family" in + case "$test_name" in test-motd.sh) suite_shard=motd ;; test-push.sh|test-push-worker-registration.sh) suite_shard=push ;; *) suite_shard=other ;; @@ -76,14 +63,46 @@ for test_file in "$ROOT_DIR"/tests/test-*.sh; do plan+=("$test_file") done +# Bounded parallelism: 1 keeps the original serial output byte for byte. +jobs=${TEST_JOBS:-1} +[[ "$jobs" =~ ^[1-9][0-9]*$ ]] || { printf 'Invalid TEST_JOBS: %s\n' "$jobs" >&2; exit 2; } + +# Suites carrying CI segment guards run TEST_SEGMENTS processes, each taking a +# disjoint share of the cases. Every other suite runs once. +segments=${TEST_SEGMENTS:-1} +[[ "$segments" =~ ^[1-9][0-9]*$ ]] || { printf 'Invalid TEST_SEGMENTS: %s\n' "$segments" >&2; exit 2; } +segmented_families=(test-motd.sh) + +job_files=() +job_segs=() +for test_file in "${plan[@]}"; do + test_name=$(basename "$test_file") + if (( segments > 1 )) && [[ " ${segmented_families[*]} " == *" $test_name "* ]]; then + for (( k = 0; k < segments; k++ )); do + job_files+=("$test_file") + job_segs+=("$k") + done + else + job_files+=("$test_file") + job_segs+=("") + fi +done + run_suite() { - local test_file=$1 test_name started status + local test_file=$1 seg=${2:-} test_name started status test_name=$(basename "$test_file") + if [[ -n "$seg" ]]; then + test_name+=" [segment $((seg + 1))/$segments]" + fi started=$SECONDS printf '\n==> %s\n' "$test_name" # Keep the child shell standalone: an if/|| wrapper can change errexit behavior. set +e - bash "$test_file" + if [[ -n "$seg" ]]; then + TEST_SEGMENT="$seg" TEST_SEGMENTS="$segments" bash "$test_file" + else + bash "$test_file" + fi status=$? set -e local elapsed=$((SECONDS - started)) @@ -94,25 +113,28 @@ run_suite() { return "$status" } -if (( ${#plan[@]} == 0 )); then +if (( ${#job_files[@]} == 0 )); then printf '\nAll selected shell tests passed.\n' exit 0 fi if (( jobs == 1 )); then - for test_file in "${plan[@]}"; do - run_suite "$test_file" || exit "$?" + for idx in "${!job_files[@]}"; do + run_suite "${job_files[idx]}" "${job_segs[idx]}" || exit "$?" done else log_dir=$(mktemp -d) trap 'rm -rf "$log_dir"' EXIT running=0 - for test_file in "${plan[@]}"; do - test_name=$(basename "$test_file") + for idx in "${!job_files[@]}"; do + test_file=${job_files[idx]} + seg=${job_segs[idx]} + key=$(basename "$test_file") + [[ -n "$seg" ]] && key+=".$seg" ( rc=0 - run_suite "$test_file" > "$log_dir/$test_name.log" 2>&1 || rc=$? - printf '%s' "$rc" > "$log_dir/$test_name.rc" + run_suite "$test_file" "$seg" > "$log_dir/$key.log" 2>&1 || rc=$? + printf '%s' "$rc" > "$log_dir/$key.rc" ) & running=$((running + 1)) if (( running >= jobs )); then @@ -122,10 +144,13 @@ else done wait || true failed=0 - for test_file in "${plan[@]}"; do - test_name=$(basename "$test_file") - cat "$log_dir/$test_name.log" - rc=$(cat "$log_dir/$test_name.rc" 2>/dev/null || printf 1) + for idx in "${!job_files[@]}"; do + test_file=${job_files[idx]} + seg=${job_segs[idx]} + key=$(basename "$test_file") + [[ -n "$seg" ]] && key+=".$seg" + cat "$log_dir/$key.log" + rc=$(cat "$log_dir/$key.rc" 2>/dev/null || printf 1) (( rc == 0 )) || failed=1 done (( failed == 0 )) || exit 1 diff --git a/tests/test-cloudflare-tunnel-01.sh b/tests/test-cloudflare-tunnel-01.sh deleted file mode 100644 index c1e38b5..0000000 --- a/tests/test-cloudflare-tunnel-01.sh +++ /dev/null @@ -1,349 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-cloudflare-tunnel-common.sh" -pass "source rendering has zero side effects and exact fields" - -( - new_case - source_cwd=$PWD - source_before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) - cloudflare_tunnel_source() { printf 'sentinel:%s\n' "$*"; } - sentinel_before=$(declare -f cloudflare_tunnel_source) - trap 'sentinel-trap' HUP - set -o noclobber - umask 027 - unset CLOUDFLARED_TEST_INTERNALS - source "$ROOT_DIR/tools/cloudflare_tunnel.sh" - sentinel_after=$(declare -f cloudflare_tunnel_source) - [[ "$sentinel_before" == "$sentinel_after" ]] || fail "ordinary source replaced sentinel function" - [[ "$(cloudflare_tunnel_source probe)" == sentinel:probe ]] || fail "ordinary source changed sentinel behavior" - [[ "$(trap -p HUP)" == *sentinel-trap* ]] || fail "ordinary source changed caller trap" - [[ "$-" == *C* ]] || fail "ordinary source changed caller shell options" - [[ "$(umask)" == 0027 ]] || fail "ordinary source changed caller umask" - [[ "$PWD" == "$source_cwd" ]] || fail "ordinary source changed caller cwd" - [[ "${CLOUDFLARED_STATE_DIR+x}" == x ]] || fail "test setup lost state variable" - after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) - [[ "$source_before" == "$after" ]] || fail "ordinary source changed filesystem" -) -pass "ordinary source preserves sentinels, shell state, and filesystem" - -new_case -before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\\n' | sort) -bash "$ROOT_DIR/tools/cloudflare_tunnel.sh" status >/dev/null 2>&1 || fail "standalone status failed in fake environment" -after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\\n' | sort) -[[ "$before" == "$after" ]] || fail "ordinary status created state, lock, or temp files" -pass "ordinary help/status paths have zero state and lock side effects" - -cat > "$TEST_DIR/source-contract.sh" </dev/null' _ \ - "$ROOT_DIR/tools/cloudflare_tunnel.sh" || fail "internal source mode unavailable" -pass "explicit internal source mode exposes test functions" - -new_case -configure_repository || fail "configure before real uninstall failed" -require_root() { :; } -check_platform() { :; } -uninstall_cloudflared --confirmed || fail "uninstall normal path failed" -[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall left repository lock" -grep -Fxq 'apt-get:remove -y cloudflared' "$FAKE_LOG" || fail "uninstall skipped package removal" -assert_absent "$CLOUDFLARED_SOURCE_FILE" "real uninstall retained source" -assert_absent "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "real uninstall retained current" -[[ -f "$CLOUDFLARED_KEYRING" ]] || fail "real uninstall removed keyring" -pass "configure then real uninstall preserves keyring and releases lock" -init_runtime_config - -new_case -mkdir -p "$CLOUDFLARED_STATE_DIR.lock" -mkdir -p "$CLOUDFLARED_AUTO_UPDATE_SCRIPT" -printf marker > "$CASE_DIR/marker" -require_root() { :; } -check_platform() { :; } -if uninstall_cloudflared --confirmed >/dev/null 2>&1; then fail "uninstall lock competition unexpectedly succeeded"; fi -[[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall lock competition removed lock" -[[ -d "$CLOUDFLARED_AUTO_UPDATE_SCRIPT" ]] || fail "lock competition touched auto-update path" -! grep -Eq 'remove|disable|daemon-reload|apt-get:' "$FAKE_LOG" || fail "lock competition performed mutation" -pass "uninstall lock competition fails before auto-update, systemctl, and APT" - -new_case -mkdir -p "$CLOUDFLARED_STATE_DIR.lock" -if disable_auto_update --confirmed >/dev/null 2>&1; then fail "disable-auto-update lock competition unexpectedly succeeded"; fi -[[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "disable-auto-update removed competing lock" -pass "auto-update lock competition fails before state mutation" -chmod 0700 "$CLOUDFLARED_STATE_DIR.lock" -if configure_repository >/dev/null 2>&1; then fail "lock competition unexpectedly succeeded"; fi -pass "key/source lock competition" - -run_failure_case "key URL download failure" 'export FAKE_CURL_FAIL=1' -run_failure_case "gpg parser failure" 'export FAKE_GPG_FAIL=1' -run_failure_case "fingerprint mismatch" 'export FAKE_GPG_CASE=fingerprint' -run_failure_case "UID mismatch" 'export FAKE_GPG_CASE=uid' -run_failure_case "multiple primary keys" 'export FAKE_GPG_CASE=multi' - -for target in key source; do - for kind in symlink directory; do - new_case - case "$target" in key) path=$CLOUDFLARED_KEYRING ;; source) path=$CLOUDFLARED_SOURCE_FILE ;; esac - case "$kind" in - symlink) ln -s "$CASE_DIR/missing" "$path" ;; - directory) mkdir "$path" ;; - esac - if configure_repository >/dev/null 2>&1; then fail "$target $kind accepted"; fi - [[ -L "$path" || -d "$path" ]] || fail "$target $kind was changed" - pass "$target rejects $kind" - done - for bad in owner gid mode; do - new_case - case "$target" in key) path=$CLOUDFLARED_KEYRING ;; source) path=$CLOUDFLARED_SOURCE_FILE ;; esac - if [[ "$target" == source ]]; then - repository_source_content > "$path" - else - printf key > "$path" - fi - chmod 0644 "$path" - case "$bad" in - owner) chown 65534:0 "$path" ;; - gid) chown 0:65534 "$path" ;; - mode) chmod 0664 "$path" ;; - esac - if configure_repository >/dev/null 2>&1; then fail "$target wrong $bad accepted"; fi - pass "$target rejects wrong $bad" - done -done - -new_case -printf 'deb https://example.invalid stable main\n' > "$CASE_DIR/root/etc/apt/sources.list.d/extra.list" -chmod 0644 "$CASE_DIR/root/etc/apt/sources.list.d/extra.list" -configure_repository -pass "unrelated APT source remains untouched" - -new_case -printf 'deb https://pkg.cloudflare.com/cloudflared any main\n' > "$CASE_DIR/root/etc/apt/sources.list.d/duplicate.list" -chmod 0644 "$CASE_DIR/root/etc/apt/sources.list.d/duplicate.list" -if configure_repository >/dev/null 2>&1; then fail "duplicate Cloudflare source accepted"; fi -assert_absent "$CLOUDFLARED_KEYRING" "duplicate source changed key" -assert_absent "$CLOUDFLARED_SOURCE_FILE" "duplicate source wrote managed source" -pass "reject duplicate or extra Cloudflare source before commit" - -new_case -printf 'deb [signed-by=%s trusted=yes] https://pkg.cloudflare.com/cloudflared any main\n' \ - "$CLOUDFLARED_KEYRING" > "$CLOUDFLARED_SOURCE_FILE" -chmod 0644 "$CLOUDFLARED_SOURCE_FILE" -if configure_repository >/dev/null 2>&1; then fail "source injection accepted"; fi -pass "reject unknown source fields and injection content" - -( -new_case -set_old_generation -INSTALL_CALL=0 -FAIL_INSTALL_CALL=1 -repository_install_file() { - INSTALL_CALL=$((INSTALL_CALL + 1)) - (( INSTALL_CALL != FAIL_INSTALL_CALL )) || return 1 - command install -o 0 -g 0 -m "$1" -- "$2" "$3" -} -if configure_repository >/dev/null 2>&1; then fail "key stage write failure succeeded"; fi -assert_old_generation -) -pass "formal key write failure rolls back" - -( -new_case -set_old_generation -INSTALL_CALL=0 -FAIL_INSTALL_CALL=2 -repository_install_file() { - INSTALL_CALL=$((INSTALL_CALL + 1)) - (( INSTALL_CALL != FAIL_INSTALL_CALL )) || return 1 - command install -o 0 -g 0 -m "$1" -- "$2" "$3" -} -if configure_repository >/dev/null 2>&1; then fail "source stage write failure succeeded"; fi -assert_old_generation -) -pass "formal source write failure rolls back" - -( -new_case -set_old_generation -RENAME_CALL=0 -FAIL_RENAME_CALL=2 -repository_rename() { - RENAME_CALL=$((RENAME_CALL + 1)) - (( RENAME_CALL != FAIL_RENAME_CALL )) || return 1 - command mv -fT -- "$1" "$2" -} -if configure_repository >/dev/null 2>&1; then fail "source commit failure succeeded"; fi -assert_old_generation -) -pass "key success/source failure rolls back" - -( -new_case -set_old_generation -RENAME_CALL=0 -FAIL_RENAME_CALL=1 -repository_rename() { - RENAME_CALL=$((RENAME_CALL + 1)) - (( RENAME_CALL != FAIL_RENAME_CALL )) || return 1 - command mv -fT -- "$1" "$2" -} -if configure_repository >/dev/null 2>&1; then fail "key commit failure succeeded"; fi -assert_old_generation -) -pass "source remains old when key commit fails" - -run_failure_case "APT probe failure rollback" 'export FAKE_APT_UPDATE_FAIL=1' 'run_repository_apt_transaction install' -run_failure_case "APT install failure rollback" 'export FAKE_APT_INSTALL_FAIL=1' 'run_repository_apt_transaction install' - -for signal in HUP INT TERM; do - case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - for phase in download validate stage key-commit source-commit apt-probe apt-install; do - new_case - set_old_generation - cat > "$CASE_DIR/signal-runner.sh" <"$CASE_DIR/out" 2>"$CASE_DIR/err" - status=$? - set -e - [[ "$status" == "$expected" ]] || fail "$signal at $phase returned $status, expected $expected" - assert_old_generation - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "$signal at $phase left lock" - pass "$signal=$expected rolls back at $phase" - done -done - -( -new_case -set_old_generation -RENAME_CALL=0 -repository_rename() { - RENAME_CALL=$((RENAME_CALL + 1)) - case "$RENAME_CALL" in 2|3) return 1 ;; esac - command mv -fT -- "$1" "$2" -} -if configure_repository >/dev/null 2>&1; then fail "rollback failure unexpectedly succeeded"; fi -find "$CLOUDFLARED_REPOSITORY_STATE_DIR" -maxdepth 2 -type f -name rollback.log -print -quit | grep -q . || - fail "rollback failure did not preserve evidence" -) -pass "rollback failure returns nonzero and preserves evidence" - -new_case -set_old_generation -configure_repository -current="$CLOUDFLARED_REPOSITORY_STATE_DIR/current" -[[ -f "$current" ]] || fail "generation marker missing" -key_hash=$(sha256sum "$CLOUDFLARED_KEYRING" | awk '{print $1}') -printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflared any main\n' "$CLOUDFLARED_KEYRING" > "$CASE_DIR/independent-source" -assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/independent-source" "source differs from official literal" -source_hash=$(sha256sum "$CASE_DIR/independent-source" | awk '{print $1}') -grep -Fxq "key_sha256=$key_hash" "$current" || fail "key generation hash mismatch" -grep -Fxq "source_sha256=$source_hash" "$current" || fail "source generation hash mismatch" -pass "key/source commit as one recorded generation" - -new_case -set_old_generation -configure_repository -validate_current_repository_manifest || fail "trusted current manifest validation failed" -remove_managed_repository -init_runtime_config -assert_absent "$CLOUDFLARED_SOURCE_FILE" "trusted current did not remove managed source" -[[ -f "$CLOUDFLARED_KEYRING" ]] || fail "managed repository removal deleted keyring" -[[ ! -f "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" ]] || fail "managed repository removal retained current" -find "$CLOUDFLARED_STATE_DIR" -maxdepth 1 -type d -name 'uninstall-*' -print -quit | grep -q . || - fail "managed source backup missing" -[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "managed repository removal left lock" -pass "configure then real repository uninstall removes source and preserves keyring" - -for signal in HUP INT TERM; do - case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - for phase in before-snapshot-create capture-auto_update_script capture-auto_update_timer capture-legacy_marker lock-acquired before-disable-auto-update after-disable-auto-update before-apt-remove after-apt-remove before-source-remove after-source-remove final-apt-update before-lock-release; do - new_case - set_old_generation - configure_repository - cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" - cp "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "$CASE_DIR/expected-current" - set +e - CLOUDFLARED_TEST_INTERNALS=1 SIGNAL_PHASE="$phase" SIGNAL_NAME="$signal" \ - bash -c ' - source "$1" - require_root() { :; } - check_platform() { :; } - uninstall_transaction_hook() { - [[ "$1" != "$SIGNAL_PHASE" ]] || kill -s "$SIGNAL_NAME" "$BASHPID" - } - uninstall_cloudflared --confirmed - ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" >"$CASE_DIR/out" 2>"$CASE_DIR/err" - status=$? - set -e - [[ "$status" == "$expected" ]] || fail "uninstall $signal at $phase returned $status" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall $signal at $phase left lock" - assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" "uninstall $signal at $phase failed source restore" - assert_same "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "$CASE_DIR/expected-current" "uninstall $signal at $phase failed current restore" - acquire_repository_lock || fail "uninstall $signal at $phase lock not reusable" - release_repository_lock || fail "uninstall $signal at $phase lock cleanup failed" - pass "uninstall $signal=$expected cleans $phase" - done -done - -new_case -set_old_generation -configure_repository -uninstall_transaction_hook() { [[ "$1" != after-disable-auto-update ]] || exit 0; } -set +e -( uninstall_cloudflared --confirmed ) >"$CASE_DIR/out" 2>"$CASE_DIR/err" -status=$? -set -e -[[ "$status" == 1 ]] || fail "active uninstall exit 0 returned $status" -[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "active uninstall exit 0 left lock" -grep -R -Fq '活动卸载事务异常退出' "$CLOUDFLARED_REPOSITORY_STATE_DIR" || fail "active uninstall exit 0 lacked evidence" \ No newline at end of file diff --git a/tests/test-cloudflare-tunnel-02.sh b/tests/test-cloudflare-tunnel-02.sh deleted file mode 100644 index 1b0cec1..0000000 --- a/tests/test-cloudflare-tunnel-02.sh +++ /dev/null @@ -1,380 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-cloudflare-tunnel-common.sh" - -pass "active uninstall exit 0 returns 1 and cleans transaction" - -for manifest_case in missing duplicate unknown generation key-digest source-digest source-content symlink directory fifo owner gid mode; do - new_case - set_old_generation - configure_repository - current="$CLOUDFLARED_REPOSITORY_STATE_DIR/current" - case "$manifest_case" in - missing) sed -i '/^source_sha256=/d' "$current" ;; - duplicate) cat "$current" >> "$current.copy"; cat "$current.copy" >> "$current"; rm "$current.copy" ;; - unknown) printf 'unknown=value\n' >> "$current" ;; - generation) sed -i 's/^generation=.*/generation=invalid/' "$current" ;; - key-digest) sed -i 's/^key_sha256=.*/key_sha256=0000000000000000000000000000000000000000000000000000000000000000/' "$current" ;; - source-digest) sed -i 's/^source_sha256=.*/source_sha256=0000000000000000000000000000000000000000000000000000000000000000/' "$current" ;; - source-content) printf '# tampered\n' >> "$CLOUDFLARED_SOURCE_FILE" ;; - symlink) rm "$current"; ln -s "$CASE_DIR/external" "$current"; printf external > "$CASE_DIR/external" ;; - directory) rm "$current"; mkdir "$current" ;; - fifo) rm "$current"; mkfifo "$current" ;; - owner) chown 65534:0 "$current" ;; - gid) chown 0:65534 "$current" ;; - mode) chmod 0644 "$current" ;; - esac - if remove_managed_repository >/dev/null 2>&1; then fail "current $manifest_case unexpectedly trusted"; fi - [[ -f "$CLOUDFLARED_KEYRING" ]] || fail "current $manifest_case deleted keyring" - [[ -e "$CLOUDFLARED_SOURCE_FILE" ]] || fail "current $manifest_case deleted source" - [[ ! -e "$CASE_DIR/external" || "$(cat "$CASE_DIR/external")" == external ]] || fail "current symlink target changed" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "current $manifest_case left lock" - pass "current manifest rejects $manifest_case" -done - -new_case -set_old_generation -export FAKE_APT_INSTALL_FAIL=1 -if run_repository_apt_transaction install >/dev/null 2>&1; then fail "old generation rollback test succeeded"; fi -assert_old_generation -pass "old key/source preserved on transaction failure" - -new_case -run_repository_apt_transaction install -grep -Fxq 'apt-get:update' "$FAKE_LOG" || fail "APT probe not called" -grep -Fxq 'apt-get:install -y cloudflared' "$FAKE_LOG" || fail "cloudflared package selection changed" -if grep -Eqi 'remove|purge|autoremove|linux-image|linux-headers' "$FAKE_LOG"; then - fail "repository transaction performed unrelated APT changes" -fi -pass "no kernel deletion or unrelated APT change" - -script_hashes_before=$(sha256sum "$ROOT_DIR/linux_setup.sh" "$ROOT_DIR"/modules/*.sh "$ROOT_DIR"/tools/push.sh \ - "$ROOT_DIR"/tools/setup-motd.sh "$ROOT_DIR"/tools/xanmod-install.sh) -script_hashes_after=$(sha256sum "$ROOT_DIR/linux_setup.sh" "$ROOT_DIR"/modules/*.sh "$ROOT_DIR"/tools/push.sh \ - "$ROOT_DIR"/tools/setup-motd.sh "$ROOT_DIR"/tools/xanmod-install.sh) -[[ "$script_hashes_before" == "$script_hashes_after" ]] || fail "payload or unrelated scripts changed during tests" -pass "payload and unrelated scripts remain unchanged" - -# Keep prior lifecycle coverage with fake systemctl only. -new_case -cat > "$CLOUDFLARED_SERVICE_FILE" < "$CLOUDFLARED_LEGACY_BIN" <<'EOF' -#!/usr/bin/env bash -echo 'cloudflared version 2025.1.0' -EOF -chmod 0755 "$CLOUDFLARED_LEGACY_BIN" -migrate_legacy_binary -assert_absent "$CLOUDFLARED_LEGACY_BIN" "recognized legacy binary was not migrated" -grep -Fq "ExecStart=$CLOUDFLARED_APT_BIN --no-autoupdate --token-file /etc/cloudflared/token" "$CLOUDFLARED_SERVICE_FILE" || - fail "legacy service path was not preserved" -pass "legacy binary and service migration behavior remains" - - -# Each malformed snapshot runs in a separate process: transaction traps and -# intentional corrupt fixtures must not leak into subsequent cases. -for schema_case in version generation duplicate unknown order state mode uid gid digest created payload-name payload-missing payload-extra payload-fifo payload-symlink manifest-fifo ancestor; do - ( - new_case - configure_repository || fail "schema fixture configure" - acquire_repository_lock || fail "schema fixture lock" - begin_uninstall_transaction || fail "schema fixture capture" - manifest="$UNINSTALL_SNAPSHOT_DIR/manifest" - case "$schema_case" in - version) sed -i '1d' "$manifest" ;; - generation) sed -i '2s/.*/generation=20000101T000000Z-1-1/' "$manifest" ;; - duplicate) sed -i '3s/.*/version=1/' "$manifest" ;; - unknown) sed -i '10s/.*/path=\/tmp\/forbidden/' "$manifest" ;; - order) sed -i '3s/auto_update_script/auto_update_timer/' "$manifest" ;; - state) sed -i '4s/.*/state=invalid/' "$manifest" ;; - mode) sed -i '5s/.*/mode=777/' "$manifest" ;; - uid) sed -i '6s/.*/uid=1/' "$manifest" ;; - gid) sed -i '7s/.*/gid=1/' "$manifest" ;; - digest) sed -i '8s/.*/sha256=invalid/' "$manifest" ;; - created) sed -i '0,/^transaction_created=false$/s//transaction_created=invalid/' "$manifest" ;; - payload-name) sed -i 's/^snapshot=target-source$/snapshot=target-current/' "$manifest" ;; - payload-missing) rm -- "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ;; - payload-extra) printf extra > "$UNINSTALL_SNAPSHOT_DIR/files/target-extra"; chmod 600 "$UNINSTALL_SNAPSHOT_DIR/files/target-extra" ;; - payload-fifo) mkfifo "$UNINSTALL_SNAPSHOT_DIR/files/.fifo" ;; - payload-symlink) ln -s target-source "$UNINSTALL_SNAPSHOT_DIR/files/.link" ;; - manifest-fifo) rm -- "$manifest"; mkfifo "$manifest" ;; - ancestor) chmod 0777 "$UNINSTALL_SNAPSHOT_DIR/files" ;; - esac - if uninstall_manifest_valid "$manifest"; then fail "schema accepted $schema_case"; fi - # No rollback is appropriate for this read-only parser test. - UNINSTALL_TRANSACTION_ACTIVE=false - UNINSTALL_SNAPSHOT_BUILDING=false - UNINSTALL_TRANSACTION_STATE=NONE - restore_repository_traps - # Parent alone owns TEST_DIR cleanup; do not run its EXIT trap here. - trap - EXIT - release_repository_lock || fail "schema fixture release" - ) > "$TEST_DIR/schema-$schema_case.log" 2>&1 || fail "schema $schema_case (see $TEST_DIR/schema-$schema_case.log)" - pass "uninstall manifest rejects $schema_case" -done - - -for ownership_case in owned foreign inode mode owner gid symlink directory fifo; do - ( - new_case - uninstall_transaction_hook() { :; } - trap - EXIT - configure_repository || fail "ownership configure" - acquire_repository_lock || fail "ownership lock" - begin_uninstall_transaction || fail "ownership capture" - target="$AUTO_UPDATE_SCRIPT" - printf 'owned-content\n' > "$CASE_DIR/payload" - chmod 0600 "$CASE_DIR/payload" - if [[ "$ownership_case" == foreign ]]; then - printf foreign > "$target" - chmod 0755 "$target" - else - uninstall_create_absent_target auto_update_script "$CASE_DIR/payload" || fail "exclusive creation" - case "$ownership_case" in - inode) mv "$target" "$target.original"; cp "$target.original" "$target" ;; - mode) chmod 0644 "$target" ;; - owner) chown 65534:0 "$target" ;; - gid) chown 0:65534 "$target" ;; - symlink) rm "$target"; ln -s "$CASE_DIR/payload" "$target" ;; - directory) rm "$target"; mkdir "$target" ;; - fifo) rm "$target"; mkfifo "$target" ;; - esac - fi - if [[ "$ownership_case" == owned ]]; then - uninstall_cleanup ownership-test || fail "owned rollback" - assert_absent "$target" "owned file survived" - else - if uninstall_cleanup ownership-test; then fail "foreign identity accepted"; fi - [[ -e "$target" || -L "$target" ]] || fail "foreign object deleted" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" ]] || fail "manifest lost" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ]] || fail "payload lost" - fi - trap - EXIT - ) > "$TEST_DIR/ownership-$ownership_case.log" 2>&1 || { cat "$TEST_DIR/ownership-$ownership_case.log"; fail "ownership $ownership_case"; } - pass "absent ownership $ownership_case" -done - -for journal_case in generation duplicate state unknown reason; do - ( - new_case - trap - EXIT - uninstall_transaction_hook() { :; } - configure_repository || fail "journal configure" - acquire_repository_lock || fail "journal lock" - begin_uninstall_transaction || fail "journal capture" - journal="$UNINSTALL_SNAPSHOT_DIR/journal" - case "$journal_case" in - generation) sed -i '2s/.*/generation=wrong/' "$journal" ;; - duplicate) printf 'state=ACTIVE\n' >> "$journal" ;; - state) sed -i '3s/.*/state=COMMITTED/' "$journal" ;; - unknown) sed -i '1s/.*/unknown=1/' "$journal" ;; - reason) sed -i '4s/.*/path=invalid/' "$journal" ;; - esac - if uninstall_journal_valid "$journal" ACTIVE; then fail "bad journal accepted"; fi - UNINSTALL_TRANSACTION_STATE=FAILED - UNINSTALL_TRANSACTION_ACTIVE=false - UNINSTALL_SNAPSHOT_BUILDING=false - restore_repository_traps - release_repository_lock || fail "journal release" - ) > "$TEST_DIR/journal-$journal_case.log" 2>&1 || fail "journal $journal_case" - pass "journal rejects $journal_case" -done - -for finalization_case in journal archive release restore; do - ( - new_case - trap - EXIT - uninstall_transaction_hook() { :; } - configure_repository || fail "finalization configure" - acquire_repository_lock || fail "finalization lock" - begin_uninstall_transaction || fail "finalization capture" - snapshot_before="$UNINSTALL_SNAPSHOT_DIR" - case "$finalization_case" in - journal|archive) - repository_rename() { - if [[ "$finalization_case" == journal && "$2" == */journal ]] || - [[ "$finalization_case" == archive && "$1" == "$snapshot_before" ]]; then return 1; fi - command mv -fT -- "$1" "$2" - } - ;; - release) - # A nonempty lock is an actual rmdir failure, not a core stub. - printf retained > "$REPOSITORY_LOCK_DIR/injected" - ;; - restore) - repository_install_file() { - [[ "$2" != */target-source ]] || return 1 - command install -o 0 -g 0 -m "$1" -- "$2" "$3" - } - ;; - esac - if uninstall_cleanup "injected-$finalization_case"; then fail "finalization failure accepted"; fi - [[ "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "failed guard state" - [[ "$UNINSTALL_TRANSACTION_ACTIVE" == false ]] || fail "active guard survived" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" ]] || fail "manifest deleted" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ]] || fail "source payload deleted" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-current" ]] || fail "current payload deleted" - cmp "$SOURCE_FILE" "$UNINSTALL_SNAPSHOT_DIR/files/target-source" || fail "source changed" - [[ -f "$KEYRING" ]] || fail "keyring deleted" - if [[ "$finalization_case" == release ]]; then - [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "lock evidence missing" - fi - ) > "$TEST_DIR/finalization-$finalization_case.log" 2>&1 || { - cat "$TEST_DIR/finalization-$finalization_case.log" - fail "finalization $finalization_case" - } - pass "finalization $finalization_case failure preserves complete evidence" -done - -for capture_id in auto_update_script auto_update_timer legacy_marker; do - ( - new_case - trap - EXIT - configure_repository || fail "capture configure" - cp "$SOURCE_FILE" "$CASE_DIR/source-before" - cp "$KEYRING" "$CASE_DIR/key-before" - acquire_repository_lock || fail "capture lock" - uninstall_transaction_hook() { [[ "$1" != "capture-$capture_id" ]]; } - if begin_uninstall_transaction; then fail "capture fault ignored"; fi - [[ "$UNINSTALL_TRANSACTION_STATE" == BUILDING ]] || fail "partial capture activated" - if uninstall_cleanup capture-failed; then fail "partial capture reported recovery"; fi - cmp "$SOURCE_FILE" "$CASE_DIR/source-before" || fail "partial source changed" - cmp "$KEYRING" "$CASE_DIR/key-before" || fail "partial key changed" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/journal" ]] || fail "partial journal missing" - [[ "$UNINSTALL_TRANSACTION_ACTIVE" == false ]] || fail "partial guard active" - ) > "$TEST_DIR/capture-$capture_id.log" 2>&1 || { - cat "$TEST_DIR/capture-$capture_id.log" - fail "capture $capture_id" - } - pass "capture failure $capture_id retains partial evidence without restoration" -done - -for archive_case in intact payload journal; do - ( - new_case - trap - EXIT - uninstall_transaction_hook() { :; } - configure_repository || fail "archive configure" - acquire_repository_lock || fail "archive lock" - begin_uninstall_transaction || fail "archive capture" - uninstall_transaction_hook() { - [[ "$1" == after-evidence-rename ]] || return 0 - case "$archive_case" in - payload) printf corrupt >> "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ;; - journal) printf 'unknown=1\n' >> "$UNINSTALL_SNAPSHOT_DIR/journal" ;; - esac - return 0 - } - rc=0 - uninstall_cleanup archive-verification || rc=$? - if [[ "$archive_case" == intact ]]; then - [[ "$rc" == 0 && "$UNINSTALL_TRANSACTION_STATE" == ROLLED_BACK ]] || fail "valid archive rejected" - else - [[ "$rc" != 0 && "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "corrupt archive accepted" - fi - [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" && -f "$UNINSTALL_SNAPSHOT_DIR/journal" ]] || fail "archive evidence lost" - [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-current" ]] || fail "archive payload lost" - ) > "$TEST_DIR/archive-$archive_case.log" 2>&1 || { - cat "$TEST_DIR/archive-$archive_case.log" - fail "archive $archive_case" - } - pass "post-rename archive verification $archive_case" -done - -for interrupted_phase in before-snapshot-create after-snapshot-active after-evidence-rename after-evidence-seal before-pending-clear; do - ( - new_case - trap - EXIT - configure_repository || fail "kill configure" - cp "$SOURCE_FILE" "$CASE_DIR/source-before" - ( - trap - EXIT - acquire_repository_lock || exit 1 - uninstall_transaction_hook() { - if [[ "$1" == "$interrupted_phase" ]]; then kill -KILL "$BASHPID"; fi - } - begin_uninstall_transaction || exit 1 - uninstall_cleanup kill-test - ) > "$CASE_DIR/killed.log" 2>&1 & - child=$! - rc=0 - wait "$child" || rc=$? - [[ "$rc" == 137 ]] || fail "SIGKILL did not reach $interrupted_phase" - # Simulate operator removing only the stale lock; pending must still block. - rmdir "$CLOUDFLARED_STATE_DIR.lock" || fail "stale lock remove" - : > "$FAKE_LOG" - for operation in install_cloudflared upgrade_cloudflared uninstall_cloudflared disable_auto_update; do - ( - trap - EXIT - "$operation" --confirmed - ) > "$CASE_DIR/retry-$operation.log" 2>&1 && fail "pending allowed $operation" - done - [[ ! -s "$FAKE_LOG" ]] || fail "pending executed external command" - cmp "$SOURCE_FILE" "$CASE_DIR/source-before" || fail "pending changed source" - compgen -G "$CLOUDFLARED_STATE_DIR/repository/pending-uninstall-*" >/dev/null || fail "pending evidence missing" - ) > "$TEST_DIR/kill-$interrupted_phase.log" 2>&1 || { - cat "$TEST_DIR/kill-$interrupted_phase.log" - fail "SIGKILL $interrupted_phase" - } - pass "SIGKILL $interrupted_phase fails closed in independent entrypoints" -done - -for stage_case in owned foreign; do - ( - new_case - trap - EXIT - configure_repository || fail "stage configure" - acquire_repository_lock || fail "stage lock" - begin_uninstall_transaction || fail "stage capture" - stage=$(mktemp "$CASE_DIR/root/.cloudflared-create.XXXXXX") - uninstall_register_stage "$stage" || fail "stage register" - if [[ "$stage_case" == foreign ]]; then - mv "$stage" "$stage.original" - printf foreign > "$stage" - fi - rc=0 - uninstall_cleanup stage-test || rc=$? - if [[ "$stage_case" == owned ]]; then - [[ "$rc" == 0 && ! -e "$stage" ]] || fail "owned stage not cleaned" - [[ "$(stat -c %a "$UNINSTALL_SNAPSHOT_DIR")" == 500 ]] || fail "archive writable" - [[ "$(stat -c %a "$UNINSTALL_SNAPSHOT_DIR/manifest")" == 400 ]] || fail "manifest writable" - validate_uninstall_archive "$UNINSTALL_SNAPSHOT_DIR" "$UNINSTALL_GENERATION" || fail "sealed archive invalid" - else - [[ "$rc" != 0 && "$(cat "$stage")" == foreign ]] || fail "foreign stage removed" - [[ -f "$UNINSTALL_PENDING" ]] || fail "stage failure lost pending" - fi - ) > "$TEST_DIR/stage-$stage_case.log" 2>&1 || { - cat "$TEST_DIR/stage-$stage_case.log" - fail "stage $stage_case" - } - pass "stage cleanup verifies $stage_case identity" -done - -for terminal_signal in HUP INT TERM; do - case "$terminal_signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - ( - new_case - trap - EXIT - configure_repository || fail "terminal signal configure" - rc=0 - ( - trap - EXIT - acquire_repository_lock || exit 1 - uninstall_transaction_hook() { - if [[ "$1" == after-evidence-seal ]]; then kill -s "$terminal_signal" "$BASHPID"; fi - } - begin_uninstall_transaction || exit 1 - uninstall_cleanup terminal-signal - ) > "$CASE_DIR/signal.log" 2>&1 || rc=$? - [[ "$rc" == "$expected" ]] || fail "terminal signal status $rc" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "terminal signal lock retained" - compgen -G "$CLOUDFLARED_STATE_DIR/repository/pending-uninstall-*" >/dev/null || fail "terminal signal pending missing" - [[ -f "$SOURCE_FILE" ]] || fail "terminal signal source lost" - ) > "$TEST_DIR/terminal-$terminal_signal.log" 2>&1 || { - cat "$TEST_DIR/terminal-$terminal_signal.log" - fail "terminal signal $terminal_signal" - } - pass "terminal $terminal_signal=$expected retains evidence without repeated rollback" -done diff --git a/tests/test-cloudflare-tunnel-03.sh b/tests/test-cloudflare-tunnel-03.sh deleted file mode 100644 index 0ec9437..0000000 --- a/tests/test-cloudflare-tunnel-03.sh +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-cloudflare-tunnel-common.sh" - -for restore_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do - ( - new_case - trap - EXIT - configure_repository || fail "six target configure" - mkdir -p "$(dirname "$AUTO_UPDATE_SCRIPT")" "$(dirname "$AUTO_UPDATE_SERVICE")" - write_auto_update_files || fail "six target auto update" - printf managed > "$STATE_DIR/repository-managed" - chmod 0600 "$STATE_DIR/repository-managed" - acquire_repository_lock || fail "six target lock" - begin_uninstall_transaction || fail "six target capture" - snapshot=$UNINSTALL_SNAPSHOT_DIR - while IFS= read -r id; do rm -- "$(uninstall_target_path "$id")"; done < <(uninstall_snapshot_targets) - repository_install_file() { - [[ "$2" != "$snapshot/files/target-$restore_id" ]] || return 1 - command install -o 0 -g 0 -m "$1" -- "$2" "$3" - } - if uninstall_cleanup six-target-failure; then fail "restore fault accepted"; fi - while IFS= read -r id; do - [[ -f "$snapshot/files/target-$id" ]] || fail "snapshot payload lost $id" - if [[ "$id" != "$restore_id" ]]; then - cmp "$(uninstall_target_path "$id")" "$snapshot/files/target-$id" || fail "restore skipped $id" - validate_secure_file "$(uninstall_target_path "$id")" "$(uninstall_target_mode "$id")" || fail "restore metadata $id" - fi - done < <(uninstall_snapshot_targets) - [[ -f "$UNINSTALL_PENDING" ]] || fail "restore pending lost" - uninstall_journal_valid "$snapshot/journal" ACTIVE || fail "failure overwrote original journal" - ) > "$TEST_DIR/restore-$restore_id.log" 2>&1 || { - cat "$TEST_DIR/restore-$restore_id.log" - fail "restore failure $restore_id" - } - pass "restore failure $restore_id continues remaining five targets and preserves original evidence" -done - -for target_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do - for bad_metadata in mode owner gid symlink directory fifo; do - ( - new_case - trap - EXIT - configure_repository || fail "metadata configure" - mkdir -p "$(dirname "$AUTO_UPDATE_SCRIPT")" "$(dirname "$AUTO_UPDATE_SERVICE")" - write_auto_update_files || fail "metadata auto update" - printf managed > "$STATE_DIR/repository-managed" - chmod 0600 "$STATE_DIR/repository-managed" - target=$(uninstall_target_path "$target_id") - case "$bad_metadata" in - mode) chmod 0666 "$target" ;; - owner) chown 1:0 "$target" ;; - gid) chown 0:1 "$target" ;; - symlink) mv "$target" "$target.external"; ln -s "$target.external" "$target" ;; - directory) rm "$target"; mkdir "$target" ;; - fifo) rm "$target"; mkfifo "$target" ;; - esac - acquire_repository_lock || fail "metadata lock" - if begin_uninstall_transaction; then fail "bad target captured"; fi - uninstall_cleanup bad-metadata >/dev/null 2>&1 || : - [[ -e "$target" || -L "$target" ]] || fail "untrusted target deleted" - [[ "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "capture failure state lost" - ) > "$TEST_DIR/meta-$target_id-$bad_metadata.log" 2>&1 || { - cat "$TEST_DIR/meta-$target_id-$bad_metadata.log" - fail "metadata $target_id $bad_metadata" - } - pass "capture rejects $target_id $bad_metadata without deleting target" - done -done - -( - unset CLOUDFLARED_TRUST_ANCHOR - init_runtime_config - [[ "$TRUST_ANCHOR" == / ]] || exit 1 - path_is_beneath_anchor / || exit 1 - path_is_beneath_anchor /var/lib || exit 1 - CLOUDFLARED_TRUST_ANCHOR=/var - path_is_beneath_anchor /var || exit 1 - path_is_beneath_anchor /var/lib || exit 1 - if path_is_beneath_anchor /variable; then exit 1; fi - if path_is_beneath_anchor /var/../etc; then exit 1; fi - path_is_beneath_anchor /var/lib/../lib || exit 1 -) || fail "pure anchor boundaries" -pass "pure paths validate default root anchor and normalized non-root boundaries" - -for creation_phase in before-create-marker before-create-manifest-stage before-create-manifest-rename before-create-publish after-create-publish; do - for creation_fault in failure HUP INT TERM; do - ( - new_case - trap - EXIT - configure_repository || fail "creation configure" - printf payload > "$CASE_DIR/payload" - chmod 0600 "$CASE_DIR/payload" - rc=0 - ( - trap - EXIT - acquire_repository_lock || exit 1 - begin_uninstall_transaction || exit 1 - uninstall_transaction_hook() { - [[ "$1" == "$creation_phase" ]] || return 0 - if [[ "$creation_fault" == failure ]]; then return 1; fi - kill -s "$creation_fault" "$BASHPID" - } - if uninstall_create_absent_target auto_update_script "$CASE_DIR/payload"; then exit 90; fi - uninstall_cleanup creation-failure || : - exit 1 - ) > "$CASE_DIR/creation.log" 2>&1 || rc=$? - case "$creation_fault" in failure) expected=1 ;; HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - [[ "$rc" == "$expected" ]] || fail "creation status $rc" - [[ ! -e "$AUTO_UPDATE_SCRIPT" && ! -L "$AUTO_UPDATE_SCRIPT" ]] || fail "owned publication not recovered" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "creation lock leaked" - ) > "$TEST_DIR/create-$creation_phase-$creation_fault.log" 2>&1 || { - cat "$TEST_DIR/create-$creation_phase-$creation_fault.log" - fail "creation $creation_phase $creation_fault" - } - pass "creation $creation_phase $creation_fault preserves ownership across publication" - done -done - -for bookkeeping_fault in marker stage rename; do - ( - new_case - trap - EXIT - configure_repository || fail "bookkeeping configure" - printf payload > "$CASE_DIR/payload"; chmod 0600 "$CASE_DIR/payload" - acquire_repository_lock || fail "bookkeeping lock" - begin_uninstall_transaction || fail "bookkeeping capture" - uninstall_transaction_hook() { - if [[ "$bookkeeping_fault" == marker && "$1" == before-create-marker ]]; then - mkdir "$UNINSTALL_SNAPSHOT_DIR/created-auto_update_script" - fi - return 0 - } - mktemp() { - if [[ "$bookkeeping_fault" == stage && "$1" == */manifest.stage.* ]]; then return 1; fi - command mktemp "$@" - } - repository_rename() { - if [[ "$bookkeeping_fault" == rename && "$1" == */manifest.stage.* ]]; then return 1; fi - command mv -fT -- "$1" "$2" - } - if uninstall_create_absent_target auto_update_script "$CASE_DIR/payload"; then fail "bookkeeping fault ignored"; fi - [[ ! -e "$AUTO_UPDATE_SCRIPT" ]] || fail "published before durable bookkeeping" - uninstall_cleanup bookkeeping-failure || : - ) > "$TEST_DIR/bookkeeping-$bookkeeping_fault.log" 2>&1 || { - cat "$TEST_DIR/bookkeeping-$bookkeeping_fault.log" - fail "bookkeeping $bookkeeping_fault" - } - pass "actual $bookkeeping_fault failure precedes ln publication" -done - -new_case -configure_repository || fail "URL configure" -grep '^curl:' "$FAKE_LOG" | grep -Fq 'https://pkg.cloudflare.com/cloudflare-main.gpg' || fail "key download URL changed" -printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflared any main\n' "$CLOUDFLARED_KEYRING" > "$CASE_DIR/official-source" -assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/official-source" "APT repository URL incorrect" -validate_current_repository_manifest || fail "official current rejected" -remove_managed_repository || fail "official source removal failed" -assert_absent "$CLOUDFLARED_SOURCE_FILE" "official source not removed" -pass "literal key URL and repository source validate current and real removal" - -new_case -printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflare-main.gpg any main\n' "$CLOUDFLARED_KEYRING" > "$CLOUDFLARED_SOURCE_FILE" -chmod 0644 "$CLOUDFLARED_SOURCE_FILE" -cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/wrong-source" -if configure_repository > "$CASE_DIR/rejected.log" 2>&1; then fail "gpg repository accepted"; fi -assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/wrong-source" "unowned wrong source overwritten" -[[ ! -s "$FAKE_LOG" ]] || fail "invalid source triggered external operation" -pass "gpg repository URL rejected without overwrite or download" - -for capture_file in old-key old-source old-current; do - for capture_fault in copy partial chmod HUP INT TERM exit0; do - ( - new_case - trap - EXIT - configure_repository || fail "capture fixture" - for object in "$KEYRING" "$SOURCE_FILE" "$REPOSITORY_STATE_DIR/current"; do - cp -p "$object" "$CASE_DIR/$(basename "$object").before" - done - : > "$FAKE_LOG" - rc=0 - ( - trap - EXIT - repository_copy_file() { - if [[ "$2" == */"$capture_file" ]]; then - case "$capture_fault" in - copy) return 1 ;; - partial) printf partial > "$2"; return 1 ;; - HUP|INT|TERM) kill -s "$capture_fault" "$BASHPID" ;; - exit0) exit 0 ;; - esac - fi - command cp -- "$1" "$2" - } - chmod() { - if [[ "$capture_fault" == chmod && "$2" == */"$capture_file" ]]; then return 1; fi - command chmod "$@" - } - configure_repository - ) > "$CASE_DIR/capture.log" 2>&1 || rc=$? - case "$capture_fault" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; *) expected=1 ;; esac - [[ "$rc" == "$expected" ]] || fail "capture status $rc" - for object in "$KEYRING" "$SOURCE_FILE" "$REPOSITORY_STATE_DIR/current"; do - cmp "$object" "$CASE_DIR/$(basename "$object").before" || fail "capture modified formal bytes" - [[ "$(stat -c '%u:%g:%a' "$object")" == "$(stat -c '%u:%g:%a' "$CASE_DIR/$(basename "$object").before")" ]] || fail "capture modified metadata" - done - [[ ! -s "$FAKE_LOG" ]] || fail "capture downloaded or called APT" - grep -q '未恢复或删除正式文件' "$CASE_DIR/capture.log" || fail "capture diagnostic missing" - acquire_repository_lock || fail "capture lock retained" - release_repository_lock || fail "capture lock release" - ) > "$TEST_DIR/capture-boundary-$capture_file-$capture_fault.log" 2>&1 || { - cat "$TEST_DIR/capture-boundary-$capture_file-$capture_fault.log" - fail "capture boundary $capture_file $capture_fault" - } - pass "repository capture $capture_file $capture_fault preserves formal bytes and metadata" - done -done - -( - new_case - trap - EXIT - set_old_generation - rc=0 - ( - trap - EXIT - begin_repository_transaction || exit 1 - exit 0 - ) > "$CASE_DIR/active-exit.log" 2>&1 || rc=$? - [[ "$rc" == 1 ]] || fail "active exit zero accepted" - assert_old_generation - [[ ! -e "$REPOSITORY_STATE_DIR/current" ]] || fail "absent current restored incorrectly" - acquire_repository_lock || fail "active exit lock retained" - release_repository_lock || fail "active exit release" -) || fail "active exit zero" -pass "complete repository transaction exit zero returns nonzero and restores absent current" - -for build_case in absent archive-failure; do - ( - new_case - trap - EXIT - if [[ "$build_case" == archive-failure ]]; then set_old_generation; fi - trap ':' HUP - before_traps=$(trap -p HUP INT TERM EXIT) - repository_copy_file() { printf partial > "$2"; return 1; } - if [[ "$build_case" == absent ]]; then - # No formal object exists; fail before any download via snapshot wrapper. - chmod() { - [[ "$2" != */transaction-* ]] || return 1 - command chmod "$@" - } - repository_transaction_hook() { exit 0; } - rc=0 - (begin_repository_transaction) > "$CASE_DIR/absent.log" 2>&1 || rc=$? - [[ "$rc" != 0 ]] || fail "absent abnormal exit accepted" - [[ ! -e "$KEYRING" && ! -e "$SOURCE_FILE" && ! -e "$REPOSITORY_STATE_DIR/current" ]] || fail "absent formal object created" - else - repository_rename() { return 1; } - if configure_repository > "$CASE_DIR/build.log" 2>&1; then fail "failed capture accepted"; fi - assert_old_generation - [[ -f "$REPOSITORY_TRANSACTION_DIR/old-key" ]] || fail "partial evidence lost" - [[ -f "$REPOSITORY_TRANSACTION_DIR/rollback.log" ]] || fail "failure diagnostic lost" - grep -q '回滚不完整' "$CASE_DIR/build.log" || fail "cleanup error not reported" - fi - [[ "$(trap -p HUP INT TERM EXIT)" == "$before_traps" ]] || fail "build traps changed" - acquire_repository_lock || fail "build cleanup lock retained" - release_repository_lock || fail "build cleanup lock release" - ) > "$TEST_DIR/build-$build_case.log" 2>&1 || { - cat "$TEST_DIR/build-$build_case.log" - fail "build $build_case" - } - pass "repository building $build_case preserves objects traps and evidence" -done diff --git a/tests/test-cloudflare-tunnel-04.sh b/tests/test-cloudflare-tunnel-04.sh deleted file mode 100644 index bc9ed84..0000000 --- a/tests/test-cloudflare-tunnel-04.sh +++ /dev/null @@ -1,375 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-cloudflare-tunnel-common.sh" - -for restore_object in key source current; do - for restore_fault in partial wrong-content wrong-mode rename post-content post-mode; do - ( - new_case - trap - EXIT - configure_repository || fail "restore fixture" - begin_repository_transaction || fail "restore transaction" - snapshot=$REPOSITORY_TRANSACTION_DIR - printf historical > "$REPOSITORY_STATE_DIR/current.rollback" - cp "$REPOSITORY_STATE_DIR/current.rollback" "$CASE_DIR/historical" - case "$restore_object" in key) target=$KEYRING ;; source) target=$SOURCE_FILE ;; current) target=$REPOSITORY_STATE_DIR/current ;; esac - cp "$target" "$CASE_DIR/pre-restore" - : > "$CASE_DIR/attempts" - repository_install_file() { - printf '%s\n' "$2" >> "$CASE_DIR/attempts" - if [[ "$2" == "$snapshot/old-$restore_object" ]]; then - case "$restore_fault" in - partial) printf partial > "$3"; return 1 ;; - wrong-content) command install -o 0 -g 0 -m "$1" "$2" "$3"; printf wrong > "$3"; return 0 ;; - wrong-mode) command install -o 0 -g 0 -m 0777 "$2" "$3"; return 0 ;; - esac - fi - command install -o 0 -g 0 -m "$1" "$2" "$3" - } - repository_rename() { - if [[ "$2" == "$target" && "$1" == */.cloudflared-rollback.* ]]; then - [[ "$restore_fault" != rename ]] || return 1 - command mv -fT -- "$1" "$2" || return 1 - case "$restore_fault" in post-content) printf corrupt > "$2" ;; post-mode) chmod 0777 "$2" ;; esac - return 0 - fi - command mv -fT -- "$1" "$2" - } - if rollback_repository_transaction injected > "$CASE_DIR/restore.log" 2>&1; then fail "restore fault accepted"; fi - for item in key source current; do - [[ -f "$REPOSITORY_TRANSACTION_DIR/old-$item" ]] || fail "backup lost $item" - grep -Fxq "$snapshot/old-$item" "$CASE_DIR/attempts" || fail "restore skipped $item" - done - case "$restore_fault" in partial|wrong-content|wrong-mode|rename) cmp "$target" "$CASE_DIR/pre-restore" || fail "invalid stage published" ;; esac - cmp "$REPOSITORY_STATE_DIR/current.rollback" "$CASE_DIR/historical" || fail "historical rollback touched" - if grep -q '旧 key/source 已恢复' "$CASE_DIR/restore.log"; then fail "false recovery success"; fi - grep -q '回滚不完整' "$CASE_DIR/restore.log" || fail "missing failure diagnosis" - acquire_repository_lock || fail "restore lock retained" - release_repository_lock || fail "restore lock release" - ) > "$TEST_DIR/publish-$restore_object-$restore_fault.log" 2>&1 || { - cat "$TEST_DIR/publish-$restore_object-$restore_fault.log" - fail "restore publish $restore_object $restore_fault" - } - pass "repository restore $restore_object $restore_fault rejects corruption and retains all backups" - done -done - -for operation in install_cloudflared upgrade_cloudflared; do - ( - new_case - trap - EXIT - cp "$CASE_DIR/bin/cloudflared" "$APT_BIN" - dpkg-query() { [[ "$*" == *'db:Status-Status'* ]] && { printf installed; return 0; }; return 1; } - timer_enabled=true - systemctl() { - printf 'systemctl:%s\n' "$*" >> "$FAKE_LOG" - if [[ "$*" == *cloudflared-apt-update.timer* && ( "$1" == is-enabled || "$1" == is-active ) ]]; then - [[ "$timer_enabled" == true ]] - else - return 0 - fi - } - write_auto_update_files || fail "timer fixture" - auto_update_file_is_managed "$AUTO_UPDATE_SCRIPT" || fail "main script template rejected" - auto_update_file_is_managed "$AUTO_UPDATE_SERVICE" || fail "main service template rejected" - auto_update_file_is_managed "$AUTO_UPDATE_TIMER" || fail "main timer template rejected" - "$operation" || fail "enabled operation" - [[ "$PRESERVE_AUTO_UPDATE" == true ]] || fail "strategy reset by init" - grep -Fxq 'systemctl:enable --now cloudflared-apt-update.timer' "$FAKE_LOG" || fail "timer not enabled" - timer_enabled=false - : > "$FAKE_LOG" - "$operation" || fail "disabled operation" - [[ "$PRESERVE_AUTO_UPDATE" == false ]] || fail "strategy leaked between operations" - if grep -Fxq 'systemctl:enable --now cloudflared-apt-update.timer' "$FAKE_LOG"; then fail "disabled timer enabled"; fi - timer_enabled=true - export FAKE_APT_UPDATE_FAIL=1 - : > "$FAKE_LOG" - if "$operation"; then fail "repository failure ignored"; fi - if grep -Eq 'systemctl:(disable|stop|enable|daemon-reload)' "$FAKE_LOG"; then fail "repository failure continued lifecycle"; fi - ) > "$TEST_DIR/policy-$operation.log" 2>&1 || { - cat "$TEST_DIR/policy-$operation.log" - fail "policy $operation" - } - pass "policy $operation preserves enabled main timer without cross-operation leakage" -done - -( - new_case - trap - EXIT - write_auto_update_files || fail "template fixture" - printf '# changed\n' >> "$AUTO_UPDATE_TIMER" - if auto_update_file_is_managed "$AUTO_UPDATE_TIMER"; then fail "partial marker trusted"; fi - cp "$AUTO_UPDATE_TIMER" "$CASE_DIR/foreign-timer" - if disable_auto_update --confirmed; then fail "unknown timer deleted"; fi - cmp "$AUTO_UPDATE_TIMER" "$CASE_DIR/foreign-timer" || fail "unknown timer changed" -) || fail "strict main template" -pass "main updater templates reject extra content without deleting unknown files" - -( - new_case - trap - EXIT - begin_repository_transaction || fail "absent transaction" - mv "$SOURCE_FILE" "$CASE_DIR/owned-source" - printf foreign > "$SOURCE_FILE"; chmod 0644 "$SOURCE_FILE" - if rollback_repository_transaction foreign; then fail "foreign absent accepted"; fi - [[ "$(cat "$SOURCE_FILE")" == foreign ]] || fail "foreign absent deleted" - [[ -f "$REPOSITORY_TRANSACTION_DIR/pending" ]] || fail "rollback pending missing" - : > "$FAKE_LOG" - if configure_repository; then fail "pending allowed restart"; fi - [[ ! -s "$FAKE_LOG" ]] || fail "pending performed external work" -) || fail "absent ownership" -pass "repository absent foreign object survives rollback and blocks restart" - -( - new_case - trap - EXIT - acquire_repository_lock || fail "lock fixture" - release_repository_lock || fail "first release" - mkdir "$CLOUDFLARED_STATE_DIR.lock" - release_repository_lock || fail "idempotent release" - [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "released another operation lock" -) || fail "lock identity" -pass "repeated lock release preserves next operation lock" - -( - new_case - trap - EXIT - set_old_generation - mkdir -p "$STATE_DIR" - chmod 0700 "$STATE_DIR" - printf unknown > "$STATE_DIR/repository-managed" - chmod 0600 "$STATE_DIR/repository-managed" - : > "$FAKE_LOG" - if uninstall_cloudflared --confirmed; then fail "bad legacy marker accepted"; fi - [[ ! -s "$FAKE_LOG" ]] || fail "legacy ownership checked after destructive operations" -) || fail "legacy preflight" -pass "legacy uninstall rejects unknown ownership before systemctl or APT" - -for review_fault in timer-enable stage-identity trap-restore; do - ( - new_case - trap - EXIT - case "$review_fault" in - timer-enable) - dpkg-query() { printf installed; } - systemctl() { - printf '%s\n' "$*" >> "$FAKE_LOG" - [[ "$*" != 'enable --now cloudflared-apt-update.timer' ]] - } - if enable_auto_update > "$CASE_DIR/error.log" 2>&1; then fail "timer enable failure hidden"; fi - grep -Fxq 'enable --now cloudflared-apt-update.timer' "$FAKE_LOG" || fail "timer injection not hit" - ;; - stage-identity) - begin_repository_transaction || fail "stage fixture" - stage=$(mktemp "$CASE_DIR/.stage.XXXXXX") - repository_track_stage "$stage" || fail "track stage" - mv "$stage" "$stage.owned" - printf foreign > "$stage" - if rollback_repository_transaction injected; then fail "foreign stage ignored"; fi - [[ "$(cat "$stage")" == foreign ]] || fail "foreign stage deleted" - ;; - trap-restore) - begin_repository_transaction || fail "trap fixture" - restore_one_trap() { printf '%s\n' "$1" >> "$CASE_DIR/trap-attempts"; return 1; } - if rollback_repository_transaction injected; then fail "trap failure hidden"; fi - [[ "$(wc -l < "$CASE_DIR/trap-attempts")" == 4 ]] || fail "trap attempts skipped" - [[ -f "$REPOSITORY_TRANSACTION_DIR/pending" ]] || fail "trap failure pending missing" - ;; - esac - ) > "$TEST_DIR/review-$review_fault.log" 2>&1 || { - cat "$TEST_DIR/review-$review_fault.log" - fail "review $review_fault" - } - pass "review $review_fault injection reached and failure preserved" -done - -for phase in disable-systemctl disable-backup disable-delete; do - for fault in HUP INT TERM zero failure; do - ( - new_case - trap - EXIT - configure_repository || fail "disable fixture" - write_auto_update_files || fail "disable files" - rc=0 - ( - trap - EXIT - updater_transaction_hook() { - [[ "$1" == "$phase" ]] || return 0 - case "$fault" in zero) exit 0 ;; failure) return 1 ;; *) kill -s "$fault" "$BASHPID" ;; esac - } - disable_auto_update --confirmed - ) > "$CASE_DIR/disable.log" 2>&1 || rc=$? - case "$fault" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; *) expected=1 ;; esac - [[ "$rc" == "$expected" ]] || fail "disable exit $rc" - [[ -f "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "premature file deletion" - compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "disable pending missing" - grep -q '自动更新操作未完成' "$CASE_DIR/disable.log" || fail "disable diagnostic missing" - acquire_repository_lock || fail "disable lock retained" - release_repository_lock || fail "disable lock release" - ) > "$TEST_DIR/disable-$phase-$fault.log" 2>&1 || { - cat "$TEST_DIR/disable-$phase-$fault.log" - fail "disable $phase $fault" - } - pass "independent disable $phase $fault preserves pending and exit contract" - done -done - -for phase in enable-validate enable-write enable-reload enable-timer disable-systemctl; do - ( - new_case - trap - EXIT - configure_repository || fail "barrier configure" - write_auto_update_files || fail "barrier files" - mkfifo "$CASE_DIR/ready" "$CASE_DIR/release" - ( - trap - EXIT - dpkg-query() { printf installed; } - updater_transaction_hook() { - [[ "$1" == "$phase" ]] || return 0 - printf ready > "$CASE_DIR/ready" - read -r _ < "$CASE_DIR/release" - } - if [[ "$phase" == disable-* ]]; then disable_auto_update --confirmed; else enable_auto_update; fi - ) > "$CASE_DIR/holder.log" 2>&1 & - holder=$! - read -r ready < "$CASE_DIR/ready" || [[ "$ready" == ready ]] || fail "barrier readiness" - for contender in enable_auto_update disable_auto_update uninstall_cloudflared; do - if ( "$contender" --confirmed ) > "$CASE_DIR/contender.log" 2>&1; then fail "contender bypassed lock"; fi - done - printf 'release\n' > "$CASE_DIR/release" - wait "$holder" || { cat "$CASE_DIR/holder.log"; fail "holder failed"; } - ) > "$TEST_DIR/barrier-$phase.log" 2>&1 || { cat "$TEST_DIR/barrier-$phase.log"; fail "barrier $phase"; } - pass "real updater barrier $phase excludes all competing mutation entrypoints" -done - -for fault in backup delete release; do - ( - new_case - trap - EXIT - configure_repository || fail "file fault fixture" - write_auto_update_files || fail "file fault files" - hit=false - cp() { - if [[ "$fault" == backup && "$*" == *auto-update-* ]]; then hit=true; return 1; fi - command cp "$@" - } - rm() { - if [[ "$fault" == delete && "$*" == *"$AUTO_UPDATE_SERVICE"* ]]; then hit=true; return 1; fi - command rm "$@" - } - rmdir() { - if [[ "$fault" == release ]]; then hit=true; return 1; fi - command rmdir "$@" - } - before=$(trap -p HUP INT TERM EXIT) - if disable_auto_update --confirmed > "$CASE_DIR/error.log" 2>&1; then fail "disable file fault accepted"; fi - [[ "$hit" == true ]] || fail "file fault not hit" - [[ "$(trap -p HUP INT TERM EXIT)" == "$before" ]] || fail "disable traps not restored" - compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "file fault pending missing" - case "$fault" in - backup) [[ -f "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "backup failure deleted files" ;; - delete) [[ ! -e "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "partial deletion group wrong" ;; - release) [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "failed release lost lock" ;; - esac - ) > "$TEST_DIR/disable-file-$fault.log" 2>&1 || { cat "$TEST_DIR/disable-file-$fault.log"; fail "disable file $fault"; } - pass "independent disable actual $fault failure retains group evidence and traps" -done - -# Separate shells load the implementation independently; only file/FIFO state is shared. -( - new_case - trap - EXIT - configure_repository || fail "uninstall barrier configure" - write_auto_update_files || fail "uninstall barrier updater" - mkfifo "$CASE_DIR/uninstall-ready" "$CASE_DIR/uninstall-release" - bash -c ' - source "$1" - check_platform() { :; } - uninstall_transaction_hook() { - [[ "$1" == after-snapshot-active ]] || return 0 - printf "%s\n" "$UNINSTALL_SNAPSHOT_DIR" > "$CASE_DIR/snapshot-path" - printf "ready\n" > "$CASE_DIR/uninstall-ready" - read -r _ < "$CASE_DIR/uninstall-release" - } - uninstall_cloudflared --confirmed - ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" > "$CASE_DIR/holder.log" 2>&1 & - holder=$! - # Timeout bounds a broken readiness handshake; no scheduling sleeps. - timeout 30 bash -c 'read -r ready < "$CASE_DIR/uninstall-ready"; [[ "$ready" == ready ]]' || fail "uninstall never reached active snapshot" - snapshot=$(cat "$CASE_DIR/snapshot-path") - [[ -f "$snapshot/manifest" ]] || fail "snapshot not ready" - lock_before=$(stat -c '%d:%i:%u:%g:%a' "$CLOUDFLARED_STATE_DIR.lock") - find "$REPOSITORY_STATE_DIR" -type f -exec sha256sum {} + | sort > "$CASE_DIR/state-before" - sha256sum "$AUTO_UPDATE_SCRIPT" "$AUTO_UPDATE_SERVICE" "$AUTO_UPDATE_TIMER" > "$CASE_DIR/updater-before" - for contender in enable_auto_update disable_auto_update; do - : > "$CASE_DIR/contender-external.log" - rc=0 - FAKE_LOG="$CASE_DIR/contender-external.log" bash -c ' - source "$1" - check_platform() { :; } - "$2" --confirmed - ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" "$contender" > "$CASE_DIR/contender.log" 2>&1 || rc=$? - [[ "$rc" != 0 ]] || fail "contender entered uninstall lock" - grep -q '无法取得 Cloudflare 仓库事务锁' "$CASE_DIR/contender.log" || fail "competition not reached" - [[ ! -s "$CASE_DIR/contender-external.log" ]] || fail "contender executed external command" - [[ "$(stat -c '%d:%i:%u:%g:%a' "$CLOUDFLARED_STATE_DIR.lock")" == "$lock_before" ]] || fail "holder lock changed" - find "$REPOSITORY_STATE_DIR" -type f -exec sha256sum {} + | sort > "$CASE_DIR/state-after" - cmp "$CASE_DIR/state-before" "$CASE_DIR/state-after" || fail "snapshot or pending changed" - sha256sum -c "$CASE_DIR/updater-before" >/dev/null || fail "contender changed updater" - done - printf 'release\n' > "$CASE_DIR/uninstall-release" - wait "$holder" || { cat "$CASE_DIR/holder.log"; fail "uninstall holder failed"; } - [[ ! -e "$SOURCE_FILE" && ! -e "$AUTO_UPDATE_TIMER" ]] || fail "uninstall did not complete" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall retained lock" -) > "$TEST_DIR/uninstall-holder.log" 2>&1 || { cat "$TEST_DIR/uninstall-holder.log"; fail "uninstall holder barrier"; } -pass "independent uninstall holder excludes enable and disable without changing snapshot pending or lock" - -for dependency_fault in update install; do - ( - new_case - trap - EXIT - write_auto_update_files || fail "dependency updater fixture" - sha256sum "$AUTO_UPDATE_SCRIPT" "$AUTO_UPDATE_SERVICE" "$AUTO_UPDATE_TIMER" > "$CASE_DIR/updater-before" - command() { - if [[ "$*" == '-v flock' ]]; then - printf 'flock-missing\n' >> "$CASE_DIR/dependency-hits" - return 1 - fi - builtin command "$@" - } - dpkg-query() { printf installed; } - apt-get() { - printf 'apt-get:%s\n' "$*" >> "$FAKE_LOG" - if [[ "$1" == update && "$dependency_fault" == update ]]; then return 41; fi - if [[ "$*" == 'install -y util-linux' ]]; then return 42; fi - return 0 - } - trap ':' HUP - before_traps=$(trap -p HUP INT TERM EXIT) - rc=0 - enable_auto_update > "$CASE_DIR/dependency.log" 2>&1 || rc=$? - case "$dependency_fault" in update) expected=41 ;; install) expected=42 ;; esac - [[ "$rc" == "$expected" ]] || fail "dependency status $rc" - grep -Fxq flock-missing "$CASE_DIR/dependency-hits" || fail "flock branch not reached" - grep -Fxq apt-get:update "$FAKE_LOG" || fail "dependency update not reached" - if [[ "$dependency_fault" == update ]]; then - if grep -q 'apt-get:install' "$FAKE_LOG"; then fail "install continued after update failure"; fi - else - grep -Fxq 'apt-get:install -y util-linux' "$FAKE_LOG" || fail "util install not reached" - fi - if grep -Eq 'systemctl:.*(reload|enable)' "$FAKE_LOG"; then fail "dependency failure continued timer work"; fi - sha256sum -c "$CASE_DIR/updater-before" >/dev/null || fail "dependency failure changed updater" - validate_current_repository_manifest || fail "prior repository configuration not recorded" - compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "dependency pending missing" - [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "dependency lock retained" - [[ "$(trap -p HUP INT TERM EXIT)" == "$before_traps" ]] || fail "dependency traps changed" - grep -q '自动更新操作未完成' "$CASE_DIR/dependency.log" || fail "dependency diagnostic missing" - ) > "$TEST_DIR/dependency-$dependency_fault.log" 2>&1 || { cat "$TEST_DIR/dependency-$dependency_fault.log"; fail "dependency $dependency_fault"; } - pass "missing flock $dependency_fault failure stops updater after recorded repository commit" -done - -entrypoint_output=$(bash -c "$(cat "$ROOT_DIR/tools/cloudflare_tunnel.sh")" cloudflare_tunnel.sh help) -grep -Fq 'cloudflare_tunnel.sh install' <<< "$entrypoint_output" || fail "bash -c entrypoint broken" -pass "bash -c entrypoint remains compatible" - -printf 'All cloudflare wrapper tests passed. PASS=%d\n' "$pass_count" diff --git a/tests/test-cloudflare-tunnel.sh b/tests/test-cloudflare-tunnel.sh new file mode 100644 index 0000000..c23689a --- /dev/null +++ b/tests/test-cloudflare-tunnel.sh @@ -0,0 +1,1544 @@ +#!/usr/bin/env bash +set -euo pipefail + +if (( EUID != 0 )); then + exec sudo --preserve-env=PATH bash "$0" "$@" +fi + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +readonly ROOT_DIR +TEST_DIR=$(mktemp -d -p "$ROOT_DIR") +readonly TEST_DIR +trap 'rm -rf "$TEST_DIR"' EXIT + +pass_count=0 +fail() { printf 'FAIL: %s\n' "$*" >&2; exit 1; } +pass() { pass_count=$((pass_count + 1)); printf 'PASS: %s\n' "$*"; } +assert_same() { cmp -s -- "$1" "$2" || fail "$3"; } +assert_absent() { [[ ! -e "$1" && ! -L "$1" ]] || fail "$2"; } + +make_fake_commands() { + local bin="$1" + mkdir -p "$bin" + cat > "$bin/curl" <<'FAKE' +#!/usr/bin/env bash +set -eu +printf 'curl:%s\n' "$*" >> "$FAKE_LOG" +[[ "${FAKE_CURL_FAIL:-}" != 1 ]] || exit 22 +output="" +while (( $# )); do + if [[ "$1" == -o ]]; then output="$2"; shift 2; continue; fi + shift +done +[[ -n "$output" ]] +printf 'fake-cloudflare-key\n' > "$output" +FAKE + cat > "$bin/gpg" <<'FAKE' +#!/usr/bin/env bash +set -eu +printf 'gpg:%s\n' "$*" >> "$FAKE_LOG" +[[ "${FAKE_GPG_FAIL:-}" != 1 ]] || exit 2 +case "${FAKE_GPG_CASE:-good}" in + good) + cat <<'EOF' +pub:-:4096:1:8A682D308D4E5E73:1761229688:::-:::scESC::::::23::0: +fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: +uid:-::::1761229688::HASH::CloudFlare Software Packaging 2025 ::::::::::0: +sub:-:4096:1:029E1444B7D9F50F:1761229688::::::e::::::23: +fpr:::::::::06C89DB3B80A8F4349697C76029E1444B7D9F50F: +EOF + ;; + fingerprint) + cat <<'EOF' +pub:-:4096:1:BAD:0:::-:::scESC: +fpr:::::::::0000000000000000000000000000000000000000: +uid:-::::0::HASH::CloudFlare Software Packaging 2025 ::: +EOF + ;; + uid) + cat <<'EOF' +pub:-:4096:1:8A682D308D4E5E73:0:::-:::scESC: +fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: +uid:-::::0::HASH::Attacker ::: +EOF + ;; + multi) + cat <<'EOF' +pub:-:4096:1:8A682D308D4E5E73:0:::-:::scESC: +fpr:::::::::CC94B39C77AE7342A68B89628A682D308D4E5E73: +uid:-::::0::HASH::CloudFlare Software Packaging 2025 ::: +pub:-:4096:1:BAD:0:::-:::scESC: +fpr:::::::::0000000000000000000000000000000000000000: +uid:-::::0::HASH::Other ::: +EOF + ;; +esac +FAKE + cat > "$bin/apt-get" <<'FAKE' +#!/usr/bin/env bash +set -eu +printf 'apt-get:%s\n' "$*" >> "$FAKE_LOG" +case "${1:-}" in + update) [[ "${FAKE_APT_UPDATE_FAIL:-}" != 1 ]] ;; + install) [[ "${FAKE_APT_INSTALL_FAIL:-}" != 1 ]] ;; + *) exit 0 ;; +esac +FAKE + cat > "$bin/systemctl" <<'FAKE' +#!/usr/bin/env bash +printf 'systemctl:%s\n' "$*" >> "$FAKE_LOG" +exit 0 +FAKE + cat > "$bin/cloudflared" <<'FAKE' +#!/usr/bin/env bash +printf 'cloudflared:%s\n' "$*" >> "$FAKE_LOG" +exit 0 +FAKE + chmod 0755 "$bin"/* +} + +new_case() { + CASE_DIR="$TEST_DIR/case" + rm -rf "$CASE_DIR" + mkdir -p "$CASE_DIR" + export CASE_DIR FAKE_LOG="$CASE_DIR/fake.log" + : > "$FAKE_LOG" + mkdir -p "$CASE_DIR/root/usr/share/keyrings" "$CASE_DIR/root/etc/apt/sources.list.d" \ + "$CASE_DIR/root/var/lib" + chmod 0755 "$CASE_DIR/root" "$CASE_DIR/root/usr" "$CASE_DIR/root/usr/share" \ + "$CASE_DIR/root/usr/share/keyrings" "$CASE_DIR/root/etc" "$CASE_DIR/root/etc/apt" \ + "$CASE_DIR/root/etc/apt/sources.list.d" "$CASE_DIR/root/var" "$CASE_DIR/root/var/lib" + make_fake_commands "$CASE_DIR/bin" + export PATH="$CASE_DIR/bin:$ORIGINAL_PATH" + export CLOUDFLARED_TRUST_ANCHOR="$CASE_DIR" + export CLOUDFLARED_KEYRING="$CASE_DIR/root/usr/share/keyrings/cloudflare-main.gpg" + export CLOUDFLARED_SOURCE_FILE="$CASE_DIR/root/etc/apt/sources.list.d/cloudflared.list" + export CLOUDFLARED_APT_SOURCE_ROOT="$CASE_DIR/root/etc/apt" + export CLOUDFLARED_STATE_DIR="$CASE_DIR/root/var/lib/cloudflared-wrapper" + export CLOUDFLARED_REPOSITORY_STATE_DIR="$CLOUDFLARED_STATE_DIR/repository" + export CLOUDFLARED_LEGACY_BIN="$CASE_DIR/legacy-cloudflared" + export CLOUDFLARED_APT_BIN="$CASE_DIR/usr-bin-cloudflared" + export CLOUDFLARED_LEGACY_UPDATER="$CASE_DIR/cloudflared-update" + export CLOUDFLARED_LEGACY_SERVICE="$CASE_DIR/cloudflared-updater.service" + export CLOUDFLARED_LEGACY_TIMER="$CASE_DIR/cloudflared-updater.timer" + export CLOUDFLARED_AUTO_UPDATE_SCRIPT="$CASE_DIR/cloudflared-apt-update" + export CLOUDFLARED_AUTO_UPDATE_SERVICE="$CASE_DIR/cloudflared-apt-update.service" + export CLOUDFLARED_AUTO_UPDATE_TIMER="$CASE_DIR/cloudflared-apt-update.timer" + export CLOUDFLARED_SERVICE_FILE="$CASE_DIR/cloudflared.service" + export CLOUDFLARED_BINARY_UPDATE_SERVICE="$CASE_DIR/cloudflared-update.service" + export CLOUDFLARED_BINARY_UPDATE_TIMER="$CASE_DIR/cloudflared-update.timer" + export CLOUDFLARED_TEST_INTERNALS=1 + unset FAKE_CURL_FAIL FAKE_GPG_FAIL FAKE_GPG_CASE FAKE_APT_UPDATE_FAIL FAKE_APT_INSTALL_FAIL + unset FAIL_INSTALL_CALL FAIL_RENAME_CALL INSTALL_CALL RENAME_CALL SIGNAL_PHASE + if [[ "${SCRIPT_SOURCED:-}" != 1 ]]; then + # shellcheck source=../tools/cloudflare_tunnel.sh + source "$ROOT_DIR/tools/cloudflare_tunnel.sh" + SCRIPT_SOURCED=1 + fi + init_runtime_config +} + +set_old_generation() { + printf 'old-key\n' > "$CLOUDFLARED_KEYRING" + repository_legacy_source_content > "$CLOUDFLARED_SOURCE_FILE" + chmod 0644 "$CLOUDFLARED_KEYRING" "$CLOUDFLARED_SOURCE_FILE" + cp "$CLOUDFLARED_KEYRING" "$CASE_DIR/expected-key" + cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" +} + +assert_old_generation() { + assert_same "$CLOUDFLARED_KEYRING" "$CASE_DIR/expected-key" "old key was not restored" + assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" "old source was not restored" +} + +run_failure_case() { + local name="$1" setup="$2" action="${3:-configure_repository}" + new_case + set_old_generation + eval "$setup" + set +e + eval "$action" >"$CASE_DIR/out" 2>"$CASE_DIR/err" + status=$? + set -e + if (( status == 0 )); then + fail "$name unexpectedly succeeded" + fi + assert_old_generation + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "$name left lock" + find "$CLOUDFLARED_REPOSITORY_STATE_DIR" -maxdepth 1 -type d -name 'failure-*' -print -quit | grep -q . || + fail "$name did not preserve failure evidence" + pass "$name" +} + +ORIGINAL_PATH=$PATH + +new_case +before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) +repository_source_content > "$CASE_DIR/source-output" +after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) +[[ "$before" == "$after" ]] || fail "source rendering had side effects" +[[ "$(cat "$CASE_DIR/source-output")" == "deb [signed-by=$CLOUDFLARED_KEYRING] https://pkg.cloudflare.com/cloudflared any main" ]] || + fail "source is not exact official definition" +pass "source rendering has zero side effects and exact fields" + +( + new_case + source_cwd=$PWD + source_before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) + cloudflare_tunnel_source() { printf 'sentinel:%s\n' "$*"; } + sentinel_before=$(declare -f cloudflare_tunnel_source) + trap 'sentinel-trap' HUP + set -o noclobber + umask 027 + unset CLOUDFLARED_TEST_INTERNALS + source "$ROOT_DIR/tools/cloudflare_tunnel.sh" + sentinel_after=$(declare -f cloudflare_tunnel_source) + [[ "$sentinel_before" == "$sentinel_after" ]] || fail "ordinary source replaced sentinel function" + [[ "$(cloudflare_tunnel_source probe)" == sentinel:probe ]] || fail "ordinary source changed sentinel behavior" + [[ "$(trap -p HUP)" == *sentinel-trap* ]] || fail "ordinary source changed caller trap" + [[ "$-" == *C* ]] || fail "ordinary source changed caller shell options" + [[ "$(umask)" == 0027 ]] || fail "ordinary source changed caller umask" + [[ "$PWD" == "$source_cwd" ]] || fail "ordinary source changed caller cwd" + [[ "${CLOUDFLARED_STATE_DIR+x}" == x ]] || fail "test setup lost state variable" + after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) + [[ "$source_before" == "$after" ]] || fail "ordinary source changed filesystem" +) +pass "ordinary source preserves sentinels, shell state, and filesystem" + +new_case +before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\\n' | sort) +bash "$ROOT_DIR/tools/cloudflare_tunnel.sh" status >/dev/null 2>&1 || fail "standalone status failed in fake environment" +after=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\\n' | sort) +[[ "$before" == "$after" ]] || fail "ordinary status created state, lock, or temp files" +pass "ordinary help/status paths have zero state and lock side effects" + +cat > "$TEST_DIR/source-contract.sh" </dev/null' _ \ + "$ROOT_DIR/tools/cloudflare_tunnel.sh" || fail "internal source mode unavailable" +pass "explicit internal source mode exposes test functions" + +new_case +configure_repository || fail "configure before real uninstall failed" +require_root() { :; } +check_platform() { :; } +uninstall_cloudflared --confirmed || fail "uninstall normal path failed" +[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall left repository lock" +grep -Fxq 'apt-get:remove -y cloudflared' "$FAKE_LOG" || fail "uninstall skipped package removal" +assert_absent "$CLOUDFLARED_SOURCE_FILE" "real uninstall retained source" +assert_absent "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "real uninstall retained current" +[[ -f "$CLOUDFLARED_KEYRING" ]] || fail "real uninstall removed keyring" +pass "configure then real uninstall preserves keyring and releases lock" +init_runtime_config + +new_case +mkdir -p "$CLOUDFLARED_STATE_DIR.lock" +mkdir -p "$CLOUDFLARED_AUTO_UPDATE_SCRIPT" +printf marker > "$CASE_DIR/marker" +require_root() { :; } +check_platform() { :; } +if uninstall_cloudflared --confirmed >/dev/null 2>&1; then fail "uninstall lock competition unexpectedly succeeded"; fi +[[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall lock competition removed lock" +[[ -d "$CLOUDFLARED_AUTO_UPDATE_SCRIPT" ]] || fail "lock competition touched auto-update path" +! grep -Eq 'remove|disable|daemon-reload|apt-get:' "$FAKE_LOG" || fail "lock competition performed mutation" +pass "uninstall lock competition fails before auto-update, systemctl, and APT" + +new_case +mkdir -p "$CLOUDFLARED_STATE_DIR.lock" +if disable_auto_update --confirmed >/dev/null 2>&1; then fail "disable-auto-update lock competition unexpectedly succeeded"; fi +[[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "disable-auto-update removed competing lock" +pass "auto-update lock competition fails before state mutation" +chmod 0700 "$CLOUDFLARED_STATE_DIR.lock" +if configure_repository >/dev/null 2>&1; then fail "lock competition unexpectedly succeeded"; fi +pass "key/source lock competition" + +run_failure_case "key URL download failure" 'export FAKE_CURL_FAIL=1' +run_failure_case "gpg parser failure" 'export FAKE_GPG_FAIL=1' +run_failure_case "fingerprint mismatch" 'export FAKE_GPG_CASE=fingerprint' +run_failure_case "UID mismatch" 'export FAKE_GPG_CASE=uid' +run_failure_case "multiple primary keys" 'export FAKE_GPG_CASE=multi' + +for target in key source; do + for kind in symlink directory; do + new_case + case "$target" in key) path=$CLOUDFLARED_KEYRING ;; source) path=$CLOUDFLARED_SOURCE_FILE ;; esac + case "$kind" in + symlink) ln -s "$CASE_DIR/missing" "$path" ;; + directory) mkdir "$path" ;; + esac + if configure_repository >/dev/null 2>&1; then fail "$target $kind accepted"; fi + [[ -L "$path" || -d "$path" ]] || fail "$target $kind was changed" + pass "$target rejects $kind" + done + for bad in owner gid mode; do + new_case + case "$target" in key) path=$CLOUDFLARED_KEYRING ;; source) path=$CLOUDFLARED_SOURCE_FILE ;; esac + if [[ "$target" == source ]]; then + repository_source_content > "$path" + else + printf key > "$path" + fi + chmod 0644 "$path" + case "$bad" in + owner) chown 65534:0 "$path" ;; + gid) chown 0:65534 "$path" ;; + mode) chmod 0664 "$path" ;; + esac + if configure_repository >/dev/null 2>&1; then fail "$target wrong $bad accepted"; fi + pass "$target rejects wrong $bad" + done +done + +new_case +printf 'deb https://example.invalid stable main\n' > "$CASE_DIR/root/etc/apt/sources.list.d/extra.list" +chmod 0644 "$CASE_DIR/root/etc/apt/sources.list.d/extra.list" +configure_repository +pass "unrelated APT source remains untouched" + +new_case +printf 'deb https://pkg.cloudflare.com/cloudflared any main\n' > "$CASE_DIR/root/etc/apt/sources.list.d/duplicate.list" +chmod 0644 "$CASE_DIR/root/etc/apt/sources.list.d/duplicate.list" +if configure_repository >/dev/null 2>&1; then fail "duplicate Cloudflare source accepted"; fi +assert_absent "$CLOUDFLARED_KEYRING" "duplicate source changed key" +assert_absent "$CLOUDFLARED_SOURCE_FILE" "duplicate source wrote managed source" +pass "reject duplicate or extra Cloudflare source before commit" + +new_case +printf 'deb [signed-by=%s trusted=yes] https://pkg.cloudflare.com/cloudflared any main\n' \ + "$CLOUDFLARED_KEYRING" > "$CLOUDFLARED_SOURCE_FILE" +chmod 0644 "$CLOUDFLARED_SOURCE_FILE" +if configure_repository >/dev/null 2>&1; then fail "source injection accepted"; fi +pass "reject unknown source fields and injection content" + +( +new_case +set_old_generation +INSTALL_CALL=0 +FAIL_INSTALL_CALL=1 +repository_install_file() { + INSTALL_CALL=$((INSTALL_CALL + 1)) + (( INSTALL_CALL != FAIL_INSTALL_CALL )) || return 1 + command install -o 0 -g 0 -m "$1" -- "$2" "$3" +} +if configure_repository >/dev/null 2>&1; then fail "key stage write failure succeeded"; fi +assert_old_generation +) +pass "formal key write failure rolls back" + +( +new_case +set_old_generation +INSTALL_CALL=0 +FAIL_INSTALL_CALL=2 +repository_install_file() { + INSTALL_CALL=$((INSTALL_CALL + 1)) + (( INSTALL_CALL != FAIL_INSTALL_CALL )) || return 1 + command install -o 0 -g 0 -m "$1" -- "$2" "$3" +} +if configure_repository >/dev/null 2>&1; then fail "source stage write failure succeeded"; fi +assert_old_generation +) +pass "formal source write failure rolls back" + +( +new_case +set_old_generation +RENAME_CALL=0 +FAIL_RENAME_CALL=2 +repository_rename() { + RENAME_CALL=$((RENAME_CALL + 1)) + (( RENAME_CALL != FAIL_RENAME_CALL )) || return 1 + command mv -fT -- "$1" "$2" +} +if configure_repository >/dev/null 2>&1; then fail "source commit failure succeeded"; fi +assert_old_generation +) +pass "key success/source failure rolls back" + +( +new_case +set_old_generation +RENAME_CALL=0 +FAIL_RENAME_CALL=1 +repository_rename() { + RENAME_CALL=$((RENAME_CALL + 1)) + (( RENAME_CALL != FAIL_RENAME_CALL )) || return 1 + command mv -fT -- "$1" "$2" +} +if configure_repository >/dev/null 2>&1; then fail "key commit failure succeeded"; fi +assert_old_generation +) +pass "source remains old when key commit fails" + +run_failure_case "APT probe failure rollback" 'export FAKE_APT_UPDATE_FAIL=1' 'run_repository_apt_transaction install' +run_failure_case "APT install failure rollback" 'export FAKE_APT_INSTALL_FAIL=1' 'run_repository_apt_transaction install' + +for signal in HUP INT TERM; do + case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + for phase in download validate stage key-commit source-commit apt-probe apt-install; do + new_case + set_old_generation + cat > "$CASE_DIR/signal-runner.sh" <"$CASE_DIR/out" 2>"$CASE_DIR/err" + status=$? + set -e + [[ "$status" == "$expected" ]] || fail "$signal at $phase returned $status, expected $expected" + assert_old_generation + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "$signal at $phase left lock" + pass "$signal=$expected rolls back at $phase" + done +done + +( +new_case +set_old_generation +RENAME_CALL=0 +repository_rename() { + RENAME_CALL=$((RENAME_CALL + 1)) + case "$RENAME_CALL" in 2|3) return 1 ;; esac + command mv -fT -- "$1" "$2" +} +if configure_repository >/dev/null 2>&1; then fail "rollback failure unexpectedly succeeded"; fi +find "$CLOUDFLARED_REPOSITORY_STATE_DIR" -maxdepth 2 -type f -name rollback.log -print -quit | grep -q . || + fail "rollback failure did not preserve evidence" +) +pass "rollback failure returns nonzero and preserves evidence" + +new_case +set_old_generation +configure_repository +current="$CLOUDFLARED_REPOSITORY_STATE_DIR/current" +[[ -f "$current" ]] || fail "generation marker missing" +key_hash=$(sha256sum "$CLOUDFLARED_KEYRING" | awk '{print $1}') +printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflared any main\n' "$CLOUDFLARED_KEYRING" > "$CASE_DIR/independent-source" +assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/independent-source" "source differs from official literal" +source_hash=$(sha256sum "$CASE_DIR/independent-source" | awk '{print $1}') +grep -Fxq "key_sha256=$key_hash" "$current" || fail "key generation hash mismatch" +grep -Fxq "source_sha256=$source_hash" "$current" || fail "source generation hash mismatch" +pass "key/source commit as one recorded generation" + +new_case +set_old_generation +configure_repository +validate_current_repository_manifest || fail "trusted current manifest validation failed" +remove_managed_repository +init_runtime_config +assert_absent "$CLOUDFLARED_SOURCE_FILE" "trusted current did not remove managed source" +[[ -f "$CLOUDFLARED_KEYRING" ]] || fail "managed repository removal deleted keyring" +[[ ! -f "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" ]] || fail "managed repository removal retained current" +find "$CLOUDFLARED_STATE_DIR" -maxdepth 1 -type d -name 'uninstall-*' -print -quit | grep -q . || + fail "managed source backup missing" +[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "managed repository removal left lock" +pass "configure then real repository uninstall removes source and preserves keyring" + +for signal in HUP INT TERM; do + case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + for phase in before-snapshot-create capture-auto_update_script capture-auto_update_timer capture-legacy_marker lock-acquired before-disable-auto-update after-disable-auto-update before-apt-remove after-apt-remove before-source-remove after-source-remove final-apt-update before-lock-release; do + new_case + set_old_generation + configure_repository + cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" + cp "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "$CASE_DIR/expected-current" + set +e + CLOUDFLARED_TEST_INTERNALS=1 SIGNAL_PHASE="$phase" SIGNAL_NAME="$signal" \ + bash -c ' + source "$1" + require_root() { :; } + check_platform() { :; } + uninstall_transaction_hook() { + [[ "$1" != "$SIGNAL_PHASE" ]] || kill -s "$SIGNAL_NAME" "$BASHPID" + } + uninstall_cloudflared --confirmed + ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" >"$CASE_DIR/out" 2>"$CASE_DIR/err" + status=$? + set -e + [[ "$status" == "$expected" ]] || fail "uninstall $signal at $phase returned $status" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall $signal at $phase left lock" + assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/expected-source" "uninstall $signal at $phase failed source restore" + assert_same "$CLOUDFLARED_REPOSITORY_STATE_DIR/current" "$CASE_DIR/expected-current" "uninstall $signal at $phase failed current restore" + acquire_repository_lock || fail "uninstall $signal at $phase lock not reusable" + release_repository_lock || fail "uninstall $signal at $phase lock cleanup failed" + pass "uninstall $signal=$expected cleans $phase" + done +done + +new_case +set_old_generation +configure_repository +uninstall_transaction_hook() { [[ "$1" != after-disable-auto-update ]] || exit 0; } +set +e +( uninstall_cloudflared --confirmed ) >"$CASE_DIR/out" 2>"$CASE_DIR/err" +status=$? +set -e +[[ "$status" == 1 ]] || fail "active uninstall exit 0 returned $status" +[[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "active uninstall exit 0 left lock" +grep -R -Fq '活动卸载事务异常退出' "$CLOUDFLARED_REPOSITORY_STATE_DIR" || fail "active uninstall exit 0 lacked evidence" +pass "active uninstall exit 0 returns 1 and cleans transaction" + +for manifest_case in missing duplicate unknown generation key-digest source-digest source-content symlink directory fifo owner gid mode; do + new_case + set_old_generation + configure_repository + current="$CLOUDFLARED_REPOSITORY_STATE_DIR/current" + case "$manifest_case" in + missing) sed -i '/^source_sha256=/d' "$current" ;; + duplicate) cat "$current" >> "$current.copy"; cat "$current.copy" >> "$current"; rm "$current.copy" ;; + unknown) printf 'unknown=value\n' >> "$current" ;; + generation) sed -i 's/^generation=.*/generation=invalid/' "$current" ;; + key-digest) sed -i 's/^key_sha256=.*/key_sha256=0000000000000000000000000000000000000000000000000000000000000000/' "$current" ;; + source-digest) sed -i 's/^source_sha256=.*/source_sha256=0000000000000000000000000000000000000000000000000000000000000000/' "$current" ;; + source-content) printf '# tampered\n' >> "$CLOUDFLARED_SOURCE_FILE" ;; + symlink) rm "$current"; ln -s "$CASE_DIR/external" "$current"; printf external > "$CASE_DIR/external" ;; + directory) rm "$current"; mkdir "$current" ;; + fifo) rm "$current"; mkfifo "$current" ;; + owner) chown 65534:0 "$current" ;; + gid) chown 0:65534 "$current" ;; + mode) chmod 0644 "$current" ;; + esac + if remove_managed_repository >/dev/null 2>&1; then fail "current $manifest_case unexpectedly trusted"; fi + [[ -f "$CLOUDFLARED_KEYRING" ]] || fail "current $manifest_case deleted keyring" + [[ -e "$CLOUDFLARED_SOURCE_FILE" ]] || fail "current $manifest_case deleted source" + [[ ! -e "$CASE_DIR/external" || "$(cat "$CASE_DIR/external")" == external ]] || fail "current symlink target changed" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "current $manifest_case left lock" + pass "current manifest rejects $manifest_case" +done + +new_case +set_old_generation +export FAKE_APT_INSTALL_FAIL=1 +if run_repository_apt_transaction install >/dev/null 2>&1; then fail "old generation rollback test succeeded"; fi +assert_old_generation +pass "old key/source preserved on transaction failure" + +new_case +run_repository_apt_transaction install +grep -Fxq 'apt-get:update' "$FAKE_LOG" || fail "APT probe not called" +grep -Fxq 'apt-get:install -y cloudflared' "$FAKE_LOG" || fail "cloudflared package selection changed" +if grep -Eqi 'remove|purge|autoremove|linux-image|linux-headers' "$FAKE_LOG"; then + fail "repository transaction performed unrelated APT changes" +fi +pass "no kernel deletion or unrelated APT change" + +script_hashes_before=$(sha256sum "$ROOT_DIR/linux_setup.sh" "$ROOT_DIR"/modules/*.sh "$ROOT_DIR"/tools/push.sh \ + "$ROOT_DIR"/tools/setup-motd.sh "$ROOT_DIR"/tools/xanmod-install.sh) +script_hashes_after=$(sha256sum "$ROOT_DIR/linux_setup.sh" "$ROOT_DIR"/modules/*.sh "$ROOT_DIR"/tools/push.sh \ + "$ROOT_DIR"/tools/setup-motd.sh "$ROOT_DIR"/tools/xanmod-install.sh) +[[ "$script_hashes_before" == "$script_hashes_after" ]] || fail "payload or unrelated scripts changed during tests" +pass "payload and unrelated scripts remain unchanged" + +# Keep prior lifecycle coverage with fake systemctl only. +new_case +cat > "$CLOUDFLARED_SERVICE_FILE" < "$CLOUDFLARED_LEGACY_BIN" <<'EOF' +#!/usr/bin/env bash +echo 'cloudflared version 2025.1.0' +EOF +chmod 0755 "$CLOUDFLARED_LEGACY_BIN" +migrate_legacy_binary +assert_absent "$CLOUDFLARED_LEGACY_BIN" "recognized legacy binary was not migrated" +grep -Fq "ExecStart=$CLOUDFLARED_APT_BIN --no-autoupdate --token-file /etc/cloudflared/token" "$CLOUDFLARED_SERVICE_FILE" || + fail "legacy service path was not preserved" +pass "legacy binary and service migration behavior remains" + + +# Each malformed snapshot runs in a separate process: transaction traps and +# intentional corrupt fixtures must not leak into subsequent cases. +for schema_case in version generation duplicate unknown order state mode uid gid digest created payload-name payload-missing payload-extra payload-fifo payload-symlink manifest-fifo ancestor; do + ( + new_case + configure_repository || fail "schema fixture configure" + acquire_repository_lock || fail "schema fixture lock" + begin_uninstall_transaction || fail "schema fixture capture" + manifest="$UNINSTALL_SNAPSHOT_DIR/manifest" + case "$schema_case" in + version) sed -i '1d' "$manifest" ;; + generation) sed -i '2s/.*/generation=20000101T000000Z-1-1/' "$manifest" ;; + duplicate) sed -i '3s/.*/version=1/' "$manifest" ;; + unknown) sed -i '10s/.*/path=\/tmp\/forbidden/' "$manifest" ;; + order) sed -i '3s/auto_update_script/auto_update_timer/' "$manifest" ;; + state) sed -i '4s/.*/state=invalid/' "$manifest" ;; + mode) sed -i '5s/.*/mode=777/' "$manifest" ;; + uid) sed -i '6s/.*/uid=1/' "$manifest" ;; + gid) sed -i '7s/.*/gid=1/' "$manifest" ;; + digest) sed -i '8s/.*/sha256=invalid/' "$manifest" ;; + created) sed -i '0,/^transaction_created=false$/s//transaction_created=invalid/' "$manifest" ;; + payload-name) sed -i 's/^snapshot=target-source$/snapshot=target-current/' "$manifest" ;; + payload-missing) rm -- "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ;; + payload-extra) printf extra > "$UNINSTALL_SNAPSHOT_DIR/files/target-extra"; chmod 600 "$UNINSTALL_SNAPSHOT_DIR/files/target-extra" ;; + payload-fifo) mkfifo "$UNINSTALL_SNAPSHOT_DIR/files/.fifo" ;; + payload-symlink) ln -s target-source "$UNINSTALL_SNAPSHOT_DIR/files/.link" ;; + manifest-fifo) rm -- "$manifest"; mkfifo "$manifest" ;; + ancestor) chmod 0777 "$UNINSTALL_SNAPSHOT_DIR/files" ;; + esac + if uninstall_manifest_valid "$manifest"; then fail "schema accepted $schema_case"; fi + # No rollback is appropriate for this read-only parser test. + UNINSTALL_TRANSACTION_ACTIVE=false + UNINSTALL_SNAPSHOT_BUILDING=false + UNINSTALL_TRANSACTION_STATE=NONE + restore_repository_traps + # Parent alone owns TEST_DIR cleanup; do not run its EXIT trap here. + trap - EXIT + release_repository_lock || fail "schema fixture release" + ) > "$TEST_DIR/schema-$schema_case.log" 2>&1 || fail "schema $schema_case (see $TEST_DIR/schema-$schema_case.log)" + pass "uninstall manifest rejects $schema_case" +done + + +for ownership_case in owned foreign inode mode owner gid symlink directory fifo; do + ( + new_case + uninstall_transaction_hook() { :; } + trap - EXIT + configure_repository || fail "ownership configure" + acquire_repository_lock || fail "ownership lock" + begin_uninstall_transaction || fail "ownership capture" + target="$AUTO_UPDATE_SCRIPT" + printf 'owned-content\n' > "$CASE_DIR/payload" + chmod 0600 "$CASE_DIR/payload" + if [[ "$ownership_case" == foreign ]]; then + printf foreign > "$target" + chmod 0755 "$target" + else + uninstall_create_absent_target auto_update_script "$CASE_DIR/payload" || fail "exclusive creation" + case "$ownership_case" in + inode) mv "$target" "$target.original"; cp "$target.original" "$target" ;; + mode) chmod 0644 "$target" ;; + owner) chown 65534:0 "$target" ;; + gid) chown 0:65534 "$target" ;; + symlink) rm "$target"; ln -s "$CASE_DIR/payload" "$target" ;; + directory) rm "$target"; mkdir "$target" ;; + fifo) rm "$target"; mkfifo "$target" ;; + esac + fi + if [[ "$ownership_case" == owned ]]; then + uninstall_cleanup ownership-test || fail "owned rollback" + assert_absent "$target" "owned file survived" + else + if uninstall_cleanup ownership-test; then fail "foreign identity accepted"; fi + [[ -e "$target" || -L "$target" ]] || fail "foreign object deleted" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" ]] || fail "manifest lost" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ]] || fail "payload lost" + fi + trap - EXIT + ) > "$TEST_DIR/ownership-$ownership_case.log" 2>&1 || { cat "$TEST_DIR/ownership-$ownership_case.log"; fail "ownership $ownership_case"; } + pass "absent ownership $ownership_case" +done + +for journal_case in generation duplicate state unknown reason; do + ( + new_case + trap - EXIT + uninstall_transaction_hook() { :; } + configure_repository || fail "journal configure" + acquire_repository_lock || fail "journal lock" + begin_uninstall_transaction || fail "journal capture" + journal="$UNINSTALL_SNAPSHOT_DIR/journal" + case "$journal_case" in + generation) sed -i '2s/.*/generation=wrong/' "$journal" ;; + duplicate) printf 'state=ACTIVE\n' >> "$journal" ;; + state) sed -i '3s/.*/state=COMMITTED/' "$journal" ;; + unknown) sed -i '1s/.*/unknown=1/' "$journal" ;; + reason) sed -i '4s/.*/path=invalid/' "$journal" ;; + esac + if uninstall_journal_valid "$journal" ACTIVE; then fail "bad journal accepted"; fi + UNINSTALL_TRANSACTION_STATE=FAILED + UNINSTALL_TRANSACTION_ACTIVE=false + UNINSTALL_SNAPSHOT_BUILDING=false + restore_repository_traps + release_repository_lock || fail "journal release" + ) > "$TEST_DIR/journal-$journal_case.log" 2>&1 || fail "journal $journal_case" + pass "journal rejects $journal_case" +done + +for finalization_case in journal archive release restore; do + ( + new_case + trap - EXIT + uninstall_transaction_hook() { :; } + configure_repository || fail "finalization configure" + acquire_repository_lock || fail "finalization lock" + begin_uninstall_transaction || fail "finalization capture" + snapshot_before="$UNINSTALL_SNAPSHOT_DIR" + case "$finalization_case" in + journal|archive) + repository_rename() { + if [[ "$finalization_case" == journal && "$2" == */journal ]] || + [[ "$finalization_case" == archive && "$1" == "$snapshot_before" ]]; then return 1; fi + command mv -fT -- "$1" "$2" + } + ;; + release) + # A nonempty lock is an actual rmdir failure, not a core stub. + printf retained > "$REPOSITORY_LOCK_DIR/injected" + ;; + restore) + repository_install_file() { + [[ "$2" != */target-source ]] || return 1 + command install -o 0 -g 0 -m "$1" -- "$2" "$3" + } + ;; + esac + if uninstall_cleanup "injected-$finalization_case"; then fail "finalization failure accepted"; fi + [[ "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "failed guard state" + [[ "$UNINSTALL_TRANSACTION_ACTIVE" == false ]] || fail "active guard survived" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" ]] || fail "manifest deleted" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ]] || fail "source payload deleted" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-current" ]] || fail "current payload deleted" + cmp "$SOURCE_FILE" "$UNINSTALL_SNAPSHOT_DIR/files/target-source" || fail "source changed" + [[ -f "$KEYRING" ]] || fail "keyring deleted" + if [[ "$finalization_case" == release ]]; then + [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "lock evidence missing" + fi + ) > "$TEST_DIR/finalization-$finalization_case.log" 2>&1 || { + cat "$TEST_DIR/finalization-$finalization_case.log" + fail "finalization $finalization_case" + } + pass "finalization $finalization_case failure preserves complete evidence" +done + +for capture_id in auto_update_script auto_update_timer legacy_marker; do + ( + new_case + trap - EXIT + configure_repository || fail "capture configure" + cp "$SOURCE_FILE" "$CASE_DIR/source-before" + cp "$KEYRING" "$CASE_DIR/key-before" + acquire_repository_lock || fail "capture lock" + uninstall_transaction_hook() { [[ "$1" != "capture-$capture_id" ]]; } + if begin_uninstall_transaction; then fail "capture fault ignored"; fi + [[ "$UNINSTALL_TRANSACTION_STATE" == BUILDING ]] || fail "partial capture activated" + if uninstall_cleanup capture-failed; then fail "partial capture reported recovery"; fi + cmp "$SOURCE_FILE" "$CASE_DIR/source-before" || fail "partial source changed" + cmp "$KEYRING" "$CASE_DIR/key-before" || fail "partial key changed" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/journal" ]] || fail "partial journal missing" + [[ "$UNINSTALL_TRANSACTION_ACTIVE" == false ]] || fail "partial guard active" + ) > "$TEST_DIR/capture-$capture_id.log" 2>&1 || { + cat "$TEST_DIR/capture-$capture_id.log" + fail "capture $capture_id" + } + pass "capture failure $capture_id retains partial evidence without restoration" +done + +for archive_case in intact payload journal; do + ( + new_case + trap - EXIT + uninstall_transaction_hook() { :; } + configure_repository || fail "archive configure" + acquire_repository_lock || fail "archive lock" + begin_uninstall_transaction || fail "archive capture" + uninstall_transaction_hook() { + [[ "$1" == after-evidence-rename ]] || return 0 + case "$archive_case" in + payload) printf corrupt >> "$UNINSTALL_SNAPSHOT_DIR/files/target-source" ;; + journal) printf 'unknown=1\n' >> "$UNINSTALL_SNAPSHOT_DIR/journal" ;; + esac + return 0 + } + rc=0 + uninstall_cleanup archive-verification || rc=$? + if [[ "$archive_case" == intact ]]; then + [[ "$rc" == 0 && "$UNINSTALL_TRANSACTION_STATE" == ROLLED_BACK ]] || fail "valid archive rejected" + else + [[ "$rc" != 0 && "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "corrupt archive accepted" + fi + [[ -f "$UNINSTALL_SNAPSHOT_DIR/manifest" && -f "$UNINSTALL_SNAPSHOT_DIR/journal" ]] || fail "archive evidence lost" + [[ -f "$UNINSTALL_SNAPSHOT_DIR/files/target-current" ]] || fail "archive payload lost" + ) > "$TEST_DIR/archive-$archive_case.log" 2>&1 || { + cat "$TEST_DIR/archive-$archive_case.log" + fail "archive $archive_case" + } + pass "post-rename archive verification $archive_case" +done + +for interrupted_phase in before-snapshot-create after-snapshot-active after-evidence-rename after-evidence-seal before-pending-clear; do + ( + new_case + trap - EXIT + configure_repository || fail "kill configure" + cp "$SOURCE_FILE" "$CASE_DIR/source-before" + ( + trap - EXIT + acquire_repository_lock || exit 1 + uninstall_transaction_hook() { + if [[ "$1" == "$interrupted_phase" ]]; then kill -KILL "$BASHPID"; fi + } + begin_uninstall_transaction || exit 1 + uninstall_cleanup kill-test + ) > "$CASE_DIR/killed.log" 2>&1 & + child=$! + rc=0 + wait "$child" || rc=$? + [[ "$rc" == 137 ]] || fail "SIGKILL did not reach $interrupted_phase" + # Simulate operator removing only the stale lock; pending must still block. + rmdir "$CLOUDFLARED_STATE_DIR.lock" || fail "stale lock remove" + : > "$FAKE_LOG" + for operation in install_cloudflared upgrade_cloudflared uninstall_cloudflared disable_auto_update; do + ( + trap - EXIT + "$operation" --confirmed + ) > "$CASE_DIR/retry-$operation.log" 2>&1 && fail "pending allowed $operation" + done + [[ ! -s "$FAKE_LOG" ]] || fail "pending executed external command" + cmp "$SOURCE_FILE" "$CASE_DIR/source-before" || fail "pending changed source" + compgen -G "$CLOUDFLARED_STATE_DIR/repository/pending-uninstall-*" >/dev/null || fail "pending evidence missing" + ) > "$TEST_DIR/kill-$interrupted_phase.log" 2>&1 || { + cat "$TEST_DIR/kill-$interrupted_phase.log" + fail "SIGKILL $interrupted_phase" + } + pass "SIGKILL $interrupted_phase fails closed in independent entrypoints" +done + +for stage_case in owned foreign; do + ( + new_case + trap - EXIT + configure_repository || fail "stage configure" + acquire_repository_lock || fail "stage lock" + begin_uninstall_transaction || fail "stage capture" + stage=$(mktemp "$CASE_DIR/root/.cloudflared-create.XXXXXX") + uninstall_register_stage "$stage" || fail "stage register" + if [[ "$stage_case" == foreign ]]; then + mv "$stage" "$stage.original" + printf foreign > "$stage" + fi + rc=0 + uninstall_cleanup stage-test || rc=$? + if [[ "$stage_case" == owned ]]; then + [[ "$rc" == 0 && ! -e "$stage" ]] || fail "owned stage not cleaned" + [[ "$(stat -c %a "$UNINSTALL_SNAPSHOT_DIR")" == 500 ]] || fail "archive writable" + [[ "$(stat -c %a "$UNINSTALL_SNAPSHOT_DIR/manifest")" == 400 ]] || fail "manifest writable" + validate_uninstall_archive "$UNINSTALL_SNAPSHOT_DIR" "$UNINSTALL_GENERATION" || fail "sealed archive invalid" + else + [[ "$rc" != 0 && "$(cat "$stage")" == foreign ]] || fail "foreign stage removed" + [[ -f "$UNINSTALL_PENDING" ]] || fail "stage failure lost pending" + fi + ) > "$TEST_DIR/stage-$stage_case.log" 2>&1 || { + cat "$TEST_DIR/stage-$stage_case.log" + fail "stage $stage_case" + } + pass "stage cleanup verifies $stage_case identity" +done + +for terminal_signal in HUP INT TERM; do + case "$terminal_signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + ( + new_case + trap - EXIT + configure_repository || fail "terminal signal configure" + rc=0 + ( + trap - EXIT + acquire_repository_lock || exit 1 + uninstall_transaction_hook() { + if [[ "$1" == after-evidence-seal ]]; then kill -s "$terminal_signal" "$BASHPID"; fi + } + begin_uninstall_transaction || exit 1 + uninstall_cleanup terminal-signal + ) > "$CASE_DIR/signal.log" 2>&1 || rc=$? + [[ "$rc" == "$expected" ]] || fail "terminal signal status $rc" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "terminal signal lock retained" + compgen -G "$CLOUDFLARED_STATE_DIR/repository/pending-uninstall-*" >/dev/null || fail "terminal signal pending missing" + [[ -f "$SOURCE_FILE" ]] || fail "terminal signal source lost" + ) > "$TEST_DIR/terminal-$terminal_signal.log" 2>&1 || { + cat "$TEST_DIR/terminal-$terminal_signal.log" + fail "terminal signal $terminal_signal" + } + pass "terminal $terminal_signal=$expected retains evidence without repeated rollback" +done + +for restore_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do + ( + new_case + trap - EXIT + configure_repository || fail "six target configure" + mkdir -p "$(dirname "$AUTO_UPDATE_SCRIPT")" "$(dirname "$AUTO_UPDATE_SERVICE")" + write_auto_update_files || fail "six target auto update" + printf managed > "$STATE_DIR/repository-managed" + chmod 0600 "$STATE_DIR/repository-managed" + acquire_repository_lock || fail "six target lock" + begin_uninstall_transaction || fail "six target capture" + snapshot=$UNINSTALL_SNAPSHOT_DIR + while IFS= read -r id; do rm -- "$(uninstall_target_path "$id")"; done < <(uninstall_snapshot_targets) + repository_install_file() { + [[ "$2" != "$snapshot/files/target-$restore_id" ]] || return 1 + command install -o 0 -g 0 -m "$1" -- "$2" "$3" + } + if uninstall_cleanup six-target-failure; then fail "restore fault accepted"; fi + while IFS= read -r id; do + [[ -f "$snapshot/files/target-$id" ]] || fail "snapshot payload lost $id" + if [[ "$id" != "$restore_id" ]]; then + cmp "$(uninstall_target_path "$id")" "$snapshot/files/target-$id" || fail "restore skipped $id" + validate_secure_file "$(uninstall_target_path "$id")" "$(uninstall_target_mode "$id")" || fail "restore metadata $id" + fi + done < <(uninstall_snapshot_targets) + [[ -f "$UNINSTALL_PENDING" ]] || fail "restore pending lost" + uninstall_journal_valid "$snapshot/journal" ACTIVE || fail "failure overwrote original journal" + ) > "$TEST_DIR/restore-$restore_id.log" 2>&1 || { + cat "$TEST_DIR/restore-$restore_id.log" + fail "restore failure $restore_id" + } + pass "restore failure $restore_id continues remaining five targets and preserves original evidence" +done + +for target_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do + for bad_metadata in mode owner gid symlink directory fifo; do + ( + new_case + trap - EXIT + configure_repository || fail "metadata configure" + mkdir -p "$(dirname "$AUTO_UPDATE_SCRIPT")" "$(dirname "$AUTO_UPDATE_SERVICE")" + write_auto_update_files || fail "metadata auto update" + printf managed > "$STATE_DIR/repository-managed" + chmod 0600 "$STATE_DIR/repository-managed" + target=$(uninstall_target_path "$target_id") + case "$bad_metadata" in + mode) chmod 0666 "$target" ;; + owner) chown 1:0 "$target" ;; + gid) chown 0:1 "$target" ;; + symlink) mv "$target" "$target.external"; ln -s "$target.external" "$target" ;; + directory) rm "$target"; mkdir "$target" ;; + fifo) rm "$target"; mkfifo "$target" ;; + esac + acquire_repository_lock || fail "metadata lock" + if begin_uninstall_transaction; then fail "bad target captured"; fi + uninstall_cleanup bad-metadata >/dev/null 2>&1 || : + [[ -e "$target" || -L "$target" ]] || fail "untrusted target deleted" + [[ "$UNINSTALL_TRANSACTION_STATE" == FAILED ]] || fail "capture failure state lost" + ) > "$TEST_DIR/meta-$target_id-$bad_metadata.log" 2>&1 || { + cat "$TEST_DIR/meta-$target_id-$bad_metadata.log" + fail "metadata $target_id $bad_metadata" + } + pass "capture rejects $target_id $bad_metadata without deleting target" + done +done + +( + unset CLOUDFLARED_TRUST_ANCHOR + init_runtime_config + [[ "$TRUST_ANCHOR" == / ]] || exit 1 + path_is_beneath_anchor / || exit 1 + path_is_beneath_anchor /var/lib || exit 1 + CLOUDFLARED_TRUST_ANCHOR=/var + path_is_beneath_anchor /var || exit 1 + path_is_beneath_anchor /var/lib || exit 1 + if path_is_beneath_anchor /variable; then exit 1; fi + if path_is_beneath_anchor /var/../etc; then exit 1; fi + path_is_beneath_anchor /var/lib/../lib || exit 1 +) || fail "pure anchor boundaries" +pass "pure paths validate default root anchor and normalized non-root boundaries" + +for creation_phase in before-create-marker before-create-manifest-stage before-create-manifest-rename before-create-publish after-create-publish; do + for creation_fault in failure HUP INT TERM; do + ( + new_case + trap - EXIT + configure_repository || fail "creation configure" + printf payload > "$CASE_DIR/payload" + chmod 0600 "$CASE_DIR/payload" + rc=0 + ( + trap - EXIT + acquire_repository_lock || exit 1 + begin_uninstall_transaction || exit 1 + uninstall_transaction_hook() { + [[ "$1" == "$creation_phase" ]] || return 0 + if [[ "$creation_fault" == failure ]]; then return 1; fi + kill -s "$creation_fault" "$BASHPID" + } + if uninstall_create_absent_target auto_update_script "$CASE_DIR/payload"; then exit 90; fi + uninstall_cleanup creation-failure || : + exit 1 + ) > "$CASE_DIR/creation.log" 2>&1 || rc=$? + case "$creation_fault" in failure) expected=1 ;; HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + [[ "$rc" == "$expected" ]] || fail "creation status $rc" + [[ ! -e "$AUTO_UPDATE_SCRIPT" && ! -L "$AUTO_UPDATE_SCRIPT" ]] || fail "owned publication not recovered" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "creation lock leaked" + ) > "$TEST_DIR/create-$creation_phase-$creation_fault.log" 2>&1 || { + cat "$TEST_DIR/create-$creation_phase-$creation_fault.log" + fail "creation $creation_phase $creation_fault" + } + pass "creation $creation_phase $creation_fault preserves ownership across publication" + done +done + +for bookkeeping_fault in marker stage rename; do + ( + new_case + trap - EXIT + configure_repository || fail "bookkeeping configure" + printf payload > "$CASE_DIR/payload"; chmod 0600 "$CASE_DIR/payload" + acquire_repository_lock || fail "bookkeeping lock" + begin_uninstall_transaction || fail "bookkeeping capture" + uninstall_transaction_hook() { + if [[ "$bookkeeping_fault" == marker && "$1" == before-create-marker ]]; then + mkdir "$UNINSTALL_SNAPSHOT_DIR/created-auto_update_script" + fi + return 0 + } + mktemp() { + if [[ "$bookkeeping_fault" == stage && "$1" == */manifest.stage.* ]]; then return 1; fi + command mktemp "$@" + } + repository_rename() { + if [[ "$bookkeeping_fault" == rename && "$1" == */manifest.stage.* ]]; then return 1; fi + command mv -fT -- "$1" "$2" + } + if uninstall_create_absent_target auto_update_script "$CASE_DIR/payload"; then fail "bookkeeping fault ignored"; fi + [[ ! -e "$AUTO_UPDATE_SCRIPT" ]] || fail "published before durable bookkeeping" + uninstall_cleanup bookkeeping-failure || : + ) > "$TEST_DIR/bookkeeping-$bookkeeping_fault.log" 2>&1 || { + cat "$TEST_DIR/bookkeeping-$bookkeeping_fault.log" + fail "bookkeeping $bookkeeping_fault" + } + pass "actual $bookkeeping_fault failure precedes ln publication" +done + +new_case +configure_repository || fail "URL configure" +grep '^curl:' "$FAKE_LOG" | grep -Fq 'https://pkg.cloudflare.com/cloudflare-main.gpg' || fail "key download URL changed" +printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflared any main\n' "$CLOUDFLARED_KEYRING" > "$CASE_DIR/official-source" +assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/official-source" "APT repository URL incorrect" +validate_current_repository_manifest || fail "official current rejected" +remove_managed_repository || fail "official source removal failed" +assert_absent "$CLOUDFLARED_SOURCE_FILE" "official source not removed" +pass "literal key URL and repository source validate current and real removal" + +new_case +printf 'deb [signed-by=%s] https://pkg.cloudflare.com/cloudflare-main.gpg any main\n' "$CLOUDFLARED_KEYRING" > "$CLOUDFLARED_SOURCE_FILE" +chmod 0644 "$CLOUDFLARED_SOURCE_FILE" +cp "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/wrong-source" +if configure_repository > "$CASE_DIR/rejected.log" 2>&1; then fail "gpg repository accepted"; fi +assert_same "$CLOUDFLARED_SOURCE_FILE" "$CASE_DIR/wrong-source" "unowned wrong source overwritten" +[[ ! -s "$FAKE_LOG" ]] || fail "invalid source triggered external operation" +pass "gpg repository URL rejected without overwrite or download" + +for capture_file in old-key old-source old-current; do + for capture_fault in copy partial chmod HUP INT TERM exit0; do + ( + new_case + trap - EXIT + configure_repository || fail "capture fixture" + for object in "$KEYRING" "$SOURCE_FILE" "$REPOSITORY_STATE_DIR/current"; do + cp -p "$object" "$CASE_DIR/$(basename "$object").before" + done + : > "$FAKE_LOG" + rc=0 + ( + trap - EXIT + repository_copy_file() { + if [[ "$2" == */"$capture_file" ]]; then + case "$capture_fault" in + copy) return 1 ;; + partial) printf partial > "$2"; return 1 ;; + HUP|INT|TERM) kill -s "$capture_fault" "$BASHPID" ;; + exit0) exit 0 ;; + esac + fi + command cp -- "$1" "$2" + } + chmod() { + if [[ "$capture_fault" == chmod && "$2" == */"$capture_file" ]]; then return 1; fi + command chmod "$@" + } + configure_repository + ) > "$CASE_DIR/capture.log" 2>&1 || rc=$? + case "$capture_fault" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; *) expected=1 ;; esac + [[ "$rc" == "$expected" ]] || fail "capture status $rc" + for object in "$KEYRING" "$SOURCE_FILE" "$REPOSITORY_STATE_DIR/current"; do + cmp "$object" "$CASE_DIR/$(basename "$object").before" || fail "capture modified formal bytes" + [[ "$(stat -c '%u:%g:%a' "$object")" == "$(stat -c '%u:%g:%a' "$CASE_DIR/$(basename "$object").before")" ]] || fail "capture modified metadata" + done + [[ ! -s "$FAKE_LOG" ]] || fail "capture downloaded or called APT" + grep -q '未恢复或删除正式文件' "$CASE_DIR/capture.log" || fail "capture diagnostic missing" + acquire_repository_lock || fail "capture lock retained" + release_repository_lock || fail "capture lock release" + ) > "$TEST_DIR/capture-boundary-$capture_file-$capture_fault.log" 2>&1 || { + cat "$TEST_DIR/capture-boundary-$capture_file-$capture_fault.log" + fail "capture boundary $capture_file $capture_fault" + } + pass "repository capture $capture_file $capture_fault preserves formal bytes and metadata" + done +done + +( + new_case + trap - EXIT + set_old_generation + rc=0 + ( + trap - EXIT + begin_repository_transaction || exit 1 + exit 0 + ) > "$CASE_DIR/active-exit.log" 2>&1 || rc=$? + [[ "$rc" == 1 ]] || fail "active exit zero accepted" + assert_old_generation + [[ ! -e "$REPOSITORY_STATE_DIR/current" ]] || fail "absent current restored incorrectly" + acquire_repository_lock || fail "active exit lock retained" + release_repository_lock || fail "active exit release" +) || fail "active exit zero" +pass "complete repository transaction exit zero returns nonzero and restores absent current" + +for build_case in absent archive-failure; do + ( + new_case + trap - EXIT + if [[ "$build_case" == archive-failure ]]; then set_old_generation; fi + trap ':' HUP + before_traps=$(trap -p HUP INT TERM EXIT) + repository_copy_file() { printf partial > "$2"; return 1; } + if [[ "$build_case" == absent ]]; then + # No formal object exists; fail before any download via snapshot wrapper. + chmod() { + [[ "$2" != */transaction-* ]] || return 1 + command chmod "$@" + } + repository_transaction_hook() { exit 0; } + rc=0 + (begin_repository_transaction) > "$CASE_DIR/absent.log" 2>&1 || rc=$? + [[ "$rc" != 0 ]] || fail "absent abnormal exit accepted" + [[ ! -e "$KEYRING" && ! -e "$SOURCE_FILE" && ! -e "$REPOSITORY_STATE_DIR/current" ]] || fail "absent formal object created" + else + repository_rename() { return 1; } + if configure_repository > "$CASE_DIR/build.log" 2>&1; then fail "failed capture accepted"; fi + assert_old_generation + [[ -f "$REPOSITORY_TRANSACTION_DIR/old-key" ]] || fail "partial evidence lost" + [[ -f "$REPOSITORY_TRANSACTION_DIR/rollback.log" ]] || fail "failure diagnostic lost" + grep -q '回滚不完整' "$CASE_DIR/build.log" || fail "cleanup error not reported" + fi + [[ "$(trap -p HUP INT TERM EXIT)" == "$before_traps" ]] || fail "build traps changed" + acquire_repository_lock || fail "build cleanup lock retained" + release_repository_lock || fail "build cleanup lock release" + ) > "$TEST_DIR/build-$build_case.log" 2>&1 || { + cat "$TEST_DIR/build-$build_case.log" + fail "build $build_case" + } + pass "repository building $build_case preserves objects traps and evidence" +done + +for restore_object in key source current; do + for restore_fault in partial wrong-content wrong-mode rename post-content post-mode; do + ( + new_case + trap - EXIT + configure_repository || fail "restore fixture" + begin_repository_transaction || fail "restore transaction" + snapshot=$REPOSITORY_TRANSACTION_DIR + printf historical > "$REPOSITORY_STATE_DIR/current.rollback" + cp "$REPOSITORY_STATE_DIR/current.rollback" "$CASE_DIR/historical" + case "$restore_object" in key) target=$KEYRING ;; source) target=$SOURCE_FILE ;; current) target=$REPOSITORY_STATE_DIR/current ;; esac + cp "$target" "$CASE_DIR/pre-restore" + : > "$CASE_DIR/attempts" + repository_install_file() { + printf '%s\n' "$2" >> "$CASE_DIR/attempts" + if [[ "$2" == "$snapshot/old-$restore_object" ]]; then + case "$restore_fault" in + partial) printf partial > "$3"; return 1 ;; + wrong-content) command install -o 0 -g 0 -m "$1" "$2" "$3"; printf wrong > "$3"; return 0 ;; + wrong-mode) command install -o 0 -g 0 -m 0777 "$2" "$3"; return 0 ;; + esac + fi + command install -o 0 -g 0 -m "$1" "$2" "$3" + } + repository_rename() { + if [[ "$2" == "$target" && "$1" == */.cloudflared-rollback.* ]]; then + [[ "$restore_fault" != rename ]] || return 1 + command mv -fT -- "$1" "$2" || return 1 + case "$restore_fault" in post-content) printf corrupt > "$2" ;; post-mode) chmod 0777 "$2" ;; esac + return 0 + fi + command mv -fT -- "$1" "$2" + } + if rollback_repository_transaction injected > "$CASE_DIR/restore.log" 2>&1; then fail "restore fault accepted"; fi + for item in key source current; do + [[ -f "$REPOSITORY_TRANSACTION_DIR/old-$item" ]] || fail "backup lost $item" + grep -Fxq "$snapshot/old-$item" "$CASE_DIR/attempts" || fail "restore skipped $item" + done + case "$restore_fault" in partial|wrong-content|wrong-mode|rename) cmp "$target" "$CASE_DIR/pre-restore" || fail "invalid stage published" ;; esac + cmp "$REPOSITORY_STATE_DIR/current.rollback" "$CASE_DIR/historical" || fail "historical rollback touched" + if grep -q '旧 key/source 已恢复' "$CASE_DIR/restore.log"; then fail "false recovery success"; fi + grep -q '回滚不完整' "$CASE_DIR/restore.log" || fail "missing failure diagnosis" + acquire_repository_lock || fail "restore lock retained" + release_repository_lock || fail "restore lock release" + ) > "$TEST_DIR/publish-$restore_object-$restore_fault.log" 2>&1 || { + cat "$TEST_DIR/publish-$restore_object-$restore_fault.log" + fail "restore publish $restore_object $restore_fault" + } + pass "repository restore $restore_object $restore_fault rejects corruption and retains all backups" + done +done + +for operation in install_cloudflared upgrade_cloudflared; do + ( + new_case + trap - EXIT + cp "$CASE_DIR/bin/cloudflared" "$APT_BIN" + dpkg-query() { [[ "$*" == *'db:Status-Status'* ]] && { printf installed; return 0; }; return 1; } + timer_enabled=true + systemctl() { + printf 'systemctl:%s\n' "$*" >> "$FAKE_LOG" + if [[ "$*" == *cloudflared-apt-update.timer* && ( "$1" == is-enabled || "$1" == is-active ) ]]; then + [[ "$timer_enabled" == true ]] + else + return 0 + fi + } + write_auto_update_files || fail "timer fixture" + auto_update_file_is_managed "$AUTO_UPDATE_SCRIPT" || fail "main script template rejected" + auto_update_file_is_managed "$AUTO_UPDATE_SERVICE" || fail "main service template rejected" + auto_update_file_is_managed "$AUTO_UPDATE_TIMER" || fail "main timer template rejected" + "$operation" || fail "enabled operation" + [[ "$PRESERVE_AUTO_UPDATE" == true ]] || fail "strategy reset by init" + grep -Fxq 'systemctl:enable --now cloudflared-apt-update.timer' "$FAKE_LOG" || fail "timer not enabled" + timer_enabled=false + : > "$FAKE_LOG" + "$operation" || fail "disabled operation" + [[ "$PRESERVE_AUTO_UPDATE" == false ]] || fail "strategy leaked between operations" + if grep -Fxq 'systemctl:enable --now cloudflared-apt-update.timer' "$FAKE_LOG"; then fail "disabled timer enabled"; fi + timer_enabled=true + export FAKE_APT_UPDATE_FAIL=1 + : > "$FAKE_LOG" + if "$operation"; then fail "repository failure ignored"; fi + if grep -Eq 'systemctl:(disable|stop|enable|daemon-reload)' "$FAKE_LOG"; then fail "repository failure continued lifecycle"; fi + ) > "$TEST_DIR/policy-$operation.log" 2>&1 || { + cat "$TEST_DIR/policy-$operation.log" + fail "policy $operation" + } + pass "policy $operation preserves enabled main timer without cross-operation leakage" +done + +( + new_case + trap - EXIT + write_auto_update_files || fail "template fixture" + printf '# changed\n' >> "$AUTO_UPDATE_TIMER" + if auto_update_file_is_managed "$AUTO_UPDATE_TIMER"; then fail "partial marker trusted"; fi + cp "$AUTO_UPDATE_TIMER" "$CASE_DIR/foreign-timer" + if disable_auto_update --confirmed; then fail "unknown timer deleted"; fi + cmp "$AUTO_UPDATE_TIMER" "$CASE_DIR/foreign-timer" || fail "unknown timer changed" +) || fail "strict main template" +pass "main updater templates reject extra content without deleting unknown files" + +( + new_case + trap - EXIT + begin_repository_transaction || fail "absent transaction" + mv "$SOURCE_FILE" "$CASE_DIR/owned-source" + printf foreign > "$SOURCE_FILE"; chmod 0644 "$SOURCE_FILE" + if rollback_repository_transaction foreign; then fail "foreign absent accepted"; fi + [[ "$(cat "$SOURCE_FILE")" == foreign ]] || fail "foreign absent deleted" + [[ -f "$REPOSITORY_TRANSACTION_DIR/pending" ]] || fail "rollback pending missing" + : > "$FAKE_LOG" + if configure_repository; then fail "pending allowed restart"; fi + [[ ! -s "$FAKE_LOG" ]] || fail "pending performed external work" +) || fail "absent ownership" +pass "repository absent foreign object survives rollback and blocks restart" + +( + new_case + trap - EXIT + acquire_repository_lock || fail "lock fixture" + release_repository_lock || fail "first release" + mkdir "$CLOUDFLARED_STATE_DIR.lock" + release_repository_lock || fail "idempotent release" + [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "released another operation lock" +) || fail "lock identity" +pass "repeated lock release preserves next operation lock" + +( + new_case + trap - EXIT + set_old_generation + mkdir -p "$STATE_DIR" + chmod 0700 "$STATE_DIR" + printf unknown > "$STATE_DIR/repository-managed" + chmod 0600 "$STATE_DIR/repository-managed" + : > "$FAKE_LOG" + if uninstall_cloudflared --confirmed; then fail "bad legacy marker accepted"; fi + [[ ! -s "$FAKE_LOG" ]] || fail "legacy ownership checked after destructive operations" +) || fail "legacy preflight" +pass "legacy uninstall rejects unknown ownership before systemctl or APT" + +for review_fault in timer-enable stage-identity trap-restore; do + ( + new_case + trap - EXIT + case "$review_fault" in + timer-enable) + dpkg-query() { printf installed; } + systemctl() { + printf '%s\n' "$*" >> "$FAKE_LOG" + [[ "$*" != 'enable --now cloudflared-apt-update.timer' ]] + } + if enable_auto_update > "$CASE_DIR/error.log" 2>&1; then fail "timer enable failure hidden"; fi + grep -Fxq 'enable --now cloudflared-apt-update.timer' "$FAKE_LOG" || fail "timer injection not hit" + ;; + stage-identity) + begin_repository_transaction || fail "stage fixture" + stage=$(mktemp "$CASE_DIR/.stage.XXXXXX") + repository_track_stage "$stage" || fail "track stage" + mv "$stage" "$stage.owned" + printf foreign > "$stage" + if rollback_repository_transaction injected; then fail "foreign stage ignored"; fi + [[ "$(cat "$stage")" == foreign ]] || fail "foreign stage deleted" + ;; + trap-restore) + begin_repository_transaction || fail "trap fixture" + restore_one_trap() { printf '%s\n' "$1" >> "$CASE_DIR/trap-attempts"; return 1; } + if rollback_repository_transaction injected; then fail "trap failure hidden"; fi + [[ "$(wc -l < "$CASE_DIR/trap-attempts")" == 4 ]] || fail "trap attempts skipped" + [[ -f "$REPOSITORY_TRANSACTION_DIR/pending" ]] || fail "trap failure pending missing" + ;; + esac + ) > "$TEST_DIR/review-$review_fault.log" 2>&1 || { + cat "$TEST_DIR/review-$review_fault.log" + fail "review $review_fault" + } + pass "review $review_fault injection reached and failure preserved" +done + +for phase in disable-systemctl disable-backup disable-delete; do + for fault in HUP INT TERM zero failure; do + ( + new_case + trap - EXIT + configure_repository || fail "disable fixture" + write_auto_update_files || fail "disable files" + rc=0 + ( + trap - EXIT + updater_transaction_hook() { + [[ "$1" == "$phase" ]] || return 0 + case "$fault" in zero) exit 0 ;; failure) return 1 ;; *) kill -s "$fault" "$BASHPID" ;; esac + } + disable_auto_update --confirmed + ) > "$CASE_DIR/disable.log" 2>&1 || rc=$? + case "$fault" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; *) expected=1 ;; esac + [[ "$rc" == "$expected" ]] || fail "disable exit $rc" + [[ -f "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "premature file deletion" + compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "disable pending missing" + grep -q '自动更新操作未完成' "$CASE_DIR/disable.log" || fail "disable diagnostic missing" + acquire_repository_lock || fail "disable lock retained" + release_repository_lock || fail "disable lock release" + ) > "$TEST_DIR/disable-$phase-$fault.log" 2>&1 || { + cat "$TEST_DIR/disable-$phase-$fault.log" + fail "disable $phase $fault" + } + pass "independent disable $phase $fault preserves pending and exit contract" + done +done + +for phase in enable-validate enable-write enable-reload enable-timer disable-systemctl; do + ( + new_case + trap - EXIT + configure_repository || fail "barrier configure" + write_auto_update_files || fail "barrier files" + mkfifo "$CASE_DIR/ready" "$CASE_DIR/release" + ( + trap - EXIT + dpkg-query() { printf installed; } + updater_transaction_hook() { + [[ "$1" == "$phase" ]] || return 0 + printf ready > "$CASE_DIR/ready" + read -r _ < "$CASE_DIR/release" + } + if [[ "$phase" == disable-* ]]; then disable_auto_update --confirmed; else enable_auto_update; fi + ) > "$CASE_DIR/holder.log" 2>&1 & + holder=$! + read -r ready < "$CASE_DIR/ready" || [[ "$ready" == ready ]] || fail "barrier readiness" + for contender in enable_auto_update disable_auto_update uninstall_cloudflared; do + if ( "$contender" --confirmed ) > "$CASE_DIR/contender.log" 2>&1; then fail "contender bypassed lock"; fi + done + printf 'release\n' > "$CASE_DIR/release" + wait "$holder" || { cat "$CASE_DIR/holder.log"; fail "holder failed"; } + ) > "$TEST_DIR/barrier-$phase.log" 2>&1 || { cat "$TEST_DIR/barrier-$phase.log"; fail "barrier $phase"; } + pass "real updater barrier $phase excludes all competing mutation entrypoints" +done + +for fault in backup delete release; do + ( + new_case + trap - EXIT + configure_repository || fail "file fault fixture" + write_auto_update_files || fail "file fault files" + hit=false + cp() { + if [[ "$fault" == backup && "$*" == *auto-update-* ]]; then hit=true; return 1; fi + command cp "$@" + } + rm() { + if [[ "$fault" == delete && "$*" == *"$AUTO_UPDATE_SERVICE"* ]]; then hit=true; return 1; fi + command rm "$@" + } + rmdir() { + if [[ "$fault" == release ]]; then hit=true; return 1; fi + command rmdir "$@" + } + before=$(trap -p HUP INT TERM EXIT) + if disable_auto_update --confirmed > "$CASE_DIR/error.log" 2>&1; then fail "disable file fault accepted"; fi + [[ "$hit" == true ]] || fail "file fault not hit" + [[ "$(trap -p HUP INT TERM EXIT)" == "$before" ]] || fail "disable traps not restored" + compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "file fault pending missing" + case "$fault" in + backup) [[ -f "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "backup failure deleted files" ;; + delete) [[ ! -e "$AUTO_UPDATE_SCRIPT" && -f "$AUTO_UPDATE_SERVICE" && -f "$AUTO_UPDATE_TIMER" ]] || fail "partial deletion group wrong" ;; + release) [[ -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "failed release lost lock" ;; + esac + ) > "$TEST_DIR/disable-file-$fault.log" 2>&1 || { cat "$TEST_DIR/disable-file-$fault.log"; fail "disable file $fault"; } + pass "independent disable actual $fault failure retains group evidence and traps" +done + +# Separate shells load the implementation independently; only file/FIFO state is shared. +( + new_case + trap - EXIT + configure_repository || fail "uninstall barrier configure" + write_auto_update_files || fail "uninstall barrier updater" + mkfifo "$CASE_DIR/uninstall-ready" "$CASE_DIR/uninstall-release" + bash -c ' + source "$1" + check_platform() { :; } + uninstall_transaction_hook() { + [[ "$1" == after-snapshot-active ]] || return 0 + printf "%s\n" "$UNINSTALL_SNAPSHOT_DIR" > "$CASE_DIR/snapshot-path" + printf "ready\n" > "$CASE_DIR/uninstall-ready" + read -r _ < "$CASE_DIR/uninstall-release" + } + uninstall_cloudflared --confirmed + ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" > "$CASE_DIR/holder.log" 2>&1 & + holder=$! + # Timeout bounds a broken readiness handshake; no scheduling sleeps. + timeout 30 bash -c 'read -r ready < "$CASE_DIR/uninstall-ready"; [[ "$ready" == ready ]]' || fail "uninstall never reached active snapshot" + snapshot=$(cat "$CASE_DIR/snapshot-path") + [[ -f "$snapshot/manifest" ]] || fail "snapshot not ready" + lock_before=$(stat -c '%d:%i:%u:%g:%a' "$CLOUDFLARED_STATE_DIR.lock") + find "$REPOSITORY_STATE_DIR" -type f -exec sha256sum {} + | sort > "$CASE_DIR/state-before" + sha256sum "$AUTO_UPDATE_SCRIPT" "$AUTO_UPDATE_SERVICE" "$AUTO_UPDATE_TIMER" > "$CASE_DIR/updater-before" + for contender in enable_auto_update disable_auto_update; do + : > "$CASE_DIR/contender-external.log" + rc=0 + FAKE_LOG="$CASE_DIR/contender-external.log" bash -c ' + source "$1" + check_platform() { :; } + "$2" --confirmed + ' _ "$ROOT_DIR/tools/cloudflare_tunnel.sh" "$contender" > "$CASE_DIR/contender.log" 2>&1 || rc=$? + [[ "$rc" != 0 ]] || fail "contender entered uninstall lock" + grep -q '无法取得 Cloudflare 仓库事务锁' "$CASE_DIR/contender.log" || fail "competition not reached" + [[ ! -s "$CASE_DIR/contender-external.log" ]] || fail "contender executed external command" + [[ "$(stat -c '%d:%i:%u:%g:%a' "$CLOUDFLARED_STATE_DIR.lock")" == "$lock_before" ]] || fail "holder lock changed" + find "$REPOSITORY_STATE_DIR" -type f -exec sha256sum {} + | sort > "$CASE_DIR/state-after" + cmp "$CASE_DIR/state-before" "$CASE_DIR/state-after" || fail "snapshot or pending changed" + sha256sum -c "$CASE_DIR/updater-before" >/dev/null || fail "contender changed updater" + done + printf 'release\n' > "$CASE_DIR/uninstall-release" + wait "$holder" || { cat "$CASE_DIR/holder.log"; fail "uninstall holder failed"; } + [[ ! -e "$SOURCE_FILE" && ! -e "$AUTO_UPDATE_TIMER" ]] || fail "uninstall did not complete" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "uninstall retained lock" +) > "$TEST_DIR/uninstall-holder.log" 2>&1 || { cat "$TEST_DIR/uninstall-holder.log"; fail "uninstall holder barrier"; } +pass "independent uninstall holder excludes enable and disable without changing snapshot pending or lock" + +for dependency_fault in update install; do + ( + new_case + trap - EXIT + write_auto_update_files || fail "dependency updater fixture" + sha256sum "$AUTO_UPDATE_SCRIPT" "$AUTO_UPDATE_SERVICE" "$AUTO_UPDATE_TIMER" > "$CASE_DIR/updater-before" + command() { + if [[ "$*" == '-v flock' ]]; then + printf 'flock-missing\n' >> "$CASE_DIR/dependency-hits" + return 1 + fi + builtin command "$@" + } + dpkg-query() { printf installed; } + apt-get() { + printf 'apt-get:%s\n' "$*" >> "$FAKE_LOG" + if [[ "$1" == update && "$dependency_fault" == update ]]; then return 41; fi + if [[ "$*" == 'install -y util-linux' ]]; then return 42; fi + return 0 + } + trap ':' HUP + before_traps=$(trap -p HUP INT TERM EXIT) + rc=0 + enable_auto_update > "$CASE_DIR/dependency.log" 2>&1 || rc=$? + case "$dependency_fault" in update) expected=41 ;; install) expected=42 ;; esac + [[ "$rc" == "$expected" ]] || fail "dependency status $rc" + grep -Fxq flock-missing "$CASE_DIR/dependency-hits" || fail "flock branch not reached" + grep -Fxq apt-get:update "$FAKE_LOG" || fail "dependency update not reached" + if [[ "$dependency_fault" == update ]]; then + if grep -q 'apt-get:install' "$FAKE_LOG"; then fail "install continued after update failure"; fi + else + grep -Fxq 'apt-get:install -y util-linux' "$FAKE_LOG" || fail "util install not reached" + fi + if grep -Eq 'systemctl:.*(reload|enable)' "$FAKE_LOG"; then fail "dependency failure continued timer work"; fi + sha256sum -c "$CASE_DIR/updater-before" >/dev/null || fail "dependency failure changed updater" + validate_current_repository_manifest || fail "prior repository configuration not recorded" + compgen -G "$REPOSITORY_STATE_DIR/pending-updater-*" >/dev/null || fail "dependency pending missing" + [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "dependency lock retained" + [[ "$(trap -p HUP INT TERM EXIT)" == "$before_traps" ]] || fail "dependency traps changed" + grep -q '自动更新操作未完成' "$CASE_DIR/dependency.log" || fail "dependency diagnostic missing" + ) > "$TEST_DIR/dependency-$dependency_fault.log" 2>&1 || { cat "$TEST_DIR/dependency-$dependency_fault.log"; fail "dependency $dependency_fault"; } + pass "missing flock $dependency_fault failure stops updater after recorded repository commit" +done + +entrypoint_output=$(bash -c "$(cat "$ROOT_DIR/tools/cloudflare_tunnel.sh")" cloudflare_tunnel.sh help) +grep -Fq 'cloudflare_tunnel.sh install' <<< "$entrypoint_output" || fail "bash -c entrypoint broken" +pass "bash -c entrypoint remains compatible" + +printf 'All cloudflare wrapper tests passed. PASS=%d\n' "$pass_count" diff --git a/tests/test-motd-01.sh b/tests/test-motd-01.sh deleted file mode 100644 index 423ed60..0000000 --- a/tests/test-motd-01.sh +++ /dev/null @@ -1,555 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-motd-common.sh" -pass "MOTD payloads stay identical"; -assert_eq cf2a4f8dfe2fe26bff5509cdaee56d9e0816fafd0594251b02e892201af5c4d5 "$(sha256sum "$TEST_DIR/t"|awk '{print $1}')" "payload hash" - -( - r="$TEST_DIR/source"; -mkdir -m700 "$r" "$r/lock"; -export_paths "$r"; -b=$(trap -p EXIT HUP INT TERM ERR); -load_motd; -[[ "$b" == "$(trap -p EXIT HUP INT TERM ERR)"&&! -e "$MOTD_STATE_DIR"&&! -e "$MOTD_LOCK_FILE" ]]||fail source -); -pass "source zero side effects" -( - r="$TEST_DIR/basic"; -create_fixture "$r"; -export_paths "$r"; -before=$(target_hash "$r"); -load_motd; -motd_run_locked_operation install; -[[ -f "$MOTD_ETC_ROOT/motd"&&! -s "$MOTD_ETC_ROOT/motd"&&-x "$MOTD_SCRIPT" ]]||fail install; -motd_run_locked_operation restore previous; -assert_eq "$before" "$(target_hash "$r")" "restore previous group"; -motd_run_locked_operation install; -printf changed>"$MOTD_ETC_ROOT/motd"; -motd_run_locked_operation restore initial; -assert_eq "$before" "$(target_hash "$r")" "restore initial group" -) -( - r="$TEST_DIR/types"; -create_fixture "$r"; -export_paths "$r"; -unlink "$MOTD_ETC_ROOT/issue"; -ln -s "$r/link" "$MOTD_ETC_ROOT/issue"; -before=$(target_hash "$r"); -load_motd; -motd_run_locked_operation install; -g=$(motd_read_pointer previous); -motd_validate_snapshot "$MOTD_STATE_DIR/generations/$g" previous "$g"; -assert_eq symlink "${MOTD_SNAPSHOT_STATE[issue]}" "symlink snapshot"; -assert_eq absent "${MOTD_SNAPSHOT_STATE[issue_net]}" "absent snapshot"; -motd_run_locked_operation restore previous; -assert_eq "$before" "$(target_hash "$r")" "symlink restore" -) -( - r="$TEST_DIR/initial"; -create_fixture "$r"; -export_paths "$r"; -before=$(target_hash "$r"); -load_motd; -motd_snapshot_capture_hook(){ [[ "$1" != issue_net ]]; -}; -assert_fail "initial capture atomic" motd_run_locked_operation install; -[[ ! -e "$MOTD_STATE_DIR/initial.current" ]]||fail partial; -assert_eq "$before" "$(target_hash "$r")" "initial failure no target change" -) -( - r="$TEST_DIR/previous"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -old=$(motd_read_pointer previous); -for id in "${MOTD_TARGET_IDS[@]}"; -do motd_snapshot_capture_hook(){ [[ "$1" != "$id" ]]; -}; -assert_fail "previous capture $id" motd_run_locked_operation install; -assert_eq "$old" "$(motd_read_pointer previous)" "previous pointer $id"; -done -) -for failing_id in motd issue_net motd_news; -do -( - r="$TEST_DIR/i-$failing_id"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -printf current>"$MOTD_ETC_ROOT/motd"; -before=$(target_hash "$r"); -motd_target_commit_hook(){ [[ "$2" != "$failing_id" ]]; -}; -assert_fail "install failure $failing_id" motd_run_locked_operation install; -assert_eq "$before" "$(target_hash "$r")" "install rollback $failing_id" -) -( - r="$TEST_DIR/r-$failing_id"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -printf current>"$MOTD_ETC_ROOT/motd"; -before=$(target_hash "$r"); -motd_target_commit_hook(){ [[ "$1" != restore||"$2" != "$failing_id" ]]; -}; -assert_fail "restore failure $failing_id" motd_run_locked_operation restore previous; -assert_eq "$before" "$(target_hash "$r")" "restore rollback $failing_id" -) -done -( - r="$TEST_DIR/manifest"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -g=$(motd_read_pointer previous); -mf="$MOTD_STATE_DIR/generations/$g/manifest"; -cp "$mf" "$r/o"; -for k in missing duplicate unknown digest generation; -do cp "$r/o" "$mf"; -case $k in missing)sed -i '$d' "$mf";; -duplicate)tail -n1 "$mf">>"$mf";; -unknown)echo x=y>>"$mf";; -digest)sed -i 's/[0-9a-f]\{64\}$/0000000000000000000000000000000000000000000000000000000000000000/' "$mf";; -generation)sed -i 's/^generation=.*/generation=00000000000000000000000000000000/' "$mf";; -esac; -before=$(target_hash "$r"); -assert_fail "manifest $k rejected" motd_run_locked_operation restore previous; -assert_eq "$before" "$(target_hash "$r")" "manifest $k no target change"; -done -) -( - r="$TEST_DIR/lock"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_lock_acquired_hook(){ : >"$r/ready"; -sleep 2; -}; -motd_take_lock & h=$!; -while [[ ! -e "$r/ready" ]]; -do sleep .02; -done; -assert_fail "install lock contention" motd_run_locked_operation install; -assert_fail "restore lock contention" motd_run_locked_operation restore previous; -[[ ! -e "$MOTD_STATE_DIR" ]]||fail lockstate; -wait "$h"; -motd_lock_acquired_hook(){ :; -}; -motd_take_lock; -motd_release_lock -); -pass "shared lock nonblocking and reacquirable" -for k in symlink writable fifo; -do( - r="$TEST_DIR/l-$k"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -case $k in symlink)printf x>"$r/x"; -ln -s "$r/x" "$MOTD_LOCK_FILE";; -writable)printf x>"$MOTD_LOCK_FILE"; -chmod 666 "$MOTD_LOCK_FILE";; -fifo)mkfifo "$MOTD_LOCK_FILE";; -esac; -assert_fail "lock rejects $k" motd_take_lock -); -done -for k in symlink writable; -do( - r="$TEST_DIR/s-$k"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -case $k in symlink)mkdir -m700 "$r/s"; -ln -s "$r/s" "$MOTD_STATE_DIR";; -writable)mkdir -m770 "$MOTD_STATE_DIR";; -esac; -motd_take_lock; -assert_fail "state rejects $k" motd_prepare_state_layout; -motd_release_lock -); -done -create_legacy(){ printf legacy>"$MOTD_ETC_ROOT/motd.initial-backup"; -: >"$MOTD_ETC_ROOT/issue.initial-absent"; -printf net>"$MOTD_ETC_ROOT/issue.net.initial-backup"; -: >"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.initial-unknown"; -printf uname>"$MOTD_ETC_ROOT/update-motd.d/10-uname.initial-backup"; -: >"$MOTD_ETC_ROOT/update-motd.d/50-motd-news.initial-absent"; -} -( - r="$TEST_DIR/legacy"; -create_fixture "$r"; -export_paths "$r"; -create_legacy; -load_motd; -motd_run_locked_operation restore initial; -assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "legacy complete restore"; -[[ -e "$MOTD_ETC_ROOT/motd.initial-backup" ]]||fail legacymove -) -for k in partial conflict commit; -do( - r="$TEST_DIR/lg-$k"; -create_fixture "$r"; -export_paths "$r"; -if [[ $k == partial ]]; -then printf x>"$MOTD_ETC_ROOT/motd.initial-backup"; -else create_legacy; -[[ $k == conflict ]]&&: >"$MOTD_ETC_ROOT/motd.initial-absent"; -fi; -load_motd; -[[ $k == commit ]]&&motd_snapshot_commit_hook(){ [[ "$3" != legacy-directory ]]; -}; -assert_fail "legacy $k rejected" motd_run_locked_operation restore initial -); -done -( - r="$TEST_DIR/interop"; -create_fixture "$r"; -export_paths "$r"; -o=$(target_hash "$r"); -bash -c '. "$1/modules/system-customize.sh";ask_yes_no(){ return 0;};motd_preview_hook(){ :;};configure_motd' _ "$ROOT_DIR">/dev/null; -printf x>"$MOTD_ETC_ROOT/motd"; -bash -c '. "$1/tools/setup-motd.sh";motd_run_locked_operation restore previous' _ "$ROOT_DIR"; -assert_eq "$o" "$(target_hash "$r")" "module state tool restore" -) -( - r="$TEST_DIR/exit"; -create_fixture "$r"; -export_paths "$r"; -before=$(target_hash "$r"); -n=0; -bash -c '. "$1";motd_take_lock;motd_install_transaction_traps;motd_prepare_state_layout;motd_validate_target_parents;motd_begin_transaction install;motd_prepare_install_plan;motd_commit_target_plan install false;exit 0' _ "$ROOT_DIR/tools/setup-motd.sh">/dev/null 2>&1||n=$?; -assert_eq 1 "$n" "active exit zero maps one"; -assert_eq "$before" "$(target_hash "$r")" "active exit rollback" -) -( - r="$TEST_DIR/rbf"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -motd_target_commit_hook(){ [[ "$2" != issue_net ]]; -}; -motd_rollback_hook(){ echo "$1">>"$r/a"; -[[ "$1" != motd&&"$1" != custom_welcome ]]; -}; -assert_fail "rollback failure nonzero" motd_run_locked_operation install; -assert_eq 6 "$(wc -l<"$r/a")" "rollback attempts six"; -[[ -e "$MOTD_STATE_DIR/pending" ]]||fail journal -) -cat >"$TEST_DIR/sig.sh"<<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1"; -hit=false -send(){ [[ $hit == false ]]||return; -hit=true; -kill -"$SIGNAL_NAME" "$BASHPID"; -} -motd_snapshot_capture_hook(){ [[ $PHASE == snapshot-building&&$1 == issue ]]&&send; -return 0; -} -motd_transaction_phase_hook(){ [[ $PHASE == snapshot-commit&&$1 == snapshot-commit ]]&&send; -return 0; -} -motd_snapshot_commit_hook(){ [[ $PHASE == snapshot-publish&&$1 == initial&&$3 == pointer ]]&&send; -return 0; -} -motd_target_commit_hook(){ case "$PHASE:$1:$2" in install-first:install:motd|install-middle:install:issue_net|restore-first:restore:motd|restore-middle:restore:issue_net)send;; -esac; -return 0; -} -case $PHASE in restore-*)motd_run_locked_operation restore previous;; -*)motd_run_locked_operation install;; -esac -CHILD -chmod 700 "$TEST_DIR/sig.sh" -for ph in snapshot-building snapshot-commit snapshot-publish install-first install-middle restore-first restore-middle; -do for sig in HUP INT TERM; -do( - r="$TEST_DIR/$ph-$sig"; -create_fixture "$r"; -export_paths "$r"; -if [[ $ph == restore-* ]]; -then bash -c '. "$1";motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh"; -printf current>"$MOTD_ETC_ROOT/motd"; -fi; -before=$(target_hash "$r"); -n=0; -env --default-signal=HUP,INT,TERM PHASE="$ph" SIGNAL_NAME="$sig" bash "$TEST_DIR/sig.sh" "$ROOT_DIR/tools/setup-motd.sh">/dev/null 2>&1||n=$?; -case $sig in HUP)e=129;; -INT)e=130;; -TERM)e=143;; -esac; -assert_eq "$e" "$n" "$sig $ph status"; -assert_eq "$before" "$(target_hash "$r")" "$sig $ph rollback" -); -done; -done -( - r="$TEST_DIR/unknown"; -create_fixture "$r"; -export_paths "$r"; -printf '# linux-setup:managed-motd'>"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; -load_motd; -motd_run_locked_operation install; -g=$(motd_read_pointer initial); -motd_validate_snapshot "$MOTD_STATE_DIR/generations/$g" initial "$g"; -assert_eq unknown "${MOTD_SNAPSHOT_STATE[motd]}" "unknown initial state" -) -( - r="$TEST_DIR/readonly"; -create_fixture "$r"; -export_paths "$r"; -bash "$ROOT_DIR/tools/setup-motd.sh" status>/dev/null; -bash "$ROOT_DIR/tools/setup-motd.sh" help>/dev/null; -[[ ! -e "$MOTD_STATE_DIR"&&! -e "$MOTD_LOCK_FILE" ]]||fail readonly -); -pass "status help no lock" -( - r="$TEST_DIR/publish"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_run_locked_operation install; -old=$(motd_read_pointer previous); -printf current>"$MOTD_ETC_ROOT/motd"; -before=$(target_hash "$r"); -motd_snapshot_manifest_hook(){ [[ "$1" != previous ]]; -}; -assert_fail "previous manifest failure" motd_run_locked_operation install; -assert_eq "$old" "$(motd_read_pointer previous)" "manifest preserves previous"; -assert_eq "$before" "$(target_hash "$r")" "manifest failure rolls back"; -motd_snapshot_manifest_hook(){ :; -}; -motd_snapshot_commit_hook(){ [[ "$1" != previous||"$3" != directory ]]; -}; -assert_fail "previous directory commit failure" motd_run_locked_operation install; -assert_eq "$old" "$(motd_read_pointer previous)" "directory failure preserves previous" -) -for k in owner gid; -do( - r="$TEST_DIR/lmeta-$k"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -printf x>"$MOTD_LOCK_FILE"; -chmod 600 "$MOTD_LOCK_FILE"; -stat(){ if [[ "${*: -1}" == "$MOTD_LOCK_FILE"&&"$*" == *"%u:%g:%a"* ]]; -then [[ $k == owner ]]&&printf '99999:%s:600\n' "$MOTD_TRUSTED_GID"||printf '%s:99999:600\n' "$MOTD_TRUSTED_UID"; -else command stat "$@"; -fi; -}; -assert_fail "lock wrong $k" motd_take_lock -); -done -for k in owner gid; -do( - r="$TEST_DIR/smeta-$k"; -create_fixture "$r"; -export_paths "$r"; -mkdir -m700 "$MOTD_STATE_DIR"; -load_motd; -stat(){ if [[ "${*: -1}" == "$MOTD_STATE_DIR"&&"$*" == *"%u:%g:%a"* ]]; -then [[ $k == owner ]]&&printf '99999:%s:700\n' "$MOTD_TRUSTED_GID"||printf '%s:99999:700\n' "$MOTD_TRUSTED_UID"; -else command stat "$@"; -fi; -}; -motd_take_lock; -assert_fail "state wrong $k" motd_prepare_state_layout; -motd_release_lock -); -done -( - r="$TEST_DIR/state-create"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -chown(){ [[ "${*: -1}" == "$MOTD_STATE_DIR" ]]&&return 1; -command chown "$@"; -}; -assert_fail "state create failure" motd_run_locked_operation install; -[[ ! -e "$MOTD_STATE_DIR" ]]||fail state-residue -) -( - r="$TEST_DIR/close"; -create_fixture "$r"; -export_paths "$r"; -load_motd; -motd_take_lock; -motd_lock_close_hook(){ return 1; -}; -assert_fail "lock close failure" motd_release_lock -) -( - r="$TEST_DIR/device"; -create_fixture "$r"; -export_paths "$r"; -if mknod "$MOTD_ETC_ROOT/issue.net" c 1 3 2>/dev/null; -then before=$(target_hash "$r"); -load_motd; -assert_fail "device rejected" motd_run_locked_operation install; -assert_eq "$before" "$(target_hash "$r")" "device no target change"; -else pass "device test skipped"; -fi -) -( - r="$TEST_DIR/legacy-state"; -create_fixture "$r"; -export_paths "$r"; -mkdir -m700 "$MOTD_STATE_DIR"; -printf legacy>"$MOTD_STATE_DIR/motd.previous-backup"; -chmod 600 "$MOTD_STATE_DIR/motd.previous-backup"; -for x in issue issue.net 00-custom-welcome 10-uname 50-motd-news; -do : >"$MOTD_STATE_DIR/$x.previous-absent"; -chmod 600 "$MOTD_STATE_DIR/$x.previous-absent"; -done; -load_motd; -motd_run_locked_operation restore previous; -assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "state legacy restore" -) - -state_fingerprint() { - local root="$1" path - [[ -d "$root" ]] || { printf 'absent\n'; return; } - while IFS= read -r path; do - if [[ -L "$path" ]]; then printf 'L|%s|%s\n' "${path#$root/}" "$(readlink "$path")" - elif [[ -f "$path" ]]; then printf 'F|%s|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" "$(sha256sum "$path" | awk '{print $1}')" - elif [[ -d "$path" ]]; then printf 'D|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" - fi - done < <(find "$root" -mindepth 1 -print | sort) -} - -generation_count() { - local directory="$1" - [[ -d "$directory" ]] || { printf '0\n'; return; } - find "$directory" -mindepth 1 -maxdepth 1 -type d ! -name '.*.stage' | wc -l -} - -assert_no_motd_residue() { - local root="$1" residue - residue=$(find "$root" -name '.linux-setup-motd.*' -o -name '.*.pointer.*' -o -name '.pending.*' -o -name '.*.stage' 2>/dev/null) - [[ -z "$residue" ]] || fail "unexpected MOTD residue: $residue" - if [[ -d "$root/state/transactions" ]]; then [[ -z "$(find "$root/state/transactions" -mindepth 1 -print -quit)" ]] || fail "transaction residue remains"; fi -} - -create_complete_legacy_scope() { - local scope="$1" prefix - cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_ETC_ROOT/motd.${scope}-backup" - cp -a "$MOTD_ETC_ROOT/issue" "$MOTD_ETC_ROOT/issue.${scope}-backup" - : > "$MOTD_ETC_ROOT/issue.net.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/issue.net.${scope}-absent" - cp -a "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.${scope}-backup" - cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_ETC_ROOT/update-motd.d/10-uname.${scope}-backup" - : > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent" -} - -for legacy_phase in random-id stat open create cat chown chmod sha256 manifest mv; do -( - root="$TEST_DIR/legacy-io-$legacy_phase"; create_fixture "$root"; export_paths "$root" - create_complete_legacy_scope initial - before_targets=$(target_hash "$root") - before_legacy=$(find "$MOTD_ETC_ROOT" -name '*.initial-*' -type f -printf '%p|%s|%m\n' | sort) - load_motd - motd_legacy_io_hook() { [[ "$1" != "$legacy_phase" ]]; } - assert_fail "legacy $legacy_phase failure returns nonzero" motd_run_locked_operation restore initial - assert_eq "$before_targets" "$(target_hash "$root")" "legacy $legacy_phase keeps formal targets" - assert_eq "$before_legacy" "$(find "$MOTD_ETC_ROOT" -name '*.initial-*' -type f -printf '%p|%s|%m\n' | sort)" "legacy $legacy_phase keeps legacy files" - [[ ! -e "$MOTD_STATE_DIR/initial.current" ]] || fail "legacy $legacy_phase published initial pointer" - assert_no_motd_residue "$root" -) -done -pass "legacy I/O failure matrix preserves data and state" - -cat > "$TEST_DIR/pointer-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1" -motd_snapshot_commit_hook() { - if [[ "$1" == previous && "$3" == pointer ]]; then kill -"$SIGNAL_NAME" "$BASHPID"; fi -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/pointer-signal-child.sh" - -for signal_name in HUP INT TERM; do -( - root="$TEST_DIR/pointer-signal-$signal_name"; create_fixture "$root"; export_paths "$root" - bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" - old_pointer=$(cat "$MOTD_STATE_DIR/previous.current") - before=$(target_hash "$root") - rc=0 - env --default-signal=HUP,INT,TERM SIGNAL_NAME="$signal_name" bash "$TEST_DIR/pointer-signal-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? - case "$signal_name" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - assert_eq "$expected" "$rc" "$signal_name during previous pointer stage returns conventional status" - assert_eq "$old_pointer" "$(cat "$MOTD_STATE_DIR/previous.current")" "$signal_name preserves previous pointer" - assert_eq "$before" "$(target_hash "$root")" "$signal_name pointer stage restores targets" - assert_no_motd_residue "$root" - bash -c '. "$1"; motd_take_lock; motd_release_lock' _ "$ROOT_DIR/tools/setup-motd.sh" -) -done - -for pending_phase in pending-create pending-write pending-chown pending-chmod pending-publish; do -( - root="$TEST_DIR/pending-failure-$pending_phase"; create_fixture "$root"; export_paths "$root" - load_motd - motd_run_locked_operation install - before=$(target_hash "$root"); before_generations=$(generation_count "$MOTD_STATE_DIR/generations") - motd_transaction_phase_hook() { [[ "$1" != "$pending_phase" ]]; } - assert_fail "pending $pending_phase failure returns nonzero" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "pending $pending_phase keeps targets" - assert_eq "$before_generations" "$(generation_count "$MOTD_STATE_DIR/generations")" "pending $pending_phase leaves no generation orphan" - [[ ! -e "$MOTD_STATE_DIR/pending" && ! -L "$MOTD_STATE_DIR/pending" ]] || fail "pending $pending_phase left pending" - assert_no_motd_residue "$root" - motd_transaction_phase_hook() { :; } - motd_run_locked_operation install -) -done - -for pending_kind in broken-symlink symlink directory fifo mode owner gid; do -( - root="$TEST_DIR/pending-kind-$pending_kind"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - pending="$MOTD_STATE_DIR/pending" - case "$pending_kind" in - broken-symlink) ln -s "$root/missing" "$pending" ;; - symlink) printf x > "$root/pending-target"; chmod 0600 "$root/pending-target"; ln -s "$root/pending-target" "$pending" ;; - directory) mkdir "$pending" ;; - fifo) mkfifo "$pending" ;; - mode) printf x > "$pending"; chmod 0666 "$pending" ;; - owner|gid) printf x > "$pending"; chmod 0600 "$pending" ;; - esac - before=$(target_hash "$root") - if [[ "$pending_kind" == owner || "$pending_kind" == gid ]]; then - stat() { - if [[ "${*: -1}" == "$pending" && "$*" == *"%u:%g:%a"* ]]; then - [[ "$pending_kind" == owner ]] && printf '99999:%s:600\n' "$MOTD_TRUSTED_GID" || printf '%s:99999:600\n' "$MOTD_TRUSTED_UID" - else command stat "$@"; fi - } - fi - assert_fail "pending $pending_kind fails closed" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "pending $pending_kind changes no target" - [[ -e "$pending" || -L "$pending" ]] || fail "pending $pending_kind evidence was removed" -) -done - -cat > "$TEST_DIR/kill-pending-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_transaction_phase_hook() { [[ "$1" == pending-active ]] && kill -KILL "$BASHPID"; return 0; } -motd_run_locked_operation install -CHILD -cat > "$TEST_DIR/kill-previous-pointer-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_snapshot_commit_hook() { [[ "$1" == previous && "$3" == pointer ]] && kill -KILL "$BASHPID"; return 0; } -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/kill-pending-child.sh" "$TEST_DIR/kill-previous-pointer-child.sh" diff --git a/tests/test-motd-02.sh b/tests/test-motd-02.sh deleted file mode 100644 index 4656502..0000000 --- a/tests/test-motd-02.sh +++ /dev/null @@ -1,267 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-motd-common.sh" - -for journal_case in pending-directory journal-transaction generation-scope generation-missing old-new-mix; do -( - root="$TEST_DIR/journal-$journal_case"; create_fixture "$root"; export_paths "$root" - bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" - rc=0 - if [[ "$journal_case" == pending-directory || "$journal_case" == journal-transaction ]]; then - bash "$TEST_DIR/kill-pending-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? - else - bash "$TEST_DIR/kill-previous-pointer-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? - fi - assert_eq 137 "$rc" "$journal_case fixture stops with SIGKILL" - transaction=$(cat "$MOTD_STATE_DIR/pending") - journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" - case "$journal_case" in - pending-directory) printf '%032x\n' 1 > "$MOTD_STATE_DIR/pending" ;; - journal-transaction) sed -i 's/^transaction=.*/transaction=00000000000000000000000000000001/' "$journal" ;; - generation-scope) initial=$(cat "$MOTD_STATE_DIR/initial.current"); sed -i "s/^previous_new=.*/previous_new=$initial/" "$journal" ;; - generation-missing) sed -i 's/^previous_new=.*/previous_new=ffffffffffffffffffffffffffffffff/' "$journal" ;; - old-new-mix) initial=$(cat "$MOTD_STATE_DIR/initial.current"); sed -i "s/^previous_old=.*/previous_old=$initial/" "$journal" ;; - esac - before=$(target_hash "$root") - load_motd - assert_fail "$journal_case pending/journal mismatch rejected" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$journal_case mismatch changes no target" - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$journal_case evidence removed" -) -done - -for tamper_case in content mode inode regular-to-symlink; do -( - root="$TEST_DIR/stage-$tamper_case"; create_fixture "$root"; export_paths "$root" - printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - before=$(target_hash "$root") - load_motd - motd_transaction_phase_hook() { - [[ "$1" == install-plan-ready ]] || return 0 - case "$tamper_case" in - content) printf tampered > "${MOTD_TARGET_STAGE[motd]}" ;; - mode) chmod 0600 "${MOTD_TARGET_STAGE[issue_net]}" ;; - inode) cp "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" "${MOTD_TARGET_STAGE[motd_news]}.replacement"; chmod 0644 "${MOTD_TARGET_STAGE[motd_news]}.replacement"; mv -Tf "${MOTD_TARGET_STAGE[motd_news]}.replacement" "${MOTD_TARGET_STAGE[motd_news]}" ;; - regular-to-symlink) rm -f "${MOTD_TARGET_STAGE[issue_net]}"; ln -s "$root/elsewhere" "${MOTD_TARGET_STAGE[issue_net]}" ;; - esac - } - motd_target_commit_hook() { : > "$root/commit-called"; } - assert_fail "$tamper_case stage tamper rejected before commit" motd_run_locked_operation install - [[ ! -e "$root/commit-called" ]] || fail "$tamper_case reached first formal commit" - assert_eq "$before" "$(target_hash "$root")" "$tamper_case stage tamper changes no target" - assert_no_motd_residue "$root" -) -done - -( - root="$TEST_DIR/stage-symlink-target"; create_fixture "$root"; export_paths "$root" - printf external > "$root/external"; ln -sf "$root/external" "$MOTD_ETC_ROOT/issue" - load_motd; motd_run_locked_operation install - before=$(target_hash "$root") - motd_transaction_phase_hook() { - [[ "$1" == restore-plan-ready ]] || return 0 - rm -f "${MOTD_TARGET_STAGE[issue]}"; ln -s "$root/other" "${MOTD_TARGET_STAGE[issue]}" - } - motd_target_commit_hook() { : > "$root/commit-called"; } - assert_fail "symlink target stage tamper rejected before restore commit" motd_run_locked_operation restore previous - [[ ! -e "$root/commit-called" ]] || fail "symlink target tamper reached formal commit" - assert_eq "$before" "$(target_hash "$root")" "symlink target tamper changes no target" -) - -for native_id in uname motd_news; do -( - root="$TEST_DIR/native-symlink-$native_id"; create_fixture "$root"; export_paths "$root" - case "$native_id" in uname) native_path="$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; motd_news) native_path="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; esac - rm -f "$native_path" - printf 'external-%s\n' "$native_id" > "$root/external-$native_id"; chmod 0755 "$root/external-$native_id" - external_hash=$(sha256sum "$root/external-$native_id"); external_mode=$(stat -c %a "$root/external-$native_id") - ln -s "$root/external-$native_id" "$native_path" - load_motd - motd_run_locked_operation install - [[ ! -e "$native_path" && ! -L "$native_path" ]] || fail "$native_id symlink entry remains enabled" - assert_eq "$external_hash" "$(sha256sum "$root/external-$native_id")" "$native_id external content unchanged" - assert_eq "$external_mode" "$(stat -c %a "$root/external-$native_id")" "$native_id external mode unchanged" - motd_run_locked_operation restore previous - [[ -L "$native_path" && "$(readlink "$native_path")" == "$root/external-$native_id" ]] || fail "$native_id previous symlink not restored" -) -done - -cat > "$TEST_DIR/sigkill-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_transaction_phase_hook() { - [[ "$1" == "$KILL_PHASE" ]] && kill -KILL "$BASHPID" - return 0 -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/sigkill-child.sh" - -for kill_phase in transaction-snapshot-building transaction-final-moved target-committed-motd target-committed-issue_net target-committed-motd_news committed-cleanup committed-old-generation-cleaned committed-pending-removed; do -( - root="$TEST_DIR/sigkill-$kill_phase"; create_fixture "$root"; export_paths "$root" - printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" - before=$(target_hash "$root") - rc=0 - KILL_PHASE="$kill_phase" bash "$TEST_DIR/sigkill-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? - assert_eq 137 "$rc" "SIGKILL at $kill_phase terminates process" - after_kill=$(target_hash "$root") - load_motd - motd_take_lock; motd_prepare_state_layout; motd_reconcile_state; motd_release_lock - if [[ "$kill_phase" == committed-* ]]; then - assert_eq "$after_kill" "$(target_hash "$root")" "committed cleanup SIGKILL does not roll back" - else - assert_eq "$before" "$(target_hash "$root")" "SIGKILL at $kill_phase rolls back to pre-transaction state" - fi - assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "SIGKILL at $kill_phase leaves bounded generations" - assert_no_motd_residue "$root" - motd_take_lock; motd_release_lock -) -done - -( - root="$TEST_DIR/generation-bounded"; create_fixture "$root"; export_paths "$root" - load_motd - for _ in {1..6}; do motd_run_locked_operation install; done - assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "repeated install keeps two generations" -) - -( - root="$TEST_DIR/generation-commit-failure"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - before_count=$(generation_count "$MOTD_STATE_DIR/generations") - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - assert_fail "target commit failure returns nonzero for generation cleanup" motd_run_locked_operation install - assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "target commit failure leaves no orphan generation" -) - -( - root="$TEST_DIR/interop-module-tool"; create_fixture "$root"; export_paths "$root" - original=$(target_hash "$root") - bash -c '. "$1/modules/system-customize.sh"; ask_yes_no(){ return 0; }; motd_preview_hook(){ :; }; configure_motd' _ "$ROOT_DIR" >/dev/null - printf changed > "$MOTD_ETC_ROOT/motd" - bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation restore previous' _ "$ROOT_DIR" - assert_eq "$original" "$(target_hash "$root")" "module install to tool previous restore" - printf changed > "$MOTD_ETC_ROOT/motd" - bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation restore initial' _ "$ROOT_DIR" - assert_eq "$original" "$(target_hash "$root")" "module install to tool initial restore" -) - -( - root="$TEST_DIR/interop-tool-module"; create_fixture "$root"; export_paths "$root" - original=$(target_hash "$root") - bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation install' _ "$ROOT_DIR" - printf between > "$MOTD_ETC_ROOT/motd"; between=$(target_hash "$root") - bash -c '. "$1/modules/system-customize.sh"; ask_yes_no(){ return 0; }; motd_preview_hook(){ :; }; configure_motd' _ "$ROOT_DIR" >/dev/null - bash -c '. "$1/modules/system-customize.sh"; motd_run_locked_operation restore previous' _ "$ROOT_DIR" - assert_eq "$between" "$(target_hash "$root")" "tool install to module previous restore" - printf changed > "$MOTD_ETC_ROOT/motd" - bash -c '. "$1/modules/system-customize.sh"; motd_run_locked_operation restore initial' _ "$ROOT_DIR" - assert_eq "$original" "$(target_hash "$root")" "tool install to module initial restore" -) - - -( - root="$TEST_DIR/cleanup-old-generation-failure"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - old_previous=$(cat "$MOTD_STATE_DIR/previous.current"); blocked="$MOTD_STATE_DIR/generations/$old_previous" - motd_cleanup_path_hook() { [[ "$1" != "$blocked" ]]; } - assert_fail "old previous generation delete failure returns nonzero" motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "old generation delete failure lost pending journal" - transaction=$(cat "$MOTD_STATE_DIR/pending"); grep -Fq 'phase=committed' "$MOTD_TRANSACTION_PARENT/$transaction/journal" || fail "old generation failure journal not committed" - [[ -d "$blocked" ]] || fail "blocked old generation unexpectedly removed" - motd_cleanup_path_hook() { :; } - motd_run_locked_operation install - [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "old generation cleanup retry left pending" -) - -( - root="$TEST_DIR/cleanup-rollback-generation-failure"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - current_initial=$(cat "$MOTD_STATE_DIR/initial.current"); current_previous=$(cat "$MOTD_STATE_DIR/previous.current") - motd_cleanup_path_hook() { - local name=${1##*/} - if [[ "$1" == "$MOTD_STATE_DIR/generations/"* && "$name" != "$current_initial" && "$name" != "$current_previous" ]]; then return 1; fi - return 0 - } - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - assert_fail "rollback generation delete failure returns nonzero" motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback generation failure lost pending" - transaction=$(cat "$MOTD_STATE_DIR/pending"); grep -Fq 'phase=rolledback' "$MOTD_TRANSACTION_PARENT/$transaction/journal" || fail "rollback generation failure journal not rolledback" - motd_cleanup_path_hook() { :; }; motd_target_commit_hook() { :; } - motd_run_locked_operation install - [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "rolledback cleanup retry left pending" - assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "rolledback cleanup retry restores bounded generations" -) - - -for hook_case in content-motd mode-issue_net inode-motd_news type-issue_net; do -( - root="$TEST_DIR/hook-target-$hook_case"; create_fixture "$root"; export_paths "$root" - printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - before=$(target_hash "$root") - load_motd - motd_target_commit_hook() { - local id="$2" stage replacement - case "$hook_case:$id" in - content-motd:motd) printf changed > "${MOTD_TARGET_STAGE[motd]}" ;; - mode-issue_net:issue_net) chmod 0600 "${MOTD_TARGET_STAGE[issue_net]}" ;; - inode-motd_news:motd_news) - stage=${MOTD_TARGET_STAGE[motd_news]}; replacement="${stage}.replacement" - cp "$stage" "$replacement"; chmod "${MOTD_STAGE_MODE[motd_news]}" "$replacement"; mv -Tf "$replacement" "$stage" - ;; - type-issue_net:issue_net) rm -f "${MOTD_TARGET_STAGE[issue_net]}"; ln -s "$root/other" "${MOTD_TARGET_STAGE[issue_net]}" ;; - esac - return 0 - } - assert_fail "$hook_case target hook tamper rejected" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$hook_case leaves formal targets unchanged" - assert_no_motd_residue "$root" -) -done - -( - root="$TEST_DIR/hook-rollback-stage"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - before=$(target_hash "$root") - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - motd_rollback_hook() { [[ "$1" != motd ]] || printf tampered > "${MOTD_TARGET_STAGE[motd]}"; return 0; } - assert_fail "rollback hook stage tamper rejected" motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback hook tamper did not preserve pending journal" - motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } - motd_run_locked_operation install - [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback hook retry left pending" - assert_eq "$before" "$(target_hash "$root")" "rollback hook retry restores original group before next install" -) - -for pointer_case in old-generation other-generation extra-line inode; do -( - root="$TEST_DIR/pointer-hook-$pointer_case"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - before=$(target_hash "$root"); old_pointer=$(cat "$MOTD_STATE_DIR/previous.current"); before_count=$(generation_count "$MOTD_STATE_DIR/generations") - motd_snapshot_commit_hook() { - local scope="$1" generation="$2" kind="$3" stage other source - [[ "$scope" == previous && "$kind" == pointer ]] || return 0 - stage=${MOTD_POINTER_STAGE_PATHS[${#MOTD_POINTER_STAGE_PATHS[@]}-1]} - case "$pointer_case" in - old-generation) printf '%s\n' "$old_pointer" > "$stage" ;; - other-generation) - other=11111111111111111111111111111111; source="$MOTD_STATE_DIR/generations/$generation" - cp -a "$source" "$MOTD_STATE_DIR/generations/$other" - sed -i "s/^generation=.*/generation=$other/" "$MOTD_STATE_DIR/generations/$other/manifest" - MOTD_NEW_GENERATIONS+=("$other") - printf '%s\n' "$other" > "$stage" - ;; - extra-line) printf 'extra\n' >> "$stage" ;; - inode) cp "$stage" "${stage}.replacement"; chmod 0600 "${stage}.replacement"; mv -Tf "${stage}.replacement" "$stage" ;; - esac - } - assert_fail "pointer hook $pointer_case rejected" motd_run_locked_operation install - assert_eq "$old_pointer" "$(cat "$MOTD_STATE_DIR/previous.current")" "pointer hook $pointer_case preserves pointer" - assert_eq "$before" "$(target_hash "$root")" "pointer hook $pointer_case preserves targets" - assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "pointer hook $pointer_case preserves generation count" - assert_no_motd_residue "$root" -) -done diff --git a/tests/test-motd-03.sh b/tests/test-motd-03.sh deleted file mode 100644 index e6b7ece..0000000 --- a/tests/test-motd-03.sh +++ /dev/null @@ -1,233 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-motd-common.sh" - -for pending_hook_case in id extra-line inode; do -( - root="$TEST_DIR/pending-hook-$pending_hook_case"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - before=$(target_hash "$root") - motd_transaction_phase_hook() { - local phase="$1" stage - [[ "$phase" == pending-publish ]] || return 0 - stage=${MOTD_PENDING_STAGE_PATHS[${#MOTD_PENDING_STAGE_PATHS[@]}-1]} - case "$pending_hook_case" in - id) printf '%032x\n' 1 > "$stage" ;; - extra-line) printf 'extra\n' >> "$stage" ;; - inode) cp "$stage" "${stage}.replacement"; chmod 0600 "${stage}.replacement"; mv -Tf "${stage}.replacement" "$stage" ;; - esac - } - motd_target_commit_hook() { : > "$root/commit-called"; } - assert_fail "pending publish hook $pending_hook_case rejected" motd_run_locked_operation install - [[ ! -e "$root/commit-called" ]] || fail "pending hook $pending_hook_case reached formal target commit" - assert_eq "$before" "$(target_hash "$root")" "pending hook $pending_hook_case preserves targets" - [[ ! -e "$MOTD_STATE_DIR/pending" && ! -L "$MOTD_STATE_DIR/pending" ]] || fail "pending hook $pending_hook_case left pending" - assert_no_motd_residue "$root" -) -done - -cat > "$TEST_DIR/building-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1" -send_building_signal() { kill -"$SIGNAL_NAME" "$BASHPID"; } -motd_transaction_phase_hook() { - case "$FAIL_PHASE:$1" in - transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) send_building_signal ;; - esac - return 0 -} -motd_snapshot_capture_hook() { - [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0 - case "$FAIL_PHASE:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) send_building_signal ;; esac - return 0 -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/building-signal-child.sh" - -for building_phase in transaction-snapshot-building capture-motd capture-issue_net capture-motd_news snapshot-commit transaction-final-move; do -( - root="$TEST_DIR/building-fail-$building_phase"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - before=$(target_hash "$root"); before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")"; before_count=$(generation_count "$MOTD_STATE_DIR/generations") - motd_transaction_phase_hook() { case "$building_phase:$1" in transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) return 1 ;; esac; return 0; } - motd_snapshot_capture_hook() { [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0; case "$building_phase:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) return 1 ;; esac; return 0; } - assert_fail "$building_phase ordinary failure returns nonzero" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$building_phase ordinary failure preserves targets" - assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$building_phase ordinary failure preserves pointers" - assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "$building_phase ordinary failure preserves generations" - assert_no_motd_residue "$root" - motd_take_lock; motd_release_lock -) - for signal_name in HUP INT TERM; do - ( - root="$TEST_DIR/building-$building_phase-$signal_name"; create_fixture "$root"; export_paths "$root" - bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" - before=$(target_hash "$root"); before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")"; before_count=$(generation_count "$MOTD_STATE_DIR/generations") - rc=0 - env --default-signal=HUP,INT,TERM FAIL_PHASE="$building_phase" SIGNAL_NAME="$signal_name" bash "$TEST_DIR/building-signal-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? - case "$signal_name" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac - assert_eq "$expected" "$rc" "$building_phase $signal_name status" - assert_eq "$before" "$(target_hash "$root")" "$building_phase $signal_name preserves targets" - assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$building_phase $signal_name preserves pointers" - assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "$building_phase $signal_name preserves generations" - assert_no_motd_residue "$root" - bash -c '. "$1"; motd_take_lock; motd_release_lock' _ "$ROOT_DIR/tools/setup-motd.sh" - ) - done -done - -for previous_impl in tool module; do -( - root="$TEST_DIR/previous-only-$previous_impl"; create_fixture "$root"; export_paths "$root" - create_complete_legacy_scope previous - case "$previous_impl" in tool) script="$ROOT_DIR/tools/setup-motd.sh" ;; module) script="$ROOT_DIR/modules/system-customize.sh" ;; esac - # shellcheck source=/dev/null - . "$script" - motd_run_locked_operation restore previous - [[ ! -e "$MOTD_STATE_DIR/initial.current" && ! -L "$MOTD_STATE_DIR/initial.current" ]] || fail "$previous_impl previous-only created initial pointer" - [[ -e "$MOTD_STATE_DIR/previous.current" ]] || fail "$previous_impl previous-only missing previous pointer" - motd_run_locked_operation restore previous - [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$previous_impl repeated previous restore left transaction" - motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/initial.current" && -e "$MOTD_STATE_DIR/previous.current" ]] || fail "$previous_impl install did not complete initial/previous pointers" - assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$previous_impl previous-only lifecycle bounded" -) -done - -for time_direction in tool-module module-tool; do -( - root="$TEST_DIR/timestamps-$time_direction"; create_fixture "$root"; export_paths "$root" - printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - for native in 10-uname 50-motd-news; do touch -a -d @1000000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1000200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done - expected_uname=$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname"); expected_news=$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news") - case "$time_direction" in - tool-module) install_script="$ROOT_DIR/tools/setup-motd.sh"; restore_script="$ROOT_DIR/modules/system-customize.sh" ;; - module-tool) install_script="$ROOT_DIR/modules/system-customize.sh"; restore_script="$ROOT_DIR/tools/setup-motd.sh" ;; - esac - # shellcheck source=/dev/null - . "$install_script"; motd_run_locked_operation install - assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction install preserves uname timestamps" - assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction install preserves news timestamps" - for native in 10-uname 50-motd-news; do touch -a -d @1100000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1100200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done - bash -c '. "$1"; motd_run_locked_operation restore previous' _ "$restore_script" - assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction previous restores uname timestamps" - assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction previous restores news timestamps" - for native in 10-uname 50-motd-news; do touch -a -d @1200000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1200200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done - bash -c '. "$1"; motd_run_locked_operation restore initial' _ "$restore_script" - assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction initial restores uname timestamps" - assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction initial restores news timestamps" -) -done - -for hidden_location in transactions generations root; do - for hidden_kind in file symlink fifo dir; do - ( - root="$TEST_DIR/hidden-$hidden_location-$hidden_kind"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - case "$hidden_location" in transactions) parent="$MOTD_TRANSACTION_PARENT" ;; generations) parent="$MOTD_STATE_DIR/generations" ;; root) parent="$MOTD_STATE_DIR" ;; esac - object="$parent/.evil-$hidden_kind" - case "$hidden_kind" in file) printf evil > "$object" ;; symlink) ln -s "$root/missing" "$object" ;; fifo) mkfifo "$object" ;; dir) mkdir "$object" ;; esac - before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - if [[ "$hidden_kind" == file || "$hidden_kind" == fifo ]]; then operation=(install); else operation=(restore previous); fi - assert_fail "$hidden_location hidden $hidden_kind rejected" motd_run_locked_operation "${operation[@]}" - assert_eq "$before" "$(target_hash "$root")" "$hidden_location hidden $hidden_kind preserves targets" - assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$hidden_location hidden $hidden_kind preserves pointers" - [[ -e "$object" || -L "$object" ]] || fail "$hidden_location hidden $hidden_kind evidence removed" - motd_take_lock; motd_release_lock - ) - done -done - -( - root="$TEST_DIR/hidden-transaction-short"; create_fixture "$root"; export_paths "$root" - load_motd; motd_run_locked_operation install - printf evil > "$MOTD_TRANSACTION_PARENT/.123" - before=$(target_hash "$root") - assert_fail "transactions .123 rejected" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "transactions .123 preserves targets" - [[ -e "$MOTD_TRANSACTION_PARENT/.123" ]] || fail "transactions .123 evidence removed" - motd_take_lock; motd_release_lock -) - -( - transaction_block="$TEST_DIR/transaction-block" - sed -n '/# === MOTD group transaction ===/,/^configure_motd()/p' "$ROOT_DIR/tools/setup-motd.sh" > "$transaction_block" - if grep -Eq "stat .*%F|regular file|regular empty file|symbolic link" "$transaction_block"; then fail "MOTD transaction depends on localized stat type text"; fi - pass "MOTD transaction avoids localized stat type text" -) - -( - root="$TEST_DIR/locale-types"; create_fixture "$root"; export_paths "$root" - printf outside > "$root/outside"; rm -f "$MOTD_ETC_ROOT/issue"; ln -s "$root/outside" "$MOTD_ETC_ROOT/issue" - create_complete_legacy_scope initial - load_motd - stat() { - if [[ "$*" == *"%F"* ]]; then printf '本地化文件类型\n'; else command stat "$@"; fi - } - LANG=zh_CN.UTF-8 motd_run_locked_operation restore initial - [[ -L "$MOTD_ETC_ROOT/issue" ]] || fail "localized stat wrapper broke symlink restore" - assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "localized stat wrapper preserves regular legacy import" - LANG=zh_CN.UTF-8 bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/modules/system-customize.sh" -) - -for missing_command in bash chown flock ln od readlink sha256sum touch wc; do -( - root="$TEST_DIR/module-missing-$missing_command"; create_fixture "$root"; export_paths "$root" - before=$(target_hash "$root"); rc=0; mkdir "$root/bin" - for available_command in apt-get apt-cache awk bash basename cat chmod chown cp curl df dirname dpkg flock grep hostname id install ln mkdir mktemp mv od readlink rm sed sha256sum sleep sort stat touch tr uname uptime wc; do - [[ "$available_command" == "$missing_command" ]] && continue - available_path=$(command -v "$available_command" || true) - if [[ -n "$available_path" ]]; then ln -s "$available_path" "$root/bin/$available_command" - else printf '#!/bin/sh\nexit 0\n' > "$root/bin/$available_command"; chmod 0755 "$root/bin/$available_command"; fi - done - PATH="$root/bin" /bin/bash -c ' - . "$1" - require_root() { :; } - main motd - ' _ "$ROOT_DIR/modules/system-customize.sh" > "$root/output" 2>&1 || rc=$? - assert_eq 1 "$rc" "module missing $missing_command fails before action" - grep -Fq "缺少必要命令: $missing_command" "$root/output" || fail "module missing $missing_command not reported" - pass "module preflight reports missing $missing_command" - assert_eq "$before" "$(target_hash "$root")" "module missing $missing_command preserves MOTD" - [[ ! -e "$MOTD_STATE_DIR" && ! -e "$MOTD_LOCK_FILE" ]] || fail "module missing $missing_command created lock/state" -) -done - -grep -Fq 'error "缺少必要命令: $required_command"' "$ROOT_DIR/modules/system-customize.sh" || fail "module missing-command error output branch absent" -pass "module dependency preflight retains missing-command output" - - -for noop_impl in tool module; do - for noop_created_type in regular symlink fifo; do - ( - root="$TEST_DIR/noop-absent-$noop_impl-$noop_created_type"; create_fixture "$root"; export_paths "$root" - path="$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; before=$(target_hash "$root") - if [[ "$noop_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_target_commit_hook() { if [[ "$2" == motd_news ]]; then case "$noop_created_type" in regular) printf created > "$3"; chmod 0755 "$3" ;; symlink) ln -s "$root/external" "$3" ;; fifo) mkfifo "$3" ;; esac; fi; return 0; } - assert_fail "$noop_impl absent noop rejects $noop_created_type creation" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$noop_impl absent noop $noop_created_type rolls back group" - [[ ! -e "$path" && ! -L "$path" ]] || fail "$noop_impl absent noop $noop_created_type left created object" - assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock - ) - done -done - -for noop_impl in tool module; do - for noop_change in regular-content regular-mode regular-inode regular-symlink symlink-target absent-regular; do - ( - root="$TEST_DIR/noop-unknown-$noop_impl-$noop_change"; create_fixture "$root"; export_paths "$root" - printf '# linux-setup:managed-motd\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" - if [[ "$noop_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_run_locked_operation install - case "$noop_change" in regular-content|regular-symlink) target_id=motd ;; regular-mode) target_id=issue ;; regular-inode) target_id=custom_welcome ;; symlink-target) target_id="uname"; rm -f "${MOTD_PATH_BY_ID[$target_id]}"; ln -s "$root/link-one" "${MOTD_PATH_BY_ID[$target_id]}" ;; absent-regular) target_id=motd_news ;; esac - before=$(target_hash "$root") - motd_target_commit_hook() { local id="$2" path="$3" replacement; [[ "$id" == "$target_id" ]] || return 0; case "$noop_change" in regular-content) printf changed > "$path" ;; regular-mode) chmod 0600 "$path" ;; regular-inode) replacement="${path}.replacement"; cp "$path" "$replacement"; chmod 0755 "$replacement"; mv -Tf "$replacement" "$path" ;; regular-symlink) rm -f "$path"; ln -s "$root/other" "$path" ;; symlink-target) rm -f "$path"; ln -s "$root/link-two" "$path" ;; absent-regular) printf created > "$path"; chmod 0755 "$path" ;; esac; return 0; } - assert_fail "$noop_impl unknown noop rejects $noop_change" motd_run_locked_operation restore initial - assert_eq "$before" "$(target_hash "$root")" "$noop_impl unknown noop $noop_change rolls back group" - assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock - ) - done -done diff --git a/tests/test-motd-04.sh b/tests/test-motd-04.sh deleted file mode 100644 index 8f0116a..0000000 --- a/tests/test-motd-04.sh +++ /dev/null @@ -1,155 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-motd-common.sh" - -for group_impl in tool module; do - for group_change in motd-content issue-mode custom-inode; do - ( - root="$TEST_DIR/final-group-$group_impl-$group_change"; create_fixture "$root"; export_paths "$root"; before=$(target_hash "$root") - if [[ "$group_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_transaction_phase_hook() { local phase="$1" path replacement; [[ "$phase" == target-committed-motd_news ]] || return 0; case "$group_change" in motd-content) printf changed > "$MOTD_ETC_ROOT/motd" ;; issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; custom-inode) path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; replacement="${path}.replacement"; cp "$path" "$replacement"; chmod 0755 "$replacement"; mv -Tf "$replacement" "$path" ;; esac; } - assert_fail "$group_impl final group rejects $group_change" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$group_impl final group $group_change rolls back all targets" - [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "$group_impl final group $group_change left pending" - assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock - ) - done -done - -for illegal_legacy_name in motd.previous-unknown issue.previous-unknown motd.invalid-backup evil.initial-backup; do -( - root="$TEST_DIR/legacy-name-${illegal_legacy_name//[^a-zA-Z0-9]/-}"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install - object="$MOTD_STATE_DIR/$illegal_legacy_name"; printf invalid > "$object"; chmod 0600 "$object"; before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - assert_fail "illegal legacy name $illegal_legacy_name rejected" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "illegal legacy name $illegal_legacy_name preserves targets" - assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "illegal legacy name $illegal_legacy_name preserves pointers" - [[ -e "$object" || -L "$object" ]] || fail "illegal legacy name $illegal_legacy_name evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "illegal legacy name $illegal_legacy_name created pending"; motd_take_lock; motd_release_lock -) -done - -for legacy_backup_kind in directory fifo socket writable wrong-owner wrong-gid device; do -( - root="$TEST_DIR/legacy-backup-$legacy_backup_kind"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install; object="$MOTD_STATE_DIR/motd.initial-backup" - case "$legacy_backup_kind" in - directory) mkdir "$object" ;; fifo) mkfifo "$object" ;; - socket) if command -v python3 >/dev/null 2>&1; then python3 - "$object" <<'PY' -import socket,sys -s=socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.close() -PY - else pass "socket legacy test skipped without python3"; exit 0; fi ;; - writable) printf bad > "$object"; chmod 0666 "$object" ;; - wrong-owner) if (( EUID == 0 )); then printf bad > "$object"; chmod 0600 "$object"; chown "$((MOTD_TRUSTED_UID+1)):$MOTD_TRUSTED_GID" "$object"; else pass "wrong-owner legacy test skipped without privilege"; exit 0; fi ;; - wrong-gid) if (( EUID == 0 )); then printf bad > "$object"; chmod 0600 "$object"; chown "$MOTD_TRUSTED_UID:$((MOTD_TRUSTED_GID+1))" "$object"; else pass "wrong-gid legacy test skipped without privilege"; exit 0; fi ;; - device) if (( EUID == 0 )) && mknod "$object" c 1 3 2>/dev/null; then :; else pass "device legacy test skipped without privilege"; exit 0; fi ;; - esac - before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - assert_fail "legacy backup $legacy_backup_kind rejected" motd_run_locked_operation restore initial - assert_eq "$before" "$(target_hash "$root")" "legacy backup $legacy_backup_kind preserves targets" - assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "legacy backup $legacy_backup_kind preserves pointers" - [[ -e "$object" || -L "$object" ]] || fail "legacy backup $legacy_backup_kind evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "legacy backup $legacy_backup_kind created pending"; motd_take_lock; motd_release_lock -) -done - -for legacy_marker_name in motd.initial-absent issue.initial-unknown motd.previous-absent; do - for legacy_marker_kind in symlink directory fifo writable; do - ( - root="$TEST_DIR/legacy-marker-${legacy_marker_name//[^a-zA-Z0-9]/-}-$legacy_marker_kind"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install; object="$MOTD_STATE_DIR/$legacy_marker_name" - case "$legacy_marker_kind" in symlink) ln -s "$root/missing" "$object" ;; directory) mkdir "$object" ;; fifo) mkfifo "$object" ;; writable) printf bad > "$object"; chmod 0666 "$object" ;; esac - before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - assert_fail "legacy marker $legacy_marker_name $legacy_marker_kind rejected" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "legacy marker $legacy_marker_name $legacy_marker_kind preserves targets" - assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "legacy marker $legacy_marker_name $legacy_marker_kind preserves pointers" - [[ -e "$object" || -L "$object" ]] || fail "legacy marker $legacy_marker_name $legacy_marker_kind evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "legacy marker $legacy_marker_name $legacy_marker_kind created pending"; motd_take_lock; motd_release_lock - ) - done -done - -( - root="$TEST_DIR/legal-state-root-legacy"; create_fixture "$root"; export_paths "$root"; mkdir -p "$MOTD_STATE_DIR"; chmod 0700 "$MOTD_STATE_DIR" - cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_STATE_DIR/motd.initial-backup"; printf target > "$root/symlink-target"; ln -s "$root/symlink-target" "$MOTD_STATE_DIR/issue.initial-backup" - : > "$MOTD_STATE_DIR/issue.net.initial-absent"; chmod 0600 "$MOTD_STATE_DIR/issue.net.initial-absent"; : > "$MOTD_STATE_DIR/00-custom-welcome.initial-unknown"; chmod 0600 "$MOTD_STATE_DIR/00-custom-welcome.initial-unknown" - cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_STATE_DIR/10-uname.initial-backup"; : > "$MOTD_STATE_DIR/50-motd-news.initial-absent"; chmod 0600 "$MOTD_STATE_DIR/50-motd-news.initial-absent" - touch -a -d @1000000000 "$MOTD_STATE_DIR/motd.initial-backup"; touch -m -d @1000200000 "$MOTD_STATE_DIR/motd.initial-backup"; legacy_times=$(stat -c %X:%Y "$MOTD_STATE_DIR/motd.initial-backup") - load_motd; motd_run_locked_operation restore initial - [[ -L "$MOTD_ETC_ROOT/issue" && "$(readlink "$MOTD_ETC_ROOT/issue")" == "$root/symlink-target" ]] || fail "legal state root symlink backup not imported" - assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "legal state root regular backup imported"; [[ -e "$MOTD_STATE_DIR/initial.current" ]] || fail "legal state root import did not publish initial pointer" - assert_eq "$legacy_times" "$(stat -c %X:%Y "$MOTD_STATE_DIR/motd.initial-backup")" "legacy root metadata validation/import preserves timestamps"; pass "legal state root regular, symlink, and markers import" -) - - -for rollback_impl in tool module; do - for rollback_damage in motd-content issue-mode custom-inode absent-regular absent-symlink absent-fifo symlink-target; do - ( - root="$TEST_DIR/rollback-final-$rollback_impl-$rollback_damage"; create_fixture "$root"; export_paths "$root" - if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_run_locked_operation install - case "$rollback_damage" in - motd-content) printf 'pre-run-motd\n' > "$MOTD_ETC_ROOT/motd" ;; - issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; - custom-inode) printf 'pre-run-custom\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0700 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" ;; - absent-regular|absent-symlink|absent-fifo) rm -f "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; - symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/original-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; - esac - before=$(target_hash "$root") - before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - motd_rollback_hook() { - local id="$1" path target - [[ "$id" == motd_news ]] || return 0 - case "$rollback_damage" in - motd-content) printf 'rollback-corrupt\n' > "$MOTD_ETC_ROOT/motd" ;; - issue-mode) chmod 0777 "$MOTD_ETC_ROOT/issue" ;; - custom-inode) - path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" - cp "$path" "${path}.replacement"; chmod 0700 "${path}.replacement"; mv -Tf "${path}.replacement" "$path" - ;; - absent-regular|absent-symlink|absent-fifo) - ROLLBACK_RECREATE_TARGET="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" - rm() { - local status - command rm "$@"; status=$?; unset -f rm - case "$rollback_damage" in - absent-regular) printf recreated > "$ROLLBACK_RECREATE_TARGET"; chmod 0755 "$ROLLBACK_RECREATE_TARGET" ;; - absent-symlink) ln -s "$root/recreated-link" "$ROLLBACK_RECREATE_TARGET" ;; - absent-fifo) mkfifo "$ROLLBACK_RECREATE_TARGET" ;; - esac - return "$status" - } - ;; - symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/changed-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; - esac - return 0 - } - assert_fail "$rollback_impl rollback final group rejects $rollback_damage" motd_run_locked_operation install - [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$rollback_impl rollback $rollback_damage lost pending" - transaction=$(cat "$MOTD_STATE_DIR/pending") - journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" - grep -Fxq 'phase=active' "$journal" || fail "$rollback_impl rollback $rollback_damage journal not active" - [[ -d "$MOTD_TRANSACTION_PARENT/$transaction/rollback" ]] || fail "$rollback_impl rollback $rollback_damage lost rollback snapshot" - pass "$rollback_impl rollback $rollback_damage preserves active recovery evidence" - motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } - motd_take_lock; motd_reconcile_state; motd_release_lock - assert_eq "$before" "$(target_hash "$root")" "$rollback_impl rollback $rollback_damage retry restores targets" - assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$rollback_impl rollback $rollback_damage retry restores pointers" - [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl rollback $rollback_damage retry left journal" - assert_no_motd_residue "$root"; assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$rollback_impl rollback $rollback_damage retry keeps bounded generations" - motd_take_lock; motd_release_lock - ) - done - -done - -for rollback_impl in tool module; do -( - root="$TEST_DIR/rollback-final-success-$rollback_impl"; create_fixture "$root"; export_paths "$root" - if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi - motd_run_locked_operation install; printf 'normal-pre-run\n' > "$MOTD_ETC_ROOT/motd"; before=$(target_hash "$root") - motd_target_commit_hook() { [[ "$2" != issue_net ]]; } - assert_fail "$rollback_impl normal rollback still returns original operation failure" motd_run_locked_operation install - assert_eq "$before" "$(target_hash "$root")" "$rollback_impl normal rollback final group succeeds" - [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl normal rollback left journal" - assert_no_motd_residue "$root" -) -done - -printf 'All MOTD rollback final validation tests passed.\n' diff --git a/tests/test-motd.sh b/tests/test-motd.sh new file mode 100644 index 0000000..a51a5e7 --- /dev/null +++ b/tests/test-motd.sh @@ -0,0 +1,1337 @@ +#!/usr/bin/env bash +set -euo pipefail +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."&&pwd); +readonly ROOT_DIR +TEST_DIR=$(mktemp -d); +readonly TEST_DIR +fail(){ echo "FAIL: $*" >&2; +exit 1; +}; +pass(){ echo "PASS: $*"; +}; +assert_eq(){ [[ "$1" == "$2" ]]||fail "$3: $1 != $2"; +pass "$3"; +}; +assert_fail(){ local l="$1"; +shift; +if "$@"; +then fail "$l"; +fi; +pass "$l"; +} +extract_payload(){ awk '/install -m 0755 \/dev\/stdin .*<<.SCRIPT./{c=1;next}c&&$0=="SCRIPT"{exit}c{print}' "$1"; +} +load_motd() { + . "$ROOT_DIR/tools/setup-motd.sh" +} + +create_fixture(){ local r="$1" e + e="$r/e"; +e+="tc"; +mkdir -m700 "$r" "$e" "$e/update-motd.d" "$r/lock"; +printf original>"$e/motd"; +chmod 640 "$e/motd"; +printf issue>"$e/issue"; +chmod 644 "$e/issue"; +printf old>"$e/update-motd.d/00-custom-welcome"; +chmod 700 "$e/update-motd.d/00-custom-welcome"; +printf uname>"$e/update-motd.d/10-uname"; +chmod 754 "$e/update-motd.d/10-uname"; +} +export_paths(){ local r="$1" e + e="$r/e"; +e+="tc"; +export MOTD_TEST_MODE=1 MOTD_ETC_ROOT="$e" MOTD_STATE_DIR="$r/state" MOTD_LOCK_FILE="$r/lock/motd.lock" MOTD_TRANSACTION_PARENT="$r/state/transactions"; +} +target_hash(){ local r="$1" e p + e="$r/e"; +e+="tc"; +for p in "$e/motd" "$e/issue" "$e/issue.net" "$e/update-motd.d/00-custom-welcome" "$e/update-motd.d/10-uname" "$e/update-motd.d/50-motd-news"; +do if [[ -L "$p" ]]; +then echo "L|$p|$(readlink "$p")"; +elif [[ -f "$p" ]]; +then echo "F|$p|$(stat -c %u:%g:%a "$p")|$(sha256sum "$p"|awk '{print $1}')"; +elif [[ ! -e "$p" ]]; +then echo "A|$p"; +else echo "O|$p|$(stat -c %F "$p")"; +fi; +done; +} +extract_payload "$ROOT_DIR/tools/setup-motd.sh">"$TEST_DIR/t"; +extract_payload "$ROOT_DIR/modules/system-customize.sh">"$TEST_DIR/m"; +cmp -s "$TEST_DIR/t" "$TEST_DIR/m"||fail payload; +# --- CI segment filter (tools/segmentize.py) --- +# TEST_SEGMENT unset: every case runs, exactly like the unsplit suite. +# TEST_SEGMENT=k: the k-th process of TEST_SEGMENTS runs its share only. +__seg_plan=(1,3,3,3,3,1,2,2,2,1,2,2,3,1,1,3,1,2,3,3,1,1,1,3,3,3,1,2,3,1,2,0) +__seg_n=0 +seg_skip() { + local slot=${__seg_plan[__seg_n]:-0} + __seg_n=$((__seg_n + 1)) + [[ -z ${TEST_SEGMENT:-} ]] && return 1 + (( slot != TEST_SEGMENT )) +} +# --- end CI segment filter --- + +if seg_skip; then :; else +pass "MOTD payloads stay identical"; +fi +if seg_skip; then :; else +assert_eq cf2a4f8dfe2fe26bff5509cdaee56d9e0816fafd0594251b02e892201af5c4d5 "$(sha256sum "$TEST_DIR/t"|awk '{print $1}')" "payload hash" +fi +if seg_skip; then :; else +( + r="$TEST_DIR/source"; +mkdir -m700 "$r" "$r/lock"; +export_paths "$r"; +b=$(trap -p EXIT HUP INT TERM ERR); +load_motd; +[[ "$b" == "$(trap -p EXIT HUP INT TERM ERR)"&&! -e "$MOTD_STATE_DIR"&&! -e "$MOTD_LOCK_FILE" ]]||fail source +); +fi +if seg_skip; then :; else +pass "source zero side effects" +fi +if seg_skip; then :; else +( + r="$TEST_DIR/basic"; +create_fixture "$r"; +export_paths "$r"; +before=$(target_hash "$r"); +load_motd; +motd_run_locked_operation install; +[[ -f "$MOTD_ETC_ROOT/motd"&&! -s "$MOTD_ETC_ROOT/motd"&&-x "$MOTD_SCRIPT" ]]||fail install; +motd_run_locked_operation restore previous; +assert_eq "$before" "$(target_hash "$r")" "restore previous group"; +motd_run_locked_operation install; +printf changed>"$MOTD_ETC_ROOT/motd"; +motd_run_locked_operation restore initial; +assert_eq "$before" "$(target_hash "$r")" "restore initial group" +) +fi +if seg_skip; then :; else +( + r="$TEST_DIR/types"; +create_fixture "$r"; +export_paths "$r"; +unlink "$MOTD_ETC_ROOT/issue"; +ln -s "$r/link" "$MOTD_ETC_ROOT/issue"; +before=$(target_hash "$r"); +load_motd; +motd_run_locked_operation install; +g=$(motd_read_pointer previous); +motd_validate_snapshot "$MOTD_STATE_DIR/generations/$g" previous "$g"; +assert_eq symlink "${MOTD_SNAPSHOT_STATE[issue]}" "symlink snapshot"; +assert_eq absent "${MOTD_SNAPSHOT_STATE[issue_net]}" "absent snapshot"; +motd_run_locked_operation restore previous; +assert_eq "$before" "$(target_hash "$r")" "symlink restore" +) +fi +if seg_skip; then :; else +( + r="$TEST_DIR/initial"; +create_fixture "$r"; +export_paths "$r"; +before=$(target_hash "$r"); +load_motd; +motd_snapshot_capture_hook(){ [[ "$1" != issue_net ]]; +}; +assert_fail "initial capture atomic" motd_run_locked_operation install; +[[ ! -e "$MOTD_STATE_DIR/initial.current" ]]||fail partial; +assert_eq "$before" "$(target_hash "$r")" "initial failure no target change" +) +fi +if seg_skip; then :; else +( + r="$TEST_DIR/previous"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +old=$(motd_read_pointer previous); +for id in "${MOTD_TARGET_IDS[@]}"; +do motd_snapshot_capture_hook(){ [[ "$1" != "$id" ]]; +}; +assert_fail "previous capture $id" motd_run_locked_operation install; +assert_eq "$old" "$(motd_read_pointer previous)" "previous pointer $id"; +done +) +fi +if seg_skip; then :; else +for failing_id in motd issue_net motd_news; +do +( + r="$TEST_DIR/i-$failing_id"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +printf current>"$MOTD_ETC_ROOT/motd"; +before=$(target_hash "$r"); +motd_target_commit_hook(){ [[ "$2" != "$failing_id" ]]; +}; +assert_fail "install failure $failing_id" motd_run_locked_operation install; +assert_eq "$before" "$(target_hash "$r")" "install rollback $failing_id" +) +( + r="$TEST_DIR/r-$failing_id"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +printf current>"$MOTD_ETC_ROOT/motd"; +before=$(target_hash "$r"); +motd_target_commit_hook(){ [[ "$1" != restore||"$2" != "$failing_id" ]]; +}; +assert_fail "restore failure $failing_id" motd_run_locked_operation restore previous; +assert_eq "$before" "$(target_hash "$r")" "restore rollback $failing_id" +) +done +fi +if seg_skip; then :; else +( + r="$TEST_DIR/manifest"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +g=$(motd_read_pointer previous); +mf="$MOTD_STATE_DIR/generations/$g/manifest"; +cp "$mf" "$r/o"; +for k in missing duplicate unknown digest generation; +do cp "$r/o" "$mf"; +case $k in missing)sed -i '$d' "$mf";; +duplicate)tail -n1 "$mf">>"$mf";; +unknown)echo x=y>>"$mf";; +digest)sed -i 's/[0-9a-f]\{64\}$/0000000000000000000000000000000000000000000000000000000000000000/' "$mf";; +generation)sed -i 's/^generation=.*/generation=00000000000000000000000000000000/' "$mf";; +esac; +before=$(target_hash "$r"); +assert_fail "manifest $k rejected" motd_run_locked_operation restore previous; +assert_eq "$before" "$(target_hash "$r")" "manifest $k no target change"; +done +) +fi +if seg_skip; then :; else +( + r="$TEST_DIR/lock"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_lock_acquired_hook(){ : >"$r/ready"; +sleep 2; +}; +motd_take_lock & h=$!; +while [[ ! -e "$r/ready" ]]; +do sleep .02; +done; +assert_fail "install lock contention" motd_run_locked_operation install; +assert_fail "restore lock contention" motd_run_locked_operation restore previous; +[[ ! -e "$MOTD_STATE_DIR" ]]||fail lockstate; +wait "$h"; +motd_lock_acquired_hook(){ :; +}; +motd_take_lock; +motd_release_lock +); +fi +if seg_skip; then :; else +pass "shared lock nonblocking and reacquirable" +fi +if seg_skip; then :; else +for k in symlink writable fifo; +do( + r="$TEST_DIR/l-$k"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +case $k in symlink)printf x>"$r/x"; +ln -s "$r/x" "$MOTD_LOCK_FILE";; +writable)printf x>"$MOTD_LOCK_FILE"; +chmod 666 "$MOTD_LOCK_FILE";; +fifo)mkfifo "$MOTD_LOCK_FILE";; +esac; +assert_fail "lock rejects $k" motd_take_lock +); +done +fi +if seg_skip; then :; else +for k in symlink writable; +do( + r="$TEST_DIR/s-$k"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +case $k in symlink)mkdir -m700 "$r/s"; +ln -s "$r/s" "$MOTD_STATE_DIR";; +writable)mkdir -m770 "$MOTD_STATE_DIR";; +esac; +motd_take_lock; +assert_fail "state rejects $k" motd_prepare_state_layout; +motd_release_lock +); +done +fi +create_legacy(){ printf legacy>"$MOTD_ETC_ROOT/motd.initial-backup"; +: >"$MOTD_ETC_ROOT/issue.initial-absent"; +printf net>"$MOTD_ETC_ROOT/issue.net.initial-backup"; +: >"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.initial-unknown"; +printf uname>"$MOTD_ETC_ROOT/update-motd.d/10-uname.initial-backup"; +: >"$MOTD_ETC_ROOT/update-motd.d/50-motd-news.initial-absent"; +} +if seg_skip; then :; else +( + r="$TEST_DIR/legacy"; +create_fixture "$r"; +export_paths "$r"; +create_legacy; +load_motd; +motd_run_locked_operation restore initial; +assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "legacy complete restore"; +[[ -e "$MOTD_ETC_ROOT/motd.initial-backup" ]]||fail legacymove +) +fi +if seg_skip; then :; else +for k in partial conflict commit; +do( + r="$TEST_DIR/lg-$k"; +create_fixture "$r"; +export_paths "$r"; +if [[ $k == partial ]]; +then printf x>"$MOTD_ETC_ROOT/motd.initial-backup"; +else create_legacy; +[[ $k == conflict ]]&&: >"$MOTD_ETC_ROOT/motd.initial-absent"; +fi; +load_motd; +[[ $k == commit ]]&&motd_snapshot_commit_hook(){ [[ "$3" != legacy-directory ]]; +}; +assert_fail "legacy $k rejected" motd_run_locked_operation restore initial +); +done +fi +if seg_skip; then :; else +( + r="$TEST_DIR/interop"; +create_fixture "$r"; +export_paths "$r"; +o=$(target_hash "$r"); +bash -c '. "$1/modules/system-customize.sh";ask_yes_no(){ return 0;};motd_preview_hook(){ :;};configure_motd' _ "$ROOT_DIR">/dev/null; +printf x>"$MOTD_ETC_ROOT/motd"; +bash -c '. "$1/tools/setup-motd.sh";motd_run_locked_operation restore previous' _ "$ROOT_DIR"; +assert_eq "$o" "$(target_hash "$r")" "module state tool restore" +) +fi +if seg_skip; then :; else +( + r="$TEST_DIR/exit"; +create_fixture "$r"; +export_paths "$r"; +before=$(target_hash "$r"); +n=0; +bash -c '. "$1";motd_take_lock;motd_install_transaction_traps;motd_prepare_state_layout;motd_validate_target_parents;motd_begin_transaction install;motd_prepare_install_plan;motd_commit_target_plan install false;exit 0' _ "$ROOT_DIR/tools/setup-motd.sh">/dev/null 2>&1||n=$?; +assert_eq 1 "$n" "active exit zero maps one"; +assert_eq "$before" "$(target_hash "$r")" "active exit rollback" +) +fi +if seg_skip; then :; else +( + r="$TEST_DIR/rbf"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +motd_target_commit_hook(){ [[ "$2" != issue_net ]]; +}; +motd_rollback_hook(){ echo "$1">>"$r/a"; +[[ "$1" != motd&&"$1" != custom_welcome ]]; +}; +assert_fail "rollback failure nonzero" motd_run_locked_operation install; +assert_eq 6 "$(wc -l<"$r/a")" "rollback attempts six"; +[[ -e "$MOTD_STATE_DIR/pending" ]]||fail journal +) +fi +cat >"$TEST_DIR/sig.sh"<<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1"; +hit=false +send(){ [[ $hit == false ]]||return; +hit=true; +kill -"$SIGNAL_NAME" "$BASHPID"; +} +motd_snapshot_capture_hook(){ [[ $PHASE == snapshot-building&&$1 == issue ]]&&send; +return 0; +} +motd_transaction_phase_hook(){ [[ $PHASE == snapshot-commit&&$1 == snapshot-commit ]]&&send; +return 0; +} +motd_snapshot_commit_hook(){ [[ $PHASE == snapshot-publish&&$1 == initial&&$3 == pointer ]]&&send; +return 0; +} +motd_target_commit_hook(){ case "$PHASE:$1:$2" in install-first:install:motd|install-middle:install:issue_net|restore-first:restore:motd|restore-middle:restore:issue_net)send;; +esac; +return 0; +} +case $PHASE in restore-*)motd_run_locked_operation restore previous;; +*)motd_run_locked_operation install;; +esac +CHILD +chmod 700 "$TEST_DIR/sig.sh" +for ph in snapshot-building snapshot-commit snapshot-publish install-first install-middle restore-first restore-middle; +do for sig in HUP INT TERM; +do( + r="$TEST_DIR/$ph-$sig"; +create_fixture "$r"; +export_paths "$r"; +if [[ $ph == restore-* ]]; +then bash -c '. "$1";motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh"; +printf current>"$MOTD_ETC_ROOT/motd"; +fi; +before=$(target_hash "$r"); +n=0; +env --default-signal=HUP,INT,TERM PHASE="$ph" SIGNAL_NAME="$sig" bash "$TEST_DIR/sig.sh" "$ROOT_DIR/tools/setup-motd.sh">/dev/null 2>&1||n=$?; +case $sig in HUP)e=129;; +INT)e=130;; +TERM)e=143;; +esac; +assert_eq "$e" "$n" "$sig $ph status"; +assert_eq "$before" "$(target_hash "$r")" "$sig $ph rollback" +); +done; +done +if seg_skip; then :; else +( + r="$TEST_DIR/unknown"; +create_fixture "$r"; +export_paths "$r"; +printf '# linux-setup:managed-motd'>"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; +load_motd; +motd_run_locked_operation install; +g=$(motd_read_pointer initial); +motd_validate_snapshot "$MOTD_STATE_DIR/generations/$g" initial "$g"; +assert_eq unknown "${MOTD_SNAPSHOT_STATE[motd]}" "unknown initial state" +) +fi +if seg_skip; then :; else +( + r="$TEST_DIR/readonly"; +create_fixture "$r"; +export_paths "$r"; +bash "$ROOT_DIR/tools/setup-motd.sh" status>/dev/null; +bash "$ROOT_DIR/tools/setup-motd.sh" help>/dev/null; +[[ ! -e "$MOTD_STATE_DIR"&&! -e "$MOTD_LOCK_FILE" ]]||fail readonly +); +fi +if seg_skip; then :; else +pass "status help no lock" +fi +if seg_skip; then :; else +( + r="$TEST_DIR/publish"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_run_locked_operation install; +old=$(motd_read_pointer previous); +printf current>"$MOTD_ETC_ROOT/motd"; +before=$(target_hash "$r"); +motd_snapshot_manifest_hook(){ [[ "$1" != previous ]]; +}; +assert_fail "previous manifest failure" motd_run_locked_operation install; +assert_eq "$old" "$(motd_read_pointer previous)" "manifest preserves previous"; +assert_eq "$before" "$(target_hash "$r")" "manifest failure rolls back"; +motd_snapshot_manifest_hook(){ :; +}; +motd_snapshot_commit_hook(){ [[ "$1" != previous||"$3" != directory ]]; +}; +assert_fail "previous directory commit failure" motd_run_locked_operation install; +assert_eq "$old" "$(motd_read_pointer previous)" "directory failure preserves previous" +) +fi +for k in owner gid; +do( + r="$TEST_DIR/lmeta-$k"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +printf x>"$MOTD_LOCK_FILE"; +chmod 600 "$MOTD_LOCK_FILE"; +stat(){ if [[ "${*: -1}" == "$MOTD_LOCK_FILE"&&"$*" == *"%u:%g:%a"* ]]; +then [[ $k == owner ]]&&printf '99999:%s:600\n' "$MOTD_TRUSTED_GID"||printf '%s:99999:600\n' "$MOTD_TRUSTED_UID"; +else command stat "$@"; +fi; +}; +if seg_skip; then :; else +assert_fail "lock wrong $k" motd_take_lock +fi +); +done +for k in owner gid; +do( + r="$TEST_DIR/smeta-$k"; +create_fixture "$r"; +export_paths "$r"; +mkdir -m700 "$MOTD_STATE_DIR"; +load_motd; +stat(){ if [[ "${*: -1}" == "$MOTD_STATE_DIR"&&"$*" == *"%u:%g:%a"* ]]; +then [[ $k == owner ]]&&printf '99999:%s:700\n' "$MOTD_TRUSTED_GID"||printf '%s:99999:700\n' "$MOTD_TRUSTED_UID"; +else command stat "$@"; +fi; +}; +motd_take_lock; +if seg_skip; then :; else +assert_fail "state wrong $k" motd_prepare_state_layout; +fi +motd_release_lock +); +done +if seg_skip; then :; else +( + r="$TEST_DIR/state-create"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +chown(){ [[ "${*: -1}" == "$MOTD_STATE_DIR" ]]&&return 1; +command chown "$@"; +}; +assert_fail "state create failure" motd_run_locked_operation install; +[[ ! -e "$MOTD_STATE_DIR" ]]||fail state-residue +) +fi +if seg_skip; then :; else +( + r="$TEST_DIR/close"; +create_fixture "$r"; +export_paths "$r"; +load_motd; +motd_take_lock; +motd_lock_close_hook(){ return 1; +}; +assert_fail "lock close failure" motd_release_lock +) +fi +if seg_skip; then :; else +( + r="$TEST_DIR/device"; +create_fixture "$r"; +export_paths "$r"; +if mknod "$MOTD_ETC_ROOT/issue.net" c 1 3 2>/dev/null; +then before=$(target_hash "$r"); +load_motd; +assert_fail "device rejected" motd_run_locked_operation install; +assert_eq "$before" "$(target_hash "$r")" "device no target change"; +else pass "device test skipped"; +fi +) +fi +if seg_skip; then :; else +( + r="$TEST_DIR/legacy-state"; +create_fixture "$r"; +export_paths "$r"; +mkdir -m700 "$MOTD_STATE_DIR"; +printf legacy>"$MOTD_STATE_DIR/motd.previous-backup"; +chmod 600 "$MOTD_STATE_DIR/motd.previous-backup"; +for x in issue issue.net 00-custom-welcome 10-uname 50-motd-news; +do : >"$MOTD_STATE_DIR/$x.previous-absent"; +chmod 600 "$MOTD_STATE_DIR/$x.previous-absent"; +done; +load_motd; +motd_run_locked_operation restore previous; +assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "state legacy restore" +) +fi + +state_fingerprint() { + local root="$1" path + [[ -d "$root" ]] || { printf 'absent\n'; return; } + while IFS= read -r path; do + if [[ -L "$path" ]]; then printf 'L|%s|%s\n' "${path#$root/}" "$(readlink "$path")" + elif [[ -f "$path" ]]; then printf 'F|%s|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" "$(sha256sum "$path" | awk '{print $1}')" + elif [[ -d "$path" ]]; then printf 'D|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" + fi + done < <(find "$root" -mindepth 1 -print | sort) +} + +generation_count() { + local directory="$1" + [[ -d "$directory" ]] || { printf '0\n'; return; } + find "$directory" -mindepth 1 -maxdepth 1 -type d ! -name '.*.stage' | wc -l +} + +assert_no_motd_residue() { + local root="$1" residue + residue=$(find "$root" -name '.linux-setup-motd.*' -o -name '.*.pointer.*' -o -name '.pending.*' -o -name '.*.stage' 2>/dev/null) + [[ -z "$residue" ]] || fail "unexpected MOTD residue: $residue" + if [[ -d "$root/state/transactions" ]]; then [[ -z "$(find "$root/state/transactions" -mindepth 1 -print -quit)" ]] || fail "transaction residue remains"; fi +} + +create_complete_legacy_scope() { + local scope="$1" prefix + cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_ETC_ROOT/motd.${scope}-backup" + cp -a "$MOTD_ETC_ROOT/issue" "$MOTD_ETC_ROOT/issue.${scope}-backup" + : > "$MOTD_ETC_ROOT/issue.net.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/issue.net.${scope}-absent" + cp -a "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.${scope}-backup" + cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_ETC_ROOT/update-motd.d/10-uname.${scope}-backup" + : > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent" +} + +if seg_skip; then :; else +for legacy_phase in random-id stat open create cat chown chmod sha256 manifest mv; do +( + root="$TEST_DIR/legacy-io-$legacy_phase"; create_fixture "$root"; export_paths "$root" + create_complete_legacy_scope initial + before_targets=$(target_hash "$root") + before_legacy=$(find "$MOTD_ETC_ROOT" -name '*.initial-*' -type f -printf '%p|%s|%m\n' | sort) + load_motd + motd_legacy_io_hook() { [[ "$1" != "$legacy_phase" ]]; } + assert_fail "legacy $legacy_phase failure returns nonzero" motd_run_locked_operation restore initial + assert_eq "$before_targets" "$(target_hash "$root")" "legacy $legacy_phase keeps formal targets" + assert_eq "$before_legacy" "$(find "$MOTD_ETC_ROOT" -name '*.initial-*' -type f -printf '%p|%s|%m\n' | sort)" "legacy $legacy_phase keeps legacy files" + [[ ! -e "$MOTD_STATE_DIR/initial.current" ]] || fail "legacy $legacy_phase published initial pointer" + assert_no_motd_residue "$root" +) +done +fi +if seg_skip; then :; else +pass "legacy I/O failure matrix preserves data and state" +fi + +cat > "$TEST_DIR/pointer-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1" +motd_snapshot_commit_hook() { + if [[ "$1" == previous && "$3" == pointer ]]; then kill -"$SIGNAL_NAME" "$BASHPID"; fi +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/pointer-signal-child.sh" + +if seg_skip; then :; else +for signal_name in HUP INT TERM; do +( + root="$TEST_DIR/pointer-signal-$signal_name"; create_fixture "$root"; export_paths "$root" + bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" + old_pointer=$(cat "$MOTD_STATE_DIR/previous.current") + before=$(target_hash "$root") + rc=0 + env --default-signal=HUP,INT,TERM SIGNAL_NAME="$signal_name" bash "$TEST_DIR/pointer-signal-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? + case "$signal_name" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + assert_eq "$expected" "$rc" "$signal_name during previous pointer stage returns conventional status" + assert_eq "$old_pointer" "$(cat "$MOTD_STATE_DIR/previous.current")" "$signal_name preserves previous pointer" + assert_eq "$before" "$(target_hash "$root")" "$signal_name pointer stage restores targets" + assert_no_motd_residue "$root" + bash -c '. "$1"; motd_take_lock; motd_release_lock' _ "$ROOT_DIR/tools/setup-motd.sh" +) +done + +for pending_phase in pending-create pending-write pending-chown pending-chmod pending-publish; do +( + root="$TEST_DIR/pending-failure-$pending_phase"; create_fixture "$root"; export_paths "$root" + load_motd + motd_run_locked_operation install + before=$(target_hash "$root"); before_generations=$(generation_count "$MOTD_STATE_DIR/generations") + motd_transaction_phase_hook() { [[ "$1" != "$pending_phase" ]]; } + assert_fail "pending $pending_phase failure returns nonzero" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "pending $pending_phase keeps targets" + assert_eq "$before_generations" "$(generation_count "$MOTD_STATE_DIR/generations")" "pending $pending_phase leaves no generation orphan" + [[ ! -e "$MOTD_STATE_DIR/pending" && ! -L "$MOTD_STATE_DIR/pending" ]] || fail "pending $pending_phase left pending" + assert_no_motd_residue "$root" + motd_transaction_phase_hook() { :; } + motd_run_locked_operation install +) +done + +for pending_kind in broken-symlink symlink directory fifo mode owner gid; do +( + root="$TEST_DIR/pending-kind-$pending_kind"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + pending="$MOTD_STATE_DIR/pending" + case "$pending_kind" in + broken-symlink) ln -s "$root/missing" "$pending" ;; + symlink) printf x > "$root/pending-target"; chmod 0600 "$root/pending-target"; ln -s "$root/pending-target" "$pending" ;; + directory) mkdir "$pending" ;; + fifo) mkfifo "$pending" ;; + mode) printf x > "$pending"; chmod 0666 "$pending" ;; + owner|gid) printf x > "$pending"; chmod 0600 "$pending" ;; + esac + before=$(target_hash "$root") + if [[ "$pending_kind" == owner || "$pending_kind" == gid ]]; then + stat() { + if [[ "${*: -1}" == "$pending" && "$*" == *"%u:%g:%a"* ]]; then + [[ "$pending_kind" == owner ]] && printf '99999:%s:600\n' "$MOTD_TRUSTED_GID" || printf '%s:99999:600\n' "$MOTD_TRUSTED_UID" + else command stat "$@"; fi + } + fi + assert_fail "pending $pending_kind fails closed" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "pending $pending_kind changes no target" + [[ -e "$pending" || -L "$pending" ]] || fail "pending $pending_kind evidence was removed" +) +done + +cat > "$TEST_DIR/kill-pending-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_transaction_phase_hook() { [[ "$1" == pending-active ]] && kill -KILL "$BASHPID"; return 0; } +motd_run_locked_operation install +CHILD +cat > "$TEST_DIR/kill-previous-pointer-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_snapshot_commit_hook() { [[ "$1" == previous && "$3" == pointer ]] && kill -KILL "$BASHPID"; return 0; } +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/kill-pending-child.sh" "$TEST_DIR/kill-previous-pointer-child.sh" + +for journal_case in pending-directory journal-transaction generation-scope generation-missing old-new-mix; do +( + root="$TEST_DIR/journal-$journal_case"; create_fixture "$root"; export_paths "$root" + bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" + rc=0 + if [[ "$journal_case" == pending-directory || "$journal_case" == journal-transaction ]]; then + bash "$TEST_DIR/kill-pending-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? + else + bash "$TEST_DIR/kill-previous-pointer-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? + fi + assert_eq 137 "$rc" "$journal_case fixture stops with SIGKILL" + transaction=$(cat "$MOTD_STATE_DIR/pending") + journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" + case "$journal_case" in + pending-directory) printf '%032x\n' 1 > "$MOTD_STATE_DIR/pending" ;; + journal-transaction) sed -i 's/^transaction=.*/transaction=00000000000000000000000000000001/' "$journal" ;; + generation-scope) initial=$(cat "$MOTD_STATE_DIR/initial.current"); sed -i "s/^previous_new=.*/previous_new=$initial/" "$journal" ;; + generation-missing) sed -i 's/^previous_new=.*/previous_new=ffffffffffffffffffffffffffffffff/' "$journal" ;; + old-new-mix) initial=$(cat "$MOTD_STATE_DIR/initial.current"); sed -i "s/^previous_old=.*/previous_old=$initial/" "$journal" ;; + esac + before=$(target_hash "$root") + load_motd + assert_fail "$journal_case pending/journal mismatch rejected" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$journal_case mismatch changes no target" + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$journal_case evidence removed" +) +done + +for tamper_case in content mode inode regular-to-symlink; do +( + root="$TEST_DIR/stage-$tamper_case"; create_fixture "$root"; export_paths "$root" + printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + before=$(target_hash "$root") + load_motd + motd_transaction_phase_hook() { + [[ "$1" == install-plan-ready ]] || return 0 + case "$tamper_case" in + content) printf tampered > "${MOTD_TARGET_STAGE[motd]}" ;; + mode) chmod 0600 "${MOTD_TARGET_STAGE[issue_net]}" ;; + inode) cp "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" "${MOTD_TARGET_STAGE[motd_news]}.replacement"; chmod 0644 "${MOTD_TARGET_STAGE[motd_news]}.replacement"; mv -Tf "${MOTD_TARGET_STAGE[motd_news]}.replacement" "${MOTD_TARGET_STAGE[motd_news]}" ;; + regular-to-symlink) rm -f "${MOTD_TARGET_STAGE[issue_net]}"; ln -s "$root/elsewhere" "${MOTD_TARGET_STAGE[issue_net]}" ;; + esac + } + motd_target_commit_hook() { : > "$root/commit-called"; } + assert_fail "$tamper_case stage tamper rejected before commit" motd_run_locked_operation install + [[ ! -e "$root/commit-called" ]] || fail "$tamper_case reached first formal commit" + assert_eq "$before" "$(target_hash "$root")" "$tamper_case stage tamper changes no target" + assert_no_motd_residue "$root" +) +done + +( + root="$TEST_DIR/stage-symlink-target"; create_fixture "$root"; export_paths "$root" + printf external > "$root/external"; ln -sf "$root/external" "$MOTD_ETC_ROOT/issue" + load_motd; motd_run_locked_operation install + before=$(target_hash "$root") + motd_transaction_phase_hook() { + [[ "$1" == restore-plan-ready ]] || return 0 + rm -f "${MOTD_TARGET_STAGE[issue]}"; ln -s "$root/other" "${MOTD_TARGET_STAGE[issue]}" + } + motd_target_commit_hook() { : > "$root/commit-called"; } + assert_fail "symlink target stage tamper rejected before restore commit" motd_run_locked_operation restore previous + [[ ! -e "$root/commit-called" ]] || fail "symlink target tamper reached formal commit" + assert_eq "$before" "$(target_hash "$root")" "symlink target tamper changes no target" +) + +for native_id in uname motd_news; do +( + root="$TEST_DIR/native-symlink-$native_id"; create_fixture "$root"; export_paths "$root" + case "$native_id" in uname) native_path="$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; motd_news) native_path="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; esac + rm -f "$native_path" + printf 'external-%s\n' "$native_id" > "$root/external-$native_id"; chmod 0755 "$root/external-$native_id" + external_hash=$(sha256sum "$root/external-$native_id"); external_mode=$(stat -c %a "$root/external-$native_id") + ln -s "$root/external-$native_id" "$native_path" + load_motd + motd_run_locked_operation install + [[ ! -e "$native_path" && ! -L "$native_path" ]] || fail "$native_id symlink entry remains enabled" + assert_eq "$external_hash" "$(sha256sum "$root/external-$native_id")" "$native_id external content unchanged" + assert_eq "$external_mode" "$(stat -c %a "$root/external-$native_id")" "$native_id external mode unchanged" + motd_run_locked_operation restore previous + [[ -L "$native_path" && "$(readlink "$native_path")" == "$root/external-$native_id" ]] || fail "$native_id previous symlink not restored" +) +done + +cat > "$TEST_DIR/sigkill-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_transaction_phase_hook() { + [[ "$1" == "$KILL_PHASE" ]] && kill -KILL "$BASHPID" + return 0 +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/sigkill-child.sh" + +for kill_phase in transaction-snapshot-building transaction-final-moved target-committed-motd target-committed-issue_net target-committed-motd_news committed-cleanup committed-old-generation-cleaned committed-pending-removed; do +( + root="$TEST_DIR/sigkill-$kill_phase"; create_fixture "$root"; export_paths "$root" + printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" + before=$(target_hash "$root") + rc=0 + KILL_PHASE="$kill_phase" bash "$TEST_DIR/sigkill-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? + assert_eq 137 "$rc" "SIGKILL at $kill_phase terminates process" + after_kill=$(target_hash "$root") + load_motd + motd_take_lock; motd_prepare_state_layout; motd_reconcile_state; motd_release_lock + if [[ "$kill_phase" == committed-* ]]; then + assert_eq "$after_kill" "$(target_hash "$root")" "committed cleanup SIGKILL does not roll back" + else + assert_eq "$before" "$(target_hash "$root")" "SIGKILL at $kill_phase rolls back to pre-transaction state" + fi + assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "SIGKILL at $kill_phase leaves bounded generations" + assert_no_motd_residue "$root" + motd_take_lock; motd_release_lock +) +done + +( + root="$TEST_DIR/generation-bounded"; create_fixture "$root"; export_paths "$root" + load_motd + for _ in {1..6}; do motd_run_locked_operation install; done + assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "repeated install keeps two generations" +) + +( + root="$TEST_DIR/generation-commit-failure"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + before_count=$(generation_count "$MOTD_STATE_DIR/generations") + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + assert_fail "target commit failure returns nonzero for generation cleanup" motd_run_locked_operation install + assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "target commit failure leaves no orphan generation" +) + +( + root="$TEST_DIR/interop-module-tool"; create_fixture "$root"; export_paths "$root" + original=$(target_hash "$root") + bash -c '. "$1/modules/system-customize.sh"; ask_yes_no(){ return 0; }; motd_preview_hook(){ :; }; configure_motd' _ "$ROOT_DIR" >/dev/null + printf changed > "$MOTD_ETC_ROOT/motd" + bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation restore previous' _ "$ROOT_DIR" + assert_eq "$original" "$(target_hash "$root")" "module install to tool previous restore" + printf changed > "$MOTD_ETC_ROOT/motd" + bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation restore initial' _ "$ROOT_DIR" + assert_eq "$original" "$(target_hash "$root")" "module install to tool initial restore" +) + +( + root="$TEST_DIR/interop-tool-module"; create_fixture "$root"; export_paths "$root" + original=$(target_hash "$root") + bash -c '. "$1/tools/setup-motd.sh"; motd_run_locked_operation install' _ "$ROOT_DIR" + printf between > "$MOTD_ETC_ROOT/motd"; between=$(target_hash "$root") + bash -c '. "$1/modules/system-customize.sh"; ask_yes_no(){ return 0; }; motd_preview_hook(){ :; }; configure_motd' _ "$ROOT_DIR" >/dev/null + bash -c '. "$1/modules/system-customize.sh"; motd_run_locked_operation restore previous' _ "$ROOT_DIR" + assert_eq "$between" "$(target_hash "$root")" "tool install to module previous restore" + printf changed > "$MOTD_ETC_ROOT/motd" + bash -c '. "$1/modules/system-customize.sh"; motd_run_locked_operation restore initial' _ "$ROOT_DIR" + assert_eq "$original" "$(target_hash "$root")" "tool install to module initial restore" +) + + +( + root="$TEST_DIR/cleanup-old-generation-failure"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + old_previous=$(cat "$MOTD_STATE_DIR/previous.current"); blocked="$MOTD_STATE_DIR/generations/$old_previous" + motd_cleanup_path_hook() { [[ "$1" != "$blocked" ]]; } + assert_fail "old previous generation delete failure returns nonzero" motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "old generation delete failure lost pending journal" + transaction=$(cat "$MOTD_STATE_DIR/pending"); grep -Fq 'phase=committed' "$MOTD_TRANSACTION_PARENT/$transaction/journal" || fail "old generation failure journal not committed" + [[ -d "$blocked" ]] || fail "blocked old generation unexpectedly removed" + motd_cleanup_path_hook() { :; } + motd_run_locked_operation install + [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "old generation cleanup retry left pending" +) + +( + root="$TEST_DIR/cleanup-rollback-generation-failure"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + current_initial=$(cat "$MOTD_STATE_DIR/initial.current"); current_previous=$(cat "$MOTD_STATE_DIR/previous.current") + motd_cleanup_path_hook() { + local name=${1##*/} + if [[ "$1" == "$MOTD_STATE_DIR/generations/"* && "$name" != "$current_initial" && "$name" != "$current_previous" ]]; then return 1; fi + return 0 + } + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + assert_fail "rollback generation delete failure returns nonzero" motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback generation failure lost pending" + transaction=$(cat "$MOTD_STATE_DIR/pending"); grep -Fq 'phase=rolledback' "$MOTD_TRANSACTION_PARENT/$transaction/journal" || fail "rollback generation failure journal not rolledback" + motd_cleanup_path_hook() { :; }; motd_target_commit_hook() { :; } + motd_run_locked_operation install + [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "rolledback cleanup retry left pending" + assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "rolledback cleanup retry restores bounded generations" +) + + +for hook_case in content-motd mode-issue_net inode-motd_news type-issue_net; do +( + root="$TEST_DIR/hook-target-$hook_case"; create_fixture "$root"; export_paths "$root" + printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + before=$(target_hash "$root") + load_motd + motd_target_commit_hook() { + local id="$2" stage replacement + case "$hook_case:$id" in + content-motd:motd) printf changed > "${MOTD_TARGET_STAGE[motd]}" ;; + mode-issue_net:issue_net) chmod 0600 "${MOTD_TARGET_STAGE[issue_net]}" ;; + inode-motd_news:motd_news) + stage=${MOTD_TARGET_STAGE[motd_news]}; replacement="${stage}.replacement" + cp "$stage" "$replacement"; chmod "${MOTD_STAGE_MODE[motd_news]}" "$replacement"; mv -Tf "$replacement" "$stage" + ;; + type-issue_net:issue_net) rm -f "${MOTD_TARGET_STAGE[issue_net]}"; ln -s "$root/other" "${MOTD_TARGET_STAGE[issue_net]}" ;; + esac + return 0 + } + assert_fail "$hook_case target hook tamper rejected" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$hook_case leaves formal targets unchanged" + assert_no_motd_residue "$root" +) +done + +( + root="$TEST_DIR/hook-rollback-stage"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + before=$(target_hash "$root") + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + motd_rollback_hook() { [[ "$1" != motd ]] || printf tampered > "${MOTD_TARGET_STAGE[motd]}"; return 0; } + assert_fail "rollback hook stage tamper rejected" motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback hook tamper did not preserve pending journal" + motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } + motd_run_locked_operation install + [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "rollback hook retry left pending" + assert_eq "$before" "$(target_hash "$root")" "rollback hook retry restores original group before next install" +) + +for pointer_case in old-generation other-generation extra-line inode; do +( + root="$TEST_DIR/pointer-hook-$pointer_case"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + before=$(target_hash "$root"); old_pointer=$(cat "$MOTD_STATE_DIR/previous.current"); before_count=$(generation_count "$MOTD_STATE_DIR/generations") + motd_snapshot_commit_hook() { + local scope="$1" generation="$2" kind="$3" stage other source + [[ "$scope" == previous && "$kind" == pointer ]] || return 0 + stage=${MOTD_POINTER_STAGE_PATHS[${#MOTD_POINTER_STAGE_PATHS[@]}-1]} + case "$pointer_case" in + old-generation) printf '%s\n' "$old_pointer" > "$stage" ;; + other-generation) + other=11111111111111111111111111111111; source="$MOTD_STATE_DIR/generations/$generation" + cp -a "$source" "$MOTD_STATE_DIR/generations/$other" + sed -i "s/^generation=.*/generation=$other/" "$MOTD_STATE_DIR/generations/$other/manifest" + MOTD_NEW_GENERATIONS+=("$other") + printf '%s\n' "$other" > "$stage" + ;; + extra-line) printf 'extra\n' >> "$stage" ;; + inode) cp "$stage" "${stage}.replacement"; chmod 0600 "${stage}.replacement"; mv -Tf "${stage}.replacement" "$stage" ;; + esac + } + assert_fail "pointer hook $pointer_case rejected" motd_run_locked_operation install + assert_eq "$old_pointer" "$(cat "$MOTD_STATE_DIR/previous.current")" "pointer hook $pointer_case preserves pointer" + assert_eq "$before" "$(target_hash "$root")" "pointer hook $pointer_case preserves targets" + assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "pointer hook $pointer_case preserves generation count" + assert_no_motd_residue "$root" +) +done + +for pending_hook_case in id extra-line inode; do +( + root="$TEST_DIR/pending-hook-$pending_hook_case"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + before=$(target_hash "$root") + motd_transaction_phase_hook() { + local phase="$1" stage + [[ "$phase" == pending-publish ]] || return 0 + stage=${MOTD_PENDING_STAGE_PATHS[${#MOTD_PENDING_STAGE_PATHS[@]}-1]} + case "$pending_hook_case" in + id) printf '%032x\n' 1 > "$stage" ;; + extra-line) printf 'extra\n' >> "$stage" ;; + inode) cp "$stage" "${stage}.replacement"; chmod 0600 "${stage}.replacement"; mv -Tf "${stage}.replacement" "$stage" ;; + esac + } + motd_target_commit_hook() { : > "$root/commit-called"; } + assert_fail "pending publish hook $pending_hook_case rejected" motd_run_locked_operation install + [[ ! -e "$root/commit-called" ]] || fail "pending hook $pending_hook_case reached formal target commit" + assert_eq "$before" "$(target_hash "$root")" "pending hook $pending_hook_case preserves targets" + [[ ! -e "$MOTD_STATE_DIR/pending" && ! -L "$MOTD_STATE_DIR/pending" ]] || fail "pending hook $pending_hook_case left pending" + assert_no_motd_residue "$root" +) +done + +cat > "$TEST_DIR/building-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1" +send_building_signal() { kill -"$SIGNAL_NAME" "$BASHPID"; } +motd_transaction_phase_hook() { + case "$FAIL_PHASE:$1" in + transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) send_building_signal ;; + esac + return 0 +} +motd_snapshot_capture_hook() { + [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0 + case "$FAIL_PHASE:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) send_building_signal ;; esac + return 0 +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/building-signal-child.sh" + +for building_phase in transaction-snapshot-building capture-motd capture-issue_net capture-motd_news snapshot-commit transaction-final-move; do +( + root="$TEST_DIR/building-fail-$building_phase"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + before=$(target_hash "$root"); before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")"; before_count=$(generation_count "$MOTD_STATE_DIR/generations") + motd_transaction_phase_hook() { case "$building_phase:$1" in transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) return 1 ;; esac; return 0; } + motd_snapshot_capture_hook() { [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0; case "$building_phase:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) return 1 ;; esac; return 0; } + assert_fail "$building_phase ordinary failure returns nonzero" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$building_phase ordinary failure preserves targets" + assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$building_phase ordinary failure preserves pointers" + assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "$building_phase ordinary failure preserves generations" + assert_no_motd_residue "$root" + motd_take_lock; motd_release_lock +) + for signal_name in HUP INT TERM; do + ( + root="$TEST_DIR/building-$building_phase-$signal_name"; create_fixture "$root"; export_paths "$root" + bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/tools/setup-motd.sh" + before=$(target_hash "$root"); before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")"; before_count=$(generation_count "$MOTD_STATE_DIR/generations") + rc=0 + env --default-signal=HUP,INT,TERM FAIL_PHASE="$building_phase" SIGNAL_NAME="$signal_name" bash "$TEST_DIR/building-signal-child.sh" "$ROOT_DIR/tools/setup-motd.sh" >/dev/null 2>&1 || rc=$? + case "$signal_name" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac + assert_eq "$expected" "$rc" "$building_phase $signal_name status" + assert_eq "$before" "$(target_hash "$root")" "$building_phase $signal_name preserves targets" + assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$building_phase $signal_name preserves pointers" + assert_eq "$before_count" "$(generation_count "$MOTD_STATE_DIR/generations")" "$building_phase $signal_name preserves generations" + assert_no_motd_residue "$root" + bash -c '. "$1"; motd_take_lock; motd_release_lock' _ "$ROOT_DIR/tools/setup-motd.sh" + ) + done +done + +for previous_impl in tool module; do +( + root="$TEST_DIR/previous-only-$previous_impl"; create_fixture "$root"; export_paths "$root" + create_complete_legacy_scope previous + case "$previous_impl" in tool) script="$ROOT_DIR/tools/setup-motd.sh" ;; module) script="$ROOT_DIR/modules/system-customize.sh" ;; esac + # shellcheck source=/dev/null + . "$script" + motd_run_locked_operation restore previous + [[ ! -e "$MOTD_STATE_DIR/initial.current" && ! -L "$MOTD_STATE_DIR/initial.current" ]] || fail "$previous_impl previous-only created initial pointer" + [[ -e "$MOTD_STATE_DIR/previous.current" ]] || fail "$previous_impl previous-only missing previous pointer" + motd_run_locked_operation restore previous + [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$previous_impl repeated previous restore left transaction" + motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/initial.current" && -e "$MOTD_STATE_DIR/previous.current" ]] || fail "$previous_impl install did not complete initial/previous pointers" + assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$previous_impl previous-only lifecycle bounded" +) +done + +for time_direction in tool-module module-tool; do +( + root="$TEST_DIR/timestamps-$time_direction"; create_fixture "$root"; export_paths "$root" + printf news > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + for native in 10-uname 50-motd-news; do touch -a -d @1000000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1000200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done + expected_uname=$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname"); expected_news=$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news") + case "$time_direction" in + tool-module) install_script="$ROOT_DIR/tools/setup-motd.sh"; restore_script="$ROOT_DIR/modules/system-customize.sh" ;; + module-tool) install_script="$ROOT_DIR/modules/system-customize.sh"; restore_script="$ROOT_DIR/tools/setup-motd.sh" ;; + esac + # shellcheck source=/dev/null + . "$install_script"; motd_run_locked_operation install + assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction install preserves uname timestamps" + assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction install preserves news timestamps" + for native in 10-uname 50-motd-news; do touch -a -d @1100000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1100200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done + bash -c '. "$1"; motd_run_locked_operation restore previous' _ "$restore_script" + assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction previous restores uname timestamps" + assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction previous restores news timestamps" + for native in 10-uname 50-motd-news; do touch -a -d @1200000000 "$MOTD_ETC_ROOT/update-motd.d/$native"; touch -m -d @1200200000 "$MOTD_ETC_ROOT/update-motd.d/$native"; done + bash -c '. "$1"; motd_run_locked_operation restore initial' _ "$restore_script" + assert_eq "$expected_uname" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/10-uname")" "$time_direction initial restores uname timestamps" + assert_eq "$expected_news" "$(stat -c %X:%Y "$MOTD_ETC_ROOT/update-motd.d/50-motd-news")" "$time_direction initial restores news timestamps" +) +done + +for hidden_location in transactions generations root; do + for hidden_kind in file symlink fifo dir; do + ( + root="$TEST_DIR/hidden-$hidden_location-$hidden_kind"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + case "$hidden_location" in transactions) parent="$MOTD_TRANSACTION_PARENT" ;; generations) parent="$MOTD_STATE_DIR/generations" ;; root) parent="$MOTD_STATE_DIR" ;; esac + object="$parent/.evil-$hidden_kind" + case "$hidden_kind" in file) printf evil > "$object" ;; symlink) ln -s "$root/missing" "$object" ;; fifo) mkfifo "$object" ;; dir) mkdir "$object" ;; esac + before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + if [[ "$hidden_kind" == file || "$hidden_kind" == fifo ]]; then operation=(install); else operation=(restore previous); fi + assert_fail "$hidden_location hidden $hidden_kind rejected" motd_run_locked_operation "${operation[@]}" + assert_eq "$before" "$(target_hash "$root")" "$hidden_location hidden $hidden_kind preserves targets" + assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$hidden_location hidden $hidden_kind preserves pointers" + [[ -e "$object" || -L "$object" ]] || fail "$hidden_location hidden $hidden_kind evidence removed" + motd_take_lock; motd_release_lock + ) + done +done + +( + root="$TEST_DIR/hidden-transaction-short"; create_fixture "$root"; export_paths "$root" + load_motd; motd_run_locked_operation install + printf evil > "$MOTD_TRANSACTION_PARENT/.123" + before=$(target_hash "$root") + assert_fail "transactions .123 rejected" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "transactions .123 preserves targets" + [[ -e "$MOTD_TRANSACTION_PARENT/.123" ]] || fail "transactions .123 evidence removed" + motd_take_lock; motd_release_lock +) + +( + transaction_block="$TEST_DIR/transaction-block" + sed -n '/# === MOTD group transaction ===/,/^configure_motd()/p' "$ROOT_DIR/tools/setup-motd.sh" > "$transaction_block" + if grep -Eq "stat .*%F|regular file|regular empty file|symbolic link" "$transaction_block"; then fail "MOTD transaction depends on localized stat type text"; fi + pass "MOTD transaction avoids localized stat type text" +) + +( + root="$TEST_DIR/locale-types"; create_fixture "$root"; export_paths "$root" + printf outside > "$root/outside"; rm -f "$MOTD_ETC_ROOT/issue"; ln -s "$root/outside" "$MOTD_ETC_ROOT/issue" + create_complete_legacy_scope initial + load_motd + stat() { + if [[ "$*" == *"%F"* ]]; then printf '本地化文件类型\n'; else command stat "$@"; fi + } + LANG=zh_CN.UTF-8 motd_run_locked_operation restore initial + [[ -L "$MOTD_ETC_ROOT/issue" ]] || fail "localized stat wrapper broke symlink restore" + assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "localized stat wrapper preserves regular legacy import" + LANG=zh_CN.UTF-8 bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/modules/system-customize.sh" +) + +for missing_command in bash chown flock ln od readlink sha256sum touch wc; do +( + root="$TEST_DIR/module-missing-$missing_command"; create_fixture "$root"; export_paths "$root" + before=$(target_hash "$root"); rc=0; mkdir "$root/bin" + for available_command in apt-get apt-cache awk bash basename cat chmod chown cp curl df dirname dpkg flock grep hostname id install ln mkdir mktemp mv od readlink rm sed sha256sum sleep sort stat touch tr uname uptime wc; do + [[ "$available_command" == "$missing_command" ]] && continue + available_path=$(command -v "$available_command" || true) + if [[ -n "$available_path" ]]; then ln -s "$available_path" "$root/bin/$available_command" + else printf '#!/bin/sh\nexit 0\n' > "$root/bin/$available_command"; chmod 0755 "$root/bin/$available_command"; fi + done + PATH="$root/bin" /bin/bash -c ' + . "$1" + require_root() { :; } + main motd + ' _ "$ROOT_DIR/modules/system-customize.sh" > "$root/output" 2>&1 || rc=$? + assert_eq 1 "$rc" "module missing $missing_command fails before action" + grep -Fq "缺少必要命令: $missing_command" "$root/output" || fail "module missing $missing_command not reported" + pass "module preflight reports missing $missing_command" + assert_eq "$before" "$(target_hash "$root")" "module missing $missing_command preserves MOTD" + [[ ! -e "$MOTD_STATE_DIR" && ! -e "$MOTD_LOCK_FILE" ]] || fail "module missing $missing_command created lock/state" +) +done + +grep -Fq 'error "缺少必要命令: $required_command"' "$ROOT_DIR/modules/system-customize.sh" || fail "module missing-command error output branch absent" +pass "module dependency preflight retains missing-command output" + + +for noop_impl in tool module; do + for noop_created_type in regular symlink fifo; do + ( + root="$TEST_DIR/noop-absent-$noop_impl-$noop_created_type"; create_fixture "$root"; export_paths "$root" + path="$MOTD_ETC_ROOT/update-motd.d/50-motd-news"; before=$(target_hash "$root") + if [[ "$noop_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_target_commit_hook() { if [[ "$2" == motd_news ]]; then case "$noop_created_type" in regular) printf created > "$3"; chmod 0755 "$3" ;; symlink) ln -s "$root/external" "$3" ;; fifo) mkfifo "$3" ;; esac; fi; return 0; } + assert_fail "$noop_impl absent noop rejects $noop_created_type creation" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$noop_impl absent noop $noop_created_type rolls back group" + [[ ! -e "$path" && ! -L "$path" ]] || fail "$noop_impl absent noop $noop_created_type left created object" + assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock + ) + done +done + +for noop_impl in tool module; do + for noop_change in regular-content regular-mode regular-inode regular-symlink symlink-target absent-regular; do + ( + root="$TEST_DIR/noop-unknown-$noop_impl-$noop_change"; create_fixture "$root"; export_paths "$root" + printf '# linux-setup:managed-motd\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0755 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" + if [[ "$noop_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_run_locked_operation install + case "$noop_change" in regular-content|regular-symlink) target_id=motd ;; regular-mode) target_id=issue ;; regular-inode) target_id=custom_welcome ;; symlink-target) target_id="uname"; rm -f "${MOTD_PATH_BY_ID[$target_id]}"; ln -s "$root/link-one" "${MOTD_PATH_BY_ID[$target_id]}" ;; absent-regular) target_id=motd_news ;; esac + before=$(target_hash "$root") + motd_target_commit_hook() { local id="$2" path="$3" replacement; [[ "$id" == "$target_id" ]] || return 0; case "$noop_change" in regular-content) printf changed > "$path" ;; regular-mode) chmod 0600 "$path" ;; regular-inode) replacement="${path}.replacement"; cp "$path" "$replacement"; chmod 0755 "$replacement"; mv -Tf "$replacement" "$path" ;; regular-symlink) rm -f "$path"; ln -s "$root/other" "$path" ;; symlink-target) rm -f "$path"; ln -s "$root/link-two" "$path" ;; absent-regular) printf created > "$path"; chmod 0755 "$path" ;; esac; return 0; } + assert_fail "$noop_impl unknown noop rejects $noop_change" motd_run_locked_operation restore initial + assert_eq "$before" "$(target_hash "$root")" "$noop_impl unknown noop $noop_change rolls back group" + assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock + ) + done +done + +for group_impl in tool module; do + for group_change in motd-content issue-mode custom-inode; do + ( + root="$TEST_DIR/final-group-$group_impl-$group_change"; create_fixture "$root"; export_paths "$root"; before=$(target_hash "$root") + if [[ "$group_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_transaction_phase_hook() { local phase="$1" path replacement; [[ "$phase" == target-committed-motd_news ]] || return 0; case "$group_change" in motd-content) printf changed > "$MOTD_ETC_ROOT/motd" ;; issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; custom-inode) path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; replacement="${path}.replacement"; cp "$path" "$replacement"; chmod 0755 "$replacement"; mv -Tf "$replacement" "$path" ;; esac; } + assert_fail "$group_impl final group rejects $group_change" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$group_impl final group $group_change rolls back all targets" + [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "$group_impl final group $group_change left pending" + assert_no_motd_residue "$root"; motd_take_lock; motd_release_lock + ) + done +done + +for illegal_legacy_name in motd.previous-unknown issue.previous-unknown motd.invalid-backup evil.initial-backup; do +( + root="$TEST_DIR/legacy-name-${illegal_legacy_name//[^a-zA-Z0-9]/-}"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install + object="$MOTD_STATE_DIR/$illegal_legacy_name"; printf invalid > "$object"; chmod 0600 "$object"; before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + assert_fail "illegal legacy name $illegal_legacy_name rejected" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "illegal legacy name $illegal_legacy_name preserves targets" + assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "illegal legacy name $illegal_legacy_name preserves pointers" + [[ -e "$object" || -L "$object" ]] || fail "illegal legacy name $illegal_legacy_name evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "illegal legacy name $illegal_legacy_name created pending"; motd_take_lock; motd_release_lock +) +done + +for legacy_backup_kind in directory fifo socket writable wrong-owner wrong-gid device; do +( + root="$TEST_DIR/legacy-backup-$legacy_backup_kind"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install; object="$MOTD_STATE_DIR/motd.initial-backup" + case "$legacy_backup_kind" in + directory) mkdir "$object" ;; fifo) mkfifo "$object" ;; + socket) if command -v python3 >/dev/null 2>&1; then python3 - "$object" <<'PY' +import socket,sys +s=socket.socket(socket.AF_UNIX); s.bind(sys.argv[1]); s.close() +PY + else pass "socket legacy test skipped without python3"; exit 0; fi ;; + writable) printf bad > "$object"; chmod 0666 "$object" ;; + wrong-owner) if (( EUID == 0 )); then printf bad > "$object"; chmod 0600 "$object"; chown "$((MOTD_TRUSTED_UID+1)):$MOTD_TRUSTED_GID" "$object"; else pass "wrong-owner legacy test skipped without privilege"; exit 0; fi ;; + wrong-gid) if (( EUID == 0 )); then printf bad > "$object"; chmod 0600 "$object"; chown "$MOTD_TRUSTED_UID:$((MOTD_TRUSTED_GID+1))" "$object"; else pass "wrong-gid legacy test skipped without privilege"; exit 0; fi ;; + device) if (( EUID == 0 )) && mknod "$object" c 1 3 2>/dev/null; then :; else pass "device legacy test skipped without privilege"; exit 0; fi ;; + esac + before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + assert_fail "legacy backup $legacy_backup_kind rejected" motd_run_locked_operation restore initial + assert_eq "$before" "$(target_hash "$root")" "legacy backup $legacy_backup_kind preserves targets" + assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "legacy backup $legacy_backup_kind preserves pointers" + [[ -e "$object" || -L "$object" ]] || fail "legacy backup $legacy_backup_kind evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "legacy backup $legacy_backup_kind created pending"; motd_take_lock; motd_release_lock +) +done + +for legacy_marker_name in motd.initial-absent issue.initial-unknown motd.previous-absent; do + for legacy_marker_kind in symlink directory fifo writable; do + ( + root="$TEST_DIR/legacy-marker-${legacy_marker_name//[^a-zA-Z0-9]/-}-$legacy_marker_kind"; create_fixture "$root"; export_paths "$root"; load_motd; motd_run_locked_operation install; object="$MOTD_STATE_DIR/$legacy_marker_name" + case "$legacy_marker_kind" in symlink) ln -s "$root/missing" "$object" ;; directory) mkdir "$object" ;; fifo) mkfifo "$object" ;; writable) printf bad > "$object"; chmod 0666 "$object" ;; esac + before=$(target_hash "$root"); pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + assert_fail "legacy marker $legacy_marker_name $legacy_marker_kind rejected" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "legacy marker $legacy_marker_name $legacy_marker_kind preserves targets" + assert_eq "$pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "legacy marker $legacy_marker_name $legacy_marker_kind preserves pointers" + [[ -e "$object" || -L "$object" ]] || fail "legacy marker $legacy_marker_name $legacy_marker_kind evidence removed"; [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "legacy marker $legacy_marker_name $legacy_marker_kind created pending"; motd_take_lock; motd_release_lock + ) + done +done + +( + root="$TEST_DIR/legal-state-root-legacy"; create_fixture "$root"; export_paths "$root"; mkdir -p "$MOTD_STATE_DIR"; chmod 0700 "$MOTD_STATE_DIR" + cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_STATE_DIR/motd.initial-backup"; printf target > "$root/symlink-target"; ln -s "$root/symlink-target" "$MOTD_STATE_DIR/issue.initial-backup" + : > "$MOTD_STATE_DIR/issue.net.initial-absent"; chmod 0600 "$MOTD_STATE_DIR/issue.net.initial-absent"; : > "$MOTD_STATE_DIR/00-custom-welcome.initial-unknown"; chmod 0600 "$MOTD_STATE_DIR/00-custom-welcome.initial-unknown" + cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_STATE_DIR/10-uname.initial-backup"; : > "$MOTD_STATE_DIR/50-motd-news.initial-absent"; chmod 0600 "$MOTD_STATE_DIR/50-motd-news.initial-absent" + touch -a -d @1000000000 "$MOTD_STATE_DIR/motd.initial-backup"; touch -m -d @1000200000 "$MOTD_STATE_DIR/motd.initial-backup"; legacy_times=$(stat -c %X:%Y "$MOTD_STATE_DIR/motd.initial-backup") + load_motd; motd_run_locked_operation restore initial + [[ -L "$MOTD_ETC_ROOT/issue" && "$(readlink "$MOTD_ETC_ROOT/issue")" == "$root/symlink-target" ]] || fail "legal state root symlink backup not imported" + assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "legal state root regular backup imported"; [[ -e "$MOTD_STATE_DIR/initial.current" ]] || fail "legal state root import did not publish initial pointer" + assert_eq "$legacy_times" "$(stat -c %X:%Y "$MOTD_STATE_DIR/motd.initial-backup")" "legacy root metadata validation/import preserves timestamps"; pass "legal state root regular, symlink, and markers import" +) + + +for rollback_impl in tool module; do + for rollback_damage in motd-content issue-mode custom-inode absent-regular absent-symlink absent-fifo symlink-target; do + ( + root="$TEST_DIR/rollback-final-$rollback_impl-$rollback_damage"; create_fixture "$root"; export_paths "$root" + if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_run_locked_operation install + case "$rollback_damage" in + motd-content) printf 'pre-run-motd\n' > "$MOTD_ETC_ROOT/motd" ;; + issue-mode) chmod 0600 "$MOTD_ETC_ROOT/issue" ;; + custom-inode) printf 'pre-run-custom\n' > "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome"; chmod 0700 "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" ;; + absent-regular|absent-symlink|absent-fifo) rm -f "$MOTD_ETC_ROOT/update-motd.d/50-motd-news" ;; + symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/original-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; + esac + before=$(target_hash "$root") + before_pointers="$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + motd_rollback_hook() { + local id="$1" path target + [[ "$id" == motd_news ]] || return 0 + case "$rollback_damage" in + motd-content) printf 'rollback-corrupt\n' > "$MOTD_ETC_ROOT/motd" ;; + issue-mode) chmod 0777 "$MOTD_ETC_ROOT/issue" ;; + custom-inode) + path="$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" + cp "$path" "${path}.replacement"; chmod 0700 "${path}.replacement"; mv -Tf "${path}.replacement" "$path" + ;; + absent-regular|absent-symlink|absent-fifo) + ROLLBACK_RECREATE_TARGET="$MOTD_ETC_ROOT/update-motd.d/50-motd-news" + rm() { + local status + command rm "$@"; status=$?; unset -f rm + case "$rollback_damage" in + absent-regular) printf recreated > "$ROLLBACK_RECREATE_TARGET"; chmod 0755 "$ROLLBACK_RECREATE_TARGET" ;; + absent-symlink) ln -s "$root/recreated-link" "$ROLLBACK_RECREATE_TARGET" ;; + absent-fifo) mkfifo "$ROLLBACK_RECREATE_TARGET" ;; + esac + return "$status" + } + ;; + symlink-target) rm -f "$MOTD_ETC_ROOT/update-motd.d/10-uname"; ln -s "$root/changed-link" "$MOTD_ETC_ROOT/update-motd.d/10-uname" ;; + esac + return 0 + } + assert_fail "$rollback_impl rollback final group rejects $rollback_damage" motd_run_locked_operation install + [[ -e "$MOTD_STATE_DIR/pending" ]] || fail "$rollback_impl rollback $rollback_damage lost pending" + transaction=$(cat "$MOTD_STATE_DIR/pending") + journal="$MOTD_TRANSACTION_PARENT/$transaction/journal" + grep -Fxq 'phase=active' "$journal" || fail "$rollback_impl rollback $rollback_damage journal not active" + [[ -d "$MOTD_TRANSACTION_PARENT/$transaction/rollback" ]] || fail "$rollback_impl rollback $rollback_damage lost rollback snapshot" + pass "$rollback_impl rollback $rollback_damage preserves active recovery evidence" + motd_target_commit_hook() { :; }; motd_rollback_hook() { :; } + motd_take_lock; motd_reconcile_state; motd_release_lock + assert_eq "$before" "$(target_hash "$root")" "$rollback_impl rollback $rollback_damage retry restores targets" + assert_eq "$before_pointers" "$(cat "$MOTD_STATE_DIR/initial.current")|$(cat "$MOTD_STATE_DIR/previous.current")" "$rollback_impl rollback $rollback_damage retry restores pointers" + [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl rollback $rollback_damage retry left journal" + assert_no_motd_residue "$root"; assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "$rollback_impl rollback $rollback_damage retry keeps bounded generations" + motd_take_lock; motd_release_lock + ) + done + +done + +for rollback_impl in tool module; do +( + root="$TEST_DIR/rollback-final-success-$rollback_impl"; create_fixture "$root"; export_paths "$root" + if [[ "$rollback_impl" == tool ]]; then load_motd; else . "$ROOT_DIR/modules/system-customize.sh"; fi + motd_run_locked_operation install; printf 'normal-pre-run\n' > "$MOTD_ETC_ROOT/motd"; before=$(target_hash "$root") + motd_target_commit_hook() { [[ "$2" != issue_net ]]; } + assert_fail "$rollback_impl normal rollback still returns original operation failure" motd_run_locked_operation install + assert_eq "$before" "$(target_hash "$root")" "$rollback_impl normal rollback final group succeeds" + [[ ! -e "$MOTD_STATE_DIR/pending" && -z "$(find "$MOTD_TRANSACTION_PARENT" -mindepth 1 -print -quit)" ]] || fail "$rollback_impl normal rollback left journal" + assert_no_motd_residue "$root" +) +done + +printf 'All MOTD rollback final validation tests passed.\n' + +fi \ No newline at end of file diff --git a/tests/test-push-worker-registration-01.sh b/tests/test-push-worker-registration-01.sh deleted file mode 100644 index e1a8df0..0000000 --- a/tests/test-push-worker-registration-01.sh +++ /dev/null @@ -1,576 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-push-worker-registration-common.sh" -assert_eq source-safe "$source_result" "source keeps zero runtime side effects" - -( - # Inert direct-child job table: no SSH or production services. - sleep 10 & snapshot_child=$! - cleanup_snapshot_child() { kill -TERM "$snapshot_child" 2>/dev/null || true; wait "$snapshot_child" 2>/dev/null || true; } - trap cleanup_snapshot_child EXIT - job_is_active "$snapshot_child" || fail "snapshot misses active direct child" - if job_is_active 0; then fail "snapshot accepts absent PID"; fi - kill -TERM "$snapshot_child" - wait "$snapshot_child" 2>/dev/null || true - if job_is_active "$snapshot_child"; then fail "snapshot retains reaped child"; fi - trap - EXIT - pass "job snapshot preserves direct-child matching and reap semantics" -) -trap 'rm -rf "$TEST_DIR"' EXIT - -# Deterministic coordinator contract: a live worker can republish after cleanup. -( - root="$TEST_DIR/republish-tracking"; mkdir -m 0700 "$root" - state_file="$root/worker.state"; : > "$state_file" - ACTIVE_WORKERS=([424242]=12345) - ACTIVE_WORKER_STATE_FILES=([424242]="$state_file") - ACTIVE_WORKER_STATE_STARTS=([424242]=12345) - worker_session_state_maybe_cleanup_failed() { return 0; } - cleanup_calls=0 - cleanup_worker_session_state_file() { cleanup_calls=$((cleanup_calls + 1)); rm -f -- "$1"; } - cleanup_active_failed_worker_sessions - [[ ${ACTIVE_WORKER_STATE_FILES[424242]:-} == "$state_file" && ${ACTIVE_WORKER_STATE_STARTS[424242]:-} == 12345 ]] || fail "active cleanup lost live publisher tracking" - : > "$state_file" - ACTIVE_WORKERS=() - job_is_active() { return 1; } - cleanup_published_worker_sessions - assert_eq 2 "$cleanup_calls" "post-reap fallback handles republished state" - [[ ! -e $state_file && ${#ACTIVE_WORKER_STATE_FILES[@]} == 0 && ${#ACTIVE_WORKER_STATE_STARTS[@]} == 0 ]] || fail "republished state remains" - pass "live publisher tracking survives successful active cleanup" -) - -setup_fixture() { - local root="$1" - CURRENT_FIXTURE_ROOT=$root - mkdir -m 0700 "$root" "$root/runtime" "$root/capture" - TMPDIR="$root/runtime"; initialize_runtime - MAX_PARALLEL=2 - MAX_RETRIES=1 - BATCH_WORKER_FAILED=false - BATCH_WORKER_ERROR=false -} -assert_no_registration_residue() { - local root="$1" - [[ -z "$(find "$root/runtime" \( -name 'worker-registration.*' -o -name '.worker-registration.*' -o -name 'worker-session.*.state' \) -print -quit)" ]] || fail "registration/session residue remains" - (( ${#REGISTERING_WORKERS[@]} == 0 && ${#ACTIVE_WORKERS[@]} == 0 )) || fail "worker arrays remain" - (( ${#WORKER_REGISTRATION_READY_FILES[@]} == 0 && ${#WORKER_REGISTRATION_RELEASE_FILES[@]} == 0 && ${#WORKER_REGISTRATION_STARTS[@]} == 0 && ${#WORKER_REGISTRATION_STAGE_FILES[@]} == 0 )) || fail "registration arrays remain" -} - -run_delayed_parent_reap_case() ( - local depth="$1" root leader leader_start leader_pgid leader_sid - root="$TEST_DIR/delayed-reap-$depth" - local state_file elapsed_start elapsed_end rc=0 - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - setup_fixture "$root" - mkfifo "$root/reap-clock" - cat > "$root/parent.sh" <<'CHILD' -#!/usr/bin/env bash -set -u -root="$1" depth="$2" -trap '' HUP INT TERM - : > "$root/parent-open" -if (( depth == 0 )); then - sleep 300 & child=$! - printf '%s\n' "$child" > "$root/leaf.pid" -else - bash "$root/parent.sh" "$root" "$((depth - 1))" & child=$! -fi -kill -STOP "$BASHPID" -wait "$child" 2>/dev/null || true -exit 0 -CHILD - chmod 0700 "$root/parent.sh" - setsid bash "$root/parent.sh" "$root" "$depth" & leader=$! - leader_start=$(wait_test_process_start "$leader") - for _ in {1..100}; do [[ -e "$root/parent-open" ]] && break; sleep .01; done - read_process_record "$leader"; leader_pgid=$PROC_PGID; leader_sid=$PROC_SID - [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "delayed $depth leader identity invalid" - WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start - WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid - publish_worker_session_state active; state_file=$WORKER_SESSION_STATE_FILE - leaf=$(<"$root/leaf.pid"); leaf_start=$(wait_test_process_start "$leaf") - kill -KILL "$leaf" 2>/dev/null || true - old_reap_status=1 - for _ in {1..30}; do - session_process_identity_matches "$leaf" "$leaf_start" "$leader_sid" || { old_reap_status=0; break; } - sleep 0.01 - done - assert_eq 1 "$old_reap_status" "delayed $depth old 300ms window remains pending" - ( sleep 1; kill -CONT -"$leader" 2>/dev/null || true ) & releaser=$! - cleanup_clock; elapsed_start=$CLEANUP_NOW - terminate_worker_transfer || rc=$? - wait "$releaser" 2>/dev/null || true - cleanup_clock; elapsed_end=$CLEANUP_NOW - assert_eq 0 "$rc" "delayed $depth parent reap succeeds" - (( elapsed_end - elapsed_start >= 100 )) || fail "delayed $depth reap did not wait for parent" - wait_test_process_identity_gone "$leader" "$leader_start" || fail "delayed $depth left leader" - collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states - assert_eq 0 "${#session_pids[@]}" "delayed $depth managed SID clears" - [[ ! -e "$state_file" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "delayed $depth state cleanup incomplete" - cleanup_runtime - exec 9>&- - pass "delayed parent reap depth $depth preserves leaf-first order" -) -run_delayed_parent_reap_case 1 -run_delayed_parent_reap_case 4 - -( - root="$TEST_DIR/reap-deadline"; setup_fixture "$root" - cat > "$root/parent.sh" <<'CHILD' -#!/usr/bin/env bash -set -u -root="$1" -trap '' HUP INT TERM -sleep 300 & child=$! -: > "$root/ready" -while [[ ! -e "$root/release" ]]; do :; done -wait "$child" 2>/dev/null || true -exit 0 -CHILD - chmod 0700 "$root/parent.sh" - setsid bash "$root/parent.sh" "$root" & leader=$! - for _ in {1..100}; do [[ -e "$root/ready" ]] && break; sleep .01; done - leader_start=$(wait_test_process_start "$leader"); read_process_record "$leader" - WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start; WORKER_TRANSFER_PGID=$PROC_PGID; WORKER_TRANSFER_SID=$PROC_SID - publish_worker_session_state active; state_file=$WORKER_SESSION_STATE_FILE - MANAGED_SESSION_CLEANUP_TICKS=300 - rc=0; terminate_worker_transfer || rc=$? - (( rc != 0 )) || fail "deadline failure unexpectedly succeeded" - [[ -d "$TEMP_DIR" && -f "$state_file" && "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || fail "deadline failure discarded evidence" - [[ "$(stat -c %a "$TEMP_DIR")" == 700 && "$(stat -c %a "$state_file")" == 600 ]] || fail "deadline evidence trust changed" - : > "$root/release"; wait "$leader" 2>/dev/null || true - remove_worker_session_state - WORKER_TRANSFER_PID=""; WORKER_TRANSFER_START=""; WORKER_TRANSFER_PGID=""; WORKER_TRANSFER_SID=""; WORKER_TRANSFER_CLEANUP_FAILED=false - clear_managed_session_cleanup_tracking - cleanup_runtime - [[ ! -e "$TEMP_DIR" ]] || fail "deadline teardown left runtime" - exec 9>&- - pass "deadline failure is bounded and preserves then safely clears evidence" -) -if [[ "${PUSH_REAP_ONLY:-false}" == true ]]; then - exit 0 -fi -test_watchdog_process() { - local seconds="$1" marker="$2" target_pid="$3" timer="" - trap '[[ -z "$timer" ]] || kill "$timer" 2>/dev/null || true; [[ -z "$timer" ]] || wait "$timer" 2>/dev/null || true; exit 0' HUP INT TERM - sleep "$seconds" & timer=$! - wait "$timer" 2>/dev/null || exit 0 - printf fired > "$marker" - kill -KILL "$target_pid" 2>/dev/null || true -} - -run_with_watchdog() { - local name="$1" seconds="$2"; shift 2 - local marker="$TEST_DIR/watchdog-$name" pid watchdog rc=0 - "$@" & pid=$! - test_watchdog_process "$seconds" "$marker" "$pid" & watchdog=$! - wait "$pid" || rc=$? - kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$marker" ]] || fail "$name exceeded watchdog deadline" - return "$rc" -} - -root_cause_regression_case() ( - root="$TEST_DIR/root-cause"; setup_fixture "$root" - GET_START_CALLS=0 - get_process_start_time() { ((GET_START_CALLS += 1)); return 1; } - push_to_server() { - local server="$1" lock_fd - exec {lock_fd}>>"$root/capture/calls.lock"; flock -x "$lock_fd" - printf '%s\n' "$server" >> "$root/capture/calls"; : > "$root/capture/ready.$server" - flock -u "$lock_fd"; exec {lock_fd}>&- - while [[ ! -e "$root/capture/release" ]]; do sleep 0.02; done - return "$MANAGED_CLEANUP_FAILURE_STATUS" - } - ( - coordinator_status=0; deadline=$((SECONDS + 15)) - while (( SECONDS < deadline )); do - if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release"; break; fi - sleep 0.02 - done - if [[ ! -e "$root/capture/release" ]]; then : > "$root/capture/coordinator-timeout"; : > "$root/capture/release"; coordinator_status=1; fi - while [[ ! -e "$root/capture/coordinator-stop" ]]; do sleep 0.02; done - exit "$coordinator_status" - ) & coordinator=$! - rc=0; run_server_batch source destination one.example two.example three.example four.example || rc=$? - : > "$root/capture/coordinator-stop"; coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? - assert_eq 0 "$coordinator_rc" "registration coordinator sees both first-slot workers" - assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "cleanup failure remains lifecycle status" - assert_eq 2 "$(wc -l < "$root/capture/calls")" "both first-slot workers enter fake transfer" - grep -Fxq one.example "$root/capture/calls" || fail "one.example did not enter fake transfer" - grep -Fxq two.example "$root/capture/calls" || fail "two.example did not enter fake transfer" - grep -Eq 'three\.example|four\.example' "$root/capture/calls" && fail "registration failure scheduled later servers" - assert_eq 0 "$GET_START_CALLS" "parent no longer depends on short get_process_start_time window" - terminate_active_workers || true; cleanup_runtime; assert_no_registration_residue "$root" - pass "deterministic registration regression avoids worker-one/worker-two deadlock" -) -run_with_watchdog root-cause 20 root_cause_regression_case - -run_ready_failure_case() ( - kind="$1"; root="$TEST_DIR/ready-$kind"; setup_fixture "$root" - WORKER_REGISTRATION_TIMEOUT_TICKS=10 - push_to_server() { : > "$root/capture/transfer"; return 0; } - case "$kind" in - missing) worker_registration_phase_hook() { [[ "$1" != before-ready ]] || while :; do :; done; } ;; - malformed) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf malformed > "$3"; } ;; - symlink) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || { rm -f "$3"; ln -s "$root/other" "$3"; }; } ;; - pid-mismatch) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || sed -i 's/^worker_pid=.*/worker_pid=999999/' "$3"; } ;; - wrong-mode) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || chmod 0644 "$3"; } ;; - duplicate-field) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf 'worker_pid=1\n' >> "$3"; } ;; - unknown-field) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf 'unexpected=value\n' >> "$3"; } ;; - illegal-number) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || sed -i 's/^worker_start=.*/worker_start=0/' "$3"; } ;; - release-failure) publish_worker_registration_release() { return 1; } ;; - esac - rc=0; launch_worker one.example source destination 1 1 || rc=$? - (( rc != 0 )) || fail "$kind registration failure unexpectedly succeeded" - [[ ! -e "$root/capture/transfer" ]] || fail "$kind reached transfer before trusted handoff" - cleanup_runtime || true; assert_no_registration_residue "$root" - [[ -z "$(jobs -pr)" ]] || fail "$kind left worker job" - pass "$kind registration failure is bounded and side-effect free" -) -for registration_failure in missing malformed symlink pid-mismatch wrong-mode duplicate-field unknown-field illegal-number release-failure; do run_ready_failure_case "$registration_failure"; done - -( - root="$TEST_DIR/release-replaced"; setup_fixture "$root" - push_to_server() { : > "$root/capture/transfer"; return 0; } - worker_registration_phase_hook() { [[ "$1" != ready-published ]] || while [[ ! -e "$root/capture/release-mutated" ]]; do sleep 0.02; done; } - parent_worker_registration_hook() { [[ "$1" != release-published ]] || { chmod 0644 "$4"; : > "$root/capture/release-mutated"; }; } - launch_worker one.example source destination 1 1 - rc=0; wait_for_all_workers || rc=$? - assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "worker rejects replaced release before transfer" - [[ ! -e "$root/capture/transfer" ]] || fail "replaced release reached transfer" - cleanup_runtime || true; assert_no_registration_residue "$root" - pass "release replacement is bounded and leaves no residue" -) -( - root="$TEST_DIR/parent-disappeared"; setup_fixture "$root" - push_to_server() { : > "$root/capture/transfer"; return 0; } - sleep 300 & fake_parent=$!; read_process_record "$fake_parent"; fake_parent_start=$PROC_START - registered_worker_entry "$fake_parent" "$fake_parent_start" one.example source destination 1 1 & worker=$! - for _ in {1..200}; do [[ -e "$TEMP_DIR/worker-registration.$worker.ready" ]] && break; sleep 0.02; done - [[ -e "$TEMP_DIR/worker-registration.$worker.ready" ]] || fail "worker did not publish ready before parent disappearance" - kill -TERM "$fake_parent" 2>/dev/null || true; wait "$fake_parent" 2>/dev/null || true - rc=0; wait "$worker" || rc=$?; (( rc != 0 )) || fail "worker accepted disappeared parent" - [[ ! -e "$root/capture/transfer" ]] || fail "worker transferred after parent disappeared" - cleanup_worker_registration_for_pid "$worker" || true; cleanup_runtime; assert_no_registration_residue "$root" - pass "worker exits before transfer when parent identity disappears" -) - -( - sleep 60 & proc_fallback_worker=$! - proc_fallback_start=$(wait_test_process_start "$proc_fallback_worker") - eval "$(declare -f read_process_record | sed '1s/read_process_record/original_read_process_record/')" - read_process_record() { [[ "$1" != "$proc_fallback_worker" ]] && original_read_process_record "$@"; } - proc_fallback_status=0 - read_process_record_for_session_scan "$proc_fallback_worker" || proc_fallback_status=$? - assert_eq 0 "$proc_fallback_status" "proc owner fallback recovers transient status-read failure" - [[ "$PROC_START" == "$proc_fallback_start" && "$PROC_UID" == "$EUID" ]] || fail "proc owner fallback identity mismatch" - kill -TERM "$proc_fallback_worker"; wait "$proc_fallback_worker" 2>/dev/null || true - pass "proc owner fallback binds UID to stable PID start" -) - -prune_identity_failure_case() ( - mode="$1"; root="$TEST_DIR/prune-$mode"; setup_fixture "$root" - sleep 300 & worker=$!; read_process_record "$worker"; start=$PROC_START - ACTIVE_WORKERS[$worker]=$start; ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state"; ACTIVE_WORKER_STATE_STARTS[$worker]=$start - eval "$(declare -f process_identity_matches | sed '1s/process_identity_matches/original_process_identity_matches/')" - eval "$(declare -f read_process_record_for_session_scan | sed '1s/read_process_record_for_session_scan/original_read_process_record_for_session_scan/')" - failures=1; scan_failures=1; [[ "$mode" == repeated ]] && { failures=100; scan_failures=100; } - process_identity_matches() { if [[ "$1" == "$worker" && "$failures" -gt 0 ]]; then ((failures -= 1)); return 1; fi; original_process_identity_matches "$@"; } - read_process_record_for_session_scan() { - if [[ "$1" == "$worker" && "$scan_failures" -gt 0 && -r "/proc/$worker/stat" ]]; then ((scan_failures -= 1)); return 1; fi - original_read_process_record_for_session_scan "$@" - } - rc=0; prune_active_workers || rc=$?; (( rc != 0 )) || fail "$mode identity failure did not stop prune" - [[ "$BATCH_WORKER_FAILED" == true ]] || fail "$mode identity failure did not set lifecycle barrier" - process_is_running "$worker" || fail "$mode prune shortened active worker grace" - [[ -n "${ACTIVE_WORKERS[$worker]:-}" ]] || fail "$mode prune discarded active worker evidence" - rc=0; wait_for_worker_slot || rc=$?; assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "$mode identity failure propagates lifecycle status" - terminate_active_workers || true - process_is_running "$worker" && fail "$mode unified active cleanup left worker" - cleanup_runtime || true; assert_no_registration_residue "$root" - pass "$mode active identity failure defers termination to unified cleanup" -) -prune_identity_failure_case once -prune_identity_failure_case repeated -( - root="$TEST_DIR/prune-cleanup-failure"; setup_fixture "$root" - sleep 300 & worker=$!; read_process_record "$worker"; start=$PROC_START - ACTIVE_WORKERS[$worker]=$start - ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state" - ACTIVE_WORKER_STATE_STARTS[$worker]=$start - eval "$(declare -f process_identity_matches | sed '1s/process_identity_matches/original_process_identity_matches/')" - eval "$(declare -f read_process_record_for_session_scan | sed '1s/read_process_record_for_session_scan/original_read_process_record_for_session_scan/')" - process_identity_matches() { [[ "$1" != "$worker" ]] && original_process_identity_matches "$@"; } - read_process_record_for_session_scan() { [[ "$1" != "$worker" || ! -r "/proc/$worker/stat" ]] && original_read_process_record_for_session_scan "$@"; } - rc=0; wait_for_worker_slot || rc=$? - assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "prune trust failure propagates 125" - [[ -n "${ACTIVE_WORKERS[$worker]:-}" ]] || fail "prune trust failure discarded active worker evidence" - eval "$(declare -f original_process_identity_matches | sed '1s/original_process_identity_matches/process_identity_matches/')" - eval "$(declare -f original_read_process_record_for_session_scan | sed '1s/original_read_process_record_for_session_scan/read_process_record_for_session_scan/')" - eval "$(declare -f cleanup_all_worker_registration_residue | sed '1s/cleanup_all_worker_registration_residue/original_cleanup_all_worker_registration_residue/')" - cleanup_all_worker_registration_residue() { return 1; } - rc=0; stop_batch_after_lifecycle_failure || rc=$? - assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "active cleanup failure remains lifecycle status 125" - eval "$(declare -f original_cleanup_all_worker_registration_residue | sed '1s/original_cleanup_all_worker_registration_residue/cleanup_all_worker_registration_residue/')" - cleanup_runtime || true; assert_no_registration_residue "$root" - pass "active cleanup failure preserves lifecycle status" -) -( - root="$TEST_DIR/prune-exited"; setup_fixture "$root" - sleep 0.1 & worker=$!; read_process_record "$worker"; start=$PROC_START - ACTIVE_WORKERS[$worker]=$start - ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state" - ACTIVE_WORKER_STATE_STARTS[$worker]=$start - sleep 0.2; prune_active_workers - assert_eq 0 "${#ACTIVE_WORKERS[@]}" "exited worker is normally reaped" - [[ "$BATCH_WORKER_FAILED" == false ]] || fail "normal exited worker set lifecycle barrier" - cleanup_runtime; assert_no_registration_residue "$root" - pass "normal exited worker accounting remains intact" -) - -cat > "$TEST_DIR/registration-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" root="$2" phase="$3" -mkdir -p -m 0700 "$root/runtime" "$root/capture" -export TMPDIR="$root/runtime" -source "$script" -initialize_runtime; install_runtime_traps -WORKER_REGISTRATION_TIMEOUT_TICKS=200 -push_to_server() { printf 'push pid=%s root=%s\n' "$BASHPID" "$root" >> "$root/capture/transfer-calls"; : > "$root/capture/transfer"; return 0; } -worker_registration_phase_hook() { local current="$1"; [[ "$phase" == "$current" ]] || return 0; : > "$root/capture/phase"; while :; do :; done; } -parent_worker_registration_hook() { local current="$1"; [[ "$phase" == "$current" ]] || return 0; : > "$root/capture/phase"; while :; do :; done; } -launch_worker one.example source destination 1 1 -wait_for_all_workers -if [[ "$phase" == normal ]]; then [[ -e "$root/capture/transfer" ]] || { printf 'normal transfer missing\n' >&2; jobs -l >&2 || true; exit 91; }; fi -cleanup_runtime -CHILD -chmod 0700 "$TEST_DIR/registration-signal-child.sh" -run_registration_signal_case() { - local phase="$1" signal_name="$2" expected="$3" root retry_root - local pid watchdog unrelated rc=0 - root="$TEST_DIR/signal-$phase-$signal_name"; CURRENT_FIXTURE_ROOT=$root - sleep 60 & unrelated=$! - env --default-signal=HUP,INT,TERM bash "$TEST_DIR/registration-signal-child.sh" "$SCRIPT" "$root" "$phase" > "$root.log" 2>&1 & pid=$! - for _ in {1..300}; do [[ -e "$root/capture/phase" ]] && break; sleep 0.05; done - [[ -e "$root/capture/phase" ]] || { cat "$root.log"; kill "$pid" "$unrelated" 2>/dev/null || true; fail "$phase/$signal_name did not reach registration phase"; } - test_watchdog_process 30 "$root.watchdog" "$pid" & watchdog=$! - kill "-$signal_name" "$pid"; wait "$pid" || rc=$? - kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root.watchdog" ]] || fail "$phase/$signal_name exceeded watchdog" - assert_eq "$expected" "$rc" "$phase registration $signal_name status" - [[ ! -e "$root/capture/transfer" ]] || fail "$phase/$signal_name reached transfer" - [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$phase/$signal_name left runtime state" - kill -0 "$unrelated" 2>/dev/null || fail "$phase/$signal_name killed unrelated process" - kill "$unrelated" 2>/dev/null || true; wait "$unrelated" 2>/dev/null || true - retry_root="${root}-retry" - env bash "$TEST_DIR/registration-signal-child.sh" "$SCRIPT" "$retry_root" normal > "$retry_root.log" 2>&1 - if [[ ! -e "$retry_root/capture/transfer" ]]; then cat "$retry_root.log" >&2 2>/dev/null || true; fail "$phase/$signal_name retry did not transfer"; fi - [[ -z "$(find "$retry_root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$phase/$signal_name retry left runtime" - pass "$phase/$signal_name leaves registration clean and retryable" -} -for registration_phase in before-ready accepted-before-release release-validated; do - run_registration_signal_case "$registration_phase" HUP 129 - run_registration_signal_case "$registration_phase" INT 130 - run_registration_signal_case "$registration_phase" TERM 143 -done - -( - sleep 60 & reused_pid=$! - replacement_start=$(get_process_start_time "$reused_pid") - expected_start=$((replacement_start + 1)) - eval "$(declare -f collect_owned_session_records | sed '1s/collect_owned_session_records/original_collect_owned_session_records/')" - collect_owned_session_records() { fail "PID reuse path attempted to inspect or terminate replacement session"; } - terminate_managed_session "$reused_pid" "$expected_start" "$reused_pid" "$reused_pid" - test_process_identity_exists "$reused_pid" "$replacement_start" || fail "PID reuse path touched replacement process" - kill -TERM "$reused_pid"; wait "$reused_pid" 2>/dev/null || true - pass "managed leader PID reuse is treated as original session gone" -) - -cat > "$TEST_DIR/failed-reap-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" root="$2" mode="$3" -mkdir -p -m 0700 "$root/runtime" "$root/capture" -export TMPDIR="$root/runtime" -source "$script" -initialize_runtime -setsid bash -c 'trap "" HUP INT TERM; while :; do :; done' & leader=$! -for _ in {1..100}; do - leader_start=$(get_process_start_time "$leader" 2>/dev/null || true) - leader_pgid=$(get_process_group_id "$leader" 2>/dev/null || true) - leader_sid=$(get_process_session_id "$leader" 2>/dev/null || true) - [[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] && break - sleep 0.01 -done -[[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || exit 90 -printf '%s\n%s\n%s\n%s\n' "$leader" "$leader_start" "$leader_pgid" "$leader_sid" > "$root/capture/leader" -WORKER_TRANSFER_PID=$leader -WORKER_TRANSFER_START=$leader_start -WORKER_TRANSFER_PGID=$leader_pgid -WORKER_TRANSFER_SID=$leader_sid -publish_worker_session_state active -state_file=$WORKER_SESSION_STATE_FILE -eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" -terminate_managed_session() { return 1; } -start_ns=$(date +%s%N) -rc=0 -terminate_worker_transfer || rc=$? -elapsed_ns=$(( $(date +%s%N) - start_ns )) -[[ "$rc" != 0 && "$elapsed_ns" -lt 2000000000 ]] || exit 91 -[[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && - "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && - "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || exit 92 -read_worker_session_state "$state_file" -[[ "$SESSION_STATE" == cleanup_failed ]] || exit 93 -next_rc=0 -run_managed_command "$TEMP_DIR/forbidden-next-command" true || next_rc=$? -[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-next-command" ]] || exit 94 -printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\nstate_file=%s\n' "$rc" "$elapsed_ns" "$next_rc" "$state_file" > "$root/capture/result" -: > "$root/capture/returned" -if [[ "$mode" == hold ]]; then - while [[ ! -e "$root/capture/release-worker" ]]; do sleep 0.01; done - for _ in {1..400}; do - reap_status=0 - reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" || reap_status=$? - case "$reap_status" in - 0|"$MANAGED_PROCESS_REUSED_STATUS") break ;; - *) sleep 0.01 ;; - esac - done - (( reap_status == 0 || reap_status == MANAGED_PROCESS_REUSED_STATUS )) || exit 95 - : > "$root/capture/reaped" - for _ in {1..400}; do - [[ ! -e "$state_file" && ! -L "$state_file" ]] && break - sleep 0.01 - done - [[ ! -e "$state_file" && ! -L "$state_file" ]] || exit 96 - terminate_managed_session() { original_terminate_managed_session "$@"; } -else - for _ in {1..400}; do [[ -e "$root/capture/release-recovery" ]] && break; sleep 0.01; done - [[ -e "$root/capture/release-recovery" ]] || exit 97 - terminate_managed_session() { original_terminate_managed_session "$@"; } -fi -cleanup_runtime -CHILD -chmod 0700 "$TEST_DIR/failed-reap-child.sh" - -run_failed_reap_case() { - local run="$1" root="$TEST_DIR/failed-reap-$1" child watchdog rc=0 - bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" recover > "$root.log" 2>&1 & child=$! - test_watchdog_process 3 "$root.watchdog-fired" "$child" & watchdog=$! - for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done - [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "failed reap run $run did not return before watchdog"; } - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root.watchdog-fired" ]] || fail "failed reap run $run fired watchdog" - : > "$root/capture/release-recovery" - wait "$child" || rc=$? - [[ "$rc" == 0 ]] || cat "$root.log" >&2 2>/dev/null || true - assert_eq 0 "$rc" "failed managed-session reap run $run returns bounded" - [[ -z "$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null)" ]] || fail "failed reap run $run left worker-session state" -} -for failed_reap_run in $(seq 1 20); do run_failed_reap_case "$failed_reap_run"; done -pass "failed managed-session reap remains bounded for 20 deterministic runs" - -( - trap - EXIT HUP INT TERM - root="$TEST_DIR/failed-reap-parent"; mkdir -p "$root/capture"; chmod 0700 "$root" "$root/capture" - bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" hold > "$root.log" 2>&1 & worker=$! - worker_start=$(wait_test_process_start "$worker") - for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done - [[ -e "$root/capture/returned" ]] || fail "parent fallback worker did not return bounded" - mapfile -t leader_fields < "$root/capture/leader" - leader=${leader_fields[0]}; leader_start=${leader_fields[1]}; leader_sid=${leader_fields[3]} - state_file=$(sed -n 's/^state_file=//p' "$root/capture/result") - TEMP_DIR=$(dirname "$state_file") - ACTIVE_WORKERS=(); ACTIVE_WORKER_STATE_FILES=(); ACTIVE_WORKER_STATE_STARTS=() - BATCH_WORKER_FAILED=false - BATCH_WORKER_ERROR=false - MAX_PARALLEL=1 - ACTIVE_WORKERS[$worker]=$worker_start - ACTIVE_WORKER_STATE_FILES[$worker]=$state_file - ACTIVE_WORKER_STATE_STARTS[$worker]=$worker_start - : > "$root/capture/release-worker" - wait_for_all_workers - assert_eq 0 "${#ACTIVE_WORKERS[@]}" "scheduler parent fallback reaps active worker" - assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "scheduler parent fallback removes cleanup_failed state" - wait_test_process_identity_gone "$leader" "$leader_start" || fail "parent fallback left same-identity leader" - session_pids=(); session_starts=(); session_ppids=(); session_pgids=(); session_states=() - collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states - assert_eq 0 "${#session_pids[@]}" "parent fallback leaves no managed SID member" - [[ ! -e "$state_file" && -e "$root/capture/reaped" ]] || fail "parent fallback evidence incomplete" - pass "parent fallback cleans a live failed session without PPID 1 zombie" -) - -( - trap - EXIT HUP INT TERM - sleep 60 & replacement=$! - replacement_start=$(wait_test_process_start "$replacement") - wait_called=false - wait() { wait_called=true; return 1; } - reap_status=0 - reap_direct_managed_leader "$replacement" "$((replacement_start + 1))" "$replacement" "$replacement" || reap_status=$? - assert_eq "$MANAGED_PROCESS_REUSED_STATUS" "$reap_status" "direct leader PID reuse returns safe status" - [[ "$wait_called" == false ]] || fail "direct leader PID reuse waited on replacement process" - test_process_identity_exists "$replacement" "$replacement_start" || fail "direct leader PID reuse touched replacement process" - kill -TERM "$replacement"; builtin wait "$replacement" 2>/dev/null || true - pass "direct leader PID reuse sends no signal and performs no wait" -) - -( - trap - EXIT HUP INT TERM - leader=424242; leader_start=777777; wait_called=false; after_wait=false - collect_owned_session_records() { - local -n pids_ref="$2" starts_ref="$3" ppids_ref="$4" pgids_ref="$5" states_ref="$6" - pids_ref=(); starts_ref=(); ppids_ref=(); pgids_ref=(); states_ref=() - [[ "$after_wait" == false ]] || return 0 - pids_ref=("$leader"); starts_ref[$leader]=$leader_start; ppids_ref[$leader]=$BASHPID - pgids_ref[$leader]=$leader; states_ref[$leader]=Z - } - read_process_record_for_session_scan() { - [[ "$after_wait" == false ]] || return 2 - PROC_UID=$EUID; PROC_PPID=$BASHPID; PROC_START=$leader_start - PROC_STATE=Z; PROC_PGID=$leader; PROC_SID=$leader - } - wait() { wait_called=true; after_wait=true; return 0; } - reap_direct_managed_leader "$leader" "$leader_start" "$leader" "$leader" - [[ "$wait_called" == true ]] || fail "same-identity leader zombie was not wait/reaped" - pass "same-identity direct leader zombie is immediately wait/reaped" -) - -cat > "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" root="$2" -mkdir -p -m 0700 "$root/runtime" "$root/capture" -export TMPDIR="$root/runtime" -source "$script" -initialize_runtime -TOTAL_TIMEOUT=1 -TIMEOUT_KILL_AFTER=1 -eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" -failures=1 -terminate_managed_session() { - if (( failures > 0 )); then ((failures -= 1)); return 1; fi - original_terminate_managed_session "$@" -} -start_ns=$(date +%s%N) -rc=0 -run_managed_command "$TEMP_DIR/hard-output" timeout 1 bash -c \ - 'trap "" HUP INT TERM; bash -c '\''trap "" HUP INT TERM; while :; do kill -STOP "$$"; done'\'' & wait "$!"' || rc=$? -elapsed_ns=$(( $(date +%s%N) - start_ns )) -[[ "$rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && "$elapsed_ns" -lt 5000000000 ]] || exit 90 -[[ "$WORKER_TRANSFER_CLEANUP_FAILED" == true && -n "$WORKER_TRANSFER_PID" && -f "$WORKER_SESSION_STATE_FILE" ]] || exit 91 -read_worker_session_state "$WORKER_SESSION_STATE_FILE" -[[ "$SESSION_STATE" == cleanup_failed ]] || exit 92 -next_rc=0 -run_managed_command "$TEMP_DIR/forbidden-hard-next" true || next_rc=$? -[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-hard-next" ]] || exit 93 -printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\n' "$rc" "$elapsed_ns" "$next_rc" > "$root/capture/result" -: > "$root/capture/returned" -terminate_worker_transfer -cleanup_runtime -CHILD -chmod 0700 "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" \ No newline at end of file diff --git a/tests/test-push-worker-registration-02.sh b/tests/test-push-worker-registration-02.sh deleted file mode 100644 index d0fe671..0000000 --- a/tests/test-push-worker-registration-02.sh +++ /dev/null @@ -1,358 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-push-worker-registration-common.sh" - -( - trap - EXIT HUP INT TERM - root="$TEST_DIR/hard-timeout-cleanup-failure" - bash "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" "$SCRIPT" "$root" > "$root.log" 2>&1 & child=$! - test_watchdog_process 5 "$root.watchdog-fired" "$child" & watchdog=$! - for _ in {1..120}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done - [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "hard timeout cleanup failure did not return bounded"; } - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root.watchdog-fired" ]] || fail "hard timeout cleanup failure fired watchdog" - rc=0; wait "$child" || rc=$? - assert_eq 0 "$rc" "hard timeout cleanup failure recovers fixture" - grep -Fxq "rc=$MANAGED_CLEANUP_FAILURE_STATUS" "$root/capture/result" || fail "hard timeout cleanup failure did not propagate 125" - grep -Fxq "next_rc=$MANAGED_CLEANUP_FAILURE_STATUS" "$root/capture/result" || fail "hard timeout cleanup failure allowed next command" - [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "hard timeout cleanup failure left runtime" - pass "hard timeout cleanup failure is bounded and blocks subsequent command" -) - -write_test_worker_session_manifest() { - local file="$1" state="$2" worker_pid="$3" worker_start="$4" leader_pid="$5" leader_start="$6" pgid="$7" sid="$8" - printf 'version=1\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\n' \ - "$state" "$worker_pid" "$worker_start" "$leader_pid" "$leader_start" "$pgid" "$sid" > "$file" - chmod 0600 "$file" -} - - -assert_retained_runtime_trusted() { - local path="$1" expected_dev="$2" expected_inode="$3" metadata="" dev inode owner gid mode trusted_gid - [[ -d "$path" && ! -L "$path" ]] || fail "retained runtime is not a regular directory: $path" - metadata=$(stat -Lc '%d:%i:%u:%g:%a' -- "$path") || fail "cannot stat retained runtime: $path" - IFS=: read -r dev inode owner gid mode <<< "$metadata" - trusted_gid=$(current_gid) - [[ "$dev" == "$expected_dev" && "$inode" == "$expected_inode" && - "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 700 ]] || - fail "retained runtime trust mismatch: $metadata" -} - -assert_retained_state_trusted() { - local file="$1" metadata="" owner gid mode trusted_gid - [[ -f "$file" && ! -L "$file" ]] || fail "retained worker state is not a regular file: $file" - metadata=$(stat -Lc '%u:%g:%a' -- "$file") || fail "cannot stat retained worker state: $file" - IFS=: read -r owner gid mode <<< "$metadata" - trusted_gid=$(current_gid) - [[ "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 600 ]] || - fail "retained worker state trust mismatch: $metadata" -} - -run_preserved_cleanup_retry_case() ( - local run="$1" root runtime runtime_dev runtime_inode credential leader leader_start leader_pgid leader_sid state_file - local owner watchdog rc=0 metadata="" state_status=0 - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - root="$TEST_DIR/runtime-evidence-retry-$run"; setup_fixture "$root" - runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE - credential="$runtime/private-key" - printf 'fixture-credential-do-not-print\n' > "$credential"; chmod 0600 "$credential" - RUNTIME_KEY_FILE=$credential - - setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! - leader_start=$(wait_test_process_start "$leader") - for _ in {1..100}; do - read_process_record "$leader" || true - [[ "$PROC_START" == "$leader_start" && "$PROC_STATE" == T ]] && break - sleep 0.01 - done - read_process_record "$leader" - leader_pgid=$PROC_PGID; leader_sid=$PROC_SID - [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "retry run $run leader identity invalid" - WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start - WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid - publish_worker_session_state active - state_file=$WORKER_SESSION_STATE_FILE - - eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" - terminate_managed_session() { return 1; } - owner=$BASHPID - test_watchdog_process 3 "$root/watchdog-fired" "$owner" & watchdog=$! - cleanup_runtime > "$root/capture/first-cleanup.log" 2>&1 || rc=$? - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - - (( rc != 0 )) || fail "retry run $run first cleanup unexpectedly succeeded" - [[ ! -e "$root/watchdog-fired" ]] || fail "retry run $run watchdog fired" - [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" ]] || - fail "retry run $run lost runtime identity" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - assert_retained_state_trusted "$state_file" - metadata=$(stat -Lc '%u:%g:%a' -- "$credential") - [[ "$metadata" == "$EUID:$(current_gid):600" ]] || fail "retry run $run credential permissions changed: $metadata" - grep -Fq 'fixture-credential-do-not-print' "$root/capture/first-cleanup.log" && fail "retry run $run printed credential content" - state_status=0; read_worker_session_state "$state_file" || state_status=$? - assert_eq 0 "$state_status" "retry run $run retained state parses" - [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && - "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || - fail "retry run $run retained state identity mismatch" - [[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && - "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && - "$WORKER_TRANSFER_CLEANUP_FAILED" == true && "$WORKER_SESSION_STATE_FILE" == "$state_file" && - "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run lost cleanup globals" - test_process_identity_exists "$leader" "$leader_start" || fail "retry run $run lost live leader tracking" - read_process_record "$leader" - [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "retry run $run orphaned or zombified leader" - - eval "$(declare -f original_terminate_managed_session | sed '1s/original_terminate_managed_session/terminate_managed_session/')" - cleanup_runtime - wait_test_process_identity_gone "$leader" "$leader_start" || fail "retry run $run left leader identity" - collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states - assert_eq 0 "${#session_pids[@]}" "retry run $run clears managed SID" - [[ ! -e "$state_file" && ! -L "$state_file" && ! -e "$runtime" && ! -L "$runtime" ]] || - fail "retry run $run left state or runtime" - runtime_lifecycle_globals_resolved || fail "retry run $run left lifecycle globals" - [[ "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run runtime state not reset" - pass "failed cleanup evidence retry run $run preserves then clears runtime" -) - -runtime_evidence_stress=${PUSH_RUNTIME_EVIDENCE_STRESS:-20} -for runtime_evidence_run in $(seq 1 "$runtime_evidence_stress"); do - run_preserved_cleanup_retry_case "$runtime_evidence_run" -done -pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" - -( - local_cleanup_root="$TEST_DIR/active-cleanup-evidence"; setup_fixture "$local_cleanup_root" - runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE - sleep 300 & worker=$!; worker_start=$(wait_test_process_start "$worker") - sleep 300 & replacement=$!; replacement_start=$(wait_test_process_start "$replacement") - state_file="$runtime/worker-session.$worker.state" - write_test_worker_session_manifest "$state_file" cleanup_failed "$worker" "$worker_start" \ - "$replacement" "$((replacement_start + 1))" "$replacement" "$replacement" - ACTIVE_WORKERS[$worker]=$worker_start - ACTIVE_WORKER_STATE_FILES[$worker]=$state_file - ACTIVE_WORKER_STATE_STARTS[$worker]=$worker_start - eval "$(declare -f terminate_active_workers | sed '1s/terminate_active_workers/original_terminate_active_workers/')" - terminate_active_workers() { return 1; } - rc=0; cleanup_runtime > "$local_cleanup_root/capture/first-cleanup.log" 2>&1 || rc=$? - (( rc != 0 )) || fail "active cleanup evidence first call unexpectedly succeeded" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - assert_retained_state_trusted "$state_file" - [[ -n "${ACTIVE_WORKERS[$worker]:-}" && "${ACTIVE_WORKER_STATE_FILES[$worker]:-}" == "$state_file" && - "${ACTIVE_WORKER_STATE_STARTS[$worker]:-}" == "$worker_start" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || - fail "active cleanup evidence maps were discarded" - test_process_identity_exists "$worker" "$worker_start" || fail "active cleanup evidence worker disappeared" - eval "$(declare -f original_terminate_active_workers | sed '1s/original_terminate_active_workers/terminate_active_workers/')" - cleanup_runtime - wait_test_process_identity_gone "$worker" "$worker_start" || fail "active cleanup retry left worker" - [[ ! -e "$state_file" && ! -e "$runtime" && ${#ACTIVE_WORKERS[@]} == 0 && - ${#ACTIVE_WORKER_STATE_FILES[@]} == 0 && ${#ACTIVE_WORKER_STATE_STARTS[@]} == 0 ]] || - fail "active cleanup retry left runtime, state, or maps" - test_process_identity_exists "$replacement" "$replacement_start" || fail "active cleanup retry touched PID-reuse replacement" - kill -TERM "$replacement"; wait "$replacement" 2>/dev/null || true - pass "active cleanup failure preserves evidence and retry clears it" -) - -run_runtime_trust_retention_case() ( - local kind="$1" root runtime runtime_dev runtime_inode original_path replacement_path rc=0 - root="$TEST_DIR/runtime-trust-$kind"; setup_fixture "$root" - runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE - case "$kind" in - symlink) - original_path="$runtime.original"; replacement_path="$runtime.replacement" - mv "$runtime" "$original_path"; mkdir -m 0700 "$replacement_path"; : > "$replacement_path/marker" - ln -s "$replacement_path" "$runtime" - ;; - wrong-mode) chmod 0755 "$runtime" ;; - wrong-owner|wrong-gid) - eval "$(declare -f stat | sed '1s/stat/original_stat/')" 2>/dev/null || true - stat() { - local last="${!#}" metadata dev inode owner gid mode - if [[ "$last" == "$runtime" && "$1" == -Lc && "$2" == '%d:%i:%u:%g:%a' ]]; then - metadata=$(command stat -Lc '%d:%i:%u:%g:%a' -- "$runtime") || return 1 - IFS=: read -r dev inode owner gid mode <<< "$metadata" - [[ "$kind" == wrong-owner ]] && owner=$((owner + 1)) || gid=$((gid + 1)) - printf '%s:%s:%s:%s:%s\n' "$dev" "$inode" "$owner" "$gid" "$mode" - else - command stat "$@" - fi - } - ;; - *) fail "unknown runtime trust case: $kind" ;; - esac - cleanup_runtime > "$root/capture/cleanup.log" 2>&1 || rc=$? - (( rc != 0 )) || fail "$kind untrusted runtime unexpectedly deleted" - [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" && - "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "$kind untrusted runtime lost path identity" - case "$kind" in - symlink) - [[ -L "$runtime" && -f "$replacement_path/marker" && -d "$original_path" ]] || fail "symlink replacement was modified" - rm -f "$runtime"; rmdir "$replacement_path" 2>/dev/null || { rm -f "$replacement_path/marker"; rmdir "$replacement_path"; } - mv "$original_path" "$runtime" - ;; - wrong-mode) chmod 0700 "$runtime" ;; - wrong-owner|wrong-gid) - unset -f stat - ;; - esac - cleanup_runtime - [[ ! -e "$runtime" && ! -L "$runtime" ]] || fail "$kind trusted retry left runtime" - pass "$kind runtime trust failure preserves evidence and replacement" -) -for runtime_trust_kind in symlink wrong-owner wrong-gid wrong-mode; do - run_runtime_trust_retention_case "$runtime_trust_kind" -done - -for final_residue_kind in worker-session registration-ready registration-stage; do - ( - root="$TEST_DIR/runtime-final-proof-$final_residue_kind"; setup_fixture "$root" - runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE - case "$final_residue_kind" in - worker-session) residue="$runtime/worker-session.99999.state" ;; - registration-ready) residue="$runtime/worker-registration.99999.ready" ;; - registration-stage) residue="$runtime/.worker-registration.99999.ready.token.stage" ;; - esac - eval "$(declare -f terminate_active_workers | sed '1s/terminate_active_workers/original_terminate_active_workers/')" - terminate_active_workers() { - original_terminate_active_workers || return 1 - printf 'retained-lifecycle-evidence\n' > "$residue"; chmod 0600 "$residue" - } - rc=0; cleanup_runtime > "$root/capture/cleanup.log" 2>&1 || rc=$? - (( rc != 0 )) || fail "$final_residue_kind final proof unexpectedly deleted runtime" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - [[ -f "$residue" && ! -L "$residue" && "$TEMP_DIR" == "$runtime" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || - fail "$final_residue_kind final proof discarded evidence" - eval "$(declare -f original_terminate_active_workers | sed '1s/original_terminate_active_workers/terminate_active_workers/')" - rm -f -- "$residue" - cleanup_runtime - [[ ! -e "$runtime" ]] || fail "$final_residue_kind final proof retry left runtime" - pass "$final_residue_kind blocks runtime deletion until evidence is cleared" - ) -done - -( - root="$TEST_DIR/runtime-final-proof-success"; setup_fixture "$root"; runtime=$TEMP_DIR - cleanup_runtime - [[ ! -e "$runtime" && -z "$TEMP_DIR" && "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || - fail "clean runtime final proof did not clear state" - pass "clean runtime final proof preserves normal success path" -) - - -cat > "$TEST_DIR/runtime-retention-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" root="$2" leader="$3" leader_start="$4" leader_pgid="$5" leader_sid="$6" -mkdir -p "$root/runtime" "$root/capture" -chmod 0700 "$root" "$root/runtime" "$root/capture" -TMPDIR="$root/runtime" -# shellcheck source=../tools/push.sh -source "$script" -initialize_runtime -credential="$TEMP_DIR/private-key" -printf 'signal-fixture-credential-do-not-print\n' > "$credential" -chmod 0600 "$credential" -RUNTIME_KEY_FILE=$credential -WORKER_TRANSFER_PID=$leader -WORKER_TRANSFER_START=$leader_start -WORKER_TRANSFER_PGID=$leader_pgid -WORKER_TRANSFER_SID=$leader_sid -publish_worker_session_state active -state_file=$WORKER_SESSION_STATE_FILE -terminate_managed_session() { return 1; } -install_runtime_traps -worker_start=$(get_process_start_time "$BASHPID") -stage="$root/capture/ready.stage" -{ - printf 'target_pid=%s\n' "$BASHPID" - printf 'target_start=%s\n' "$worker_start" - printf 'runtime=%s\n' "$TEMP_DIR" - printf 'runtime_dev=%s\n' "$TEMP_DIR_DEV" - printf 'runtime_inode=%s\n' "$TEMP_DIR_INODE" - printf 'state_file=%s\n' "$state_file" - printf 'credential=%s\n' "$credential" -} > "$stage" -chmod 0600 "$stage" -mv -fT "$stage" "$root/capture/ready" -while :; do :; done -CHILD -chmod 0700 "$TEST_DIR/runtime-retention-signal-child.sh" - -run_runtime_retention_signal_case() ( - local signal_name="$1" expected="$2" root leader="" leader_start="" leader_pgid="" leader_sid="" target="" target_pid="" target_start="" - local runtime runtime_dev runtime_inode state_file credential watchdog="" rc=0 line key value state_status=0 reap_status=0 - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - cleanup_runtime_retention_signal_fixture() { - if [[ -n "$target" ]]; then kill -KILL "$target" 2>/dev/null || true; wait "$target" 2>/dev/null || true; fi - if [[ -n "$leader" ]]; then kill -CONT "$leader" 2>/dev/null || true; kill -KILL "$leader" 2>/dev/null || true; wait "$leader" 2>/dev/null || true; fi - if [[ -n "$watchdog" ]]; then kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true; fi - } - trap cleanup_runtime_retention_signal_fixture EXIT - root="$TEST_DIR/runtime-retention-signal-$signal_name"; CURRENT_FIXTURE_ROOT=$root - mkdir -m 0700 "$root" - setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! - leader_start=$(wait_test_process_start "$leader") - for _ in {1..100}; do read_process_record "$leader" || true; [[ "$PROC_STATE" == T ]] && break; sleep 0.01; done - read_process_record "$leader"; leader_pgid=$PROC_PGID; leader_sid=$PROC_SID - [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "$signal_name retention leader identity invalid" - - ( - exec env --default-signal=HUP,INT,TERM bash "$TEST_DIR/runtime-retention-signal-child.sh" \ - "$SCRIPT" "$root" "$leader" "$leader_start" "$leader_pgid" "$leader_sid" - ) > "$root/output.log" 2>&1 & target=$! - for _ in {1..300}; do [[ -f "$root/capture/ready" ]] && break; sleep 0.02; done - [[ -f "$root/capture/ready" ]] || { cat "$root/output.log" >&2 || true; fail "$signal_name retention child not ready"; } - while IFS= read -r line; do - key=${line%%=*}; value=${line#*=} - case "$key" in - target_pid) target_pid=$value ;; - target_start) target_start=$value ;; - runtime) runtime=$value ;; - runtime_dev) runtime_dev=$value ;; - runtime_inode) runtime_inode=$value ;; - state_file) state_file=$value ;; - credential) credential=$value ;; - esac - done < "$root/capture/ready" - [[ "$target_pid" == "$target" && "$target_start" =~ ^[1-9][0-9]*$ && - "$runtime_dev" =~ ^[0-9]+$ && "$runtime_inode" =~ ^[1-9][0-9]*$ ]] || - fail "$signal_name retention ready metadata malformed" - job_is_active "$target" || fail "$signal_name retention target job missing" - test_watchdog_process 5 "$root/watchdog-fired" "$target" & watchdog=$! - kill "-$signal_name" "$target" - wait "$target" || rc=$? - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root/watchdog-fired" ]] || fail "$signal_name retention watchdog fired" - assert_eq "$expected" "$rc" "$signal_name failed cleanup preserves signal status" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - assert_retained_state_trusted "$state_file" - [[ "$(stat -Lc '%u:%g:%a' -- "$credential")" == "$EUID:$(current_gid):600" ]] || - fail "$signal_name retained credential permissions changed" - grep -Fq "$runtime" "$root/output.log" || fail "$signal_name failure output omitted runtime path" - grep -Fq 'signal-fixture-credential-do-not-print' "$root/output.log" && fail "$signal_name failure output leaked credential" - TEMP_DIR=$runtime - state_status=0; read_worker_session_state "$state_file" || state_status=$? - assert_eq 0 "$state_status" "$signal_name retained cleanup_failed state parses" - [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && - "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || - fail "$signal_name retained cleanup_failed identity mismatch" - test_process_identity_exists "$leader" "$leader_start" || fail "$signal_name failed cleanup lost leader tracking" - read_process_record "$leader" - [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "$signal_name teardown leader lost safe parent" - - terminate_managed_session "$leader" "$leader_start" "$leader_pgid" "$leader_sid" true - reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" "$BASHPID" || reap_status=$? - assert_eq 0 "$reap_status" "$signal_name teardown safely reaps leader" - collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states - assert_eq 0 "${#session_pids[@]}" "$signal_name teardown clears managed SID" - assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" - rm -rf -- "$runtime" - TEMP_DIR="" - leader=""; target=""; watchdog="" - trap - EXIT - pass "$signal_name cleanup failure retains trusted runtime and state" -) -run_runtime_retention_signal_case HUP 129 -run_runtime_retention_signal_case INT 130 -run_runtime_retention_signal_case TERM 143 diff --git a/tests/test-push-worker-registration-03.sh b/tests/test-push-worker-registration-03.sh deleted file mode 100644 index 086aeda..0000000 --- a/tests/test-push-worker-registration-03.sh +++ /dev/null @@ -1,491 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-push-worker-registration-common.sh" - -( - root="$TEST_DIR/session-dual-publish"; setup_fixture "$root" - : > "$root/capture/terminate-calls" - terminate_managed_session() { printf '%s:%s:%s:%s:%s\n' "$@" >> "$root/capture/terminate-calls"; return 0; } - worker_session_state_publish_hook() { - local phase="$1" state="$2" - if [[ "$phase" == stage-ready && "$state" == cleanup_failed ]]; then - : > "$root/capture/stage.$BASHPID" - while [[ ! -e "$root/capture/release" ]]; do sleep 0.01; done - elif [[ "$phase" == after-rename && "$state" == cleanup_failed ]]; then - : > "$root/capture/renamed.$BASHPID" - fi - } - session_publisher() { - WORKER_TRANSFER_PID=$BASHPID; WORKER_TRANSFER_START=11 - WORKER_TRANSFER_PGID=$BASHPID; WORKER_TRANSFER_SID=$BASHPID - publish_worker_session_state active - publish_worker_session_state cleanup_failed - } - session_publisher & first=$!; session_publisher & second=$! - first_start=$(get_process_start_time "$first"); second_start=$(get_process_start_time "$second") - ACTIVE_WORKERS[$first]=$first_start; ACTIVE_WORKERS[$second]=$second_start - ACTIVE_WORKER_STATE_FILES[$first]="$TEMP_DIR/worker-session.$first.state" - ACTIVE_WORKER_STATE_FILES[$second]="$TEMP_DIR/worker-session.$second.state" - ACTIVE_WORKER_STATE_STARTS[$first]=$first_start; ACTIVE_WORKER_STATE_STARTS[$second]=$second_start - for _ in {1..300}; do [[ -e "$root/capture/stage.$first" && -e "$root/capture/stage.$second" ]] && break; sleep 0.01; done - [[ -e "$root/capture/stage.$first" && -e "$root/capture/stage.$second" ]] || fail "dual publishers did not reach pre-rename hook" - cleanup_published_worker_sessions - assert_eq 2 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback waits while both workers can still publish" - : > "$root/capture/release" - wait "$first"; wait "$second" - [[ -e "$root/capture/renamed.$first" && -e "$root/capture/renamed.$second" ]] || fail "dual publishers did not complete rename" - prune_active_workers - cleanup_published_worker_sessions - assert_eq 2 "$(wc -l < "$root/capture/terminate-calls")" "fallback processes both stable cleanup_failed manifests" - assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "dual fallback clears both state maps" - assert_eq 0 "${#ACTIVE_WORKER_STATE_STARTS[@]}" "dual fallback clears both state identities" - runtime_has_published_worker_state && fail "dual fallback left worker session state" - cleanup_runtime; assert_no_registration_residue "$root" - pass "dual worker state transitions stabilize after wait/reap" -) - -run_transient_state_case() ( - kind="$1"; root="$TEST_DIR/session-transient-$kind"; setup_fixture "$root" - worker=11111; start=22222; file="$TEMP_DIR/worker-session.$worker.state" - write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 - ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start - : > "$root/capture/terminate-calls" - terminate_managed_session() { printf 'call\n' >> "$root/capture/terminate-calls"; return 0; } - failures=2 - case "$kind" in - inode) - worker_session_state_read_hook() { - if [[ "$1" == after-open && "$failures" -gt 0 ]]; then - next="$file.next.$failures" - write_test_worker_session_manifest "$next" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 - mv -fT "$next" "$file"; ((failures -= 1)) - fi - } - ;; - open) - eval "$(declare -f open_worker_session_state_fd | sed '1s/open_worker_session_state_fd/original_open_worker_session_state_fd/')" - open_worker_session_state_fd() { - if (( failures > 0 )); then ((failures -= 1)); return "$WORKER_SESSION_STATE_TRANSIENT_STATUS"; fi - original_open_worker_session_state_fd "$@" - } - ;; - stat) - eval "$(declare -f worker_session_state_path_metadata | sed '1s/worker_session_state_path_metadata/original_worker_session_state_path_metadata/')" - worker_session_state_path_metadata() { - if (( failures > 0 )); then ((failures -= 1)); return "$WORKER_SESSION_STATE_TRANSIENT_STATUS"; fi - original_worker_session_state_path_metadata "$@" - } - ;; - esac - cleanup_published_worker_sessions - assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "$kind transient failure clears state map after stabilization" - assert_eq 1 "$(wc -l < "$root/capture/terminate-calls")" "$kind transient failure performs one trusted session cleanup" - [[ ! -e "$file" ]] || fail "$kind transient failure left state file" - cleanup_runtime; pass "$kind transient worker state failure is bounded and recoverable" -) -run_transient_state_case inode -run_transient_state_case open -run_transient_state_case stat - -( - root="$TEST_DIR/session-not-reaped"; setup_fixture "$root" - sleep 300 & worker=$!; start=$(get_process_start_time "$worker") - file="$TEMP_DIR/worker-session.$worker.state" - write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 - ACTIVE_WORKERS[$worker]=$start; ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start - : > "$root/capture/terminate-calls" - terminate_managed_session() { printf call >> "$root/capture/terminate-calls"; return 0; } - cleanup_published_worker_sessions - assert_eq 1 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback retains state while worker is not reaped" - assert_eq 0 "$(wc -c < "$root/capture/terminate-calls")" "fallback performs no session cleanup before reap" - kill -TERM "$worker"; wait "$worker" 2>/dev/null || true; prune_active_workers - cleanup_published_worker_sessions - assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback succeeds after worker reap" - cleanup_runtime; pass "worker reap gates final fallback" -) - -run_permanent_state_case() ( - kind="$1"; root="$TEST_DIR/session-permanent-$kind"; setup_fixture "$root" - worker=11111; start=22222; file="$TEMP_DIR/worker-session.$worker.state" - write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 - ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start - : > "$root/capture/terminate-calls" - terminate_managed_session() { printf call >> "$root/capture/terminate-calls"; return 0; } - case "$kind" in - symlink) mv "$file" "$file.real"; ln -s "$file.real" "$file" ;; - wrong-mode) chmod 0644 "$file" ;; - wrong-owner|wrong-gid) - stat() { - last=${!#} - if [[ "$last" == "$file" ]]; then - raw=$(command stat -Lc '%d:%i:%u:%g:%a:%s' -- "$file") - IFS=: read -r dev ino owner gid mode size <<< "$raw" - [[ "$kind" == wrong-owner ]] && owner=$((EUID + 1)) || gid=$((gid + 1)) - printf '%s:%s:%s:%s:%s:%s\n' "$dev" "$ino" "$owner" "$gid" "$mode" "$size" - else command stat "$@"; fi - } - ;; - malformed) printf malformed > "$file" ;; - duplicate) printf 'worker_pid=11111\n' >> "$file" ;; - unknown) printf 'unexpected=value\n' >> "$file" ;; - missing) sed -i '/^sid=/d' "$file" ;; - pid-mismatch) sed -i 's/^worker_pid=.*/worker_pid=99999/' "$file" ;; - stable-tamper) - worker_session_state_read_hook() { [[ "$1" != after-open ]] || printf tampered > "$file"; } - ;; - esac - rc=0; cleanup_published_worker_sessions || rc=$? - (( rc != 0 )) || fail "$kind permanent state unexpectedly succeeded" - assert_eq 1 "${#ACTIVE_WORKER_STATE_FILES[@]}" "$kind permanent state preserves map evidence" - assert_eq 0 "$(wc -c < "$root/capture/terminate-calls")" "$kind permanent state executes no session cleanup" - [[ -e "$file" || -L "$file" ]] || fail "$kind permanent state deleted evidence" - unset 'ACTIVE_WORKER_STATE_FILES[$worker]' 'ACTIVE_WORKER_STATE_STARTS[$worker]' - command rm -f -- "$file" "$file.real" 2>/dev/null || true - cleanup_runtime; pass "$kind permanent worker state remains rejected" -) -for permanent_state in symlink wrong-mode wrong-owner wrong-gid malformed duplicate unknown missing pid-mismatch stable-tamper; do - run_permanent_state_case "$permanent_state" -done - -run_parallel_fallback_iteration() ( - run="$1"; root="$TEST_DIR/session-parallel-$run"; setup_fixture "$root" - MAX_PARALLEL=2 - push_to_server() { - local server="$1" lock_fd - exec {lock_fd}>>"$root/capture/calls.lock"; flock -x "$lock_fd" - printf '%s\n' "$server" >> "$root/capture/calls"; : > "$root/capture/ready.$server" - flock -u "$lock_fd"; exec {lock_fd}>&- - while [[ ! -e "$root/capture/release" ]]; do sleep 0.01; done - WORKER_TRANSFER_PID=$BASHPID - WORKER_TRANSFER_START=11 - WORKER_TRANSFER_PGID=$BASHPID - WORKER_TRANSFER_SID=$BASHPID - publish_worker_session_state cleanup_failed - return "$MANAGED_CLEANUP_FAILURE_STATUS" - } - terminate_managed_session() { return 0; } - ( - deadline=$((SECONDS + 15)); status=0 - while (( SECONDS < deadline )); do - if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release"; break; fi - sleep 0.01 - done - [[ -e "$root/capture/release" ]] || { : > "$root/capture/coordinator-timeout"; : > "$root/capture/release"; status=1; } - while [[ ! -e "$root/capture/stop" ]]; do sleep 0.01; done - exit "$status" - ) & coordinator=$! - rc=0; run_server_batch source destination one.example two.example three.example four.example || rc=$? - : > "$root/capture/stop"; coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? - assert_eq 0 "$coordinator_rc" "parallel fallback run $run coordinator releases both workers" - assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "parallel fallback run $run returns lifecycle status" - assert_eq 2 "$(wc -l < "$root/capture/calls")" "parallel fallback run $run starts only first two workers" - assert_eq 0 "${#ACTIVE_WORKERS[@]}" "parallel fallback run $run reaps active workers" - assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "parallel fallback run $run clears state maps" - assert_eq 0 "${#ACTIVE_WORKER_STATE_STARTS[@]}" "parallel fallback run $run clears state identities" - [[ ! -e "$root/capture/ready.three.example" && ! -e "$root/capture/ready.four.example" ]] || fail "parallel fallback run $run launched later worker" - runtime_has_published_worker_state && fail "parallel fallback run $run left session state" - reset_batch_lifecycle_barrier; cleanup_runtime; assert_no_registration_residue "$root" -) -parallel_fallback_runs=${PUSH_WORKER_REGISTRATION_PARALLEL_STRESS:-1} -for parallel_run in $(seq 1 "$parallel_fallback_runs"); do run_parallel_fallback_iteration "$parallel_run"; done -pass "parallel fallback completed $parallel_fallback_runs deterministic run(s)" - -write_active_grace_fixture() { - local root="$1" - mkdir -m 0700 "$root" "$root/bin" "$root/runtime" "$root/ssh" "$root/pids" - printf source > "$root/source" - cat > "$root/config.conf" < "$root/bin/timeout" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/timeout.pid" -[[ ${1:-} == --kill-after=* ]] && shift -shift -"$@" & -wait "$!" -EOF - cat > "$root/bin/sshpass" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/sshpass.pid" -[[ ${1:-} == -e ]] && shift -"$@" & -wait "$!" -EOF - cat > "$root/bin/rsync" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -set -m -printf '%s\n' "$$" > "$PID_DIR/rsync.pid" -ssh fake-target & -wait "$!" -EOF - cat > "$root/bin/ssh" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/ssh.pid" -if [[ "${STATE_PUBLISH_MODE:-active}" == cleanup ]]; then - while [[ ! -e "$STATE_CLEANUP_RELEASE" ]]; do sleep 0.01; done - exit 0 -fi -sleep 300 & -printf '%s\n' "$!" > "$PID_DIR/leaf.pid" -wait "$!" -EOF - chmod 0700 "$root/bin"/* -} -cat > "$TEST_DIR/active-grace-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" -shift -source "$script" -worker_after_session_cleanup_hook() { - : > "$ACTIVE_GRACE_STARTED" - sleep 3.2 - : > "$ACTIVE_GRACE_FINISHED" -} -main "$@" -CHILD -chmod 0700 "$TEST_DIR/active-grace-child.sh" -run_active_grace_signal_case() { - local signal_name="$1" expected="$2" root - local main_pid watchdog unrelated rc=0 state_file file pid - root="$TEST_DIR/active-grace-$signal_name" - CURRENT_FIXTURE_ROOT=$root - write_active_grace_fixture "$root" - sleep 60 & unrelated=$! - ( - cd "$root" - exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ - ACTIVE_GRACE_STARTED="$root/cleanup-started" ACTIVE_GRACE_FINISHED="$root/cleanup-finished" \ - bash "$TEST_DIR/active-grace-child.sh" "$SCRIPT" "$root/source" /remote/path - ) > "$root/output.log" 2>&1 & - main_pid=$! - for _ in {1..400}; do - state_file=$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null || true) - [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] && break - sleep 0.05 - done - if [[ -z "${state_file:-}" || ! -f "$root/pids/leaf.pid" ]]; then - cat "$root/output.log" >&2 || true - kill "$main_pid" "$unrelated" 2>/dev/null || true - fail "$signal_name active grace fixture did not enter managed session" - fi - test_watchdog_process 20 "$root/watchdog-timeout" "$main_pid" & watchdog=$! - kill "-$signal_name" "$main_pid" - wait "$main_pid" || rc=$? - kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root/watchdog-timeout" ]] || fail "$signal_name active grace exceeded watchdog" - assert_eq "$expected" "$rc" "$signal_name active worker preserves main signal status" - [[ -e "$root/cleanup-started" && -e "$root/cleanup-finished" ]] || { cat "$root/output.log" >&2 || true; fail "$signal_name active worker was killed before cleanup hook completed"; } - for file in "$root/pids"/*.pid; do - [[ -f "$file" ]] || continue - pid=$(<"$file") - process_is_running "$pid" && fail "$signal_name active grace left $(basename "$file") pid=$pid" - done - [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$signal_name active grace left runtime state" - kill -0 "$unrelated" 2>/dev/null || fail "$signal_name active grace killed unrelated process" - kill "$unrelated" 2>/dev/null || true; wait "$unrelated" 2>/dev/null || true - pass "$signal_name active worker receives full managed-session cleanup grace" -} -run_active_grace_signal_case HUP 129 -run_active_grace_signal_case INT 130 -run_active_grace_signal_case TERM 143 - -cat > "$TEST_DIR/live-cleanup-failure-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" -shift -source "$script" -main_owner=$BASHPID -eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" -terminate_managed_session() { - if [[ "$BASHPID" != "$main_owner" ]]; then return 1; fi - original_terminate_managed_session "$@" -} -main "$@" -CHILD -chmod 0700 "$TEST_DIR/live-cleanup-failure-child.sh" - -run_live_cleanup_failure_signal_case() ( - trap - EXIT HUP INT TERM - local signal_name="$1" expected="$2" root main_pid main_start state_file="" saved_temp - local worker_pid="" worker_start="" leader_pid="" leader_start="" managed_sid="" watchdog="" unrelated="" unrelated_start="" rc=0 - cleanup_live_failure_fixture() { - [[ -z "$worker_pid" || -z "$worker_start" ]] || { test_process_identity_exists "$worker_pid" "$worker_start" && kill -KILL "$worker_pid" 2>/dev/null || true; } - [[ -z "$main_pid" || -z "$main_start" ]] || { test_process_identity_exists "$main_pid" "$main_start" && kill -KILL "$main_pid" 2>/dev/null || true; } - [[ -z "$unrelated" || -z "$unrelated_start" ]] || { test_process_identity_exists "$unrelated" "$unrelated_start" && kill -TERM "$unrelated" 2>/dev/null || true; } - [[ -z "$main_pid" ]] || wait "$main_pid" 2>/dev/null || true - [[ -z "$watchdog" ]] || { kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true; } - [[ -z "$unrelated" ]] || wait "$unrelated" 2>/dev/null || true - } - trap cleanup_live_failure_fixture EXIT - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - root="$TEST_DIR/live-cleanup-failure-$signal_name"; CURRENT_FIXTURE_ROOT=$root - write_active_grace_fixture "$root" - sleep 60 & unrelated=$!; unrelated_start=$(wait_test_process_start "$unrelated") - ( - cd "$root" - exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ - bash "$TEST_DIR/live-cleanup-failure-child.sh" "$SCRIPT" "$root/source" /remote/path - ) > "$root/output.log" 2>&1 & - main_pid=$!; main_start=$(wait_test_process_start "$main_pid") - for _ in {1..500}; do - state_file=$(find "$root/runtime" -type f -name 'worker-session.*.state' -print -quit 2>/dev/null || true) - [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] && break - sleep 0.05 - done - [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] || fail "$signal_name cleanup-failure fixture did not reach live session" - saved_temp=$TEMP_DIR; TEMP_DIR=$(dirname "$state_file") - read_worker_session_state "$state_file" || fail "$signal_name cleanup-failure state unreadable" - worker_pid=$SESSION_WORKER_PID; worker_start=$SESSION_WORKER_START - leader_pid=$SESSION_LEADER_PID; leader_start=$SESSION_LEADER_START; managed_sid=$SESSION_SID - TEMP_DIR=$saved_temp - wait_test_process_identity_present "$worker_pid" "$worker_start" || fail "$signal_name cleanup-failure worker identity missing" - wait_test_process_identity_present "$leader_pid" "$leader_start" || fail "$signal_name cleanup-failure leader identity missing" - test_watchdog_process 30 "$root/watchdog-fired" "$main_pid" & watchdog=$! - kill "-$signal_name" "$main_pid" - wait "$main_pid" || rc=$? - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root/watchdog-fired" ]] || fail "$signal_name cleanup failure fired watchdog" - assert_eq "$expected" "$rc" "$signal_name cleanup failure preserves main signal status" - wait_test_process_identity_gone "$worker_pid" "$worker_start" || fail "$signal_name cleanup failure left worker identity" - wait_test_process_identity_gone "$leader_pid" "$leader_start" || fail "$signal_name cleanup failure left leader identity" - collect_owned_session_records "$managed_sid" session_pids session_starts session_ppids session_pgids session_states || fail "$signal_name cleanup failure cannot verify managed SID" - assert_eq 0 "${#session_pids[@]}" "$signal_name cleanup failure leaves no managed SID member or zombie" - [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$signal_name cleanup failure left runtime or state" - test_process_identity_exists "$unrelated" "$unrelated_start" || fail "$signal_name cleanup failure killed unrelated process" - kill -TERM "$unrelated"; wait "$unrelated" 2>/dev/null || true - main_pid=""; main_start=""; watchdog=""; unrelated=""; unrelated_start="" - trap - EXIT - pass "$signal_name cleanup failure uses parent fallback without watchdog or zombie" -) -run_live_cleanup_failure_signal_case HUP 129 -run_live_cleanup_failure_signal_case INT 130 -run_live_cleanup_failure_signal_case TERM 143 - -cat > "$TEST_DIR/state-publication-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -script="$1" -shift -source "$script" -main_owner=$BASHPID -eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" -terminate_managed_session() { - if [[ "$STATE_PUBLISH_MODE" == cleanup && "$BASHPID" != "$main_owner" ]]; then return 1; fi - original_terminate_managed_session "$@" -} -worker_session_state_publish_hook() { - local phase="$1" state="$2" state_file="$3" hook_stage="$4" marker_stage worker_start - if [[ "$STATE_PUBLISH_MODE" == cleanup && "$state" == active && "$phase" == state-file-assigned ]]; then - : > "$STATE_CLEANUP_RELEASE" - fi - [[ "$phase" == "$STATE_PUBLISH_PHASE" && "$state" == "$STATE_PUBLISH_STATE" ]] || return 0 - [[ ! -e "$STATE_PUBLISH_MARKER" ]] || return 0 - worker_start="" - for _ in {1..20}; do - if read_process_record "$BASHPID" && [[ "$PROC_UID" == "$EUID" && "$PROC_STATE" != Z ]]; then worker_start=$PROC_START; break; fi - sleep 0.01 - done - [[ -n "$worker_start" ]] || return 1 - marker_stage="$STATE_PUBLISH_MARKER.$BASHPID.stage" - (umask 077; set -o noclobber; : > "$marker_stage") || return 1 - printf 'phase=%s\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\nstate_file=%s\nhook_stage=%s\n' \ - "$phase" "$state" "$BASHPID" "$worker_start" "$WORKER_TRANSFER_PID" "$WORKER_TRANSFER_START" \ - "$WORKER_TRANSFER_PGID" "$WORKER_TRANSFER_SID" "$state_file" "$hook_stage" > "$marker_stage" - chmod 0600 "$marker_stage"; mv -fT "$marker_stage" "$STATE_PUBLISH_MARKER" - while :; do :; done -} -main "$@" -CHILD -chmod 0700 "$TEST_DIR/state-publication-child.sh" - -run_state_publication_signal_case() ( - local state="$1" phase="$2" signal_name="$3" expected="$4" mode root main_pid="" main_start="" worker_pid="" worker_start="" leader_pid="" leader_start="" managed_sid="" - local watchdog="" watchdog_start="" unrelated="" unrelated_start="" rc=0 line key value marker state_file hook_stage - local -a session_pids=() - local -A session_starts=() session_ppids=() session_pgids=() session_states=() - cleanup_publication_fixture() { - [[ -z "$worker_pid" || -z "$worker_start" ]] || { test_process_identity_exists "$worker_pid" "$worker_start" && kill -KILL "$worker_pid" 2>/dev/null || true; } - [[ -z "$main_pid" || -z "$main_start" ]] || { test_process_identity_exists "$main_pid" "$main_start" && kill -KILL "$main_pid" 2>/dev/null || true; } - [[ -z "$watchdog" || -z "$watchdog_start" ]] || { test_process_identity_exists "$watchdog" "$watchdog_start" && kill -TERM "$watchdog" 2>/dev/null || true; } - [[ -z "$unrelated" || -z "$unrelated_start" ]] || { test_process_identity_exists "$unrelated" "$unrelated_start" && kill -TERM "$unrelated" 2>/dev/null || true; } - [[ -z "$main_pid" ]] || wait "$main_pid" 2>/dev/null || true - [[ -z "$watchdog" ]] || wait "$watchdog" 2>/dev/null || true - [[ -z "$unrelated" ]] || wait "$unrelated" 2>/dev/null || true - } - trap cleanup_publication_fixture EXIT - mode=active; [[ "$state" == cleanup_failed ]] && mode=cleanup - root="$TEST_DIR/state-publish-$state-$phase-$signal_name${5:-}"; CURRENT_FIXTURE_ROOT=$root - write_active_grace_fixture "$root" - mkdir -m 0700 "$root/capture" - marker="$root/capture/publish-marker" - sleep 60 & unrelated=$!; unrelated_start=$(wait_test_process_start "$unrelated") - ( - cd "$root" - exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ - STATE_PUBLISH_MODE="$mode" STATE_PUBLISH_STATE="$state" STATE_PUBLISH_PHASE="$phase" STATE_PUBLISH_MARKER="$marker" \ - STATE_CLEANUP_RELEASE="$root/capture/cleanup-release" bash "$TEST_DIR/state-publication-child.sh" "$SCRIPT" "$root/source" /remote/path - ) > "$root/output.log" 2>&1 & - main_pid=$!; main_start=$(wait_test_process_start "$main_pid") - for _ in {1..600}; do [[ -f "$marker" ]] && break; sleep 0.05; done - [[ -f "$marker" && ! -L "$marker" && "$(stat -Lc %a "$marker")" == 600 ]] || { cat "$root/output.log" >&2 || true; fail "$state/$phase/$signal_name did not reach publish hook"; } - while IFS= read -r line; do key=${line%%=*}; value=${line#*=}; case "$key" in worker_pid) worker_pid=$value;;worker_start) worker_start=$value;;leader_pid) leader_pid=$value;;leader_start) leader_start=$value;;sid) managed_sid=$value;;state_file) state_file=$value;;hook_stage) hook_stage=$value;;esac; done < "$marker" - [[ "$worker_pid" =~ ^[1-9][0-9]*$ && "$worker_start" =~ ^[1-9][0-9]*$ && "$managed_sid" =~ ^[1-9][0-9]*$ ]] || fail "$state/$phase/$signal_name marker identity malformed" - wait_test_process_identity_present "$worker_pid" "$worker_start" || { cat "$marker" >&2; fail "$state/$phase/$signal_name worker identity missing at hook"; } - test_watchdog_process 30 "$root/watchdog-timeout" "$main_pid" & watchdog=$!; watchdog_start=$(wait_test_process_start "$watchdog") - printf "DIAG: publication state=%s phase=%s signal=%s main=%s/%s worker=%s/%s sid=%s\n" \ - "$state" "$phase" "$signal_name" "$main_pid" "$main_start" "$worker_pid" "$worker_start" "$managed_sid" - kill "-$signal_name" "$main_pid"; wait "$main_pid" || rc=$? - kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$root/watchdog-timeout" ]] || fail "$state/$phase/$signal_name watchdog fired" - assert_eq "$expected" "$rc" "$state/$phase preserves $signal_name status" - wait_test_process_identity_gone "$worker_pid" "$worker_start" || fail "$state/$phase/$signal_name left worker identity" - if [[ "$leader_pid" =~ ^[1-9][0-9]*$ && "$leader_start" =~ ^[1-9][0-9]*$ ]]; then - if ! wait_test_process_identity_gone "$leader_pid" "$leader_start"; then - read_process_record "$leader_pid" || true - printf 'leader remains pid=%s expected_start=%s actual_start=%s state=%s ppid=%s pgid=%s sid=%s\n' \ - "$leader_pid" "$leader_start" "${PROC_START:-unknown}" "${PROC_STATE:-unknown}" "${PROC_PPID:-unknown}" "${PROC_PGID:-unknown}" "${PROC_SID:-unknown}" >&2 - cat "/proc/$leader_pid/stat" >&2 2>/dev/null || true - fail "$state/$phase/$signal_name left leader identity" - fi - fi - collect_owned_session_records "$managed_sid" session_pids session_starts session_ppids session_pgids session_states || fail "$state/$phase/$signal_name cannot verify managed SID" - (( ${#session_pids[@]} == 0 )) || fail "$state/$phase/$signal_name left managed SID member: ${session_pids[*]}" - [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$state/$phase/$signal_name left runtime/session/registration state" - [[ ! -e "$state_file" && ! -e "$hook_stage" && ! -L "$state_file" && ! -L "$hook_stage" ]] || fail "$state/$phase/$signal_name left publication path" - test_process_identity_exists "$unrelated" "$unrelated_start" || fail "$state/$phase/$signal_name killed unrelated process" - kill -TERM "$unrelated"; wait "$unrelated" 2>/dev/null || true - unrelated=""; unrelated_start=""; main_pid=""; main_start=""; watchdog=""; watchdog_start="" - trap - EXIT - pass "$state/$phase/$signal_name publication window cleans process tree and runtime" -) -publication_states=${PUSH_STATE_PUBLICATION_STATES:-"active cleanup_failed"} -publication_phases=${PUSH_STATE_PUBLICATION_PHASES:-"before-stage-create stage-ready after-rename state-file-assigned"} -publication_signals=${PUSH_STATE_PUBLICATION_SIGNALS:-"HUP INT TERM"} \ No newline at end of file diff --git a/tests/test-push-worker-registration-04.sh b/tests/test-push-worker-registration-04.sh deleted file mode 100644 index 019fb99..0000000 --- a/tests/test-push-worker-registration-04.sh +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail -source "$(dirname "${BASH_SOURCE[0]}")/lib/test-push-worker-registration-common.sh" - -for publication_state in $publication_states; do - for publication_phase in $publication_phases; do - for publication_signal in $publication_signals; do - case "$publication_signal" in HUP) publication_status=129 ;; INT) publication_status=130 ;; TERM) publication_status=143 ;; *) fail "unknown publication signal" ;; esac - run_state_publication_signal_case "$publication_state" "$publication_phase" "$publication_signal" "$publication_status" - done - done -done - -( - # Three bounded extra probes; preserve original matrix and stop at first failure. - for diagnostic_round in 1 2 3; do - printf "DIAG: extra HUP probe=%s/3\n" "$diagnostic_round" - run_state_publication_signal_case cleanup_failed after-rename HUP 129 "-probe-$diagnostic_round" - done - root="$TEST_DIR/normal-parallel"; setup_fixture "$root" - : > "$root/capture/current"; : > "$root/capture/max" - push_to_server() { - local server="$1" lock_fd current max group - case "$server" in one.example|two.example) group=first ;; *) group=second ;; esac - exec {lock_fd}>>"$root/capture/lock"; flock -x "$lock_fd" - current=$(<"$root/capture/current"); ((current += 1)); printf '%s' "$current" > "$root/capture/current" - max=$(<"$root/capture/max"); (( current > max )) && printf '%s' "$current" > "$root/capture/max" - printf '%s\n' "$server" >> "$root/capture/transfer-calls"; : > "$root/capture/ready.$server" - flock -u "$lock_fd"; exec {lock_fd}>&- - while [[ ! -e "$root/capture/release.$group" ]]; do sleep 0.02; done - exec {lock_fd}>>"$root/capture/lock"; flock -x "$lock_fd" - current=$(<"$root/capture/current"); ((current -= 1)); printf '%s' "$current" > "$root/capture/current" - flock -u "$lock_fd"; exec {lock_fd}>&- - record_result success "$server" - } - ( - deadline=$((SECONDS + 15)) - while (( SECONDS < deadline )); do - if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release.first"; break; fi - sleep 0.02 - done - [[ -e "$root/capture/release.first" ]] || { : > "$root/capture/coordinator-timeout"; : > "$root/capture/release.first"; : > "$root/capture/release.second"; exit 1; } - while (( SECONDS < deadline + 15 )); do - if [[ -e "$root/capture/ready.three.example" && -e "$root/capture/ready.four.example" ]]; then : > "$root/capture/release.second"; exit 0; fi - sleep 0.02 - done - : > "$root/capture/coordinator-timeout"; : > "$root/capture/release.second"; exit 1 - ) & coordinator=$! - run_server_batch source destination one.example two.example three.example four.example - coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? - assert_eq 0 "$coordinator_rc" "normal parallel coordinator releases both worker pairs" - assert_eq 2 "$(<"$root/capture/max")" "normal registration preserves MAX_PARALLEL=2" - assert_eq 4 "$(wc -l < "$root/capture/transfer-calls")" "normal registration completes four fake servers" - assert_no_registration_residue "$root"; cleanup_runtime; assert_no_registration_residue "$root" - pass "normal registration/session lifecycle cleans all state" -) -printf 'All push worker registration tests passed.\n' diff --git a/tests/test-push-worker-registration.sh b/tests/test-push-worker-registration.sh new file mode 100755 index 0000000..3ef0fcf --- /dev/null +++ b/tests/test-push-worker-registration.sh @@ -0,0 +1,1553 @@ +#!/usr/bin/env bash +set -euo pipefail +# Safe runtime identity only: no environment dump, credentials, or xtrace. +printf "DIAG: bash=%s kernel=%s\n" "$BASH_VERSION" "$(uname -r)" +dpkg-query -W bash libc6 2>/dev/null || true + +ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +readonly ROOT_DIR +readonly SCRIPT="$ROOT_DIR/tools/push.sh" +TEST_DIR=$(mktemp -d) +readonly TEST_DIR +CURRENT_FIXTURE_ROOT="" +trap 'rm -rf "$TEST_DIR"' EXIT + +failure_diagnostics() { + local root=${CURRENT_FIXTURE_ROOT:-} file pid + [[ -n "$root" && -d "$root" ]] || return 0 + printf '%s\n' '=== worker registration diagnostics ===' >&2 + printf '%s\n' '--- coordinator/status ---' >&2 + for file in "$root"/capture/coordinator* "$root"/capture/watchdog* "$root"/capture/timeout*; do + [[ -e "$file" || -L "$file" ]] || continue + printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true + done + printf '%s\n' '--- registration/release/stages ---' >&2 + find "$root" \( -name 'worker-registration.*' -o -name '.worker-registration.*' \) -print 2>/dev/null | sort >&2 || true + for file in "$root"/runtime/push-runtime.*/worker-registration.*; do + [[ -e "$file" || -L "$file" ]] || continue + printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true + done + printf '%s\n' '--- worker/session states ---' >&2 + for file in "$root"/runtime/push-runtime.*/worker-session.*.state; do + [[ -e "$file" || -L "$file" ]] || continue + printf '[%s]\n' "$file" >&2; cat "$file" >&2 2>/dev/null || true + done + printf '%s\n' '--- jobs ---' >&2; jobs -l >&2 || true + printf '%s\n' '--- runtime tree ---' >&2 + find "$root/runtime" -mindepth 1 -maxdepth 4 -printf '%y %m %p\n' 2>/dev/null | sort >&2 || true + printf '%s\n' '--- process output ---' >&2 + [[ -f "$root/output.log" ]] && cat "$root/output.log" >&2 2>/dev/null || true + printf '%s\n' '--- fake command calls ---' >&2 + for file in "$root"/capture/calls "$root"/capture/transfer-calls; do + [[ -e "$file" ]] && { printf '[%s]\n' "$file" >&2; cat "$file" >&2 || true; } + done + printf '%s\n' '--- recorded proc identities ---' >&2 + for file in "$root"/capture/*.pid; do + [[ -f "$file" && ! -L "$file" ]] || continue + pid=$(<"$file") + [[ "$pid" =~ ^[1-9][0-9]*$ && -r "/proc/$pid/stat" ]] && printf 'pid=%s stat=%s\n' "$pid" "$(<"/proc/$pid/stat")" >&2 + done +} +fail() { failure_diagnostics; printf 'FAIL: %s\n' "$*" >&2; exit 1; } +pass() { printf 'PASS: %s\n' "$*"; } +assert_eq() { [[ "$1" == "$2" ]] || fail "$3: expected '$1', got '$2'"; pass "$3"; } +process_is_running() { local pid="$1" state; [[ -r "/proc/$pid/stat" ]] || return 1; state=$(awk '{print $3}' "/proc/$pid/stat" 2>/dev/null) || return 1; [[ "$state" != Z ]]; } +test_process_identity_exists() { + local pid="$1" start="$2" + read_process_record "$pid" || return 1 + [[ "$PROC_UID" == "$EUID" && "$PROC_START" == "$start" ]] +} +wait_test_process_identity_gone() { + local pid="$1" start="$2" _ + for _ in {1..100}; do test_process_identity_exists "$pid" "$start" || return 0; sleep 0.05; done + return 1 +} +wait_test_process_identity_present() { + local pid="$1" start="$2" _ + for _ in {1..30}; do test_process_identity_exists "$pid" "$start" && return 0; [[ -r "/proc/$pid/stat" ]] || return 1; sleep 0.01; done + return 1 +} +wait_test_process_start() { + local pid="$1" value="" _ + for _ in {1..100}; do value=$(get_process_start_time "$pid" 2>/dev/null || true); [[ -n "$value" ]] && { printf '%s\n' "$value"; return 0; }; sleep 0.005; done + return 1 +} + +source_root="$TEST_DIR/source"; mkdir -m 0700 "$source_root" +source_result=$(env TMPDIR="$source_root" SCRIPT="$SCRIPT" bash -c ' +set -euo pipefail +before=$(find "$TMPDIR" -mindepth 1 -print -quit) +source "$SCRIPT" +[[ -z "$TEMP_DIR" && -z "$SUCCESS_FILE" && -z "$FAILED_FILE" ]] +[[ "$before" == "$(find "$TMPDIR" -mindepth 1 -print -quit)" ]] +printf source-safe +') +assert_eq source-safe "$source_result" "source keeps zero runtime side effects" +# shellcheck source=../tools/push.sh +source "$SCRIPT" +( + # Inert direct-child job table: no SSH or production services. + sleep 10 & snapshot_child=$! + cleanup_snapshot_child() { kill -TERM "$snapshot_child" 2>/dev/null || true; wait "$snapshot_child" 2>/dev/null || true; } + trap cleanup_snapshot_child EXIT + job_is_active "$snapshot_child" || fail "snapshot misses active direct child" + if job_is_active 0; then fail "snapshot accepts absent PID"; fi + kill -TERM "$snapshot_child" + wait "$snapshot_child" 2>/dev/null || true + if job_is_active "$snapshot_child"; then fail "snapshot retains reaped child"; fi + trap - EXIT + pass "job snapshot preserves direct-child matching and reap semantics" +) +trap 'rm -rf "$TEST_DIR"' EXIT + +# Deterministic coordinator contract: a live worker can republish after cleanup. +( + root="$TEST_DIR/republish-tracking"; mkdir -m 0700 "$root" + state_file="$root/worker.state"; : > "$state_file" + ACTIVE_WORKERS=([424242]=12345) + ACTIVE_WORKER_STATE_FILES=([424242]="$state_file") + ACTIVE_WORKER_STATE_STARTS=([424242]=12345) + worker_session_state_maybe_cleanup_failed() { return 0; } + cleanup_calls=0 + cleanup_worker_session_state_file() { cleanup_calls=$((cleanup_calls + 1)); rm -f -- "$1"; } + cleanup_active_failed_worker_sessions + [[ ${ACTIVE_WORKER_STATE_FILES[424242]:-} == "$state_file" && ${ACTIVE_WORKER_STATE_STARTS[424242]:-} == 12345 ]] || fail "active cleanup lost live publisher tracking" + : > "$state_file" + ACTIVE_WORKERS=() + job_is_active() { return 1; } + cleanup_published_worker_sessions + assert_eq 2 "$cleanup_calls" "post-reap fallback handles republished state" + [[ ! -e $state_file && ${#ACTIVE_WORKER_STATE_FILES[@]} == 0 && ${#ACTIVE_WORKER_STATE_STARTS[@]} == 0 ]] || fail "republished state remains" + pass "live publisher tracking survives successful active cleanup" +) + +setup_fixture() { + local root="$1" + CURRENT_FIXTURE_ROOT=$root + mkdir -m 0700 "$root" "$root/runtime" "$root/capture" + TMPDIR="$root/runtime"; initialize_runtime + MAX_PARALLEL=2 + MAX_RETRIES=1 + BATCH_WORKER_FAILED=false + BATCH_WORKER_ERROR=false +} +assert_no_registration_residue() { + local root="$1" + [[ -z "$(find "$root/runtime" \( -name 'worker-registration.*' -o -name '.worker-registration.*' -o -name 'worker-session.*.state' \) -print -quit)" ]] || fail "registration/session residue remains" + (( ${#REGISTERING_WORKERS[@]} == 0 && ${#ACTIVE_WORKERS[@]} == 0 )) || fail "worker arrays remain" + (( ${#WORKER_REGISTRATION_READY_FILES[@]} == 0 && ${#WORKER_REGISTRATION_RELEASE_FILES[@]} == 0 && ${#WORKER_REGISTRATION_STARTS[@]} == 0 && ${#WORKER_REGISTRATION_STAGE_FILES[@]} == 0 )) || fail "registration arrays remain" +} + +run_delayed_parent_reap_case() ( + local depth="$1" root leader leader_start leader_pgid leader_sid + root="$TEST_DIR/delayed-reap-$depth" + local state_file elapsed_start elapsed_end rc=0 + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + setup_fixture "$root" + mkfifo "$root/reap-clock" + cat > "$root/parent.sh" <<'CHILD' +#!/usr/bin/env bash +set -u +root="$1" depth="$2" +trap '' HUP INT TERM + : > "$root/parent-open" +if (( depth == 0 )); then + sleep 300 & child=$! + printf '%s\n' "$child" > "$root/leaf.pid" +else + bash "$root/parent.sh" "$root" "$((depth - 1))" & child=$! +fi +kill -STOP "$BASHPID" +wait "$child" 2>/dev/null || true +exit 0 +CHILD + chmod 0700 "$root/parent.sh" + setsid bash "$root/parent.sh" "$root" "$depth" & leader=$! + leader_start=$(wait_test_process_start "$leader") + for _ in {1..100}; do [[ -e "$root/parent-open" ]] && break; sleep .01; done + read_process_record "$leader"; leader_pgid=$PROC_PGID; leader_sid=$PROC_SID + [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "delayed $depth leader identity invalid" + WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start + WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid + publish_worker_session_state active; state_file=$WORKER_SESSION_STATE_FILE + leaf=$(<"$root/leaf.pid"); leaf_start=$(wait_test_process_start "$leaf") + kill -KILL "$leaf" 2>/dev/null || true + old_reap_status=1 + for _ in {1..30}; do + session_process_identity_matches "$leaf" "$leaf_start" "$leader_sid" || { old_reap_status=0; break; } + sleep 0.01 + done + assert_eq 1 "$old_reap_status" "delayed $depth old 300ms window remains pending" + ( sleep 1; kill -CONT -"$leader" 2>/dev/null || true ) & releaser=$! + cleanup_clock; elapsed_start=$CLEANUP_NOW + terminate_worker_transfer || rc=$? + wait "$releaser" 2>/dev/null || true + cleanup_clock; elapsed_end=$CLEANUP_NOW + assert_eq 0 "$rc" "delayed $depth parent reap succeeds" + (( elapsed_end - elapsed_start >= 100 )) || fail "delayed $depth reap did not wait for parent" + wait_test_process_identity_gone "$leader" "$leader_start" || fail "delayed $depth left leader" + collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states + assert_eq 0 "${#session_pids[@]}" "delayed $depth managed SID clears" + [[ ! -e "$state_file" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "delayed $depth state cleanup incomplete" + cleanup_runtime + exec 9>&- + pass "delayed parent reap depth $depth preserves leaf-first order" +) +run_delayed_parent_reap_case 1 +run_delayed_parent_reap_case 4 + +( + root="$TEST_DIR/reap-deadline"; setup_fixture "$root" + cat > "$root/parent.sh" <<'CHILD' +#!/usr/bin/env bash +set -u +root="$1" +trap '' HUP INT TERM +sleep 300 & child=$! +: > "$root/ready" +while [[ ! -e "$root/release" ]]; do :; done +wait "$child" 2>/dev/null || true +exit 0 +CHILD + chmod 0700 "$root/parent.sh" + setsid bash "$root/parent.sh" "$root" & leader=$! + for _ in {1..100}; do [[ -e "$root/ready" ]] && break; sleep .01; done + leader_start=$(wait_test_process_start "$leader"); read_process_record "$leader" + WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start; WORKER_TRANSFER_PGID=$PROC_PGID; WORKER_TRANSFER_SID=$PROC_SID + publish_worker_session_state active; state_file=$WORKER_SESSION_STATE_FILE + MANAGED_SESSION_CLEANUP_TICKS=300 + rc=0; terminate_worker_transfer || rc=$? + (( rc != 0 )) || fail "deadline failure unexpectedly succeeded" + [[ -d "$TEMP_DIR" && -f "$state_file" && "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || fail "deadline failure discarded evidence" + [[ "$(stat -c %a "$TEMP_DIR")" == 700 && "$(stat -c %a "$state_file")" == 600 ]] || fail "deadline evidence trust changed" + : > "$root/release"; wait "$leader" 2>/dev/null || true + remove_worker_session_state + WORKER_TRANSFER_PID=""; WORKER_TRANSFER_START=""; WORKER_TRANSFER_PGID=""; WORKER_TRANSFER_SID=""; WORKER_TRANSFER_CLEANUP_FAILED=false + clear_managed_session_cleanup_tracking + cleanup_runtime + [[ ! -e "$TEMP_DIR" ]] || fail "deadline teardown left runtime" + exec 9>&- + pass "deadline failure is bounded and preserves then safely clears evidence" +) +if [[ "${PUSH_REAP_ONLY:-false}" == true ]]; then + exit 0 +fi +test_watchdog_process() { + local seconds="$1" marker="$2" target_pid="$3" timer="" + trap '[[ -z "$timer" ]] || kill "$timer" 2>/dev/null || true; [[ -z "$timer" ]] || wait "$timer" 2>/dev/null || true; exit 0' HUP INT TERM + sleep "$seconds" & timer=$! + wait "$timer" 2>/dev/null || exit 0 + printf fired > "$marker" + kill -KILL "$target_pid" 2>/dev/null || true +} + +run_with_watchdog() { + local name="$1" seconds="$2"; shift 2 + local marker="$TEST_DIR/watchdog-$name" pid watchdog rc=0 + "$@" & pid=$! + test_watchdog_process "$seconds" "$marker" "$pid" & watchdog=$! + wait "$pid" || rc=$? + kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$marker" ]] || fail "$name exceeded watchdog deadline" + return "$rc" +} + +root_cause_regression_case() ( + root="$TEST_DIR/root-cause"; setup_fixture "$root" + GET_START_CALLS=0 + get_process_start_time() { ((GET_START_CALLS += 1)); return 1; } + push_to_server() { + local server="$1" lock_fd + exec {lock_fd}>>"$root/capture/calls.lock"; flock -x "$lock_fd" + printf '%s\n' "$server" >> "$root/capture/calls"; : > "$root/capture/ready.$server" + flock -u "$lock_fd"; exec {lock_fd}>&- + while [[ ! -e "$root/capture/release" ]]; do sleep 0.02; done + return "$MANAGED_CLEANUP_FAILURE_STATUS" + } + ( + coordinator_status=0; deadline=$((SECONDS + 15)) + while (( SECONDS < deadline )); do + if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release"; break; fi + sleep 0.02 + done + if [[ ! -e "$root/capture/release" ]]; then : > "$root/capture/coordinator-timeout"; : > "$root/capture/release"; coordinator_status=1; fi + while [[ ! -e "$root/capture/coordinator-stop" ]]; do sleep 0.02; done + exit "$coordinator_status" + ) & coordinator=$! + rc=0; run_server_batch source destination one.example two.example three.example four.example || rc=$? + : > "$root/capture/coordinator-stop"; coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? + assert_eq 0 "$coordinator_rc" "registration coordinator sees both first-slot workers" + assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "cleanup failure remains lifecycle status" + assert_eq 2 "$(wc -l < "$root/capture/calls")" "both first-slot workers enter fake transfer" + grep -Fxq one.example "$root/capture/calls" || fail "one.example did not enter fake transfer" + grep -Fxq two.example "$root/capture/calls" || fail "two.example did not enter fake transfer" + grep -Eq 'three\.example|four\.example' "$root/capture/calls" && fail "registration failure scheduled later servers" + assert_eq 0 "$GET_START_CALLS" "parent no longer depends on short get_process_start_time window" + terminate_active_workers || true; cleanup_runtime; assert_no_registration_residue "$root" + pass "deterministic registration regression avoids worker-one/worker-two deadlock" +) +run_with_watchdog root-cause 20 root_cause_regression_case + +run_ready_failure_case() ( + kind="$1"; root="$TEST_DIR/ready-$kind"; setup_fixture "$root" + WORKER_REGISTRATION_TIMEOUT_TICKS=10 + push_to_server() { : > "$root/capture/transfer"; return 0; } + case "$kind" in + missing) worker_registration_phase_hook() { [[ "$1" != before-ready ]] || while :; do :; done; } ;; + malformed) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf malformed > "$3"; } ;; + symlink) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || { rm -f "$3"; ln -s "$root/other" "$3"; }; } ;; + pid-mismatch) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || sed -i 's/^worker_pid=.*/worker_pid=999999/' "$3"; } ;; + wrong-mode) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || chmod 0644 "$3"; } ;; + duplicate-field) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf 'worker_pid=1\n' >> "$3"; } ;; + unknown-field) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || printf 'unexpected=value\n' >> "$3"; } ;; + illegal-number) parent_worker_registration_hook() { [[ "$1" != ready-observed ]] || sed -i 's/^worker_start=.*/worker_start=0/' "$3"; } ;; + release-failure) publish_worker_registration_release() { return 1; } ;; + esac + rc=0; launch_worker one.example source destination 1 1 || rc=$? + (( rc != 0 )) || fail "$kind registration failure unexpectedly succeeded" + [[ ! -e "$root/capture/transfer" ]] || fail "$kind reached transfer before trusted handoff" + cleanup_runtime || true; assert_no_registration_residue "$root" + [[ -z "$(jobs -pr)" ]] || fail "$kind left worker job" + pass "$kind registration failure is bounded and side-effect free" +) +for registration_failure in missing malformed symlink pid-mismatch wrong-mode duplicate-field unknown-field illegal-number release-failure; do run_ready_failure_case "$registration_failure"; done + +( + root="$TEST_DIR/release-replaced"; setup_fixture "$root" + push_to_server() { : > "$root/capture/transfer"; return 0; } + worker_registration_phase_hook() { [[ "$1" != ready-published ]] || while [[ ! -e "$root/capture/release-mutated" ]]; do sleep 0.02; done; } + parent_worker_registration_hook() { [[ "$1" != release-published ]] || { chmod 0644 "$4"; : > "$root/capture/release-mutated"; }; } + launch_worker one.example source destination 1 1 + rc=0; wait_for_all_workers || rc=$? + assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "worker rejects replaced release before transfer" + [[ ! -e "$root/capture/transfer" ]] || fail "replaced release reached transfer" + cleanup_runtime || true; assert_no_registration_residue "$root" + pass "release replacement is bounded and leaves no residue" +) +( + root="$TEST_DIR/parent-disappeared"; setup_fixture "$root" + push_to_server() { : > "$root/capture/transfer"; return 0; } + sleep 300 & fake_parent=$!; read_process_record "$fake_parent"; fake_parent_start=$PROC_START + registered_worker_entry "$fake_parent" "$fake_parent_start" one.example source destination 1 1 & worker=$! + for _ in {1..200}; do [[ -e "$TEMP_DIR/worker-registration.$worker.ready" ]] && break; sleep 0.02; done + [[ -e "$TEMP_DIR/worker-registration.$worker.ready" ]] || fail "worker did not publish ready before parent disappearance" + kill -TERM "$fake_parent" 2>/dev/null || true; wait "$fake_parent" 2>/dev/null || true + rc=0; wait "$worker" || rc=$?; (( rc != 0 )) || fail "worker accepted disappeared parent" + [[ ! -e "$root/capture/transfer" ]] || fail "worker transferred after parent disappeared" + cleanup_worker_registration_for_pid "$worker" || true; cleanup_runtime; assert_no_registration_residue "$root" + pass "worker exits before transfer when parent identity disappears" +) + +( + sleep 60 & proc_fallback_worker=$! + proc_fallback_start=$(wait_test_process_start "$proc_fallback_worker") + eval "$(declare -f read_process_record | sed '1s/read_process_record/original_read_process_record/')" + read_process_record() { [[ "$1" != "$proc_fallback_worker" ]] && original_read_process_record "$@"; } + proc_fallback_status=0 + read_process_record_for_session_scan "$proc_fallback_worker" || proc_fallback_status=$? + assert_eq 0 "$proc_fallback_status" "proc owner fallback recovers transient status-read failure" + [[ "$PROC_START" == "$proc_fallback_start" && "$PROC_UID" == "$EUID" ]] || fail "proc owner fallback identity mismatch" + kill -TERM "$proc_fallback_worker"; wait "$proc_fallback_worker" 2>/dev/null || true + pass "proc owner fallback binds UID to stable PID start" +) + +prune_identity_failure_case() ( + mode="$1"; root="$TEST_DIR/prune-$mode"; setup_fixture "$root" + sleep 300 & worker=$!; read_process_record "$worker"; start=$PROC_START + ACTIVE_WORKERS[$worker]=$start; ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state"; ACTIVE_WORKER_STATE_STARTS[$worker]=$start + eval "$(declare -f process_identity_matches | sed '1s/process_identity_matches/original_process_identity_matches/')" + eval "$(declare -f read_process_record_for_session_scan | sed '1s/read_process_record_for_session_scan/original_read_process_record_for_session_scan/')" + failures=1; scan_failures=1; [[ "$mode" == repeated ]] && { failures=100; scan_failures=100; } + process_identity_matches() { if [[ "$1" == "$worker" && "$failures" -gt 0 ]]; then ((failures -= 1)); return 1; fi; original_process_identity_matches "$@"; } + read_process_record_for_session_scan() { + if [[ "$1" == "$worker" && "$scan_failures" -gt 0 && -r "/proc/$worker/stat" ]]; then ((scan_failures -= 1)); return 1; fi + original_read_process_record_for_session_scan "$@" + } + rc=0; prune_active_workers || rc=$?; (( rc != 0 )) || fail "$mode identity failure did not stop prune" + [[ "$BATCH_WORKER_FAILED" == true ]] || fail "$mode identity failure did not set lifecycle barrier" + process_is_running "$worker" || fail "$mode prune shortened active worker grace" + [[ -n "${ACTIVE_WORKERS[$worker]:-}" ]] || fail "$mode prune discarded active worker evidence" + rc=0; wait_for_worker_slot || rc=$?; assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "$mode identity failure propagates lifecycle status" + terminate_active_workers || true + process_is_running "$worker" && fail "$mode unified active cleanup left worker" + cleanup_runtime || true; assert_no_registration_residue "$root" + pass "$mode active identity failure defers termination to unified cleanup" +) +prune_identity_failure_case once +prune_identity_failure_case repeated +( + root="$TEST_DIR/prune-cleanup-failure"; setup_fixture "$root" + sleep 300 & worker=$!; read_process_record "$worker"; start=$PROC_START + ACTIVE_WORKERS[$worker]=$start + ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state" + ACTIVE_WORKER_STATE_STARTS[$worker]=$start + eval "$(declare -f process_identity_matches | sed '1s/process_identity_matches/original_process_identity_matches/')" + eval "$(declare -f read_process_record_for_session_scan | sed '1s/read_process_record_for_session_scan/original_read_process_record_for_session_scan/')" + process_identity_matches() { [[ "$1" != "$worker" ]] && original_process_identity_matches "$@"; } + read_process_record_for_session_scan() { [[ "$1" != "$worker" || ! -r "/proc/$worker/stat" ]] && original_read_process_record_for_session_scan "$@"; } + rc=0; wait_for_worker_slot || rc=$? + assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "prune trust failure propagates 125" + [[ -n "${ACTIVE_WORKERS[$worker]:-}" ]] || fail "prune trust failure discarded active worker evidence" + eval "$(declare -f original_process_identity_matches | sed '1s/original_process_identity_matches/process_identity_matches/')" + eval "$(declare -f original_read_process_record_for_session_scan | sed '1s/original_read_process_record_for_session_scan/read_process_record_for_session_scan/')" + eval "$(declare -f cleanup_all_worker_registration_residue | sed '1s/cleanup_all_worker_registration_residue/original_cleanup_all_worker_registration_residue/')" + cleanup_all_worker_registration_residue() { return 1; } + rc=0; stop_batch_after_lifecycle_failure || rc=$? + assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "active cleanup failure remains lifecycle status 125" + eval "$(declare -f original_cleanup_all_worker_registration_residue | sed '1s/original_cleanup_all_worker_registration_residue/cleanup_all_worker_registration_residue/')" + cleanup_runtime || true; assert_no_registration_residue "$root" + pass "active cleanup failure preserves lifecycle status" +) +( + root="$TEST_DIR/prune-exited"; setup_fixture "$root" + sleep 0.1 & worker=$!; read_process_record "$worker"; start=$PROC_START + ACTIVE_WORKERS[$worker]=$start + ACTIVE_WORKER_STATE_FILES[$worker]="$TEMP_DIR/worker-session.$worker.state" + ACTIVE_WORKER_STATE_STARTS[$worker]=$start + sleep 0.2; prune_active_workers + assert_eq 0 "${#ACTIVE_WORKERS[@]}" "exited worker is normally reaped" + [[ "$BATCH_WORKER_FAILED" == false ]] || fail "normal exited worker set lifecycle barrier" + cleanup_runtime; assert_no_registration_residue "$root" + pass "normal exited worker accounting remains intact" +) + +cat > "$TEST_DIR/registration-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" root="$2" phase="$3" +mkdir -p -m 0700 "$root/runtime" "$root/capture" +export TMPDIR="$root/runtime" +source "$script" +initialize_runtime; install_runtime_traps +WORKER_REGISTRATION_TIMEOUT_TICKS=200 +push_to_server() { printf 'push pid=%s root=%s\n' "$BASHPID" "$root" >> "$root/capture/transfer-calls"; : > "$root/capture/transfer"; return 0; } +worker_registration_phase_hook() { local current="$1"; [[ "$phase" == "$current" ]] || return 0; : > "$root/capture/phase"; while :; do :; done; } +parent_worker_registration_hook() { local current="$1"; [[ "$phase" == "$current" ]] || return 0; : > "$root/capture/phase"; while :; do :; done; } +launch_worker one.example source destination 1 1 +wait_for_all_workers +if [[ "$phase" == normal ]]; then [[ -e "$root/capture/transfer" ]] || { printf 'normal transfer missing\n' >&2; jobs -l >&2 || true; exit 91; }; fi +cleanup_runtime +CHILD +chmod 0700 "$TEST_DIR/registration-signal-child.sh" +run_registration_signal_case() { + local phase="$1" signal_name="$2" expected="$3" root retry_root + local pid watchdog unrelated rc=0 + root="$TEST_DIR/signal-$phase-$signal_name"; CURRENT_FIXTURE_ROOT=$root + sleep 60 & unrelated=$! + env --default-signal=HUP,INT,TERM bash "$TEST_DIR/registration-signal-child.sh" "$SCRIPT" "$root" "$phase" > "$root.log" 2>&1 & pid=$! + for _ in {1..300}; do [[ -e "$root/capture/phase" ]] && break; sleep 0.05; done + [[ -e "$root/capture/phase" ]] || { cat "$root.log"; kill "$pid" "$unrelated" 2>/dev/null || true; fail "$phase/$signal_name did not reach registration phase"; } + test_watchdog_process 30 "$root.watchdog" "$pid" & watchdog=$! + kill "-$signal_name" "$pid"; wait "$pid" || rc=$? + kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root.watchdog" ]] || fail "$phase/$signal_name exceeded watchdog" + assert_eq "$expected" "$rc" "$phase registration $signal_name status" + [[ ! -e "$root/capture/transfer" ]] || fail "$phase/$signal_name reached transfer" + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$phase/$signal_name left runtime state" + kill -0 "$unrelated" 2>/dev/null || fail "$phase/$signal_name killed unrelated process" + kill "$unrelated" 2>/dev/null || true; wait "$unrelated" 2>/dev/null || true + retry_root="${root}-retry" + env bash "$TEST_DIR/registration-signal-child.sh" "$SCRIPT" "$retry_root" normal > "$retry_root.log" 2>&1 + if [[ ! -e "$retry_root/capture/transfer" ]]; then cat "$retry_root.log" >&2 2>/dev/null || true; fail "$phase/$signal_name retry did not transfer"; fi + [[ -z "$(find "$retry_root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$phase/$signal_name retry left runtime" + pass "$phase/$signal_name leaves registration clean and retryable" +} +for registration_phase in before-ready accepted-before-release release-validated; do + run_registration_signal_case "$registration_phase" HUP 129 + run_registration_signal_case "$registration_phase" INT 130 + run_registration_signal_case "$registration_phase" TERM 143 +done + +( + sleep 60 & reused_pid=$! + replacement_start=$(get_process_start_time "$reused_pid") + expected_start=$((replacement_start + 1)) + eval "$(declare -f collect_owned_session_records | sed '1s/collect_owned_session_records/original_collect_owned_session_records/')" + collect_owned_session_records() { fail "PID reuse path attempted to inspect or terminate replacement session"; } + terminate_managed_session "$reused_pid" "$expected_start" "$reused_pid" "$reused_pid" + test_process_identity_exists "$reused_pid" "$replacement_start" || fail "PID reuse path touched replacement process" + kill -TERM "$reused_pid"; wait "$reused_pid" 2>/dev/null || true + pass "managed leader PID reuse is treated as original session gone" +) + +cat > "$TEST_DIR/failed-reap-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" root="$2" mode="$3" +mkdir -p -m 0700 "$root/runtime" "$root/capture" +export TMPDIR="$root/runtime" +source "$script" +initialize_runtime +setsid bash -c 'trap "" HUP INT TERM; while :; do :; done' & leader=$! +for _ in {1..100}; do + leader_start=$(get_process_start_time "$leader" 2>/dev/null || true) + leader_pgid=$(get_process_group_id "$leader" 2>/dev/null || true) + leader_sid=$(get_process_session_id "$leader" 2>/dev/null || true) + [[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] && break + sleep 0.01 +done +[[ -n "$leader_start" && "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || exit 90 +printf '%s\n%s\n%s\n%s\n' "$leader" "$leader_start" "$leader_pgid" "$leader_sid" > "$root/capture/leader" +WORKER_TRANSFER_PID=$leader +WORKER_TRANSFER_START=$leader_start +WORKER_TRANSFER_PGID=$leader_pgid +WORKER_TRANSFER_SID=$leader_sid +publish_worker_session_state active +state_file=$WORKER_SESSION_STATE_FILE +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +terminate_managed_session() { return 1; } +start_ns=$(date +%s%N) +rc=0 +terminate_worker_transfer || rc=$? +elapsed_ns=$(( $(date +%s%N) - start_ns )) +[[ "$rc" != 0 && "$elapsed_ns" -lt 2000000000 ]] || exit 91 +[[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && + "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && + "$WORKER_TRANSFER_CLEANUP_FAILED" == true ]] || exit 92 +read_worker_session_state "$state_file" +[[ "$SESSION_STATE" == cleanup_failed ]] || exit 93 +next_rc=0 +run_managed_command "$TEMP_DIR/forbidden-next-command" true || next_rc=$? +[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-next-command" ]] || exit 94 +printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\nstate_file=%s\n' "$rc" "$elapsed_ns" "$next_rc" "$state_file" > "$root/capture/result" +: > "$root/capture/returned" +if [[ "$mode" == hold ]]; then + while [[ ! -e "$root/capture/release-worker" ]]; do sleep 0.01; done + for _ in {1..400}; do + reap_status=0 + reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" || reap_status=$? + case "$reap_status" in + 0|"$MANAGED_PROCESS_REUSED_STATUS") break ;; + *) sleep 0.01 ;; + esac + done + (( reap_status == 0 || reap_status == MANAGED_PROCESS_REUSED_STATUS )) || exit 95 + : > "$root/capture/reaped" + for _ in {1..400}; do + [[ ! -e "$state_file" && ! -L "$state_file" ]] && break + sleep 0.01 + done + [[ ! -e "$state_file" && ! -L "$state_file" ]] || exit 96 + terminate_managed_session() { original_terminate_managed_session "$@"; } +else + for _ in {1..400}; do [[ -e "$root/capture/release-recovery" ]] && break; sleep 0.01; done + [[ -e "$root/capture/release-recovery" ]] || exit 97 + terminate_managed_session() { original_terminate_managed_session "$@"; } +fi +cleanup_runtime +CHILD +chmod 0700 "$TEST_DIR/failed-reap-child.sh" + +run_failed_reap_case() { + local run="$1" root="$TEST_DIR/failed-reap-$1" child watchdog rc=0 + bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" recover > "$root.log" 2>&1 & child=$! + test_watchdog_process 3 "$root.watchdog-fired" "$child" & watchdog=$! + for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done + [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "failed reap run $run did not return before watchdog"; } + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root.watchdog-fired" ]] || fail "failed reap run $run fired watchdog" + : > "$root/capture/release-recovery" + wait "$child" || rc=$? + [[ "$rc" == 0 ]] || cat "$root.log" >&2 2>/dev/null || true + assert_eq 0 "$rc" "failed managed-session reap run $run returns bounded" + [[ -z "$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null)" ]] || fail "failed reap run $run left worker-session state" +} +for failed_reap_run in $(seq 1 20); do run_failed_reap_case "$failed_reap_run"; done +pass "failed managed-session reap remains bounded for 20 deterministic runs" + +( + trap - EXIT HUP INT TERM + root="$TEST_DIR/failed-reap-parent"; mkdir -p "$root/capture"; chmod 0700 "$root" "$root/capture" + bash "$TEST_DIR/failed-reap-child.sh" "$SCRIPT" "$root" hold > "$root.log" 2>&1 & worker=$! + worker_start=$(wait_test_process_start "$worker") + for _ in {1..80}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done + [[ -e "$root/capture/returned" ]] || fail "parent fallback worker did not return bounded" + mapfile -t leader_fields < "$root/capture/leader" + leader=${leader_fields[0]}; leader_start=${leader_fields[1]}; leader_sid=${leader_fields[3]} + state_file=$(sed -n 's/^state_file=//p' "$root/capture/result") + TEMP_DIR=$(dirname "$state_file") + ACTIVE_WORKERS=(); ACTIVE_WORKER_STATE_FILES=(); ACTIVE_WORKER_STATE_STARTS=() + BATCH_WORKER_FAILED=false + BATCH_WORKER_ERROR=false + MAX_PARALLEL=1 + ACTIVE_WORKERS[$worker]=$worker_start + ACTIVE_WORKER_STATE_FILES[$worker]=$state_file + ACTIVE_WORKER_STATE_STARTS[$worker]=$worker_start + : > "$root/capture/release-worker" + wait_for_all_workers + assert_eq 0 "${#ACTIVE_WORKERS[@]}" "scheduler parent fallback reaps active worker" + assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "scheduler parent fallback removes cleanup_failed state" + wait_test_process_identity_gone "$leader" "$leader_start" || fail "parent fallback left same-identity leader" + session_pids=(); session_starts=(); session_ppids=(); session_pgids=(); session_states=() + collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states + assert_eq 0 "${#session_pids[@]}" "parent fallback leaves no managed SID member" + [[ ! -e "$state_file" && -e "$root/capture/reaped" ]] || fail "parent fallback evidence incomplete" + pass "parent fallback cleans a live failed session without PPID 1 zombie" +) + +( + trap - EXIT HUP INT TERM + sleep 60 & replacement=$! + replacement_start=$(wait_test_process_start "$replacement") + wait_called=false + wait() { wait_called=true; return 1; } + reap_status=0 + reap_direct_managed_leader "$replacement" "$((replacement_start + 1))" "$replacement" "$replacement" || reap_status=$? + assert_eq "$MANAGED_PROCESS_REUSED_STATUS" "$reap_status" "direct leader PID reuse returns safe status" + [[ "$wait_called" == false ]] || fail "direct leader PID reuse waited on replacement process" + test_process_identity_exists "$replacement" "$replacement_start" || fail "direct leader PID reuse touched replacement process" + kill -TERM "$replacement"; builtin wait "$replacement" 2>/dev/null || true + pass "direct leader PID reuse sends no signal and performs no wait" +) + +( + trap - EXIT HUP INT TERM + leader=424242; leader_start=777777; wait_called=false; after_wait=false + collect_owned_session_records() { + local -n pids_ref="$2" starts_ref="$3" ppids_ref="$4" pgids_ref="$5" states_ref="$6" + pids_ref=(); starts_ref=(); ppids_ref=(); pgids_ref=(); states_ref=() + [[ "$after_wait" == false ]] || return 0 + pids_ref=("$leader"); starts_ref[$leader]=$leader_start; ppids_ref[$leader]=$BASHPID + pgids_ref[$leader]=$leader; states_ref[$leader]=Z + } + read_process_record_for_session_scan() { + [[ "$after_wait" == false ]] || return 2 + PROC_UID=$EUID; PROC_PPID=$BASHPID; PROC_START=$leader_start + PROC_STATE=Z; PROC_PGID=$leader; PROC_SID=$leader + } + wait() { wait_called=true; after_wait=true; return 0; } + reap_direct_managed_leader "$leader" "$leader_start" "$leader" "$leader" + [[ "$wait_called" == true ]] || fail "same-identity leader zombie was not wait/reaped" + pass "same-identity direct leader zombie is immediately wait/reaped" +) + +cat > "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" root="$2" +mkdir -p -m 0700 "$root/runtime" "$root/capture" +export TMPDIR="$root/runtime" +source "$script" +initialize_runtime +TOTAL_TIMEOUT=1 +TIMEOUT_KILL_AFTER=1 +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +failures=1 +terminate_managed_session() { + if (( failures > 0 )); then ((failures -= 1)); return 1; fi + original_terminate_managed_session "$@" +} +start_ns=$(date +%s%N) +rc=0 +run_managed_command "$TEMP_DIR/hard-output" timeout 1 bash -c \ + 'trap "" HUP INT TERM; bash -c '\''trap "" HUP INT TERM; while :; do kill -STOP "$$"; done'\'' & wait "$!"' || rc=$? +elapsed_ns=$(( $(date +%s%N) - start_ns )) +[[ "$rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && "$elapsed_ns" -lt 5000000000 ]] || exit 90 +[[ "$WORKER_TRANSFER_CLEANUP_FAILED" == true && -n "$WORKER_TRANSFER_PID" && -f "$WORKER_SESSION_STATE_FILE" ]] || exit 91 +read_worker_session_state "$WORKER_SESSION_STATE_FILE" +[[ "$SESSION_STATE" == cleanup_failed ]] || exit 92 +next_rc=0 +run_managed_command "$TEMP_DIR/forbidden-hard-next" true || next_rc=$? +[[ "$next_rc" == "$MANAGED_CLEANUP_FAILURE_STATUS" && ! -e "$TEMP_DIR/forbidden-hard-next" ]] || exit 93 +printf 'rc=%s\nelapsed_ns=%s\nnext_rc=%s\n' "$rc" "$elapsed_ns" "$next_rc" > "$root/capture/result" +: > "$root/capture/returned" +terminate_worker_transfer +cleanup_runtime +CHILD +chmod 0700 "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" +( + trap - EXIT HUP INT TERM + root="$TEST_DIR/hard-timeout-cleanup-failure" + bash "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" "$SCRIPT" "$root" > "$root.log" 2>&1 & child=$! + test_watchdog_process 5 "$root.watchdog-fired" "$child" & watchdog=$! + for _ in {1..120}; do [[ -e "$root/capture/returned" ]] && break; sleep 0.05; done + [[ -e "$root/capture/returned" ]] || { cat "$root.log" >&2 || true; fail "hard timeout cleanup failure did not return bounded"; } + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root.watchdog-fired" ]] || fail "hard timeout cleanup failure fired watchdog" + rc=0; wait "$child" || rc=$? + assert_eq 0 "$rc" "hard timeout cleanup failure recovers fixture" + grep -Fxq "rc=$MANAGED_CLEANUP_FAILURE_STATUS" "$root/capture/result" || fail "hard timeout cleanup failure did not propagate 125" + grep -Fxq "next_rc=$MANAGED_CLEANUP_FAILURE_STATUS" "$root/capture/result" || fail "hard timeout cleanup failure allowed next command" + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "hard timeout cleanup failure left runtime" + pass "hard timeout cleanup failure is bounded and blocks subsequent command" +) + +write_test_worker_session_manifest() { + local file="$1" state="$2" worker_pid="$3" worker_start="$4" leader_pid="$5" leader_start="$6" pgid="$7" sid="$8" + printf 'version=1\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\n' \ + "$state" "$worker_pid" "$worker_start" "$leader_pid" "$leader_start" "$pgid" "$sid" > "$file" + chmod 0600 "$file" +} + + +assert_retained_runtime_trusted() { + local path="$1" expected_dev="$2" expected_inode="$3" metadata="" dev inode owner gid mode trusted_gid + [[ -d "$path" && ! -L "$path" ]] || fail "retained runtime is not a regular directory: $path" + metadata=$(stat -Lc '%d:%i:%u:%g:%a' -- "$path") || fail "cannot stat retained runtime: $path" + IFS=: read -r dev inode owner gid mode <<< "$metadata" + trusted_gid=$(current_gid) + [[ "$dev" == "$expected_dev" && "$inode" == "$expected_inode" && + "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 700 ]] || + fail "retained runtime trust mismatch: $metadata" +} + +assert_retained_state_trusted() { + local file="$1" metadata="" owner gid mode trusted_gid + [[ -f "$file" && ! -L "$file" ]] || fail "retained worker state is not a regular file: $file" + metadata=$(stat -Lc '%u:%g:%a' -- "$file") || fail "cannot stat retained worker state: $file" + IFS=: read -r owner gid mode <<< "$metadata" + trusted_gid=$(current_gid) + [[ "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 600 ]] || + fail "retained worker state trust mismatch: $metadata" +} + +run_preserved_cleanup_retry_case() ( + local run="$1" root runtime runtime_dev runtime_inode credential leader leader_start leader_pgid leader_sid state_file + local owner watchdog rc=0 metadata="" state_status=0 + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + root="$TEST_DIR/runtime-evidence-retry-$run"; setup_fixture "$root" + runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE + credential="$runtime/private-key" + printf 'fixture-credential-do-not-print\n' > "$credential"; chmod 0600 "$credential" + RUNTIME_KEY_FILE=$credential + + setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! + leader_start=$(wait_test_process_start "$leader") + for _ in {1..100}; do + read_process_record "$leader" || true + [[ "$PROC_START" == "$leader_start" && "$PROC_STATE" == T ]] && break + sleep 0.01 + done + read_process_record "$leader" + leader_pgid=$PROC_PGID; leader_sid=$PROC_SID + [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "retry run $run leader identity invalid" + WORKER_TRANSFER_PID=$leader; WORKER_TRANSFER_START=$leader_start + WORKER_TRANSFER_PGID=$leader_pgid; WORKER_TRANSFER_SID=$leader_sid + publish_worker_session_state active + state_file=$WORKER_SESSION_STATE_FILE + + eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" + terminate_managed_session() { return 1; } + owner=$BASHPID + test_watchdog_process 3 "$root/watchdog-fired" "$owner" & watchdog=$! + cleanup_runtime > "$root/capture/first-cleanup.log" 2>&1 || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + + (( rc != 0 )) || fail "retry run $run first cleanup unexpectedly succeeded" + [[ ! -e "$root/watchdog-fired" ]] || fail "retry run $run watchdog fired" + [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" ]] || + fail "retry run $run lost runtime identity" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + assert_retained_state_trusted "$state_file" + metadata=$(stat -Lc '%u:%g:%a' -- "$credential") + [[ "$metadata" == "$EUID:$(current_gid):600" ]] || fail "retry run $run credential permissions changed: $metadata" + grep -Fq 'fixture-credential-do-not-print' "$root/capture/first-cleanup.log" && fail "retry run $run printed credential content" + state_status=0; read_worker_session_state "$state_file" || state_status=$? + assert_eq 0 "$state_status" "retry run $run retained state parses" + [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && + "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || + fail "retry run $run retained state identity mismatch" + [[ "$WORKER_TRANSFER_PID" == "$leader" && "$WORKER_TRANSFER_START" == "$leader_start" && + "$WORKER_TRANSFER_PGID" == "$leader_pgid" && "$WORKER_TRANSFER_SID" == "$leader_sid" && + "$WORKER_TRANSFER_CLEANUP_FAILED" == true && "$WORKER_SESSION_STATE_FILE" == "$state_file" && + "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run lost cleanup globals" + test_process_identity_exists "$leader" "$leader_start" || fail "retry run $run lost live leader tracking" + read_process_record "$leader" + [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "retry run $run orphaned or zombified leader" + + eval "$(declare -f original_terminate_managed_session | sed '1s/original_terminate_managed_session/terminate_managed_session/')" + cleanup_runtime + wait_test_process_identity_gone "$leader" "$leader_start" || fail "retry run $run left leader identity" + collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states + assert_eq 0 "${#session_pids[@]}" "retry run $run clears managed SID" + [[ ! -e "$state_file" && ! -L "$state_file" && ! -e "$runtime" && ! -L "$runtime" ]] || + fail "retry run $run left state or runtime" + runtime_lifecycle_globals_resolved || fail "retry run $run left lifecycle globals" + [[ "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "retry run $run runtime state not reset" + pass "failed cleanup evidence retry run $run preserves then clears runtime" +) + +runtime_evidence_stress=${PUSH_RUNTIME_EVIDENCE_STRESS:-20} +for runtime_evidence_run in $(seq 1 "$runtime_evidence_stress"); do + run_preserved_cleanup_retry_case "$runtime_evidence_run" +done +pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" + +( + local_cleanup_root="$TEST_DIR/active-cleanup-evidence"; setup_fixture "$local_cleanup_root" + runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE + sleep 300 & worker=$!; worker_start=$(wait_test_process_start "$worker") + sleep 300 & replacement=$!; replacement_start=$(wait_test_process_start "$replacement") + state_file="$runtime/worker-session.$worker.state" + write_test_worker_session_manifest "$state_file" cleanup_failed "$worker" "$worker_start" \ + "$replacement" "$((replacement_start + 1))" "$replacement" "$replacement" + ACTIVE_WORKERS[$worker]=$worker_start + ACTIVE_WORKER_STATE_FILES[$worker]=$state_file + ACTIVE_WORKER_STATE_STARTS[$worker]=$worker_start + eval "$(declare -f terminate_active_workers | sed '1s/terminate_active_workers/original_terminate_active_workers/')" + terminate_active_workers() { return 1; } + rc=0; cleanup_runtime > "$local_cleanup_root/capture/first-cleanup.log" 2>&1 || rc=$? + (( rc != 0 )) || fail "active cleanup evidence first call unexpectedly succeeded" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + assert_retained_state_trusted "$state_file" + [[ -n "${ACTIVE_WORKERS[$worker]:-}" && "${ACTIVE_WORKER_STATE_FILES[$worker]:-}" == "$state_file" && + "${ACTIVE_WORKER_STATE_STARTS[$worker]:-}" == "$worker_start" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || + fail "active cleanup evidence maps were discarded" + test_process_identity_exists "$worker" "$worker_start" || fail "active cleanup evidence worker disappeared" + eval "$(declare -f original_terminate_active_workers | sed '1s/original_terminate_active_workers/terminate_active_workers/')" + cleanup_runtime + wait_test_process_identity_gone "$worker" "$worker_start" || fail "active cleanup retry left worker" + [[ ! -e "$state_file" && ! -e "$runtime" && ${#ACTIVE_WORKERS[@]} == 0 && + ${#ACTIVE_WORKER_STATE_FILES[@]} == 0 && ${#ACTIVE_WORKER_STATE_STARTS[@]} == 0 ]] || + fail "active cleanup retry left runtime, state, or maps" + test_process_identity_exists "$replacement" "$replacement_start" || fail "active cleanup retry touched PID-reuse replacement" + kill -TERM "$replacement"; wait "$replacement" 2>/dev/null || true + pass "active cleanup failure preserves evidence and retry clears it" +) + +run_runtime_trust_retention_case() ( + local kind="$1" root runtime runtime_dev runtime_inode original_path replacement_path rc=0 + root="$TEST_DIR/runtime-trust-$kind"; setup_fixture "$root" + runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE + case "$kind" in + symlink) + original_path="$runtime.original"; replacement_path="$runtime.replacement" + mv "$runtime" "$original_path"; mkdir -m 0700 "$replacement_path"; : > "$replacement_path/marker" + ln -s "$replacement_path" "$runtime" + ;; + wrong-mode) chmod 0755 "$runtime" ;; + wrong-owner|wrong-gid) + eval "$(declare -f stat | sed '1s/stat/original_stat/')" 2>/dev/null || true + stat() { + local last="${!#}" metadata dev inode owner gid mode + if [[ "$last" == "$runtime" && "$1" == -Lc && "$2" == '%d:%i:%u:%g:%a' ]]; then + metadata=$(command stat -Lc '%d:%i:%u:%g:%a' -- "$runtime") || return 1 + IFS=: read -r dev inode owner gid mode <<< "$metadata" + [[ "$kind" == wrong-owner ]] && owner=$((owner + 1)) || gid=$((gid + 1)) + printf '%s:%s:%s:%s:%s\n' "$dev" "$inode" "$owner" "$gid" "$mode" + else + command stat "$@" + fi + } + ;; + *) fail "unknown runtime trust case: $kind" ;; + esac + cleanup_runtime > "$root/capture/cleanup.log" 2>&1 || rc=$? + (( rc != 0 )) || fail "$kind untrusted runtime unexpectedly deleted" + [[ "$TEMP_DIR" == "$runtime" && "$TEMP_DIR_DEV" == "$runtime_dev" && "$TEMP_DIR_INODE" == "$runtime_inode" && + "$RUNTIME_CLEANUP_ACTIVE" == false ]] || fail "$kind untrusted runtime lost path identity" + case "$kind" in + symlink) + [[ -L "$runtime" && -f "$replacement_path/marker" && -d "$original_path" ]] || fail "symlink replacement was modified" + rm -f "$runtime"; rmdir "$replacement_path" 2>/dev/null || { rm -f "$replacement_path/marker"; rmdir "$replacement_path"; } + mv "$original_path" "$runtime" + ;; + wrong-mode) chmod 0700 "$runtime" ;; + wrong-owner|wrong-gid) + unset -f stat + ;; + esac + cleanup_runtime + [[ ! -e "$runtime" && ! -L "$runtime" ]] || fail "$kind trusted retry left runtime" + pass "$kind runtime trust failure preserves evidence and replacement" +) +for runtime_trust_kind in symlink wrong-owner wrong-gid wrong-mode; do + run_runtime_trust_retention_case "$runtime_trust_kind" +done + +for final_residue_kind in worker-session registration-ready registration-stage; do + ( + root="$TEST_DIR/runtime-final-proof-$final_residue_kind"; setup_fixture "$root" + runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE + case "$final_residue_kind" in + worker-session) residue="$runtime/worker-session.99999.state" ;; + registration-ready) residue="$runtime/worker-registration.99999.ready" ;; + registration-stage) residue="$runtime/.worker-registration.99999.ready.token.stage" ;; + esac + eval "$(declare -f terminate_active_workers | sed '1s/terminate_active_workers/original_terminate_active_workers/')" + terminate_active_workers() { + original_terminate_active_workers || return 1 + printf 'retained-lifecycle-evidence\n' > "$residue"; chmod 0600 "$residue" + } + rc=0; cleanup_runtime > "$root/capture/cleanup.log" 2>&1 || rc=$? + (( rc != 0 )) || fail "$final_residue_kind final proof unexpectedly deleted runtime" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + [[ -f "$residue" && ! -L "$residue" && "$TEMP_DIR" == "$runtime" && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || + fail "$final_residue_kind final proof discarded evidence" + eval "$(declare -f original_terminate_active_workers | sed '1s/original_terminate_active_workers/terminate_active_workers/')" + rm -f -- "$residue" + cleanup_runtime + [[ ! -e "$runtime" ]] || fail "$final_residue_kind final proof retry left runtime" + pass "$final_residue_kind blocks runtime deletion until evidence is cleared" + ) +done + +( + root="$TEST_DIR/runtime-final-proof-success"; setup_fixture "$root"; runtime=$TEMP_DIR + cleanup_runtime + [[ ! -e "$runtime" && -z "$TEMP_DIR" && "$RUNTIME_STATE" == none && "$RUNTIME_CLEANUP_ACTIVE" == false ]] || + fail "clean runtime final proof did not clear state" + pass "clean runtime final proof preserves normal success path" +) + + +cat > "$TEST_DIR/runtime-retention-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" root="$2" leader="$3" leader_start="$4" leader_pgid="$5" leader_sid="$6" +mkdir -p "$root/runtime" "$root/capture" +chmod 0700 "$root" "$root/runtime" "$root/capture" +TMPDIR="$root/runtime" +# shellcheck source=../tools/push.sh +source "$script" +initialize_runtime +credential="$TEMP_DIR/private-key" +printf 'signal-fixture-credential-do-not-print\n' > "$credential" +chmod 0600 "$credential" +RUNTIME_KEY_FILE=$credential +WORKER_TRANSFER_PID=$leader +WORKER_TRANSFER_START=$leader_start +WORKER_TRANSFER_PGID=$leader_pgid +WORKER_TRANSFER_SID=$leader_sid +publish_worker_session_state active +state_file=$WORKER_SESSION_STATE_FILE +terminate_managed_session() { return 1; } +install_runtime_traps +worker_start=$(get_process_start_time "$BASHPID") +stage="$root/capture/ready.stage" +{ + printf 'target_pid=%s\n' "$BASHPID" + printf 'target_start=%s\n' "$worker_start" + printf 'runtime=%s\n' "$TEMP_DIR" + printf 'runtime_dev=%s\n' "$TEMP_DIR_DEV" + printf 'runtime_inode=%s\n' "$TEMP_DIR_INODE" + printf 'state_file=%s\n' "$state_file" + printf 'credential=%s\n' "$credential" +} > "$stage" +chmod 0600 "$stage" +mv -fT "$stage" "$root/capture/ready" +while :; do :; done +CHILD +chmod 0700 "$TEST_DIR/runtime-retention-signal-child.sh" + +run_runtime_retention_signal_case() ( + local signal_name="$1" expected="$2" root leader="" leader_start="" leader_pgid="" leader_sid="" target="" target_pid="" target_start="" + local runtime runtime_dev runtime_inode state_file credential watchdog="" rc=0 line key value state_status=0 reap_status=0 + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + cleanup_runtime_retention_signal_fixture() { + if [[ -n "$target" ]]; then kill -KILL "$target" 2>/dev/null || true; wait "$target" 2>/dev/null || true; fi + if [[ -n "$leader" ]]; then kill -CONT "$leader" 2>/dev/null || true; kill -KILL "$leader" 2>/dev/null || true; wait "$leader" 2>/dev/null || true; fi + if [[ -n "$watchdog" ]]; then kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true; fi + } + trap cleanup_runtime_retention_signal_fixture EXIT + root="$TEST_DIR/runtime-retention-signal-$signal_name"; CURRENT_FIXTURE_ROOT=$root + mkdir -m 0700 "$root" + setsid bash -c 'trap "" HUP INT TERM; kill -STOP "$BASHPID"; while :; do :; done' & leader=$! + leader_start=$(wait_test_process_start "$leader") + for _ in {1..100}; do read_process_record "$leader" || true; [[ "$PROC_STATE" == T ]] && break; sleep 0.01; done + read_process_record "$leader"; leader_pgid=$PROC_PGID; leader_sid=$PROC_SID + [[ "$leader_pgid" == "$leader" && "$leader_sid" == "$leader" ]] || fail "$signal_name retention leader identity invalid" + + ( + exec env --default-signal=HUP,INT,TERM bash "$TEST_DIR/runtime-retention-signal-child.sh" \ + "$SCRIPT" "$root" "$leader" "$leader_start" "$leader_pgid" "$leader_sid" + ) > "$root/output.log" 2>&1 & target=$! + for _ in {1..300}; do [[ -f "$root/capture/ready" ]] && break; sleep 0.02; done + [[ -f "$root/capture/ready" ]] || { cat "$root/output.log" >&2 || true; fail "$signal_name retention child not ready"; } + while IFS= read -r line; do + key=${line%%=*}; value=${line#*=} + case "$key" in + target_pid) target_pid=$value ;; + target_start) target_start=$value ;; + runtime) runtime=$value ;; + runtime_dev) runtime_dev=$value ;; + runtime_inode) runtime_inode=$value ;; + state_file) state_file=$value ;; + credential) credential=$value ;; + esac + done < "$root/capture/ready" + [[ "$target_pid" == "$target" && "$target_start" =~ ^[1-9][0-9]*$ && + "$runtime_dev" =~ ^[0-9]+$ && "$runtime_inode" =~ ^[1-9][0-9]*$ ]] || + fail "$signal_name retention ready metadata malformed" + job_is_active "$target" || fail "$signal_name retention target job missing" + test_watchdog_process 5 "$root/watchdog-fired" "$target" & watchdog=$! + kill "-$signal_name" "$target" + wait "$target" || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root/watchdog-fired" ]] || fail "$signal_name retention watchdog fired" + assert_eq "$expected" "$rc" "$signal_name failed cleanup preserves signal status" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + assert_retained_state_trusted "$state_file" + [[ "$(stat -Lc '%u:%g:%a' -- "$credential")" == "$EUID:$(current_gid):600" ]] || + fail "$signal_name retained credential permissions changed" + grep -Fq "$runtime" "$root/output.log" || fail "$signal_name failure output omitted runtime path" + grep -Fq 'signal-fixture-credential-do-not-print' "$root/output.log" && fail "$signal_name failure output leaked credential" + TEMP_DIR=$runtime + state_status=0; read_worker_session_state "$state_file" || state_status=$? + assert_eq 0 "$state_status" "$signal_name retained cleanup_failed state parses" + [[ "$SESSION_STATE" == cleanup_failed && "$SESSION_LEADER_PID" == "$leader" && + "$SESSION_LEADER_START" == "$leader_start" && "$SESSION_PGID" == "$leader_pgid" && "$SESSION_SID" == "$leader_sid" ]] || + fail "$signal_name retained cleanup_failed identity mismatch" + test_process_identity_exists "$leader" "$leader_start" || fail "$signal_name failed cleanup lost leader tracking" + read_process_record "$leader" + [[ "$PROC_PPID" == "$BASHPID" && "$PROC_STATE" != Z ]] || fail "$signal_name teardown leader lost safe parent" + + terminate_managed_session "$leader" "$leader_start" "$leader_pgid" "$leader_sid" true + reap_direct_managed_leader "$leader" "$leader_start" "$leader_pgid" "$leader_sid" "$BASHPID" || reap_status=$? + assert_eq 0 "$reap_status" "$signal_name teardown safely reaps leader" + collect_owned_session_records "$leader_sid" session_pids session_starts session_ppids session_pgids session_states + assert_eq 0 "${#session_pids[@]}" "$signal_name teardown clears managed SID" + assert_retained_runtime_trusted "$runtime" "$runtime_dev" "$runtime_inode" + rm -rf -- "$runtime" + TEMP_DIR="" + leader=""; target=""; watchdog="" + trap - EXIT + pass "$signal_name cleanup failure retains trusted runtime and state" +) +run_runtime_retention_signal_case HUP 129 +run_runtime_retention_signal_case INT 130 +run_runtime_retention_signal_case TERM 143 + +( + root="$TEST_DIR/session-dual-publish"; setup_fixture "$root" + : > "$root/capture/terminate-calls" + terminate_managed_session() { printf '%s:%s:%s:%s:%s\n' "$@" >> "$root/capture/terminate-calls"; return 0; } + worker_session_state_publish_hook() { + local phase="$1" state="$2" + if [[ "$phase" == stage-ready && "$state" == cleanup_failed ]]; then + : > "$root/capture/stage.$BASHPID" + while [[ ! -e "$root/capture/release" ]]; do sleep 0.01; done + elif [[ "$phase" == after-rename && "$state" == cleanup_failed ]]; then + : > "$root/capture/renamed.$BASHPID" + fi + } + session_publisher() { + WORKER_TRANSFER_PID=$BASHPID; WORKER_TRANSFER_START=11 + WORKER_TRANSFER_PGID=$BASHPID; WORKER_TRANSFER_SID=$BASHPID + publish_worker_session_state active + publish_worker_session_state cleanup_failed + } + session_publisher & first=$!; session_publisher & second=$! + first_start=$(get_process_start_time "$first"); second_start=$(get_process_start_time "$second") + ACTIVE_WORKERS[$first]=$first_start; ACTIVE_WORKERS[$second]=$second_start + ACTIVE_WORKER_STATE_FILES[$first]="$TEMP_DIR/worker-session.$first.state" + ACTIVE_WORKER_STATE_FILES[$second]="$TEMP_DIR/worker-session.$second.state" + ACTIVE_WORKER_STATE_STARTS[$first]=$first_start; ACTIVE_WORKER_STATE_STARTS[$second]=$second_start + for _ in {1..300}; do [[ -e "$root/capture/stage.$first" && -e "$root/capture/stage.$second" ]] && break; sleep 0.01; done + [[ -e "$root/capture/stage.$first" && -e "$root/capture/stage.$second" ]] || fail "dual publishers did not reach pre-rename hook" + cleanup_published_worker_sessions + assert_eq 2 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback waits while both workers can still publish" + : > "$root/capture/release" + wait "$first"; wait "$second" + [[ -e "$root/capture/renamed.$first" && -e "$root/capture/renamed.$second" ]] || fail "dual publishers did not complete rename" + prune_active_workers + cleanup_published_worker_sessions + assert_eq 2 "$(wc -l < "$root/capture/terminate-calls")" "fallback processes both stable cleanup_failed manifests" + assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "dual fallback clears both state maps" + assert_eq 0 "${#ACTIVE_WORKER_STATE_STARTS[@]}" "dual fallback clears both state identities" + runtime_has_published_worker_state && fail "dual fallback left worker session state" + cleanup_runtime; assert_no_registration_residue "$root" + pass "dual worker state transitions stabilize after wait/reap" +) + +run_transient_state_case() ( + kind="$1"; root="$TEST_DIR/session-transient-$kind"; setup_fixture "$root" + worker=11111; start=22222; file="$TEMP_DIR/worker-session.$worker.state" + write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 + ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start + : > "$root/capture/terminate-calls" + terminate_managed_session() { printf 'call\n' >> "$root/capture/terminate-calls"; return 0; } + failures=2 + case "$kind" in + inode) + worker_session_state_read_hook() { + if [[ "$1" == after-open && "$failures" -gt 0 ]]; then + next="$file.next.$failures" + write_test_worker_session_manifest "$next" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 + mv -fT "$next" "$file"; ((failures -= 1)) + fi + } + ;; + open) + eval "$(declare -f open_worker_session_state_fd | sed '1s/open_worker_session_state_fd/original_open_worker_session_state_fd/')" + open_worker_session_state_fd() { + if (( failures > 0 )); then ((failures -= 1)); return "$WORKER_SESSION_STATE_TRANSIENT_STATUS"; fi + original_open_worker_session_state_fd "$@" + } + ;; + stat) + eval "$(declare -f worker_session_state_path_metadata | sed '1s/worker_session_state_path_metadata/original_worker_session_state_path_metadata/')" + worker_session_state_path_metadata() { + if (( failures > 0 )); then ((failures -= 1)); return "$WORKER_SESSION_STATE_TRANSIENT_STATUS"; fi + original_worker_session_state_path_metadata "$@" + } + ;; + esac + cleanup_published_worker_sessions + assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "$kind transient failure clears state map after stabilization" + assert_eq 1 "$(wc -l < "$root/capture/terminate-calls")" "$kind transient failure performs one trusted session cleanup" + [[ ! -e "$file" ]] || fail "$kind transient failure left state file" + cleanup_runtime; pass "$kind transient worker state failure is bounded and recoverable" +) +run_transient_state_case inode +run_transient_state_case open +run_transient_state_case stat + +( + root="$TEST_DIR/session-not-reaped"; setup_fixture "$root" + sleep 300 & worker=$!; start=$(get_process_start_time "$worker") + file="$TEMP_DIR/worker-session.$worker.state" + write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 + ACTIVE_WORKERS[$worker]=$start; ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start + : > "$root/capture/terminate-calls" + terminate_managed_session() { printf call >> "$root/capture/terminate-calls"; return 0; } + cleanup_published_worker_sessions + assert_eq 1 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback retains state while worker is not reaped" + assert_eq 0 "$(wc -c < "$root/capture/terminate-calls")" "fallback performs no session cleanup before reap" + kill -TERM "$worker"; wait "$worker" 2>/dev/null || true; prune_active_workers + cleanup_published_worker_sessions + assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "fallback succeeds after worker reap" + cleanup_runtime; pass "worker reap gates final fallback" +) + +run_permanent_state_case() ( + kind="$1"; root="$TEST_DIR/session-permanent-$kind"; setup_fixture "$root" + worker=11111; start=22222; file="$TEMP_DIR/worker-session.$worker.state" + write_test_worker_session_manifest "$file" cleanup_failed "$worker" "$start" 33333 44444 33333 33333 + ACTIVE_WORKER_STATE_FILES[$worker]=$file; ACTIVE_WORKER_STATE_STARTS[$worker]=$start + : > "$root/capture/terminate-calls" + terminate_managed_session() { printf call >> "$root/capture/terminate-calls"; return 0; } + case "$kind" in + symlink) mv "$file" "$file.real"; ln -s "$file.real" "$file" ;; + wrong-mode) chmod 0644 "$file" ;; + wrong-owner|wrong-gid) + stat() { + last=${!#} + if [[ "$last" == "$file" ]]; then + raw=$(command stat -Lc '%d:%i:%u:%g:%a:%s' -- "$file") + IFS=: read -r dev ino owner gid mode size <<< "$raw" + [[ "$kind" == wrong-owner ]] && owner=$((EUID + 1)) || gid=$((gid + 1)) + printf '%s:%s:%s:%s:%s:%s\n' "$dev" "$ino" "$owner" "$gid" "$mode" "$size" + else command stat "$@"; fi + } + ;; + malformed) printf malformed > "$file" ;; + duplicate) printf 'worker_pid=11111\n' >> "$file" ;; + unknown) printf 'unexpected=value\n' >> "$file" ;; + missing) sed -i '/^sid=/d' "$file" ;; + pid-mismatch) sed -i 's/^worker_pid=.*/worker_pid=99999/' "$file" ;; + stable-tamper) + worker_session_state_read_hook() { [[ "$1" != after-open ]] || printf tampered > "$file"; } + ;; + esac + rc=0; cleanup_published_worker_sessions || rc=$? + (( rc != 0 )) || fail "$kind permanent state unexpectedly succeeded" + assert_eq 1 "${#ACTIVE_WORKER_STATE_FILES[@]}" "$kind permanent state preserves map evidence" + assert_eq 0 "$(wc -c < "$root/capture/terminate-calls")" "$kind permanent state executes no session cleanup" + [[ -e "$file" || -L "$file" ]] || fail "$kind permanent state deleted evidence" + unset 'ACTIVE_WORKER_STATE_FILES[$worker]' 'ACTIVE_WORKER_STATE_STARTS[$worker]' + command rm -f -- "$file" "$file.real" 2>/dev/null || true + cleanup_runtime; pass "$kind permanent worker state remains rejected" +) +for permanent_state in symlink wrong-mode wrong-owner wrong-gid malformed duplicate unknown missing pid-mismatch stable-tamper; do + run_permanent_state_case "$permanent_state" +done + +run_parallel_fallback_iteration() ( + run="$1"; root="$TEST_DIR/session-parallel-$run"; setup_fixture "$root" + MAX_PARALLEL=2 + push_to_server() { + local server="$1" lock_fd + exec {lock_fd}>>"$root/capture/calls.lock"; flock -x "$lock_fd" + printf '%s\n' "$server" >> "$root/capture/calls"; : > "$root/capture/ready.$server" + flock -u "$lock_fd"; exec {lock_fd}>&- + while [[ ! -e "$root/capture/release" ]]; do sleep 0.01; done + WORKER_TRANSFER_PID=$BASHPID + WORKER_TRANSFER_START=11 + WORKER_TRANSFER_PGID=$BASHPID + WORKER_TRANSFER_SID=$BASHPID + publish_worker_session_state cleanup_failed + return "$MANAGED_CLEANUP_FAILURE_STATUS" + } + terminate_managed_session() { return 0; } + ( + deadline=$((SECONDS + 15)); status=0 + while (( SECONDS < deadline )); do + if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release"; break; fi + sleep 0.01 + done + [[ -e "$root/capture/release" ]] || { : > "$root/capture/coordinator-timeout"; : > "$root/capture/release"; status=1; } + while [[ ! -e "$root/capture/stop" ]]; do sleep 0.01; done + exit "$status" + ) & coordinator=$! + rc=0; run_server_batch source destination one.example two.example three.example four.example || rc=$? + : > "$root/capture/stop"; coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? + assert_eq 0 "$coordinator_rc" "parallel fallback run $run coordinator releases both workers" + assert_eq "$MANAGED_CLEANUP_FAILURE_STATUS" "$rc" "parallel fallback run $run returns lifecycle status" + assert_eq 2 "$(wc -l < "$root/capture/calls")" "parallel fallback run $run starts only first two workers" + assert_eq 0 "${#ACTIVE_WORKERS[@]}" "parallel fallback run $run reaps active workers" + assert_eq 0 "${#ACTIVE_WORKER_STATE_FILES[@]}" "parallel fallback run $run clears state maps" + assert_eq 0 "${#ACTIVE_WORKER_STATE_STARTS[@]}" "parallel fallback run $run clears state identities" + [[ ! -e "$root/capture/ready.three.example" && ! -e "$root/capture/ready.four.example" ]] || fail "parallel fallback run $run launched later worker" + runtime_has_published_worker_state && fail "parallel fallback run $run left session state" + reset_batch_lifecycle_barrier; cleanup_runtime; assert_no_registration_residue "$root" +) +parallel_fallback_runs=${PUSH_WORKER_REGISTRATION_PARALLEL_STRESS:-1} +for parallel_run in $(seq 1 "$parallel_fallback_runs"); do run_parallel_fallback_iteration "$parallel_run"; done +pass "parallel fallback completed $parallel_fallback_runs deterministic run(s)" + +write_active_grace_fixture() { + local root="$1" + mkdir -m 0700 "$root" "$root/bin" "$root/runtime" "$root/ssh" "$root/pids" + printf source > "$root/source" + cat > "$root/config.conf" < "$root/bin/timeout" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/timeout.pid" +[[ ${1:-} == --kill-after=* ]] && shift +shift +"$@" & +wait "$!" +EOF + cat > "$root/bin/sshpass" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/sshpass.pid" +[[ ${1:-} == -e ]] && shift +"$@" & +wait "$!" +EOF + cat > "$root/bin/rsync" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +set -m +printf '%s\n' "$$" > "$PID_DIR/rsync.pid" +ssh fake-target & +wait "$!" +EOF + cat > "$root/bin/ssh" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/ssh.pid" +if [[ "${STATE_PUBLISH_MODE:-active}" == cleanup ]]; then + while [[ ! -e "$STATE_CLEANUP_RELEASE" ]]; do sleep 0.01; done + exit 0 +fi +sleep 300 & +printf '%s\n' "$!" > "$PID_DIR/leaf.pid" +wait "$!" +EOF + chmod 0700 "$root/bin"/* +} +cat > "$TEST_DIR/active-grace-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" +shift +source "$script" +worker_after_session_cleanup_hook() { + : > "$ACTIVE_GRACE_STARTED" + sleep 3.2 + : > "$ACTIVE_GRACE_FINISHED" +} +main "$@" +CHILD +chmod 0700 "$TEST_DIR/active-grace-child.sh" +run_active_grace_signal_case() { + local signal_name="$1" expected="$2" root + local main_pid watchdog unrelated rc=0 state_file file pid + root="$TEST_DIR/active-grace-$signal_name" + CURRENT_FIXTURE_ROOT=$root + write_active_grace_fixture "$root" + sleep 60 & unrelated=$! + ( + cd "$root" + exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ + ACTIVE_GRACE_STARTED="$root/cleanup-started" ACTIVE_GRACE_FINISHED="$root/cleanup-finished" \ + bash "$TEST_DIR/active-grace-child.sh" "$SCRIPT" "$root/source" /remote/path + ) > "$root/output.log" 2>&1 & + main_pid=$! + for _ in {1..400}; do + state_file=$(find "$root/runtime" -name 'worker-session.*.state' -print -quit 2>/dev/null || true) + [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] && break + sleep 0.05 + done + if [[ -z "${state_file:-}" || ! -f "$root/pids/leaf.pid" ]]; then + cat "$root/output.log" >&2 || true + kill "$main_pid" "$unrelated" 2>/dev/null || true + fail "$signal_name active grace fixture did not enter managed session" + fi + test_watchdog_process 20 "$root/watchdog-timeout" "$main_pid" & watchdog=$! + kill "-$signal_name" "$main_pid" + wait "$main_pid" || rc=$? + kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root/watchdog-timeout" ]] || fail "$signal_name active grace exceeded watchdog" + assert_eq "$expected" "$rc" "$signal_name active worker preserves main signal status" + [[ -e "$root/cleanup-started" && -e "$root/cleanup-finished" ]] || { cat "$root/output.log" >&2 || true; fail "$signal_name active worker was killed before cleanup hook completed"; } + for file in "$root/pids"/*.pid; do + [[ -f "$file" ]] || continue + pid=$(<"$file") + process_is_running "$pid" && fail "$signal_name active grace left $(basename "$file") pid=$pid" + done + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$signal_name active grace left runtime state" + kill -0 "$unrelated" 2>/dev/null || fail "$signal_name active grace killed unrelated process" + kill "$unrelated" 2>/dev/null || true; wait "$unrelated" 2>/dev/null || true + pass "$signal_name active worker receives full managed-session cleanup grace" +} +run_active_grace_signal_case HUP 129 +run_active_grace_signal_case INT 130 +run_active_grace_signal_case TERM 143 + +cat > "$TEST_DIR/live-cleanup-failure-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" +shift +source "$script" +main_owner=$BASHPID +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +terminate_managed_session() { + if [[ "$BASHPID" != "$main_owner" ]]; then return 1; fi + original_terminate_managed_session "$@" +} +main "$@" +CHILD +chmod 0700 "$TEST_DIR/live-cleanup-failure-child.sh" + +run_live_cleanup_failure_signal_case() ( + trap - EXIT HUP INT TERM + local signal_name="$1" expected="$2" root main_pid main_start state_file="" saved_temp + local worker_pid="" worker_start="" leader_pid="" leader_start="" managed_sid="" watchdog="" unrelated="" unrelated_start="" rc=0 + cleanup_live_failure_fixture() { + [[ -z "$worker_pid" || -z "$worker_start" ]] || { test_process_identity_exists "$worker_pid" "$worker_start" && kill -KILL "$worker_pid" 2>/dev/null || true; } + [[ -z "$main_pid" || -z "$main_start" ]] || { test_process_identity_exists "$main_pid" "$main_start" && kill -KILL "$main_pid" 2>/dev/null || true; } + [[ -z "$unrelated" || -z "$unrelated_start" ]] || { test_process_identity_exists "$unrelated" "$unrelated_start" && kill -TERM "$unrelated" 2>/dev/null || true; } + [[ -z "$main_pid" ]] || wait "$main_pid" 2>/dev/null || true + [[ -z "$watchdog" ]] || { kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true; } + [[ -z "$unrelated" ]] || wait "$unrelated" 2>/dev/null || true + } + trap cleanup_live_failure_fixture EXIT + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + root="$TEST_DIR/live-cleanup-failure-$signal_name"; CURRENT_FIXTURE_ROOT=$root + write_active_grace_fixture "$root" + sleep 60 & unrelated=$!; unrelated_start=$(wait_test_process_start "$unrelated") + ( + cd "$root" + exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ + bash "$TEST_DIR/live-cleanup-failure-child.sh" "$SCRIPT" "$root/source" /remote/path + ) > "$root/output.log" 2>&1 & + main_pid=$!; main_start=$(wait_test_process_start "$main_pid") + for _ in {1..500}; do + state_file=$(find "$root/runtime" -type f -name 'worker-session.*.state' -print -quit 2>/dev/null || true) + [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] && break + sleep 0.05 + done + [[ -n "$state_file" && -f "$root/pids/leaf.pid" ]] || fail "$signal_name cleanup-failure fixture did not reach live session" + saved_temp=$TEMP_DIR; TEMP_DIR=$(dirname "$state_file") + read_worker_session_state "$state_file" || fail "$signal_name cleanup-failure state unreadable" + worker_pid=$SESSION_WORKER_PID; worker_start=$SESSION_WORKER_START + leader_pid=$SESSION_LEADER_PID; leader_start=$SESSION_LEADER_START; managed_sid=$SESSION_SID + TEMP_DIR=$saved_temp + wait_test_process_identity_present "$worker_pid" "$worker_start" || fail "$signal_name cleanup-failure worker identity missing" + wait_test_process_identity_present "$leader_pid" "$leader_start" || fail "$signal_name cleanup-failure leader identity missing" + test_watchdog_process 30 "$root/watchdog-fired" "$main_pid" & watchdog=$! + kill "-$signal_name" "$main_pid" + wait "$main_pid" || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root/watchdog-fired" ]] || fail "$signal_name cleanup failure fired watchdog" + assert_eq "$expected" "$rc" "$signal_name cleanup failure preserves main signal status" + wait_test_process_identity_gone "$worker_pid" "$worker_start" || fail "$signal_name cleanup failure left worker identity" + wait_test_process_identity_gone "$leader_pid" "$leader_start" || fail "$signal_name cleanup failure left leader identity" + collect_owned_session_records "$managed_sid" session_pids session_starts session_ppids session_pgids session_states || fail "$signal_name cleanup failure cannot verify managed SID" + assert_eq 0 "${#session_pids[@]}" "$signal_name cleanup failure leaves no managed SID member or zombie" + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$signal_name cleanup failure left runtime or state" + test_process_identity_exists "$unrelated" "$unrelated_start" || fail "$signal_name cleanup failure killed unrelated process" + kill -TERM "$unrelated"; wait "$unrelated" 2>/dev/null || true + main_pid=""; main_start=""; watchdog=""; unrelated=""; unrelated_start="" + trap - EXIT + pass "$signal_name cleanup failure uses parent fallback without watchdog or zombie" +) +run_live_cleanup_failure_signal_case HUP 129 +run_live_cleanup_failure_signal_case INT 130 +run_live_cleanup_failure_signal_case TERM 143 + +cat > "$TEST_DIR/state-publication-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +script="$1" +shift +source "$script" +main_owner=$BASHPID +eval "$(declare -f terminate_managed_session | sed '1s/terminate_managed_session/original_terminate_managed_session/')" +terminate_managed_session() { + if [[ "$STATE_PUBLISH_MODE" == cleanup && "$BASHPID" != "$main_owner" ]]; then return 1; fi + original_terminate_managed_session "$@" +} +worker_session_state_publish_hook() { + local phase="$1" state="$2" state_file="$3" hook_stage="$4" marker_stage worker_start + if [[ "$STATE_PUBLISH_MODE" == cleanup && "$state" == active && "$phase" == state-file-assigned ]]; then + : > "$STATE_CLEANUP_RELEASE" + fi + [[ "$phase" == "$STATE_PUBLISH_PHASE" && "$state" == "$STATE_PUBLISH_STATE" ]] || return 0 + [[ ! -e "$STATE_PUBLISH_MARKER" ]] || return 0 + worker_start="" + for _ in {1..20}; do + if read_process_record "$BASHPID" && [[ "$PROC_UID" == "$EUID" && "$PROC_STATE" != Z ]]; then worker_start=$PROC_START; break; fi + sleep 0.01 + done + [[ -n "$worker_start" ]] || return 1 + marker_stage="$STATE_PUBLISH_MARKER.$BASHPID.stage" + (umask 077; set -o noclobber; : > "$marker_stage") || return 1 + printf 'phase=%s\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\nstate_file=%s\nhook_stage=%s\n' \ + "$phase" "$state" "$BASHPID" "$worker_start" "$WORKER_TRANSFER_PID" "$WORKER_TRANSFER_START" \ + "$WORKER_TRANSFER_PGID" "$WORKER_TRANSFER_SID" "$state_file" "$hook_stage" > "$marker_stage" + chmod 0600 "$marker_stage"; mv -fT "$marker_stage" "$STATE_PUBLISH_MARKER" + while :; do :; done +} +main "$@" +CHILD +chmod 0700 "$TEST_DIR/state-publication-child.sh" + +run_state_publication_signal_case() ( + local state="$1" phase="$2" signal_name="$3" expected="$4" mode root main_pid="" main_start="" worker_pid="" worker_start="" leader_pid="" leader_start="" managed_sid="" + local watchdog="" watchdog_start="" unrelated="" unrelated_start="" rc=0 line key value marker state_file hook_stage + local -a session_pids=() + local -A session_starts=() session_ppids=() session_pgids=() session_states=() + cleanup_publication_fixture() { + [[ -z "$worker_pid" || -z "$worker_start" ]] || { test_process_identity_exists "$worker_pid" "$worker_start" && kill -KILL "$worker_pid" 2>/dev/null || true; } + [[ -z "$main_pid" || -z "$main_start" ]] || { test_process_identity_exists "$main_pid" "$main_start" && kill -KILL "$main_pid" 2>/dev/null || true; } + [[ -z "$watchdog" || -z "$watchdog_start" ]] || { test_process_identity_exists "$watchdog" "$watchdog_start" && kill -TERM "$watchdog" 2>/dev/null || true; } + [[ -z "$unrelated" || -z "$unrelated_start" ]] || { test_process_identity_exists "$unrelated" "$unrelated_start" && kill -TERM "$unrelated" 2>/dev/null || true; } + [[ -z "$main_pid" ]] || wait "$main_pid" 2>/dev/null || true + [[ -z "$watchdog" ]] || wait "$watchdog" 2>/dev/null || true + [[ -z "$unrelated" ]] || wait "$unrelated" 2>/dev/null || true + } + trap cleanup_publication_fixture EXIT + mode=active; [[ "$state" == cleanup_failed ]] && mode=cleanup + root="$TEST_DIR/state-publish-$state-$phase-$signal_name${5:-}"; CURRENT_FIXTURE_ROOT=$root + write_active_grace_fixture "$root" + mkdir -m 0700 "$root/capture" + marker="$root/capture/publish-marker" + sleep 60 & unrelated=$!; unrelated_start=$(wait_test_process_start "$unrelated") + ( + cd "$root" + exec env --default-signal=HUP,INT,TERM PATH="$root/bin:$PATH" TMPDIR="$root/runtime" PID_DIR="$root/pids" \ + STATE_PUBLISH_MODE="$mode" STATE_PUBLISH_STATE="$state" STATE_PUBLISH_PHASE="$phase" STATE_PUBLISH_MARKER="$marker" \ + STATE_CLEANUP_RELEASE="$root/capture/cleanup-release" bash "$TEST_DIR/state-publication-child.sh" "$SCRIPT" "$root/source" /remote/path + ) > "$root/output.log" 2>&1 & + main_pid=$!; main_start=$(wait_test_process_start "$main_pid") + for _ in {1..600}; do [[ -f "$marker" ]] && break; sleep 0.05; done + [[ -f "$marker" && ! -L "$marker" && "$(stat -Lc %a "$marker")" == 600 ]] || { cat "$root/output.log" >&2 || true; fail "$state/$phase/$signal_name did not reach publish hook"; } + while IFS= read -r line; do key=${line%%=*}; value=${line#*=}; case "$key" in worker_pid) worker_pid=$value;;worker_start) worker_start=$value;;leader_pid) leader_pid=$value;;leader_start) leader_start=$value;;sid) managed_sid=$value;;state_file) state_file=$value;;hook_stage) hook_stage=$value;;esac; done < "$marker" + [[ "$worker_pid" =~ ^[1-9][0-9]*$ && "$worker_start" =~ ^[1-9][0-9]*$ && "$managed_sid" =~ ^[1-9][0-9]*$ ]] || fail "$state/$phase/$signal_name marker identity malformed" + wait_test_process_identity_present "$worker_pid" "$worker_start" || { cat "$marker" >&2; fail "$state/$phase/$signal_name worker identity missing at hook"; } + test_watchdog_process 30 "$root/watchdog-timeout" "$main_pid" & watchdog=$!; watchdog_start=$(wait_test_process_start "$watchdog") + printf "DIAG: publication state=%s phase=%s signal=%s main=%s/%s worker=%s/%s sid=%s\n" \ + "$state" "$phase" "$signal_name" "$main_pid" "$main_start" "$worker_pid" "$worker_start" "$managed_sid" + kill "-$signal_name" "$main_pid"; wait "$main_pid" || rc=$? + kill -TERM "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$root/watchdog-timeout" ]] || fail "$state/$phase/$signal_name watchdog fired" + assert_eq "$expected" "$rc" "$state/$phase preserves $signal_name status" + wait_test_process_identity_gone "$worker_pid" "$worker_start" || fail "$state/$phase/$signal_name left worker identity" + if [[ "$leader_pid" =~ ^[1-9][0-9]*$ && "$leader_start" =~ ^[1-9][0-9]*$ ]]; then + if ! wait_test_process_identity_gone "$leader_pid" "$leader_start"; then + read_process_record "$leader_pid" || true + printf 'leader remains pid=%s expected_start=%s actual_start=%s state=%s ppid=%s pgid=%s sid=%s\n' \ + "$leader_pid" "$leader_start" "${PROC_START:-unknown}" "${PROC_STATE:-unknown}" "${PROC_PPID:-unknown}" "${PROC_PGID:-unknown}" "${PROC_SID:-unknown}" >&2 + cat "/proc/$leader_pid/stat" >&2 2>/dev/null || true + fail "$state/$phase/$signal_name left leader identity" + fi + fi + collect_owned_session_records "$managed_sid" session_pids session_starts session_ppids session_pgids session_states || fail "$state/$phase/$signal_name cannot verify managed SID" + (( ${#session_pids[@]} == 0 )) || fail "$state/$phase/$signal_name left managed SID member: ${session_pids[*]}" + [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "$state/$phase/$signal_name left runtime/session/registration state" + [[ ! -e "$state_file" && ! -e "$hook_stage" && ! -L "$state_file" && ! -L "$hook_stage" ]] || fail "$state/$phase/$signal_name left publication path" + test_process_identity_exists "$unrelated" "$unrelated_start" || fail "$state/$phase/$signal_name killed unrelated process" + kill -TERM "$unrelated"; wait "$unrelated" 2>/dev/null || true + unrelated=""; unrelated_start=""; main_pid=""; main_start=""; watchdog=""; watchdog_start="" + trap - EXIT + pass "$state/$phase/$signal_name publication window cleans process tree and runtime" +) +publication_states=${PUSH_STATE_PUBLICATION_STATES:-"active cleanup_failed"} +publication_phases=${PUSH_STATE_PUBLICATION_PHASES:-"before-stage-create stage-ready after-rename state-file-assigned"} +publication_signals=${PUSH_STATE_PUBLICATION_SIGNALS:-"HUP INT TERM"} +for publication_state in $publication_states; do + for publication_phase in $publication_phases; do + for publication_signal in $publication_signals; do + case "$publication_signal" in HUP) publication_status=129 ;; INT) publication_status=130 ;; TERM) publication_status=143 ;; *) fail "unknown publication signal" ;; esac + run_state_publication_signal_case "$publication_state" "$publication_phase" "$publication_signal" "$publication_status" + done + done +done + +( + # Three bounded extra probes; preserve original matrix and stop at first failure. + for diagnostic_round in 1 2 3; do + printf "DIAG: extra HUP probe=%s/3\n" "$diagnostic_round" + run_state_publication_signal_case cleanup_failed after-rename HUP 129 "-probe-$diagnostic_round" + done + root="$TEST_DIR/normal-parallel"; setup_fixture "$root" + : > "$root/capture/current"; : > "$root/capture/max" + push_to_server() { + local server="$1" lock_fd current max group + case "$server" in one.example|two.example) group=first ;; *) group=second ;; esac + exec {lock_fd}>>"$root/capture/lock"; flock -x "$lock_fd" + current=$(<"$root/capture/current"); ((current += 1)); printf '%s' "$current" > "$root/capture/current" + max=$(<"$root/capture/max"); (( current > max )) && printf '%s' "$current" > "$root/capture/max" + printf '%s\n' "$server" >> "$root/capture/transfer-calls"; : > "$root/capture/ready.$server" + flock -u "$lock_fd"; exec {lock_fd}>&- + while [[ ! -e "$root/capture/release.$group" ]]; do sleep 0.02; done + exec {lock_fd}>>"$root/capture/lock"; flock -x "$lock_fd" + current=$(<"$root/capture/current"); ((current -= 1)); printf '%s' "$current" > "$root/capture/current" + flock -u "$lock_fd"; exec {lock_fd}>&- + record_result success "$server" + } + ( + deadline=$((SECONDS + 15)) + while (( SECONDS < deadline )); do + if [[ -e "$root/capture/ready.one.example" && -e "$root/capture/ready.two.example" ]]; then : > "$root/capture/release.first"; break; fi + sleep 0.02 + done + [[ -e "$root/capture/release.first" ]] || { : > "$root/capture/coordinator-timeout"; : > "$root/capture/release.first"; : > "$root/capture/release.second"; exit 1; } + while (( SECONDS < deadline + 15 )); do + if [[ -e "$root/capture/ready.three.example" && -e "$root/capture/ready.four.example" ]]; then : > "$root/capture/release.second"; exit 0; fi + sleep 0.02 + done + : > "$root/capture/coordinator-timeout"; : > "$root/capture/release.second"; exit 1 + ) & coordinator=$! + run_server_batch source destination one.example two.example three.example four.example + coordinator_rc=0; wait "$coordinator" || coordinator_rc=$? + assert_eq 0 "$coordinator_rc" "normal parallel coordinator releases both worker pairs" + assert_eq 2 "$(<"$root/capture/max")" "normal registration preserves MAX_PARALLEL=2" + assert_eq 4 "$(wc -l < "$root/capture/transfer-calls")" "normal registration completes four fake servers" + assert_no_registration_residue "$root"; cleanup_runtime; assert_no_registration_residue "$root" + pass "normal registration/session lifecycle cleans all state" +) +printf 'All push worker registration tests passed.\n' From 603c128478691fec53d94a447e45656bce34974d Mon Sep 17 00:00:00 2001 From: LUNO Date: Sat, 24 Oct 2026 13:05:41 +0800 Subject: [PATCH 03/12] ci: keep the serial contract fixtures out of the parallel pool test-shards, test-selection and test-selection-git re-enter tests/run.sh with stub suites and assert the exact serial output, so they must not inherit TEST_JOBS or TEST_SEGMENTS from the CI job. run.sh also scrubs both from every suite it starts, so a nested invocation can never re-enable them. --- tests/run.sh | 6 ++++-- tests/test-selection-git.sh | 2 ++ tests/test-selection.sh | 2 ++ tests/test-shards.sh | 2 ++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/tests/run.sh b/tests/run.sh index 15d741b..326a0ae 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -98,10 +98,12 @@ run_suite() { printf '\n==> %s\n' "$test_name" # Keep the child shell standalone: an if/|| wrapper can change errexit behavior. set +e + # Nested fixtures re-enter this script; they must not inherit the pool or + # segment settings, or a stub suite would be duplicated. if [[ -n "$seg" ]]; then - TEST_SEGMENT="$seg" TEST_SEGMENTS="$segments" bash "$test_file" + env -u TEST_JOBS -u TEST_SEGMENTS TEST_SEGMENT="$seg" bash "$test_file" else - bash "$test_file" + env -u TEST_JOBS -u TEST_SEGMENTS bash "$test_file" fi status=$? set -e diff --git a/tests/test-selection-git.sh b/tests/test-selection-git.sh index aacb939..81e6ec4 100644 --- a/tests/test-selection-git.sh +++ b/tests/test-selection-git.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +# These fixtures re-enter run.sh and require its serial, whole-shard behaviour. +unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) TEMP_DIR=$(mktemp -d) trap 'rm -rf "$TEMP_DIR"' EXIT diff --git a/tests/test-selection.sh b/tests/test-selection.sh index 591b5cc..942e523 100644 --- a/tests/test-selection.sh +++ b/tests/test-selection.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +# These fixtures re-enter run.sh and require its serial, whole-shard behaviour. +unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) TEMP_DIR=$(mktemp -d) trap 'rm -rf "$TEMP_DIR"' EXIT diff --git a/tests/test-shards.sh b/tests/test-shards.sh index 598fd82..6397696 100644 --- a/tests/test-shards.sh +++ b/tests/test-shards.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +# These fixtures re-enter run.sh and require its serial, whole-shard behaviour. +unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT # Inert fixtures only: verify partitioning without running production suites. ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) TEMP_DIR=$(mktemp -d) From c1579eeee02430abb6a099a59fd78c8a2aeb2a5a Mon Sep 17 00:00:00 2001 From: LUNO Date: Sat, 24 Oct 2026 13:16:10 +0800 Subject: [PATCH 04/12] ci: drop the motd segment guards Four segments re-ran the ~212 s of top-level code that cannot be wrapped (compound statements that do not parse standalone), so the shard went from 341 s to 390 s on Ubuntu and 317 s vs 306 s on Debian. Revert to the plain suite; the parallel runner and the other CI changes stay. --- tests/test-motd.sh | 78 ---------------------------------------------- 1 file changed, 78 deletions(-) diff --git a/tests/test-motd.sh b/tests/test-motd.sh index a51a5e7..cccdae2 100644 --- a/tests/test-motd.sh +++ b/tests/test-motd.sh @@ -60,26 +60,8 @@ done; extract_payload "$ROOT_DIR/tools/setup-motd.sh">"$TEST_DIR/t"; extract_payload "$ROOT_DIR/modules/system-customize.sh">"$TEST_DIR/m"; cmp -s "$TEST_DIR/t" "$TEST_DIR/m"||fail payload; -# --- CI segment filter (tools/segmentize.py) --- -# TEST_SEGMENT unset: every case runs, exactly like the unsplit suite. -# TEST_SEGMENT=k: the k-th process of TEST_SEGMENTS runs its share only. -__seg_plan=(1,3,3,3,3,1,2,2,2,1,2,2,3,1,1,3,1,2,3,3,1,1,1,3,3,3,1,2,3,1,2,0) -__seg_n=0 -seg_skip() { - local slot=${__seg_plan[__seg_n]:-0} - __seg_n=$((__seg_n + 1)) - [[ -z ${TEST_SEGMENT:-} ]] && return 1 - (( slot != TEST_SEGMENT )) -} -# --- end CI segment filter --- - -if seg_skip; then :; else pass "MOTD payloads stay identical"; -fi -if seg_skip; then :; else assert_eq cf2a4f8dfe2fe26bff5509cdaee56d9e0816fafd0594251b02e892201af5c4d5 "$(sha256sum "$TEST_DIR/t"|awk '{print $1}')" "payload hash" -fi -if seg_skip; then :; else ( r="$TEST_DIR/source"; mkdir -m700 "$r" "$r/lock"; @@ -88,11 +70,7 @@ b=$(trap -p EXIT HUP INT TERM ERR); load_motd; [[ "$b" == "$(trap -p EXIT HUP INT TERM ERR)"&&! -e "$MOTD_STATE_DIR"&&! -e "$MOTD_LOCK_FILE" ]]||fail source ); -fi -if seg_skip; then :; else pass "source zero side effects" -fi -if seg_skip; then :; else ( r="$TEST_DIR/basic"; create_fixture "$r"; @@ -108,8 +86,6 @@ printf changed>"$MOTD_ETC_ROOT/motd"; motd_run_locked_operation restore initial; assert_eq "$before" "$(target_hash "$r")" "restore initial group" ) -fi -if seg_skip; then :; else ( r="$TEST_DIR/types"; create_fixture "$r"; @@ -126,8 +102,6 @@ assert_eq absent "${MOTD_SNAPSHOT_STATE[issue_net]}" "absent snapshot"; motd_run_locked_operation restore previous; assert_eq "$before" "$(target_hash "$r")" "symlink restore" ) -fi -if seg_skip; then :; else ( r="$TEST_DIR/initial"; create_fixture "$r"; @@ -140,8 +114,6 @@ assert_fail "initial capture atomic" motd_run_locked_operation install; [[ ! -e "$MOTD_STATE_DIR/initial.current" ]]||fail partial; assert_eq "$before" "$(target_hash "$r")" "initial failure no target change" ) -fi -if seg_skip; then :; else ( r="$TEST_DIR/previous"; create_fixture "$r"; @@ -156,8 +128,6 @@ assert_fail "previous capture $id" motd_run_locked_operation install; assert_eq "$old" "$(motd_read_pointer previous)" "previous pointer $id"; done ) -fi -if seg_skip; then :; else for failing_id in motd issue_net motd_news; do ( @@ -187,8 +157,6 @@ assert_fail "restore failure $failing_id" motd_run_locked_operation restore prev assert_eq "$before" "$(target_hash "$r")" "restore rollback $failing_id" ) done -fi -if seg_skip; then :; else ( r="$TEST_DIR/manifest"; create_fixture "$r"; @@ -211,8 +179,6 @@ assert_fail "manifest $k rejected" motd_run_locked_operation restore previous; assert_eq "$before" "$(target_hash "$r")" "manifest $k no target change"; done ) -fi -if seg_skip; then :; else ( r="$TEST_DIR/lock"; create_fixture "$r"; @@ -234,11 +200,7 @@ motd_lock_acquired_hook(){ :; motd_take_lock; motd_release_lock ); -fi -if seg_skip; then :; else pass "shared lock nonblocking and reacquirable" -fi -if seg_skip; then :; else for k in symlink writable fifo; do( r="$TEST_DIR/l-$k"; @@ -254,8 +216,6 @@ esac; assert_fail "lock rejects $k" motd_take_lock ); done -fi -if seg_skip; then :; else for k in symlink writable; do( r="$TEST_DIR/s-$k"; @@ -271,7 +231,6 @@ assert_fail "state rejects $k" motd_prepare_state_layout; motd_release_lock ); done -fi create_legacy(){ printf legacy>"$MOTD_ETC_ROOT/motd.initial-backup"; : >"$MOTD_ETC_ROOT/issue.initial-absent"; printf net>"$MOTD_ETC_ROOT/issue.net.initial-backup"; @@ -279,7 +238,6 @@ printf net>"$MOTD_ETC_ROOT/issue.net.initial-backup"; printf uname>"$MOTD_ETC_ROOT/update-motd.d/10-uname.initial-backup"; : >"$MOTD_ETC_ROOT/update-motd.d/50-motd-news.initial-absent"; } -if seg_skip; then :; else ( r="$TEST_DIR/legacy"; create_fixture "$r"; @@ -290,8 +248,6 @@ motd_run_locked_operation restore initial; assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "legacy complete restore"; [[ -e "$MOTD_ETC_ROOT/motd.initial-backup" ]]||fail legacymove ) -fi -if seg_skip; then :; else for k in partial conflict commit; do( r="$TEST_DIR/lg-$k"; @@ -308,8 +264,6 @@ load_motd; assert_fail "legacy $k rejected" motd_run_locked_operation restore initial ); done -fi -if seg_skip; then :; else ( r="$TEST_DIR/interop"; create_fixture "$r"; @@ -320,8 +274,6 @@ printf x>"$MOTD_ETC_ROOT/motd"; bash -c '. "$1/tools/setup-motd.sh";motd_run_locked_operation restore previous' _ "$ROOT_DIR"; assert_eq "$o" "$(target_hash "$r")" "module state tool restore" ) -fi -if seg_skip; then :; else ( r="$TEST_DIR/exit"; create_fixture "$r"; @@ -332,8 +284,6 @@ bash -c '. "$1";motd_take_lock;motd_install_transaction_traps;motd_prepare_state assert_eq 1 "$n" "active exit zero maps one"; assert_eq "$before" "$(target_hash "$r")" "active exit rollback" ) -fi -if seg_skip; then :; else ( r="$TEST_DIR/rbf"; create_fixture "$r"; @@ -349,7 +299,6 @@ assert_fail "rollback failure nonzero" motd_run_locked_operation install; assert_eq 6 "$(wc -l<"$r/a")" "rollback attempts six"; [[ -e "$MOTD_STATE_DIR/pending" ]]||fail journal ) -fi cat >"$TEST_DIR/sig.sh"<<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -400,7 +349,6 @@ assert_eq "$before" "$(target_hash "$r")" "$sig $ph rollback" ); done; done -if seg_skip; then :; else ( r="$TEST_DIR/unknown"; create_fixture "$r"; @@ -412,8 +360,6 @@ g=$(motd_read_pointer initial); motd_validate_snapshot "$MOTD_STATE_DIR/generations/$g" initial "$g"; assert_eq unknown "${MOTD_SNAPSHOT_STATE[motd]}" "unknown initial state" ) -fi -if seg_skip; then :; else ( r="$TEST_DIR/readonly"; create_fixture "$r"; @@ -422,11 +368,7 @@ bash "$ROOT_DIR/tools/setup-motd.sh" status>/dev/null; bash "$ROOT_DIR/tools/setup-motd.sh" help>/dev/null; [[ ! -e "$MOTD_STATE_DIR"&&! -e "$MOTD_LOCK_FILE" ]]||fail readonly ); -fi -if seg_skip; then :; else pass "status help no lock" -fi -if seg_skip; then :; else ( r="$TEST_DIR/publish"; create_fixture "$r"; @@ -448,7 +390,6 @@ motd_snapshot_commit_hook(){ [[ "$1" != previous||"$3" != directory ]]; assert_fail "previous directory commit failure" motd_run_locked_operation install; assert_eq "$old" "$(motd_read_pointer previous)" "directory failure preserves previous" ) -fi for k in owner gid; do( r="$TEST_DIR/lmeta-$k"; @@ -462,9 +403,7 @@ then [[ $k == owner ]]&&printf '99999:%s:600\n' "$MOTD_TRUSTED_GID"||printf '%s: else command stat "$@"; fi; }; -if seg_skip; then :; else assert_fail "lock wrong $k" motd_take_lock -fi ); done for k in owner gid; @@ -480,13 +419,10 @@ else command stat "$@"; fi; }; motd_take_lock; -if seg_skip; then :; else assert_fail "state wrong $k" motd_prepare_state_layout; -fi motd_release_lock ); done -if seg_skip; then :; else ( r="$TEST_DIR/state-create"; create_fixture "$r"; @@ -498,8 +434,6 @@ command chown "$@"; assert_fail "state create failure" motd_run_locked_operation install; [[ ! -e "$MOTD_STATE_DIR" ]]||fail state-residue ) -fi -if seg_skip; then :; else ( r="$TEST_DIR/close"; create_fixture "$r"; @@ -510,8 +444,6 @@ motd_lock_close_hook(){ return 1; }; assert_fail "lock close failure" motd_release_lock ) -fi -if seg_skip; then :; else ( r="$TEST_DIR/device"; create_fixture "$r"; @@ -524,8 +456,6 @@ assert_eq "$before" "$(target_hash "$r")" "device no target change"; else pass "device test skipped"; fi ) -fi -if seg_skip; then :; else ( r="$TEST_DIR/legacy-state"; create_fixture "$r"; @@ -541,7 +471,6 @@ load_motd; motd_run_locked_operation restore previous; assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "state legacy restore" ) -fi state_fingerprint() { local root="$1" path @@ -577,7 +506,6 @@ create_complete_legacy_scope() { : > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent" } -if seg_skip; then :; else for legacy_phase in random-id stat open create cat chown chmod sha256 manifest mv; do ( root="$TEST_DIR/legacy-io-$legacy_phase"; create_fixture "$root"; export_paths "$root" @@ -593,10 +521,7 @@ for legacy_phase in random-id stat open create cat chown chmod sha256 manifest m assert_no_motd_residue "$root" ) done -fi -if seg_skip; then :; else pass "legacy I/O failure matrix preserves data and state" -fi cat > "$TEST_DIR/pointer-signal-child.sh" <<'CHILD' #!/usr/bin/env bash @@ -610,7 +535,6 @@ motd_run_locked_operation install CHILD chmod 0700 "$TEST_DIR/pointer-signal-child.sh" -if seg_skip; then :; else for signal_name in HUP INT TERM; do ( root="$TEST_DIR/pointer-signal-$signal_name"; create_fixture "$root"; export_paths "$root" @@ -1333,5 +1257,3 @@ for rollback_impl in tool module; do done printf 'All MOTD rollback final validation tests passed.\n' - -fi \ No newline at end of file From dc02fa6daffafc3e5aa5fcafb1d67fb4a5bdf48f Mon Sep 17 00:00:00 2001 From: LUNO Date: Sat, 24 Oct 2026 14:33:10 +0800 Subject: [PATCH 05/12] ci: split the three heavy suites into four semantic segments Each heavy suite keeps its original entry point and is rewritten into a shared prologue plus four contiguous semantic segments guarded by 'if seg_is N; then'. The prologue holds the definitions and the child-script fixtures that later segments consume, so every segment runs on its own; the Cloudflare segments 1-3 additionally replay the parent-shell state (new_case, the two privilege stubs, the uninstall transaction hook) that C0 used to establish for them. Ranges follow the measured cost of the original linear run, so the largest segment is about a third of the suite instead of the whole of it. Loops are not split and no assertion, timeout or stress iteration is changed. Verified on hkhkt (Debian 13, 8 vCPU), serial original vs four segments: motd 655 PASS == 119+140+202+194, 473 s -> 162 s push-worker 303 PASS == 90+70+85+58, 247 s -> 85 s cloudflare not comparable there: the original suite already fails on that host at 'uninstall INT at before-snapshot-create returned 0', so equivalence is left to CI. TEST_SEGMENTS now defaults to 1 and CI sets 4; with it unset every segment runs in order, so a local 'bash tests/test-x.sh' still runs the whole suite. --- tests/run.sh | 2 +- tests/test-cloudflare-tunnel.sh | 27 +++ tests/test-motd.sh | 269 ++++++++++++----------- tests/test-push-worker-registration.sh | 284 +++++++++++++------------ 4 files changed, 322 insertions(+), 260 deletions(-) diff --git a/tests/run.sh b/tests/run.sh index 326a0ae..5d96f84 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -71,7 +71,7 @@ jobs=${TEST_JOBS:-1} # disjoint share of the cases. Every other suite runs once. segments=${TEST_SEGMENTS:-1} [[ "$segments" =~ ^[1-9][0-9]*$ ]] || { printf 'Invalid TEST_SEGMENTS: %s\n' "$segments" >&2; exit 2; } -segmented_families=(test-motd.sh) +segmented_families=(test-motd.sh test-cloudflare-tunnel.sh test-push-worker-registration.sh) job_files=() job_segs=() diff --git a/tests/test-cloudflare-tunnel.sh b/tests/test-cloudflare-tunnel.sh index c23689a..094cd50 100644 --- a/tests/test-cloudflare-tunnel.sh +++ b/tests/test-cloudflare-tunnel.sh @@ -1,3 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +# --- CI segments (tools/segmentize_ranges.py) --- +# TEST_SEGMENT unset: every segment runs in order, exactly like the original. +seg_is() { + [[ -z ${TEST_SEGMENT:-} ]] || (( $1 == TEST_SEGMENT )) +} +# --- end CI segments --- + #!/usr/bin/env bash set -euo pipefail @@ -172,6 +181,16 @@ run_failure_case() { ORIGINAL_PATH=$PATH +# replay the state that a later segment inherited from an earlier one +if [[ -n ${TEST_SEGMENT:-} ]] && (( TEST_SEGMENT != 0 )); then +new_case +require_root() { :; } +check_platform() { :; } +uninstall_transaction_hook() { [[ "$1" != after-disable-auto-update ]] || exit 0; } +fi + +if seg_is 0; then + new_case before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) repository_source_content > "$CASE_DIR/source-output" @@ -596,6 +615,9 @@ grep -Fq "ExecStart=$CLOUDFLARED_APT_BIN --no-autoupdate --token-file /etc/cloud pass "legacy binary and service migration behavior remains" +fi + +if seg_is 1; then # Each malformed snapshot runs in a separate process: transaction traps and # intentional corrupt fixtures must not leak into subsequent cases. for schema_case in version generation duplicate unknown order state mode uid gid digest created payload-name payload-missing payload-extra payload-fifo payload-symlink manifest-fifo ancestor; do @@ -874,7 +896,9 @@ for stage_case in owned foreign; do } pass "stage cleanup verifies $stage_case identity" done +fi +if seg_is 2; then for terminal_signal in HUP INT TERM; do case "$terminal_signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac ( @@ -1222,7 +1246,9 @@ for restore_object in key source current; do pass "repository restore $restore_object $restore_fault rejects corruption and retains all backups" done done +fi +if seg_is 3; then for operation in install_cloudflared upgrade_cloudflared; do ( new_case @@ -1540,5 +1566,6 @@ done entrypoint_output=$(bash -c "$(cat "$ROOT_DIR/tools/cloudflare_tunnel.sh")" cloudflare_tunnel.sh help) grep -Fq 'cloudflare_tunnel.sh install' <<< "$entrypoint_output" || fail "bash -c entrypoint broken" pass "bash -c entrypoint remains compatible" +fi printf 'All cloudflare wrapper tests passed. PASS=%d\n' "$pass_count" diff --git a/tests/test-motd.sh b/tests/test-motd.sh index cccdae2..8a33696 100644 --- a/tests/test-motd.sh +++ b/tests/test-motd.sh @@ -1,3 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +# --- CI segments (tools/segmentize_ranges.py) --- +# TEST_SEGMENT unset: every segment runs in order, exactly like the original. +seg_is() { + [[ -z ${TEST_SEGMENT:-} ]] || (( $1 == TEST_SEGMENT )) +} +# --- end CI segments --- + #!/usr/bin/env bash set -euo pipefail ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."&&pwd); @@ -57,6 +66,133 @@ else echo "O|$p|$(stat -c %F "$p")"; fi; done; } +create_legacy(){ printf legacy>"$MOTD_ETC_ROOT/motd.initial-backup"; +: >"$MOTD_ETC_ROOT/issue.initial-absent"; +printf net>"$MOTD_ETC_ROOT/issue.net.initial-backup"; +: >"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.initial-unknown"; +printf uname>"$MOTD_ETC_ROOT/update-motd.d/10-uname.initial-backup"; +: >"$MOTD_ETC_ROOT/update-motd.d/50-motd-news.initial-absent"; +} +cat >"$TEST_DIR/sig.sh"<<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1"; +hit=false +send(){ [[ $hit == false ]]||return; +hit=true; +kill -"$SIGNAL_NAME" "$BASHPID"; +} +motd_snapshot_capture_hook(){ [[ $PHASE == snapshot-building&&$1 == issue ]]&&send; +return 0; +} +motd_transaction_phase_hook(){ [[ $PHASE == snapshot-commit&&$1 == snapshot-commit ]]&&send; +return 0; +} +motd_snapshot_commit_hook(){ [[ $PHASE == snapshot-publish&&$1 == initial&&$3 == pointer ]]&&send; +return 0; +} +motd_target_commit_hook(){ case "$PHASE:$1:$2" in install-first:install:motd|install-middle:install:issue_net|restore-first:restore:motd|restore-middle:restore:issue_net)send;; +esac; +return 0; +} +case $PHASE in restore-*)motd_run_locked_operation restore previous;; +*)motd_run_locked_operation install;; +esac +CHILD +chmod 700 "$TEST_DIR/sig.sh" +state_fingerprint() { + local root="$1" path + [[ -d "$root" ]] || { printf 'absent\n'; return; } + while IFS= read -r path; do + if [[ -L "$path" ]]; then printf 'L|%s|%s\n' "${path#$root/}" "$(readlink "$path")" + elif [[ -f "$path" ]]; then printf 'F|%s|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" "$(sha256sum "$path" | awk '{print $1}')" + elif [[ -d "$path" ]]; then printf 'D|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" + fi + done < <(find "$root" -mindepth 1 -print | sort) +} + +generation_count() { + local directory="$1" + [[ -d "$directory" ]] || { printf '0\n'; return; } + find "$directory" -mindepth 1 -maxdepth 1 -type d ! -name '.*.stage' | wc -l +} + +assert_no_motd_residue() { + local root="$1" residue + residue=$(find "$root" -name '.linux-setup-motd.*' -o -name '.*.pointer.*' -o -name '.pending.*' -o -name '.*.stage' 2>/dev/null) + [[ -z "$residue" ]] || fail "unexpected MOTD residue: $residue" + if [[ -d "$root/state/transactions" ]]; then [[ -z "$(find "$root/state/transactions" -mindepth 1 -print -quit)" ]] || fail "transaction residue remains"; fi +} + +create_complete_legacy_scope() { + local scope="$1" prefix + cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_ETC_ROOT/motd.${scope}-backup" + cp -a "$MOTD_ETC_ROOT/issue" "$MOTD_ETC_ROOT/issue.${scope}-backup" + : > "$MOTD_ETC_ROOT/issue.net.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/issue.net.${scope}-absent" + cp -a "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.${scope}-backup" + cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_ETC_ROOT/update-motd.d/10-uname.${scope}-backup" + : > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent" +} +cat > "$TEST_DIR/pointer-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1" +motd_snapshot_commit_hook() { + if [[ "$1" == previous && "$3" == pointer ]]; then kill -"$SIGNAL_NAME" "$BASHPID"; fi +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/pointer-signal-child.sh" +cat > "$TEST_DIR/kill-pending-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_transaction_phase_hook() { [[ "$1" == pending-active ]] && kill -KILL "$BASHPID"; return 0; } +motd_run_locked_operation install +CHILD +cat > "$TEST_DIR/kill-previous-pointer-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_snapshot_commit_hook() { [[ "$1" == previous && "$3" == pointer ]] && kill -KILL "$BASHPID"; return 0; } +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/kill-pending-child.sh" "$TEST_DIR/kill-previous-pointer-child.sh" +cat > "$TEST_DIR/sigkill-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_transaction_phase_hook() { + [[ "$1" == "$KILL_PHASE" ]] && kill -KILL "$BASHPID" + return 0 +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/sigkill-child.sh" +cat > "$TEST_DIR/building-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1" +send_building_signal() { kill -"$SIGNAL_NAME" "$BASHPID"; } +motd_transaction_phase_hook() { + case "$FAIL_PHASE:$1" in + transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) send_building_signal ;; + esac + return 0 +} +motd_snapshot_capture_hook() { + [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0 + case "$FAIL_PHASE:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) send_building_signal ;; esac + return 0 +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/building-signal-child.sh" + +if seg_is 0; then extract_payload "$ROOT_DIR/tools/setup-motd.sh">"$TEST_DIR/t"; extract_payload "$ROOT_DIR/modules/system-customize.sh">"$TEST_DIR/m"; cmp -s "$TEST_DIR/t" "$TEST_DIR/m"||fail payload; @@ -231,13 +367,6 @@ assert_fail "state rejects $k" motd_prepare_state_layout; motd_release_lock ); done -create_legacy(){ printf legacy>"$MOTD_ETC_ROOT/motd.initial-backup"; -: >"$MOTD_ETC_ROOT/issue.initial-absent"; -printf net>"$MOTD_ETC_ROOT/issue.net.initial-backup"; -: >"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.initial-unknown"; -printf uname>"$MOTD_ETC_ROOT/update-motd.d/10-uname.initial-backup"; -: >"$MOTD_ETC_ROOT/update-motd.d/50-motd-news.initial-absent"; -} ( r="$TEST_DIR/legacy"; create_fixture "$r"; @@ -299,34 +428,6 @@ assert_fail "rollback failure nonzero" motd_run_locked_operation install; assert_eq 6 "$(wc -l<"$r/a")" "rollback attempts six"; [[ -e "$MOTD_STATE_DIR/pending" ]]||fail journal ) -cat >"$TEST_DIR/sig.sh"<<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1"; -hit=false -send(){ [[ $hit == false ]]||return; -hit=true; -kill -"$SIGNAL_NAME" "$BASHPID"; -} -motd_snapshot_capture_hook(){ [[ $PHASE == snapshot-building&&$1 == issue ]]&&send; -return 0; -} -motd_transaction_phase_hook(){ [[ $PHASE == snapshot-commit&&$1 == snapshot-commit ]]&&send; -return 0; -} -motd_snapshot_commit_hook(){ [[ $PHASE == snapshot-publish&&$1 == initial&&$3 == pointer ]]&&send; -return 0; -} -motd_target_commit_hook(){ case "$PHASE:$1:$2" in install-first:install:motd|install-middle:install:issue_net|restore-first:restore:motd|restore-middle:restore:issue_net)send;; -esac; -return 0; -} -case $PHASE in restore-*)motd_run_locked_operation restore previous;; -*)motd_run_locked_operation install;; -esac -CHILD -chmod 700 "$TEST_DIR/sig.sh" for ph in snapshot-building snapshot-commit snapshot-publish install-first install-middle restore-first restore-middle; do for sig in HUP INT TERM; do( @@ -471,41 +572,9 @@ load_motd; motd_run_locked_operation restore previous; assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "state legacy restore" ) +fi -state_fingerprint() { - local root="$1" path - [[ -d "$root" ]] || { printf 'absent\n'; return; } - while IFS= read -r path; do - if [[ -L "$path" ]]; then printf 'L|%s|%s\n' "${path#$root/}" "$(readlink "$path")" - elif [[ -f "$path" ]]; then printf 'F|%s|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" "$(sha256sum "$path" | awk '{print $1}')" - elif [[ -d "$path" ]]; then printf 'D|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" - fi - done < <(find "$root" -mindepth 1 -print | sort) -} - -generation_count() { - local directory="$1" - [[ -d "$directory" ]] || { printf '0\n'; return; } - find "$directory" -mindepth 1 -maxdepth 1 -type d ! -name '.*.stage' | wc -l -} - -assert_no_motd_residue() { - local root="$1" residue - residue=$(find "$root" -name '.linux-setup-motd.*' -o -name '.*.pointer.*' -o -name '.pending.*' -o -name '.*.stage' 2>/dev/null) - [[ -z "$residue" ]] || fail "unexpected MOTD residue: $residue" - if [[ -d "$root/state/transactions" ]]; then [[ -z "$(find "$root/state/transactions" -mindepth 1 -print -quit)" ]] || fail "transaction residue remains"; fi -} - -create_complete_legacy_scope() { - local scope="$1" prefix - cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_ETC_ROOT/motd.${scope}-backup" - cp -a "$MOTD_ETC_ROOT/issue" "$MOTD_ETC_ROOT/issue.${scope}-backup" - : > "$MOTD_ETC_ROOT/issue.net.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/issue.net.${scope}-absent" - cp -a "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.${scope}-backup" - cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_ETC_ROOT/update-motd.d/10-uname.${scope}-backup" - : > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent" -} - +if seg_is 1; then for legacy_phase in random-id stat open create cat chown chmod sha256 manifest mv; do ( root="$TEST_DIR/legacy-io-$legacy_phase"; create_fixture "$root"; export_paths "$root" @@ -523,17 +592,6 @@ for legacy_phase in random-id stat open create cat chown chmod sha256 manifest m done pass "legacy I/O failure matrix preserves data and state" -cat > "$TEST_DIR/pointer-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1" -motd_snapshot_commit_hook() { - if [[ "$1" == previous && "$3" == pointer ]]; then kill -"$SIGNAL_NAME" "$BASHPID"; fi -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/pointer-signal-child.sh" for signal_name in HUP INT TERM; do ( @@ -596,21 +654,6 @@ for pending_kind in broken-symlink symlink directory fifo mode owner gid; do ) done -cat > "$TEST_DIR/kill-pending-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_transaction_phase_hook() { [[ "$1" == pending-active ]] && kill -KILL "$BASHPID"; return 0; } -motd_run_locked_operation install -CHILD -cat > "$TEST_DIR/kill-previous-pointer-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_snapshot_commit_hook() { [[ "$1" == previous && "$3" == pointer ]] && kill -KILL "$BASHPID"; return 0; } -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/kill-pending-child.sh" "$TEST_DIR/kill-previous-pointer-child.sh" for journal_case in pending-directory journal-transaction generation-scope generation-missing old-new-mix; do ( @@ -696,17 +739,6 @@ for native_id in uname motd_news; do ) done -cat > "$TEST_DIR/sigkill-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_transaction_phase_hook() { - [[ "$1" == "$KILL_PHASE" ]] && kill -KILL "$BASHPID" - return 0 -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/sigkill-child.sh" for kill_phase in transaction-snapshot-building transaction-final-moved target-committed-motd target-committed-issue_net target-committed-motd_news committed-cleanup committed-old-generation-cleaned committed-pending-removed; do ( @@ -831,7 +863,9 @@ for hook_case in content-motd mode-issue_net inode-motd_news type-issue_net; do assert_no_motd_residue "$root" ) done +fi +if seg_is 2; then ( root="$TEST_DIR/hook-rollback-stage"; create_fixture "$root"; export_paths "$root" load_motd; motd_run_locked_operation install @@ -900,26 +934,6 @@ for pending_hook_case in id extra-line inode; do ) done -cat > "$TEST_DIR/building-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1" -send_building_signal() { kill -"$SIGNAL_NAME" "$BASHPID"; } -motd_transaction_phase_hook() { - case "$FAIL_PHASE:$1" in - transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) send_building_signal ;; - esac - return 0 -} -motd_snapshot_capture_hook() { - [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0 - case "$FAIL_PHASE:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) send_building_signal ;; esac - return 0 -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/building-signal-child.sh" for building_phase in transaction-snapshot-building capture-motd capture-issue_net capture-motd_news snapshot-commit transaction-final-move; do ( @@ -1073,7 +1087,9 @@ done grep -Fq 'error "缺少必要命令: $required_command"' "$ROOT_DIR/modules/system-customize.sh" || fail "module missing-command error output branch absent" pass "module dependency preflight retains missing-command output" +fi +if seg_is 3; then for noop_impl in tool module; do for noop_created_type in regular symlink fifo; do ( @@ -1255,5 +1271,6 @@ for rollback_impl in tool module; do assert_no_motd_residue "$root" ) done +fi printf 'All MOTD rollback final validation tests passed.\n' diff --git a/tests/test-push-worker-registration.sh b/tests/test-push-worker-registration.sh index 3ef0fcf..7196192 100755 --- a/tests/test-push-worker-registration.sh +++ b/tests/test-push-worker-registration.sh @@ -1,3 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +# --- CI segments (tools/segmentize_ranges.py) --- +# TEST_SEGMENT unset: every segment runs in order, exactly like the original. +seg_is() { + [[ -z ${TEST_SEGMENT:-} ]] || (( $1 == TEST_SEGMENT )) +} +# --- end CI segments --- + #!/usr/bin/env bash set -euo pipefail # Safe runtime identity only: no environment dump, credentials, or xtrace. @@ -73,6 +82,139 @@ wait_test_process_start() { return 1 } +source "$SCRIPT" +trap 'rm -rf "$TEST_DIR"' EXIT +setup_fixture() { + local root="$1" + CURRENT_FIXTURE_ROOT=$root + mkdir -m 0700 "$root" "$root/runtime" "$root/capture" + TMPDIR="$root/runtime"; initialize_runtime + MAX_PARALLEL=2 + MAX_RETRIES=1 + BATCH_WORKER_FAILED=false + BATCH_WORKER_ERROR=false +} +assert_no_registration_residue() { + local root="$1" + [[ -z "$(find "$root/runtime" \( -name 'worker-registration.*' -o -name '.worker-registration.*' -o -name 'worker-session.*.state' \) -print -quit)" ]] || fail "registration/session residue remains" + (( ${#REGISTERING_WORKERS[@]} == 0 && ${#ACTIVE_WORKERS[@]} == 0 )) || fail "worker arrays remain" + (( ${#WORKER_REGISTRATION_READY_FILES[@]} == 0 && ${#WORKER_REGISTRATION_RELEASE_FILES[@]} == 0 && ${#WORKER_REGISTRATION_STARTS[@]} == 0 && ${#WORKER_REGISTRATION_STAGE_FILES[@]} == 0 )) || fail "registration arrays remain" +} +test_watchdog_process() { + local seconds="$1" marker="$2" target_pid="$3" timer="" + trap '[[ -z "$timer" ]] || kill "$timer" 2>/dev/null || true; [[ -z "$timer" ]] || wait "$timer" 2>/dev/null || true; exit 0' HUP INT TERM + sleep "$seconds" & timer=$! + wait "$timer" 2>/dev/null || exit 0 + printf fired > "$marker" + kill -KILL "$target_pid" 2>/dev/null || true +} + +run_with_watchdog() { + local name="$1" seconds="$2"; shift 2 + local marker="$TEST_DIR/watchdog-$name" pid watchdog rc=0 + "$@" & pid=$! + test_watchdog_process "$seconds" "$marker" "$pid" & watchdog=$! + wait "$pid" || rc=$? + kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$marker" ]] || fail "$name exceeded watchdog deadline" + return "$rc" +} +write_test_worker_session_manifest() { + local file="$1" state="$2" worker_pid="$3" worker_start="$4" leader_pid="$5" leader_start="$6" pgid="$7" sid="$8" + printf 'version=1\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\n' \ + "$state" "$worker_pid" "$worker_start" "$leader_pid" "$leader_start" "$pgid" "$sid" > "$file" + chmod 0600 "$file" +} + + +assert_retained_runtime_trusted() { + local path="$1" expected_dev="$2" expected_inode="$3" metadata="" dev inode owner gid mode trusted_gid + [[ -d "$path" && ! -L "$path" ]] || fail "retained runtime is not a regular directory: $path" + metadata=$(stat -Lc '%d:%i:%u:%g:%a' -- "$path") || fail "cannot stat retained runtime: $path" + IFS=: read -r dev inode owner gid mode <<< "$metadata" + trusted_gid=$(current_gid) + [[ "$dev" == "$expected_dev" && "$inode" == "$expected_inode" && + "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 700 ]] || + fail "retained runtime trust mismatch: $metadata" +} + +assert_retained_state_trusted() { + local file="$1" metadata="" owner gid mode trusted_gid + [[ -f "$file" && ! -L "$file" ]] || fail "retained worker state is not a regular file: $file" + metadata=$(stat -Lc '%u:%g:%a' -- "$file") || fail "cannot stat retained worker state: $file" + IFS=: read -r owner gid mode <<< "$metadata" + trusted_gid=$(current_gid) + [[ "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 600 ]] || + fail "retained worker state trust mismatch: $metadata" +} +write_active_grace_fixture() { + local root="$1" + mkdir -m 0700 "$root" "$root/bin" "$root/runtime" "$root/ssh" "$root/pids" + printf source > "$root/source" + cat > "$root/config.conf" < "$root/bin/timeout" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/timeout.pid" +[[ ${1:-} == --kill-after=* ]] && shift +shift +"$@" & +wait "$!" +EOF + cat > "$root/bin/sshpass" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/sshpass.pid" +[[ ${1:-} == -e ]] && shift +"$@" & +wait "$!" +EOF + cat > "$root/bin/rsync" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +set -m +printf '%s\n' "$$" > "$PID_DIR/rsync.pid" +ssh fake-target & +wait "$!" +EOF + cat > "$root/bin/ssh" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/ssh.pid" +if [[ "${STATE_PUBLISH_MODE:-active}" == cleanup ]]; then + while [[ ! -e "$STATE_CLEANUP_RELEASE" ]]; do sleep 0.01; done + exit 0 +fi +sleep 300 & +printf '%s\n' "$!" > "$PID_DIR/leaf.pid" +wait "$!" +EOF + chmod 0700 "$root/bin"/* +} + +if seg_is 0; then source_root="$TEST_DIR/source"; mkdir -m 0700 "$source_root" source_result=$(env TMPDIR="$source_root" SCRIPT="$SCRIPT" bash -c ' set -euo pipefail @@ -84,7 +226,6 @@ printf source-safe ') assert_eq source-safe "$source_result" "source keeps zero runtime side effects" # shellcheck source=../tools/push.sh -source "$SCRIPT" ( # Inert direct-child job table: no SSH or production services. sleep 10 & snapshot_child=$! @@ -98,7 +239,6 @@ source "$SCRIPT" trap - EXIT pass "job snapshot preserves direct-child matching and reap semantics" ) -trap 'rm -rf "$TEST_DIR"' EXIT # Deterministic coordinator contract: a live worker can republish after cleanup. ( @@ -121,22 +261,6 @@ trap 'rm -rf "$TEST_DIR"' EXIT pass "live publisher tracking survives successful active cleanup" ) -setup_fixture() { - local root="$1" - CURRENT_FIXTURE_ROOT=$root - mkdir -m 0700 "$root" "$root/runtime" "$root/capture" - TMPDIR="$root/runtime"; initialize_runtime - MAX_PARALLEL=2 - MAX_RETRIES=1 - BATCH_WORKER_FAILED=false - BATCH_WORKER_ERROR=false -} -assert_no_registration_residue() { - local root="$1" - [[ -z "$(find "$root/runtime" \( -name 'worker-registration.*' -o -name '.worker-registration.*' -o -name 'worker-session.*.state' \) -print -quit)" ]] || fail "registration/session residue remains" - (( ${#REGISTERING_WORKERS[@]} == 0 && ${#ACTIVE_WORKERS[@]} == 0 )) || fail "worker arrays remain" - (( ${#WORKER_REGISTRATION_READY_FILES[@]} == 0 && ${#WORKER_REGISTRATION_RELEASE_FILES[@]} == 0 && ${#WORKER_REGISTRATION_STARTS[@]} == 0 && ${#WORKER_REGISTRATION_STAGE_FILES[@]} == 0 )) || fail "registration arrays remain" -} run_delayed_parent_reap_case() ( local depth="$1" root leader leader_start leader_pgid leader_sid @@ -233,25 +357,6 @@ CHILD if [[ "${PUSH_REAP_ONLY:-false}" == true ]]; then exit 0 fi -test_watchdog_process() { - local seconds="$1" marker="$2" target_pid="$3" timer="" - trap '[[ -z "$timer" ]] || kill "$timer" 2>/dev/null || true; [[ -z "$timer" ]] || wait "$timer" 2>/dev/null || true; exit 0' HUP INT TERM - sleep "$seconds" & timer=$! - wait "$timer" 2>/dev/null || exit 0 - printf fired > "$marker" - kill -KILL "$target_pid" 2>/dev/null || true -} - -run_with_watchdog() { - local name="$1" seconds="$2"; shift 2 - local marker="$TEST_DIR/watchdog-$name" pid watchdog rc=0 - "$@" & pid=$! - test_watchdog_process "$seconds" "$marker" "$pid" & watchdog=$! - wait "$pid" || rc=$? - kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$marker" ]] || fail "$name exceeded watchdog deadline" - return "$rc" -} root_cause_regression_case() ( root="$TEST_DIR/root-cause"; setup_fixture "$root" @@ -672,36 +777,9 @@ chmod 0700 "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "hard timeout cleanup failure left runtime" pass "hard timeout cleanup failure is bounded and blocks subsequent command" ) +fi -write_test_worker_session_manifest() { - local file="$1" state="$2" worker_pid="$3" worker_start="$4" leader_pid="$5" leader_start="$6" pgid="$7" sid="$8" - printf 'version=1\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\n' \ - "$state" "$worker_pid" "$worker_start" "$leader_pid" "$leader_start" "$pgid" "$sid" > "$file" - chmod 0600 "$file" -} - - -assert_retained_runtime_trusted() { - local path="$1" expected_dev="$2" expected_inode="$3" metadata="" dev inode owner gid mode trusted_gid - [[ -d "$path" && ! -L "$path" ]] || fail "retained runtime is not a regular directory: $path" - metadata=$(stat -Lc '%d:%i:%u:%g:%a' -- "$path") || fail "cannot stat retained runtime: $path" - IFS=: read -r dev inode owner gid mode <<< "$metadata" - trusted_gid=$(current_gid) - [[ "$dev" == "$expected_dev" && "$inode" == "$expected_inode" && - "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 700 ]] || - fail "retained runtime trust mismatch: $metadata" -} - -assert_retained_state_trusted() { - local file="$1" metadata="" owner gid mode trusted_gid - [[ -f "$file" && ! -L "$file" ]] || fail "retained worker state is not a regular file: $file" - metadata=$(stat -Lc '%u:%g:%a' -- "$file") || fail "cannot stat retained worker state: $file" - IFS=: read -r owner gid mode <<< "$metadata" - trusted_gid=$(current_gid) - [[ "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 600 ]] || - fail "retained worker state trust mismatch: $metadata" -} - +if seg_is 1; then run_preserved_cleanup_retry_case() ( local run="$1" root runtime runtime_dev runtime_inode credential leader leader_start leader_pgid leader_sid state_file local owner watchdog rc=0 metadata="" state_status=0 @@ -891,8 +969,9 @@ done fail "clean runtime final proof did not clear state" pass "clean runtime final proof preserves normal success path" ) +fi - +if seg_is 2; then cat > "$TEST_DIR/runtime-retention-signal-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1199,72 +1278,6 @@ parallel_fallback_runs=${PUSH_WORKER_REGISTRATION_PARALLEL_STRESS:-1} for parallel_run in $(seq 1 "$parallel_fallback_runs"); do run_parallel_fallback_iteration "$parallel_run"; done pass "parallel fallback completed $parallel_fallback_runs deterministic run(s)" -write_active_grace_fixture() { - local root="$1" - mkdir -m 0700 "$root" "$root/bin" "$root/runtime" "$root/ssh" "$root/pids" - printf source > "$root/source" - cat > "$root/config.conf" < "$root/bin/timeout" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/timeout.pid" -[[ ${1:-} == --kill-after=* ]] && shift -shift -"$@" & -wait "$!" -EOF - cat > "$root/bin/sshpass" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/sshpass.pid" -[[ ${1:-} == -e ]] && shift -"$@" & -wait "$!" -EOF - cat > "$root/bin/rsync" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -set -m -printf '%s\n' "$$" > "$PID_DIR/rsync.pid" -ssh fake-target & -wait "$!" -EOF - cat > "$root/bin/ssh" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/ssh.pid" -if [[ "${STATE_PUBLISH_MODE:-active}" == cleanup ]]; then - while [[ ! -e "$STATE_CLEANUP_RELEASE" ]]; do sleep 0.01; done - exit 0 -fi -sleep 300 & -printf '%s\n' "$!" > "$PID_DIR/leaf.pid" -wait "$!" -EOF - chmod 0700 "$root/bin"/* -} cat > "$TEST_DIR/active-grace-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1397,7 +1410,9 @@ run_live_cleanup_failure_signal_case() ( run_live_cleanup_failure_signal_case HUP 129 run_live_cleanup_failure_signal_case INT 130 run_live_cleanup_failure_signal_case TERM 143 +fi +if seg_is 3; then cat > "$TEST_DIR/state-publication-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1550,4 +1565,7 @@ done assert_no_registration_residue "$root"; cleanup_runtime; assert_no_registration_residue "$root" pass "normal registration/session lifecycle cleans all state" ) +fi + printf 'All push worker registration tests passed.\n' + From 50b431af16d8d5ed3fedf2f644560fa3e1bfc9e4 Mon Sep 17 00:00:00 2001 From: LUNO Date: Sat, 24 Oct 2026 14:42:06 +0800 Subject: [PATCH 06/12] ci: let the segments provide CI parallelism TEST_JOBS=4 ran the four push-worker segments and test-push.sh at the same time, five processes on a four-core runner; test-push.sh then failed its transfer-failure assertion under contention. Segments already fill the runner, so run the suites of a shard one after another again (TEST_JOBS stays available, default 1). --- .github/workflows/shell-tests.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index 2f1b589..fd39188 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -85,7 +85,6 @@ jobs: TEST_BASE_SHA: ${{ github.event.pull_request.base.sha }} TEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }} TEST_SHARD: ${{ matrix.shard }} - TEST_JOBS: 4 TEST_SEGMENTS: 4 run: bash tests/run.sh "$TEST_SHARD" @@ -128,7 +127,6 @@ jobs: TEST_BASE_SHA: ${{ github.event.pull_request.base.sha }} TEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }} TEST_SHARD: ${{ matrix.shard }} - TEST_JOBS: 4 TEST_SEGMENTS: 4 run: bash tests/run.sh "$TEST_SHARD" From e5f130a99bd20e2b8de23bedd32714ee3f0f3aa6 Mon Sep 17 00:00:00 2001 From: LUNO Date: Sat, 24 Oct 2026 14:56:09 +0800 Subject: [PATCH 07/12] ci: run a suite's segments concurrently They were expanded into consecutive entries of the job list, so with the pool off they ran one after another: the motd shard took 68+110+86+126 s, worse than the unsplit suite. Keep the segments of one suite in flight together and run whole suites of a shard only after that. --- README.md | 2 +- tests/run.sh | 96 ++++++++++++++++++++-------------------------------- 2 files changed, 38 insertions(+), 60 deletions(-) diff --git a/README.md b/README.md index 8e44f0f..5e4d9bf 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,7 @@ 生产宿主机（包括 Netcup）仅执行差异检查及改动脚本的 `bash -n`，不运行全量 ShellCheck 或完整测试集； 分析和测试交由 CI：PR 按完整差异及依赖映射选择套件，README/许可证修改无需业务测试； 公共设施或未知依赖改动回退全量，main 推送仍执行全量。两个必需 Job 始终运行，输出选择和跳过记录。 -motd 套件内的用例带有 CI 分段守卫：`TEST_SEGMENTS` 个进程各执行互不重叠的一份（CI 取 4，本机默认 1，即原样串行）；同一分片内的套件由 `tests/run.sh` 以 `TEST_JOBS` 限定的并发执行。 +重套件（motd、Cloudflare Tunnel、push worker 注册）按语义分成四段，import 守卫 `seg_is`：`TEST_SEGMENTS` 个进程各跑互不重叠的一段并同时执行（CI 取 4，本机默认 1，即按原顺序串行）。 Ubuntu 使用固定 ShellCheck v0.10.0：所有受 Git 跟踪、位于 tests/ 之外的 .sh 文件执行 warning 门槛，tests/ 与 tests/lib/ 内的 .sh 文件执行 error 门槛；新目录中的生产 .sh 文件自动纳入，不维护生产文件白名单。Zsh 配置 p10k-config.zsh 不交给 Bash ShellCheck。两组 lint 都会执行并累计失败，Debian 不重复 lint。 例外仅允许局部明确的 SC 规则编号和邻近理由；禁止文件级或全局 warning 抑制。测试 warning 不阻断不等于被 disable，禁止用抑制隐藏行为缺陷。CI 工作流改动继续触发全量测试回退；warning 门槛失败时后续 Ubuntu 行为套件不会执行，不能以 Debian 通过代替双环境验收。 语法检查仍覆盖全部 .sh 脚本；文档修改无需本地全套测试，但仍须通过必需 CI。 diff --git a/tests/run.sh b/tests/run.sh index 5d96f84..82d469a 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -63,30 +63,20 @@ for test_file in "$ROOT_DIR"/tests/test-*.sh; do plan+=("$test_file") done -# Bounded parallelism: 1 keeps the original serial output byte for byte. -jobs=${TEST_JOBS:-1} -[[ "$jobs" =~ ^[1-9][0-9]*$ ]] || { printf 'Invalid TEST_JOBS: %s\n' "$jobs" >&2; exit 2; } - -# Suites carrying CI segment guards run TEST_SEGMENTS processes, each taking a -# disjoint share of the cases. Every other suite runs once. +# Suites carrying CI segment guards are listed here; TEST_SEGMENTS sets how many +# processes share the work. With TEST_SEGMENTS=1 every suite runs in one process. +readonly SEGMENTED_FAMILIES=(test-motd.sh test-cloudflare-tunnel.sh test-push-worker-registration.sh) segments=${TEST_SEGMENTS:-1} [[ "$segments" =~ ^[1-9][0-9]*$ ]] || { printf 'Invalid TEST_SEGMENTS: %s\n' "$segments" >&2; exit 2; } -segmented_families=(test-motd.sh test-cloudflare-tunnel.sh test-push-worker-registration.sh) -job_files=() -job_segs=() -for test_file in "${plan[@]}"; do - test_name=$(basename "$test_file") - if (( segments > 1 )) && [[ " ${segmented_families[*]} " == *" $test_name "* ]]; then - for (( k = 0; k < segments; k++ )); do - job_files+=("$test_file") - job_segs+=("$k") - done - else - job_files+=("$test_file") - job_segs+=("") - fi -done +is_segmented() { + local name=$1 entry + (( segments > 1 )) || return 1 + for entry in "${SEGMENTED_FAMILIES[@]}"; do + [[ "$entry" == "$name" ]] && return 0 + done + return 1 +} run_suite() { local test_file=$1 seg=${2:-} test_name started status @@ -97,9 +87,8 @@ run_suite() { started=$SECONDS printf '\n==> %s\n' "$test_name" # Keep the child shell standalone: an if/|| wrapper can change errexit behavior. + # Nested fixtures re-enter this script, so neither setting may leak to them. set +e - # Nested fixtures re-enter this script; they must not inherit the pool or - # segment settings, or a stub suite would be duplicated. if [[ -n "$seg" ]]; then env -u TEST_JOBS -u TEST_SEGMENTS TEST_SEGMENT="$seg" bash "$test_file" else @@ -115,47 +104,36 @@ run_suite() { return "$status" } -if (( ${#job_files[@]} == 0 )); then - printf '\nAll selected shell tests passed.\n' - exit 0 -fi - -if (( jobs == 1 )); then - for idx in "${!job_files[@]}"; do - run_suite "${job_files[idx]}" "${job_segs[idx]}" || exit "$?" - done -else - log_dir=$(mktemp -d) - trap 'rm -rf "$log_dir"' EXIT - running=0 - for idx in "${!job_files[@]}"; do - test_file=${job_files[idx]} - seg=${job_segs[idx]} - key=$(basename "$test_file") - [[ -n "$seg" ]] && key+=".$seg" +# Segments of one suite share no state, so they run together and fill the +# runner; the suites of a shard still run one after another. +run_segments() { + local test_file=$1 dir k rc failed=0 pids=() + dir=$(mktemp -d) + for (( k = 0; k < segments; k++ )); do ( rc=0 - run_suite "$test_file" "$seg" > "$log_dir/$key.log" 2>&1 || rc=$? - printf '%s' "$rc" > "$log_dir/$key.rc" + run_suite "$test_file" "$k" > "$dir/$k.log" 2>&1 || rc=$? + printf '%s' "$rc" > "$dir/$k.rc" ) & - running=$((running + 1)) - if (( running >= jobs )); then - wait -n || true - running=$((running - 1)) - fi + pids+=("$!") done - wait || true - failed=0 - for idx in "${!job_files[@]}"; do - test_file=${job_files[idx]} - seg=${job_segs[idx]} - key=$(basename "$test_file") - [[ -n "$seg" ]] && key+=".$seg" - cat "$log_dir/$key.log" - rc=$(cat "$log_dir/$key.rc" 2>/dev/null || printf 1) + for k in "${pids[@]}"; do wait "$k" || true; done + for (( k = 0; k < segments; k++ )); do + cat "$dir/$k.log" + rc=$(cat "$dir/$k.rc" 2>/dev/null || printf 1) (( rc == 0 )) || failed=1 done - (( failed == 0 )) || exit 1 -fi + rm -rf "$dir" + return "$failed" +} + +for test_file in "${plan[@]:-}"; do + [[ -n "$test_file" ]] || continue + if is_segmented "$(basename "$test_file")"; then + run_segments "$test_file" || exit 1 + else + run_suite "$test_file" || exit "$?" + fi +done printf '\nAll selected shell tests passed.\n' From bd5b41b7e468213e64216fb357c335872c6a935b Mon Sep 17 00:00:00 2001 From: LUNO Date: Sat, 24 Oct 2026 15:22:02 +0800 Subject: [PATCH 08/12] test(cloudflare): keep TEST_SEGMENT across the privilege re-exec The suite re-executes itself through sudo when it is not root, and sudo resets the environment, so every segment saw an empty TEST_SEGMENT and ran the whole suite: four segments of 318 s each instead of a quarter each. Preserve the variable alongside PATH. --- tests/test-cloudflare-tunnel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-cloudflare-tunnel.sh b/tests/test-cloudflare-tunnel.sh index 094cd50..a62d5f2 100644 --- a/tests/test-cloudflare-tunnel.sh +++ b/tests/test-cloudflare-tunnel.sh @@ -11,7 +11,7 @@ seg_is() { set -euo pipefail if (( EUID != 0 )); then - exec sudo --preserve-env=PATH bash "$0" "$@" + exec sudo --preserve-env=PATH --preserve-env=TEST_SEGMENT bash "$0" "$@" fi ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) From ade70c3e0499163ba1b7e3c27fe2f8d29109a701 Mon Sep 17 00:00:00 2001 From: FORGE Date: Sat, 24 Oct 2026 16:18:54 +0800 Subject: [PATCH 09/12] ci: recut motd and push-worker segments from measured unit costs Segment boundaries now come from per-unit wall times measured on the shell test host, and may not fall inside the span of a function or top-level variable defined after the prelude: a segment only sees the prelude and its own block, so a cut inside such a span broke later segments. --- tests/test-motd.sh | 14 +++++++------- tests/test-push-worker-registration.sh | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tests/test-motd.sh b/tests/test-motd.sh index 8a33696..6a17bd8 100644 --- a/tests/test-motd.sh +++ b/tests/test-motd.sh @@ -572,9 +572,7 @@ load_motd; motd_run_locked_operation restore previous; assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "state legacy restore" ) -fi -if seg_is 1; then for legacy_phase in random-id stat open create cat chown chmod sha256 manifest mv; do ( root="$TEST_DIR/legacy-io-$legacy_phase"; create_fixture "$root"; export_paths "$root" @@ -626,7 +624,9 @@ for pending_phase in pending-create pending-write pending-chown pending-chmod pe motd_run_locked_operation install ) done +fi +if seg_is 1; then for pending_kind in broken-symlink symlink directory fifo mode owner gid; do ( root="$TEST_DIR/pending-kind-$pending_kind"; create_fixture "$root"; export_paths "$root" @@ -863,9 +863,7 @@ for hook_case in content-motd mode-issue_net inode-motd_news type-issue_net; do assert_no_motd_residue "$root" ) done -fi -if seg_is 2; then ( root="$TEST_DIR/hook-rollback-stage"; create_fixture "$root"; export_paths "$root" load_motd; motd_run_locked_operation install @@ -933,8 +931,10 @@ for pending_hook_case in id extra-line inode; do assert_no_motd_residue "$root" ) done +fi +if seg_is 2; then for building_phase in transaction-snapshot-building capture-motd capture-issue_net capture-motd_news snapshot-commit transaction-final-move; do ( root="$TEST_DIR/building-fail-$building_phase"; create_fixture "$root"; export_paths "$root" @@ -1087,9 +1087,7 @@ done grep -Fq 'error "缺少必要命令: $required_command"' "$ROOT_DIR/modules/system-customize.sh" || fail "module missing-command error output branch absent" pass "module dependency preflight retains missing-command output" -fi -if seg_is 3; then for noop_impl in tool module; do for noop_created_type in regular symlink fifo; do ( @@ -1121,7 +1119,9 @@ for noop_impl in tool module; do ) done done +fi +if seg_is 3; then for group_impl in tool module; do for group_change in motd-content issue-mode custom-inode; do ( @@ -1271,6 +1271,6 @@ for rollback_impl in tool module; do assert_no_motd_residue "$root" ) done -fi printf 'All MOTD rollback final validation tests passed.\n' +fi diff --git a/tests/test-push-worker-registration.sh b/tests/test-push-worker-registration.sh index 7196192..acb194e 100755 --- a/tests/test-push-worker-registration.sh +++ b/tests/test-push-worker-registration.sh @@ -690,7 +690,9 @@ pass "failed managed-session reap remains bounded for 20 deterministic runs" [[ ! -e "$state_file" && -e "$root/capture/reaped" ]] || fail "parent fallback evidence incomplete" pass "parent fallback cleans a live failed session without PPID 1 zombie" ) +fi +if seg_is 1; then ( trap - EXIT HUP INT TERM sleep 60 & replacement=$! @@ -777,9 +779,7 @@ chmod 0700 "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" [[ -z "$(find "$root/runtime" -mindepth 1 -print -quit 2>/dev/null)" ]] || fail "hard timeout cleanup failure left runtime" pass "hard timeout cleanup failure is bounded and blocks subsequent command" ) -fi -if seg_is 1; then run_preserved_cleanup_retry_case() ( local run="$1" root runtime runtime_dev runtime_inode credential leader leader_start leader_pgid leader_sid state_file local owner watchdog rc=0 metadata="" state_status=0 @@ -852,7 +852,9 @@ for runtime_evidence_run in $(seq 1 "$runtime_evidence_stress"); do run_preserved_cleanup_retry_case "$runtime_evidence_run" done pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" +fi +if seg_is 2; then ( local_cleanup_root="$TEST_DIR/active-cleanup-evidence"; setup_fixture "$local_cleanup_root" runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE @@ -969,9 +971,7 @@ done fail "clean runtime final proof did not clear state" pass "clean runtime final proof preserves normal success path" ) -fi -if seg_is 2; then cat > "$TEST_DIR/runtime-retention-signal-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1336,7 +1336,9 @@ run_active_grace_signal_case() { run_active_grace_signal_case HUP 129 run_active_grace_signal_case INT 130 run_active_grace_signal_case TERM 143 +fi +if seg_is 3; then cat > "$TEST_DIR/live-cleanup-failure-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1410,9 +1412,7 @@ run_live_cleanup_failure_signal_case() ( run_live_cleanup_failure_signal_case HUP 129 run_live_cleanup_failure_signal_case INT 130 run_live_cleanup_failure_signal_case TERM 143 -fi -if seg_is 3; then cat > "$TEST_DIR/state-publication-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1565,7 +1565,7 @@ done assert_no_registration_residue "$root"; cleanup_runtime; assert_no_registration_residue "$root" pass "normal registration/session lifecycle cleans all state" ) -fi printf 'All push worker registration tests passed.\n' +fi From 8dc0948c2d2df4d7e15e1e131a70d1b021b6bc0e Mon Sep 17 00:00:00 2001 From: FORGE Date: Sat, 24 Oct 2026 16:50:03 +0800 Subject: [PATCH 10/12] ci: split each OS test job into five shards motd, push-worker, push and cloudflare each get their own matrix shard, so cloudflare no longer queues behind xanmod inside other and the push worker suite no longer waits for test-push. push now covers test-push.sh only; tests/run.sh all remains the local full run. --- .github/workflows/shell-tests.yml | 4 ++-- README.md | 3 +++ tests/run.sh | 10 +++++++--- tests/test-shards.sh | 16 ++++++++++------ 4 files changed, 22 insertions(+), 11 deletions(-) diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index fd39188..337ae04 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -58,7 +58,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [motd, push, other] + shard: [motd, push-worker, push, cloudflare, other] steps: - uses: actions/checkout@v7 with: @@ -96,7 +96,7 @@ jobs: strategy: fail-fast: false matrix: - shard: [motd, push, other] + shard: [motd, push-worker, push, cloudflare, other] steps: - name: Install test dependencies run: | diff --git a/README.md b/README.md index 5e4d9bf..ab2e6b6 100644 --- a/README.md +++ b/README.md @@ -23,6 +23,9 @@ 生产宿主机（包括 Netcup）仅执行差异检查及改动脚本的 `bash -n`，不运行全量 ShellCheck 或完整测试集； 分析和测试交由 CI：PR 按完整差异及依赖映射选择套件，README/许可证修改无需业务测试； 公共设施或未知依赖改动回退全量，main 推送仍执行全量。两个必需 Job 始终运行，输出选择和跳过记录。 +每个 OS 的测试作业按五个 shard 并行：`motd`（`test-motd.sh`）、`push-worker` +（`test-push-worker-registration.sh`）、`push`（仅 `test-push.sh`）、`cloudflare` +（`test-cloudflare-tunnel.sh`）、`other`（其余全部套件）；本机全量用 `tests/run.sh all`。 重套件（motd、Cloudflare Tunnel、push worker 注册）按语义分成四段，import 守卫 `seg_is`：`TEST_SEGMENTS` 个进程各跑互不重叠的一段并同时执行（CI 取 4，本机默认 1，即按原顺序串行）。 Ubuntu 使用固定 ShellCheck v0.10.0：所有受 Git 跟踪、位于 tests/ 之外的 .sh 文件执行 warning 门槛，tests/ 与 tests/lib/ 内的 .sh 文件执行 error 门槛；新目录中的生产 .sh 文件自动纳入，不维护生产文件白名单。Zsh 配置 p10k-config.zsh 不交给 Bash ShellCheck。两组 lint 都会执行并累计失败，Debian 不重复 lint。 例外仅允许局部明确的 SC 规则编号和邻近理由；禁止文件级或全局 warning 抑制。测试 warning 不阻断不等于被 disable，禁止用抑制隐藏行为缺陷。CI 工作流改动继续触发全量测试回退；warning 门槛失败时后续 Ubuntu 行为套件不会执行，不能以 Debian 通过代替双环境验收。 diff --git a/tests/run.sh b/tests/run.sh index 82d469a..f3757aa 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -6,11 +6,11 @@ readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) # Positional, not exported: nested selection fixtures must still run all suites. shard=${1:-all} if (( $# > 1 )); then - printf 'Usage: tests/run.sh [all|motd|push|other]\n' >&2 + printf 'Usage: tests/run.sh [all|motd|push-worker|push|cloudflare|other]\n' >&2 exit 2 fi case "$shard" in - all|motd|push|other) ;; + all|motd|push-worker|push|cloudflare|other) ;; *) printf 'Invalid test shard: %s\n' "$shard" >&2; exit 2 ;; esac @@ -51,9 +51,13 @@ for test_file in "$ROOT_DIR"/tests/test-*.sh; do printf 'SKIP: %s (unaffected by PR diff)\n' "$test_name" continue fi + # One suite per heavy family so CI can schedule them independently; every + # other suite shares `other`. case "$test_name" in test-motd.sh) suite_shard=motd ;; - test-push.sh|test-push-worker-registration.sh) suite_shard=push ;; + test-push-worker-registration.sh) suite_shard=push-worker ;; + test-push.sh) suite_shard=push ;; + test-cloudflare-tunnel.sh) suite_shard=cloudflare ;; *) suite_shard=other ;; esac if [[ "$shard" != all && "$shard" != "$suite_shard" ]]; then diff --git a/tests/test-shards.sh b/tests/test-shards.sh index 6397696..ce0db5a 100644 --- a/tests/test-shards.sh +++ b/tests/test-shards.sh @@ -10,7 +10,7 @@ mkdir -p "$TEMP_DIR/tests" "$TEMP_DIR/bin" cp "$ROOT_DIR/tests/run.sh" "$TEMP_DIR/tests/run.sh" export CAPTURE="$TEMP_DIR/capture" GITHUB_STEP_SUMMARY="$TEMP_DIR/summary" unset TEST_BASE_SHA TEST_HEAD_SHA -for name in motd push push-worker-registration new-suite; do +for name in cloudflare-tunnel motd push push-worker-registration new-suite; do printf '#!/usr/bin/env bash\nprintf "%%s\\n" "%s" >> "$CAPTURE"\n' "test-$name.sh" > "$TEMP_DIR/tests/test-$name.sh" done check() { @@ -20,15 +20,17 @@ check() { bash "$TEMP_DIR/tests/run.sh" "$@" > "$TEMP_DIR/output" 2>&1 [[ "$(cat "$CAPTURE")" == "$expected" ]] || { cat "$TEMP_DIR/output"; exit 1; } } -all=$'test-motd.sh\ntest-new-suite.sh\ntest-push-worker-registration.sh\ntest-push.sh' +all=$'test-cloudflare-tunnel.sh\ntest-motd.sh\ntest-new-suite.sh\ntest-push-worker-registration.sh\ntest-push.sh' check "$all" check "$all" all check test-motd.sh motd -check $'test-push-worker-registration.sh\ntest-push.sh' push +check test-push-worker-registration.sh push-worker +check test-push.sh push +check test-cloudflare-tunnel.sh cloudflare check test-new-suite.sh other # Every current and future suite belongs to exactly one shard. : > "$TEMP_DIR/union" -for shard in motd push other; do +for shard in motd push-worker push cloudflare other; do : > "$CAPTURE" bash "$TEMP_DIR/tests/run.sh" "$shard" > "$TEMP_DIR/output" 2>&1 cat "$CAPTURE" >> "$TEMP_DIR/union" @@ -53,15 +55,17 @@ STUB chmod +x "$TEMP_DIR/bin/git" export PATH="$TEMP_DIR/bin:$PATH" TEST_BASE_SHA=base TEST_HEAD_SHA=head check '' motd +check '' cloudflare check '' other -check $'test-push-worker-registration.sh\ntest-push.sh' push +check test-push.sh push +check test-push-worker-registration.sh push-worker DIFF_FAIL=1 check test-motd.sh motd DIFF_FAIL=1 check test-new-suite.sh other # Child errexit and failure propagation must survive sharding. printf '#!/usr/bin/env bash\nset -e\nfalse\nprintf unexpected >> "$CAPTURE"\n' > "$TEMP_DIR/tests/test-push-worker-registration.sh" : > "$CAPTURE" rc=0 -bash "$TEMP_DIR/tests/run.sh" push > "$TEMP_DIR/output" 2>&1 || rc=$? +bash "$TEMP_DIR/tests/run.sh" push-worker > "$TEMP_DIR/output" 2>&1 || rc=$? [[ "$rc" == 1 && ! -s "$CAPTURE" ]] grep -q 'exit=1' "$TEMP_DIR/output" printf 'All shard fixture tests passed.\n' From 9818f1b219b0a6d460b0d5b473fe7e87a81bd4e0 Mon Sep 17 00:00:00 2001 From: FORGE Date: Sat, 24 Oct 2026 17:31:54 +0800 Subject: [PATCH 11/12] ci: spread motd and cloudflare over two jobs with segment subsets Both suites were cut into four segments and run by a single job each, so their wall clock was the suite's total work divided by four cores. Cut each of them into eight segments (same single-file `seg_is` guards, prelude kept once, no overlap or gap) and add `TEST_SEGMENT_SET` to tests/run.sh so the two new shards of a suite run four segments each: motd-a / motd-b TEST_SEGMENTS=8, sets "0 1 2 3" and "4 5 6 7" cloudflare-a / -b same, two jobs Cut points come from per-statement wall times measured with a DEBUG trap on the shell test host (top-level statements and whole `for`/subshell blocks are atomic), minimising the largest segment; the reference tree in the same measurement passes 655 motd and 312 cloudflare assertions, exactly the per-segment sums after cutting. `TEST_SEGMENT_SET` accepts only distinct in-range indices and exits 2 before any suite starts; unset keeps the old behaviour. test-shards.sh now also proves that each source segment is covered by exactly one job, that the four subsets are the full segment set, and that invalid sets abort, and the README documents the seven shards. --- .github/workflows/shell-tests.yml | 72 ++++++++++++++++++++++++--- README.md | 11 +++-- tests/run.sh | 33 +++++++++++-- tests/test-cloudflare-tunnel.sh | 25 ++++++---- tests/test-motd.sh | 26 +++++----- tests/test-shards.sh | 82 ++++++++++++++++++++++++++++++- 6 files changed, 212 insertions(+), 37 deletions(-) diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index 337ae04..be5a514 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -58,7 +58,34 @@ jobs: strategy: fail-fast: false matrix: - shard: [motd, push-worker, push, cloudflare, other] + include: + # motd and cloudflare are cut into eight segments; each of their two + # jobs runs four of them, so a single suite spans two jobs. + - shard: motd-a + suite: motd + segments: 8 + segment_set: "0 1 2 3" + - shard: motd-b + suite: motd + segments: 8 + segment_set: "4 5 6 7" + - shard: push-worker + suite: push-worker + segments: 4 + - shard: push + suite: push + segments: 1 + - shard: cloudflare-a + suite: cloudflare + segments: 8 + segment_set: "0 1 2 3" + - shard: cloudflare-b + suite: cloudflare + segments: 8 + segment_set: "4 5 6 7" + - shard: other + suite: other + segments: 1 steps: - uses: actions/checkout@v7 with: @@ -84,9 +111,10 @@ jobs: env: TEST_BASE_SHA: ${{ github.event.pull_request.base.sha }} TEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - TEST_SHARD: ${{ matrix.shard }} - TEST_SEGMENTS: 4 - run: bash tests/run.sh "$TEST_SHARD" + TEST_SUITE: ${{ matrix.suite }} + TEST_SEGMENTS: ${{ matrix.segments }} + TEST_SEGMENT_SET: ${{ matrix.segment_set }} + run: bash tests/run.sh "$TEST_SUITE" debian-tests: name: Debian 13 / ${{ matrix.shard }} @@ -96,7 +124,34 @@ jobs: strategy: fail-fast: false matrix: - shard: [motd, push-worker, push, cloudflare, other] + include: + # motd and cloudflare are cut into eight segments; each of their two + # jobs runs four of them, so a single suite spans two jobs. + - shard: motd-a + suite: motd + segments: 8 + segment_set: "0 1 2 3" + - shard: motd-b + suite: motd + segments: 8 + segment_set: "4 5 6 7" + - shard: push-worker + suite: push-worker + segments: 4 + - shard: push + suite: push + segments: 1 + - shard: cloudflare-a + suite: cloudflare + segments: 8 + segment_set: "0 1 2 3" + - shard: cloudflare-b + suite: cloudflare + segments: 8 + segment_set: "4 5 6 7" + - shard: other + suite: other + segments: 1 steps: - name: Install test dependencies run: | @@ -126,9 +181,10 @@ jobs: env: TEST_BASE_SHA: ${{ github.event.pull_request.base.sha }} TEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - TEST_SHARD: ${{ matrix.shard }} - TEST_SEGMENTS: 4 - run: bash tests/run.sh "$TEST_SHARD" + TEST_SUITE: ${{ matrix.suite }} + TEST_SEGMENTS: ${{ matrix.segments }} + TEST_SEGMENT_SET: ${{ matrix.segment_set }} + run: bash tests/run.sh "$TEST_SUITE" # Preserve existing required-check names; skipped/cancelled dependencies fail closed. ubuntu: diff --git a/README.md b/README.md index ab2e6b6..76d9173 100644 --- a/README.md +++ b/README.md @@ -23,10 +23,13 @@ 生产宿主机（包括 Netcup）仅执行差异检查及改动脚本的 `bash -n`，不运行全量 ShellCheck 或完整测试集； 分析和测试交由 CI：PR 按完整差异及依赖映射选择套件，README/许可证修改无需业务测试； 公共设施或未知依赖改动回退全量，main 推送仍执行全量。两个必需 Job 始终运行，输出选择和跳过记录。 -每个 OS 的测试作业按五个 shard 并行：`motd`（`test-motd.sh`）、`push-worker` -（`test-push-worker-registration.sh`）、`push`（仅 `test-push.sh`）、`cloudflare` -（`test-cloudflare-tunnel.sh`）、`other`（其余全部套件）；本机全量用 `tests/run.sh all`。 -重套件（motd、Cloudflare Tunnel、push worker 注册）按语义分成四段，import 守卫 `seg_is`：`TEST_SEGMENTS` 个进程各跑互不重叠的一段并同时执行（CI 取 4，本机默认 1，即按原顺序串行）。 +每个 OS 的测试作业按七个 shard 并行：`motd-a`/`motd-b`（`test-motd.sh`，各跑四段）、 +`cloudflare-a`/`cloudflare-b`（`test-cloudflare-tunnel.sh`，各跑四段）、`push-worker` +（`test-push-worker-registration.sh`）、`push`（仅 `test-push.sh`）、`other`（其余全部套件）； +本机全量用 `tests/run.sh all`。 +重套件（motd、Cloudflare Tunnel、push worker 注册）按语义分段，import 守卫 `seg_is`：`TEST_SEGMENTS` 个进程各跑互不重叠的一段并同时执行（本机默认 1，即按原顺序串行）。 +motd 与 Cloudflare Tunnel 各切成八段：`-a` 作业用 `TEST_SEGMENT_SET="0 1 2 3"`、`-b` 作业用 `"4 5 6 7"`，两个作业的并集恰好覆盖全部段且互不重叠；push worker 注册仍为四段、单作业。 +`TEST_SEGMENT_SET` 只接受互不重复、且在 `TEST_SEGMENTS` 范围内的段下标，非法值时在任何套件启动前以状态 2 退出；未设置则照旧跑全部段。 Ubuntu 使用固定 ShellCheck v0.10.0：所有受 Git 跟踪、位于 tests/ 之外的 .sh 文件执行 warning 门槛，tests/ 与 tests/lib/ 内的 .sh 文件执行 error 门槛；新目录中的生产 .sh 文件自动纳入，不维护生产文件白名单。Zsh 配置 p10k-config.zsh 不交给 Bash ShellCheck。两组 lint 都会执行并累计失败，Debian 不重复 lint。 例外仅允许局部明确的 SC 规则编号和邻近理由；禁止文件级或全局 warning 抑制。测试 warning 不阻断不等于被 disable，禁止用抑制隐藏行为缺陷。CI 工作流改动继续触发全量测试回退；warning 门槛失败时后续 Ubuntu 行为套件不会执行，不能以 Debian 通过代替双环境验收。 语法检查仍覆盖全部 .sh 脚本；文档修改无需本地全套测试，但仍须通过必需 CI。 diff --git a/tests/run.sh b/tests/run.sh index f3757aa..0a22d73 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -73,6 +73,25 @@ readonly SEGMENTED_FAMILIES=(test-motd.sh test-cloudflare-tunnel.sh test-push-wo segments=${TEST_SEGMENTS:-1} [[ "$segments" =~ ^[1-9][0-9]*$ ]] || { printf 'Invalid TEST_SEGMENTS: %s\n' "$segments" >&2; exit 2; } +# TEST_SEGMENT_SET narrows one job to a subset of the segments ("0 1 2 3"), so a +# single suite can be spread over several jobs. Unset or blank keeps every +# segment; anything that is not a distinct in-range index aborts before a suite +# starts. +segment_set=() +seen_segments=" " +if [[ -n ${TEST_SEGMENT_SET:-} && -n ${TEST_SEGMENT_SET//[[:space:]]/} ]]; then + for segment_index in $TEST_SEGMENT_SET; do + [[ "$segment_index" =~ ^[0-9]+$ ]] \ + || { printf 'Invalid TEST_SEGMENT_SET entry: %s\n' "$segment_index" >&2; exit 2; } + (( segment_index < segments )) \ + || { printf 'Segment %s out of range for TEST_SEGMENTS=%s\n' "$segment_index" "$segments" >&2; exit 2; } + [[ "$seen_segments" != *" $segment_index "* ]] \ + || { printf 'Duplicate segment in TEST_SEGMENT_SET: %s\n' "$segment_index" >&2; exit 2; } + seen_segments+="$segment_index " + segment_set+=("$segment_index") + done +fi + is_segmented() { local name=$1 entry (( segments > 1 )) || return 1 @@ -94,9 +113,9 @@ run_suite() { # Nested fixtures re-enter this script, so neither setting may leak to them. set +e if [[ -n "$seg" ]]; then - env -u TEST_JOBS -u TEST_SEGMENTS TEST_SEGMENT="$seg" bash "$test_file" + env -u TEST_JOBS -u TEST_SEGMENTS -u TEST_SEGMENT_SET TEST_SEGMENT="$seg" bash "$test_file" else - env -u TEST_JOBS -u TEST_SEGMENTS bash "$test_file" + env -u TEST_JOBS -u TEST_SEGMENTS -u TEST_SEGMENT_SET bash "$test_file" fi status=$? set -e @@ -112,8 +131,14 @@ run_suite() { # runner; the suites of a shard still run one after another. run_segments() { local test_file=$1 dir k rc failed=0 pids=() + local -a queue=() dir=$(mktemp -d) - for (( k = 0; k < segments; k++ )); do + if (( ${#segment_set[@]} )); then + queue=("${segment_set[@]}") + else + for (( k = 0; k < segments; k++ )); do queue+=("$k"); done + fi + for k in "${queue[@]}"; do ( rc=0 run_suite "$test_file" "$k" > "$dir/$k.log" 2>&1 || rc=$? @@ -122,7 +147,7 @@ run_segments() { pids+=("$!") done for k in "${pids[@]}"; do wait "$k" || true; done - for (( k = 0; k < segments; k++ )); do + for k in "${queue[@]}"; do cat "$dir/$k.log" rc=$(cat "$dir/$k.rc" 2>/dev/null || printf 1) (( rc == 0 )) || failed=1 diff --git a/tests/test-cloudflare-tunnel.sh b/tests/test-cloudflare-tunnel.sh index a62d5f2..2e48726 100644 --- a/tests/test-cloudflare-tunnel.sh +++ b/tests/test-cloudflare-tunnel.sh @@ -188,7 +188,6 @@ require_root() { :; } check_platform() { :; } uninstall_transaction_hook() { [[ "$1" != after-disable-auto-update ]] || exit 0; } fi - if seg_is 0; then new_case @@ -421,7 +420,9 @@ pass "source remains old when key commit fails" run_failure_case "APT probe failure rollback" 'export FAKE_APT_UPDATE_FAIL=1' 'run_repository_apt_transaction install' run_failure_case "APT install failure rollback" 'export FAKE_APT_INSTALL_FAIL=1' 'run_repository_apt_transaction install' +fi +if seg_is 1; then for signal in HUP INT TERM; do case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac for phase in download validate stage key-commit source-commit apt-probe apt-install; do @@ -473,6 +474,9 @@ if configure_repository >/dev/null 2>&1; then fail "rollback failure unexpectedl find "$CLOUDFLARED_REPOSITORY_STATE_DIR" -maxdepth 2 -type f -name rollback.log -print -quit | grep -q . || fail "rollback failure did not preserve evidence" ) +fi + +if seg_is 2; then pass "rollback failure returns nonzero and preserves evidence" new_case @@ -501,7 +505,9 @@ find "$CLOUDFLARED_STATE_DIR" -maxdepth 1 -type d -name 'uninstall-*' -print -qu fail "managed source backup missing" [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "managed repository removal left lock" pass "configure then real repository uninstall removes source and preserves keyring" +fi +if seg_is 3; then for signal in HUP INT TERM; do case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac for phase in before-snapshot-create capture-auto_update_script capture-auto_update_timer capture-legacy_marker lock-acquired before-disable-auto-update after-disable-auto-update before-apt-remove after-apt-remove before-source-remove after-source-remove final-apt-update before-lock-release; do @@ -532,7 +538,9 @@ for signal in HUP INT TERM; do pass "uninstall $signal=$expected cleans $phase" done done +fi +if seg_is 4; then new_case set_old_generation configure_repository @@ -615,9 +623,7 @@ grep -Fq "ExecStart=$CLOUDFLARED_APT_BIN --no-autoupdate --token-file /etc/cloud pass "legacy binary and service migration behavior remains" -fi -if seg_is 1; then # Each malformed snapshot runs in a separate process: transaction traps and # intentional corrupt fixtures must not leak into subsequent cases. for schema_case in version generation duplicate unknown order state mode uid gid digest created payload-name payload-missing payload-extra payload-fifo payload-symlink manifest-fifo ancestor; do @@ -659,8 +665,9 @@ for schema_case in version generation duplicate unknown order state mode uid gid ) > "$TEST_DIR/schema-$schema_case.log" 2>&1 || fail "schema $schema_case (see $TEST_DIR/schema-$schema_case.log)" pass "uninstall manifest rejects $schema_case" done +fi - +if seg_is 5; then for ownership_case in owned foreign inode mode owner gid symlink directory fifo; do ( new_case @@ -896,9 +903,7 @@ for stage_case in owned foreign; do } pass "stage cleanup verifies $stage_case identity" done -fi -if seg_is 2; then for terminal_signal in HUP INT TERM; do case "$terminal_signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac ( @@ -959,7 +964,9 @@ for restore_id in auto_update_script auto_update_service auto_update_timer sourc } pass "restore failure $restore_id continues remaining five targets and preserves original evidence" done +fi +if seg_is 6; then for target_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do for bad_metadata in mode owner gid symlink directory fifo; do ( @@ -1040,7 +1047,9 @@ for creation_phase in before-create-marker before-create-manifest-stage before-c pass "creation $creation_phase $creation_fault preserves ownership across publication" done done +fi +if seg_is 7; then for bookkeeping_fault in marker stage rename; do ( new_case @@ -1246,9 +1255,7 @@ for restore_object in key source current; do pass "repository restore $restore_object $restore_fault rejects corruption and retains all backups" done done -fi -if seg_is 3; then for operation in install_cloudflared upgrade_cloudflared; do ( new_case @@ -1566,6 +1573,6 @@ done entrypoint_output=$(bash -c "$(cat "$ROOT_DIR/tools/cloudflare_tunnel.sh")" cloudflare_tunnel.sh help) grep -Fq 'cloudflare_tunnel.sh install' <<< "$entrypoint_output" || fail "bash -c entrypoint broken" pass "bash -c entrypoint remains compatible" -fi printf 'All cloudflare wrapper tests passed. PASS=%d\n' "$pass_count" +fi diff --git a/tests/test-motd.sh b/tests/test-motd.sh index 6a17bd8..a7ea717 100644 --- a/tests/test-motd.sh +++ b/tests/test-motd.sh @@ -191,7 +191,6 @@ motd_snapshot_capture_hook() { motd_run_locked_operation install CHILD chmod 0700 "$TEST_DIR/building-signal-child.sh" - if seg_is 0; then extract_payload "$ROOT_DIR/tools/setup-motd.sh">"$TEST_DIR/t"; extract_payload "$ROOT_DIR/modules/system-customize.sh">"$TEST_DIR/m"; @@ -428,6 +427,9 @@ assert_fail "rollback failure nonzero" motd_run_locked_operation install; assert_eq 6 "$(wc -l<"$r/a")" "rollback attempts six"; [[ -e "$MOTD_STATE_DIR/pending" ]]||fail journal ) +fi + +if seg_is 1; then for ph in snapshot-building snapshot-commit snapshot-publish install-first install-middle restore-first restore-middle; do for sig in HUP INT TERM; do( @@ -589,8 +591,9 @@ for legacy_phase in random-id stat open create cat chown chmod sha256 manifest m ) done pass "legacy I/O failure matrix preserves data and state" +fi - +if seg_is 2; then for signal_name in HUP INT TERM; do ( root="$TEST_DIR/pointer-signal-$signal_name"; create_fixture "$root"; export_paths "$root" @@ -624,9 +627,7 @@ for pending_phase in pending-create pending-write pending-chown pending-chmod pe motd_run_locked_operation install ) done -fi -if seg_is 1; then for pending_kind in broken-symlink symlink directory fifo mode owner gid; do ( root="$TEST_DIR/pending-kind-$pending_kind"; create_fixture "$root"; export_paths "$root" @@ -738,8 +739,9 @@ for native_id in uname motd_news; do [[ -L "$native_path" && "$(readlink "$native_path")" == "$root/external-$native_id" ]] || fail "$native_id previous symlink not restored" ) done +fi - +if seg_is 3; then for kill_phase in transaction-snapshot-building transaction-final-moved target-committed-motd target-committed-issue_net target-committed-motd_news committed-cleanup committed-old-generation-cleaned committed-pending-removed; do ( root="$TEST_DIR/sigkill-$kill_phase"; create_fixture "$root"; export_paths "$root" @@ -837,8 +839,9 @@ done [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "rolledback cleanup retry left pending" assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "rolledback cleanup retry restores bounded generations" ) +fi - +if seg_is 4; then for hook_case in content-motd mode-issue_net inode-motd_news type-issue_net; do ( root="$TEST_DIR/hook-target-$hook_case"; create_fixture "$root"; export_paths "$root" @@ -931,10 +934,8 @@ for pending_hook_case in id extra-line inode; do assert_no_motd_residue "$root" ) done -fi -if seg_is 2; then for building_phase in transaction-snapshot-building capture-motd capture-issue_net capture-motd_news snapshot-commit transaction-final-move; do ( root="$TEST_DIR/building-fail-$building_phase"; create_fixture "$root"; export_paths "$root" @@ -966,7 +967,9 @@ for building_phase in transaction-snapshot-building capture-motd capture-issue_n ) done done +fi +if seg_is 5; then for previous_impl in tool module; do ( root="$TEST_DIR/previous-only-$previous_impl"; create_fixture "$root"; export_paths "$root" @@ -1060,7 +1063,9 @@ done assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "localized stat wrapper preserves regular legacy import" LANG=zh_CN.UTF-8 bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/modules/system-customize.sh" ) +fi +if seg_is 6; then for missing_command in bash chown flock ln od readlink sha256sum touch wc; do ( root="$TEST_DIR/module-missing-$missing_command"; create_fixture "$root"; export_paths "$root" @@ -1119,9 +1124,7 @@ for noop_impl in tool module; do ) done done -fi -if seg_is 3; then for group_impl in tool module; do for group_change in motd-content issue-mode custom-inode; do ( @@ -1195,8 +1198,9 @@ done assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "legal state root regular backup imported"; [[ -e "$MOTD_STATE_DIR/initial.current" ]] || fail "legal state root import did not publish initial pointer" assert_eq "$legacy_times" "$(stat -c %X:%Y "$MOTD_STATE_DIR/motd.initial-backup")" "legacy root metadata validation/import preserves timestamps"; pass "legal state root regular, symlink, and markers import" ) +fi - +if seg_is 7; then for rollback_impl in tool module; do for rollback_damage in motd-content issue-mode custom-inode absent-regular absent-symlink absent-fifo symlink-target; do ( diff --git a/tests/test-shards.sh b/tests/test-shards.sh index ce0db5a..417b739 100644 --- a/tests/test-shards.sh +++ b/tests/test-shards.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail # These fixtures re-enter run.sh and require its serial, whole-shard behaviour. -unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT +unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT TEST_SEGMENT_SET # Inert fixtures only: verify partitioning without running production suites. ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) TEMP_DIR=$(mktemp -d) @@ -53,6 +53,7 @@ cat > "$TEMP_DIR/bin/git" <<'STUB' printf 'tools/push.sh\0' STUB chmod +x "$TEMP_DIR/bin/git" +ORIGINAL_PATH=$PATH export PATH="$TEMP_DIR/bin:$PATH" TEST_BASE_SHA=base TEST_HEAD_SHA=head check '' motd check '' cloudflare @@ -68,4 +69,83 @@ rc=0 bash "$TEMP_DIR/tests/run.sh" push-worker > "$TEMP_DIR/output" 2>&1 || rc=$? [[ "$rc" == 1 && ! -s "$CAPTURE" ]] grep -q 'exit=1' "$TEMP_DIR/output" +PATH=$ORIGINAL_PATH +unset TEST_BASE_SHA TEST_HEAD_SHA +# Segment subsets: one suite can be spread over several jobs, so every segment +# of a suite must be covered by exactly one job and every job must name a real +# segment of the suite's source. +guards() { + awk '/^if seg_is [0-9]+; then$/ { n = $3; sub(";", "", n); print n }' "$1" +} +workflow="$ROOT_DIR/.github/workflows/shell-tests.yml" +[[ -f "$workflow" ]] +jobs=$(awk ' + /^[[:space:]]*- shard:/ { if (suite != "") print suite "|" segments "|" set; suite = ""; segments = ""; set = ""; next } + /^[[:space:]]+suite:/ { suite = $2; next } + /^[[:space:]]+segments:/ { segments = $2; next } + /^[[:space:]]+segment_set:/ { + set = $0 + sub(/^[[:space:]]*segment_set:[[:space:]]*/, "", set) + gsub(/"/, "", set) + next + } + END { if (suite != "") print suite "|" segments "|" set } +' "$workflow") +[[ -n "$jobs" ]] +# The Ubuntu and Debian matrices are separate lists of the same jobs. +raw_jobs=$(printf '%s\n' "$jobs" | grep -c . || true) +jobs=$(printf '%s\n' "$jobs" | sort -u) +distinct=$(printf '%s\n' "$jobs" | grep -c . || true) +[[ "$raw_jobs" == "$(( distinct * 2 ))" ]] +for suite in motd cloudflare push-worker push other; do + declared="" + covered=" " + while IFS='|' read -r job_suite job_segments job_set; do + [[ "$job_suite" == "$suite" ]] || continue + [[ -n "$declared" ]] || declared=$job_segments + [[ "$declared" == "$job_segments" ]] || { printf 'inconsistent segment count for %s\n' "$suite"; exit 1; } + if [[ -z "$job_set" ]]; then + for (( index = 0; index < job_segments; index++ )); do covered+="$index "; done + else + covered+="$job_set " + fi + done <<< "$jobs" + [[ -n "$declared" ]] || { printf 'no job for %s\n' "$suite"; exit 1; } + for (( index = 0; index < declared; index++ )); do + count=$(printf '%s' "$covered" | tr ' ' '\n' | grep -c "^${index}$" || true) + [[ "$count" == 1 ]] || { printf '%s segment %s covered %s times\n' "$suite" "$index" "$count"; exit 1; } + done + total=$(printf '%s' "$covered" | tr ' ' '\n' | grep -c '^[0-9][0-9]*$' || true) + [[ "$total" == "$declared" ]] || { printf '%s covers %s segments, declared %s\n' "$suite" "$total" "$declared"; exit 1; } +done +[[ "$(guards "$ROOT_DIR/tests/test-motd.sh" | tr '\n' ' ')" == "0 1 2 3 4 5 6 7 " ]] +[[ "$(guards "$ROOT_DIR/tests/test-cloudflare-tunnel.sh" | tr '\n' ' ')" == "0 1 2 3 4 5 6 7 " ]] +[[ "$(guards "$ROOT_DIR/tests/test-push-worker-registration.sh" | tr '\n' ' ')" == "0 1 2 3 " ]] +# TEST_SEGMENTS wins over nothing: the subset must run only the named segments. +cat > "$TEMP_DIR/tests/test-motd.sh" <<'FIXTURE' +#!/usr/bin/env bash +printf 'test-motd.sh%s\n' "${TEST_SEGMENT:+-$TEST_SEGMENT}" >> "$CAPTURE" +FIXTURE +check_subset() { + local expected=$1 set=$2 + : > "$CAPTURE" + rc=0 + TEST_SEGMENTS=4 TEST_SEGMENT_SET="$set" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 || rc=$? + [[ "$rc" == 0 ]] || { cat "$TEMP_DIR/output"; exit 1; } + [[ "$(sort "$CAPTURE")" == "$(printf '%s\n' "$expected" | sort)" ]] || { cat "$TEMP_DIR/output"; exit 1; } +} +check_subset 'test-motd.sh-0 +test-motd.sh-3' '0 3' +check_subset 'test-motd.sh-2' '2' +check_subset 'test-motd.sh-0 +test-motd.sh-1 +test-motd.sh-2 +test-motd.sh-3' '' +# Anything but a distinct in-range index aborts before the suite starts. +for bad in '0 4' x '-1' '1 1' '4' '1,2'; do + : > "$CAPTURE" + rc=0 + TEST_SEGMENTS=4 TEST_SEGMENT_SET="$bad" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 || rc=$? + [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { printf 'subset %s accepted\n' "$bad"; exit 1; } +done printf 'All shard fixture tests passed.\n' From ab451ca7cf6e13100c15671beeedfd56d3f5372c Mon Sep 17 00:00:00 2001 From: FORGE Date: Sat, 24 Oct 2026 18:24:40 +0800 Subject: [PATCH 12/12] test(ci): reject impossible segment configs and prove shard coverage L60-01: accept TEST_SEGMENTS from 1..8 only and TEST_SEGMENT_SET entries as a single digit 0..7, split with `read -a` so no value is subject to pathname expansion or long-integer arithmetic. Out-of-range, duplicate, oversized and glob-like entries exit 2 before the first suite starts. L60-02: parse every matrix entry as job/shard/suite/segments/segment_set without deduplicating, assert unique shards plus exactly-once full coverage separately for ubuntu-tests and debian-tests, bind each declared count to the suite's segment guards and to the family list in run.sh, and exercise subsets for all three segmented families. L60-03: check the whole execution plan against the per-suite segment counts (motd 8, cloudflare 8, push-worker 4) before running anything, keep TEST_SEGMENTS=1 as a serial whole-suite run, and refuse a subset aimed at an unsegmented suite. L60-04: drop TEST_SEGMENT and TEST_SEGMENT_SET from nested fixture environments. --- tests/run.sh | 55 +++++-- tests/test-selection-git.sh | 2 +- tests/test-selection.sh | 2 +- tests/test-shards.sh | 277 +++++++++++++++++++++++++++++------- 4 files changed, 269 insertions(+), 67 deletions(-) diff --git a/tests/run.sh b/tests/run.sh index 0a22d73..b787d72 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -67,21 +67,34 @@ for test_file in "$ROOT_DIR"/tests/test-*.sh; do plan+=("$test_file") done -# Suites carrying CI segment guards are listed here; TEST_SEGMENTS sets how many -# processes share the work. With TEST_SEGMENTS=1 every suite runs in one process. -readonly SEGMENTED_FAMILIES=(test-motd.sh test-cloudflare-tunnel.sh test-push-worker-registration.sh) +# Suites carrying CI segment guards, with the number of segments their source +# defines. TEST_SEGMENTS sets how many processes share the work; with +# TEST_SEGMENTS=1 every suite runs in one process, all segments in order. +declare -rA SEGMENTED_SUITES=( + [test-motd.sh]=8 + [test-cloudflare-tunnel.sh]=8 + [test-push-worker-registration.sh]=4 +) segments=${TEST_SEGMENTS:-1} -[[ "$segments" =~ ^[1-9][0-9]*$ ]] || { printf 'Invalid TEST_SEGMENTS: %s\n' "$segments" >&2; exit 2; } +# One digit, at most the eight segments a suite actually has: no accepted value +# can overflow shell arithmetic and every index below stays a single character. +[[ "$segments" =~ ^[1-8]$ ]] || { printf 'Invalid TEST_SEGMENTS: %s\n' "$segments" >&2; exit 2; } # TEST_SEGMENT_SET narrows one job to a subset of the segments ("0 1 2 3"), so a # single suite can be spread over several jobs. Unset or blank keeps every # segment; anything that is not a distinct in-range index aborts before a suite -# starts. +# starts. `read -a` splits on IFS without pathname expansion, so an entry such +# as "[1]" is compared literally instead of being matched against file names. segment_set=() seen_segments=" " -if [[ -n ${TEST_SEGMENT_SET:-} && -n ${TEST_SEGMENT_SET//[[:space:]]/} ]]; then - for segment_index in $TEST_SEGMENT_SET; do - [[ "$segment_index" =~ ^[0-9]+$ ]] \ +if [[ -n "${TEST_SEGMENT_SET:-}" && -n "${TEST_SEGMENT_SET//[[:space:]]/}" ]]; then + # Flatten embedded newlines first: `read -a` stops at the first line, which + # would drop the rest of the set without a word. + segment_value=${TEST_SEGMENT_SET//$'\t'/ } + segment_value=${segment_value//$'\n'/ } + read -r -a segment_entries <<< "$segment_value" + for segment_index in "${segment_entries[@]}"; do + [[ "$segment_index" =~ ^[0-7]$ ]] \ || { printf 'Invalid TEST_SEGMENT_SET entry: %s\n' "$segment_index" >&2; exit 2; } (( segment_index < segments )) \ || { printf 'Segment %s out of range for TEST_SEGMENTS=%s\n' "$segment_index" "$segments" >&2; exit 2; } @@ -90,15 +103,29 @@ if [[ -n ${TEST_SEGMENT_SET:-} && -n ${TEST_SEGMENT_SET//[[:space:]]/} ]]; then seen_segments+="$segment_index " segment_set+=("$segment_index") done + (( segments > 1 )) \ + || { printf 'TEST_SEGMENT_SET requires TEST_SEGMENTS above 1\n' >&2; exit 2; } fi +# The whole plan must agree with the segment counts above before the first suite +# starts: a count that matches no suite (or a subset aimed at an unsegmented one) +# would otherwise silently run part of the work and still exit 0. +for test_file in "${plan[@]:-}"; do + [[ -n "$test_file" ]] || continue + test_name=$(basename "$test_file") + declared_segments=${SEGMENTED_SUITES[${test_name}]:-1} + if (( ${#segment_set[@]} )) && [[ -z "${SEGMENTED_SUITES[${test_name}]:-}" ]]; then + printf 'TEST_SEGMENT_SET does not apply to unsegmented suite: %s\n' "$test_name" >&2 + exit 2 + fi + (( segments == 1 || segments == declared_segments )) \ + || { printf 'TEST_SEGMENTS=%s but %s has %s segments\n' "$segments" "$test_name" "$declared_segments" >&2; exit 2; } +done + is_segmented() { - local name=$1 entry + local name=$1 (( segments > 1 )) || return 1 - for entry in "${SEGMENTED_FAMILIES[@]}"; do - [[ "$entry" == "$name" ]] && return 0 - done - return 1 + [[ -n "${SEGMENTED_SUITES[${name}]:-}" ]] } run_suite() { @@ -115,7 +142,7 @@ run_suite() { if [[ -n "$seg" ]]; then env -u TEST_JOBS -u TEST_SEGMENTS -u TEST_SEGMENT_SET TEST_SEGMENT="$seg" bash "$test_file" else - env -u TEST_JOBS -u TEST_SEGMENTS -u TEST_SEGMENT_SET bash "$test_file" + env -u TEST_JOBS -u TEST_SEGMENTS -u TEST_SEGMENT -u TEST_SEGMENT_SET bash "$test_file" fi status=$? set -e diff --git a/tests/test-selection-git.sh b/tests/test-selection-git.sh index 81e6ec4..c442c16 100644 --- a/tests/test-selection-git.sh +++ b/tests/test-selection-git.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail # These fixtures re-enter run.sh and require its serial, whole-shard behaviour. -unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT +unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT TEST_SEGMENT_SET ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) TEMP_DIR=$(mktemp -d) trap 'rm -rf "$TEMP_DIR"' EXIT diff --git a/tests/test-selection.sh b/tests/test-selection.sh index 942e523..e220b38 100644 --- a/tests/test-selection.sh +++ b/tests/test-selection.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail # These fixtures re-enter run.sh and require its serial, whole-shard behaviour. -unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT +unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT TEST_SEGMENT_SET ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) TEMP_DIR=$(mktemp -d) trap 'rm -rf "$TEMP_DIR"' EXIT diff --git a/tests/test-shards.sh b/tests/test-shards.sh index 417b739..c8bde8d 100644 --- a/tests/test-shards.sh +++ b/tests/test-shards.sh @@ -71,16 +71,61 @@ bash "$TEMP_DIR/tests/run.sh" push-worker > "$TEMP_DIR/output" 2>&1 || rc=$? grep -q 'exit=1' "$TEMP_DIR/output" PATH=$ORIGINAL_PATH unset TEST_BASE_SHA TEST_HEAD_SHA -# Segment subsets: one suite can be spread over several jobs, so every segment -# of a suite must be covered by exactly one job and every job must name a real -# segment of the suite's source. +# Segment coverage: one suite can be spread over several jobs, so every segment +# of a suite must be covered by exactly one job in each OS matrix, and every job +# must name real segments of the suite's source. Nothing is deduplicated before +# the checks: a repeated shard name or a repeated segment is a failure. guards() { awk '/^if seg_is [0-9]+; then$/ { n = $3; sub(";", "", n); print n }' "$1" } +# run.sh is the single source of truth for which suites are segmented and how +# many segments each one has. +segmented_suites() { + awk ' + /^declare -rA SEGMENTED_SUITES=\(/ { inside = 1; next } + inside && /^\)/ { inside = 0; next } + inside { + entry = $1 + if (entry !~ /^\[test-[^]]*\.sh\]=[0-9]+$/) next + name = entry; sub(/^\[/, "", name); sub(/\]=.*/, "", name) + count = entry; sub(/^.*\]=/, "", count) + print name, count + } + ' "$ROOT_DIR/tests/run.sh" +} +file_of_suite() { + case $1 in + motd) printf 'test-motd.sh' ;; + cloudflare) printf 'test-cloudflare-tunnel.sh' ;; + push-worker) printf 'test-push-worker-registration.sh' ;; + push) printf 'test-push.sh' ;; + *) return 1 ;; + esac +} +suite_of_file() { + case $1 in + test-motd.sh) printf 'motd' ;; + test-cloudflare-tunnel.sh) printf 'cloudflare' ;; + test-push-worker-registration.sh) printf 'push-worker' ;; + test-push.sh) printf 'push' ;; + *) return 1 ;; + esac +} +fail() { printf '%s\n' "$*" >&2; exit 1; } workflow="$ROOT_DIR/.github/workflows/shell-tests.yml" [[ -f "$workflow" ]] +# job|shard|suite|segments|segment_set, one line per matrix entry, duplicates kept. jobs=$(awk ' - /^[[:space:]]*- shard:/ { if (suite != "") print suite "|" segments "|" set; suite = ""; segments = ""; set = ""; next } + /^ [a-z][a-z0-9-]*:$/ { + if (suite != "") print job "|" shard "|" suite "|" segments "|" set + job = $1; sub(/:$/, "", job); shard = ""; suite = ""; segments = ""; set = "" + next + } + /^[[:space:]]*- shard:/ { + if (suite != "") print job "|" shard "|" suite "|" segments "|" set + shard = $3; suite = ""; segments = ""; set = "" + next + } /^[[:space:]]+suite:/ { suite = $2; next } /^[[:space:]]+segments:/ { segments = $2; next } /^[[:space:]]+segment_set:/ { @@ -89,63 +134,193 @@ jobs=$(awk ' gsub(/"/, "", set) next } - END { if (suite != "") print suite "|" segments "|" set } + END { if (suite != "") print job "|" shard "|" suite "|" segments "|" set } ' "$workflow") -[[ -n "$jobs" ]] -# The Ubuntu and Debian matrices are separate lists of the same jobs. -raw_jobs=$(printf '%s\n' "$jobs" | grep -c . || true) -jobs=$(printf '%s\n' "$jobs" | sort -u) -distinct=$(printf '%s\n' "$jobs" | grep -c . || true) -[[ "$raw_jobs" == "$(( distinct * 2 ))" ]] -for suite in motd cloudflare push-worker push other; do - declared="" - covered=" " - while IFS='|' read -r job_suite job_segments job_set; do - [[ "$job_suite" == "$suite" ]] || continue - [[ -n "$declared" ]] || declared=$job_segments - [[ "$declared" == "$job_segments" ]] || { printf 'inconsistent segment count for %s\n' "$suite"; exit 1; } - if [[ -z "$job_set" ]]; then - for (( index = 0; index < job_segments; index++ )); do covered+="$index "; done +[[ -n "$jobs" ]] || fail 'no shard matrix in the workflow' + +# One OS matrix on its own: unique shard names, every segment of every suite +# covered exactly once, union 0..N-1. +check_matrix() { + local job=$1 rows suite row_suite suite_segments segment_set_value + local declared covered index count total expected actual + rows=$(printf '%s\n' "$jobs" | awk -F'|' -v job="$job" '$1 == job') + [[ -n "$rows" ]] || fail "no matrix rows for $job" + if printf '%s\n' "$rows" | cut -d'|' -f2 | sort | uniq -d | grep -q .; then + fail "$job repeats a shard name" + fi + for suite in motd cloudflare push-worker push other; do + declared="" + covered=" " + while IFS='|' read -r _job _shard row_suite suite_segments segment_set_value; do + [[ "$row_suite" == "$suite" ]] || continue + case "$suite_segments" in + ''|*[!0-9]*) fail "$job $suite has an unsupported segment count: '$suite_segments'" ;; + esac + if [[ -z "$declared" ]]; then + declared=$suite_segments + elif [[ "$declared" != "$suite_segments" ]]; then + fail "$job $suite mixes segment counts: $declared and $suite_segments" + fi + if [[ -z "$segment_set_value" ]]; then + for (( index = 0; index < suite_segments; index++ )); do + covered+=" $index" + done + else + covered+=" $segment_set_value" + fi + done <<< "$rows" + [[ -n "$declared" ]] || fail "$job has no job for $suite" + for (( index = 0; index < declared; index++ )); do + count=$(printf '%s' "$covered" | tr ' ' '\n' | grep -cx -- "$index" || true) + [[ "$count" == 1 ]] || fail "$job $suite segment $index covered $count times" + done + total=$(printf '%s' "$covered" | tr ' ' '\n' | grep -cE '^[0-9]+$' || true) + [[ "$total" == "$declared" ]] || fail "$job $suite covers $total segments, declared $declared" + expected=$(seq -s ' ' 0 "$(( declared - 1 ))") + actual=$(printf '%s' "$covered" | tr ' ' '\n' | grep -E '^[0-9]+$' | sort -n -u | tr '\n' ' ') + actual=${actual% } + [[ "$actual" == "$expected" ]] || fail "$job $suite covers '$actual', expected '$expected'" + done +} + +# The declared counts must be the ones the sources actually implement, and both +# OS matrices must agree on them. +declared_segments() { + local job=$1 suite=$2 rows first + rows=$(printf '%s\n' "$jobs" | awk -F'|' -v job="$job" -v suite="$suite" \ + '$1 == job && $3 == suite { print $4 }') + [[ -n "$rows" ]] || fail "$job has no job for $suite" + first=$(printf '%s\n' "$rows" | head -1) + [[ "$(printf '%s\n' "$rows" | grep -cx -- "$first" || true)" == \ + "$(printf '%s\n' "$rows" | grep -c . || true)" ]] || fail "$job $suite mixes segment counts" + printf '%s' "$first" +} + +check_suites() { + local suite file declared ubuntu_declared debian_declared guards_list guard_count expected name + local -A families=() + while read -r name count; do + [[ -n "$name" ]] || continue + families[$name]=$count + done <<< "$(segmented_suites)" + (( ${#families[@]} > 0 )) || fail 'run.sh declares no segmented suite' + for name in "${!families[@]}"; do + suite_of_file "$name" > /dev/null || fail "run.sh segments $name, which is not in the matrix" + done + for suite in motd cloudflare push-worker push other; do + ubuntu_declared=$(declared_segments ubuntu-tests "$suite") + debian_declared=$(declared_segments debian-tests "$suite") + [[ "$ubuntu_declared" == "$debian_declared" ]] \ + || fail "$suite has $ubuntu_declared segments on Ubuntu but $debian_declared on Debian" + declared=$ubuntu_declared + if file=$(file_of_suite "$suite"); then + guards_list=$(guards "$ROOT_DIR/tests/$file") + guard_count=$(printf '%s' "$guards_list" | grep -c . || true) + if (( declared > 1 )); then + [[ -n "${families[$file]:-}" ]] || fail "$suite runs $declared segments but run.sh does not segment $file" + [[ "${families[$file]}" == "$declared" ]] \ + || fail "run.sh cuts $file into ${families[$file]} segments, the matrix says $declared" + [[ "$guard_count" == "$declared" ]] \ + || fail "$file has $guard_count segment guards, the matrix says $declared" + expected=$(seq -s ' ' 0 "$(( declared - 1 ))") + [[ "$(printf '%s\n' "$guards_list" | tr '\n' ' ')" == "$expected " ]] \ + || fail "$file guard numbers are not 0..$(( declared - 1 ))" + else + [[ -z "${families[$file]:-}" ]] || fail "run.sh segments $file but its jobs run it whole" + [[ "$guard_count" == 0 ]] || fail "$file has segment guards but its jobs run it whole" + fi else - covered+="$job_set " + # `other` is a bucket of suites, not a suite that can be cut up. + (( declared == 1 )) || fail "$suite is a bucket and cannot be cut into $declared segments" fi - done <<< "$jobs" - [[ -n "$declared" ]] || { printf 'no job for %s\n' "$suite"; exit 1; } - for (( index = 0; index < declared; index++ )); do - count=$(printf '%s' "$covered" | tr ' ' '\n' | grep -c "^${index}$" || true) - [[ "$count" == 1 ]] || { printf '%s segment %s covered %s times\n' "$suite" "$index" "$count"; exit 1; } done - total=$(printf '%s' "$covered" | tr ' ' '\n' | grep -c '^[0-9][0-9]*$' || true) - [[ "$total" == "$declared" ]] || { printf '%s covers %s segments, declared %s\n' "$suite" "$total" "$declared"; exit 1; } -done -[[ "$(guards "$ROOT_DIR/tests/test-motd.sh" | tr '\n' ' ')" == "0 1 2 3 4 5 6 7 " ]] -[[ "$(guards "$ROOT_DIR/tests/test-cloudflare-tunnel.sh" | tr '\n' ' ')" == "0 1 2 3 4 5 6 7 " ]] -[[ "$(guards "$ROOT_DIR/tests/test-push-worker-registration.sh" | tr '\n' ' ')" == "0 1 2 3 " ]] -# TEST_SEGMENTS wins over nothing: the subset must run only the named segments. -cat > "$TEMP_DIR/tests/test-motd.sh" <<'FIXTURE' -#!/usr/bin/env bash -printf 'test-motd.sh%s\n' "${TEST_SEGMENT:+-$TEST_SEGMENT}" >> "$CAPTURE" -FIXTURE +} + +check_matrix ubuntu-tests +check_matrix debian-tests +check_suites + +# Subset execution: every segmented family must run only the segments its job +# names, and a segment count or subset the suite cannot honour must abort before +# the first suite starts. TEST_SEGMENTS=1 still runs a suite whole in one +# process. +make_segment_fixture() { + printf '#!/usr/bin/env bash\nprintf "%%s\\n" "%s${TEST_SEGMENT:+-$TEST_SEGMENT}" >> "$CAPTURE"\n' "$1" \ + > "$TEMP_DIR/tests/$1" +} +make_segment_fixture test-motd.sh +make_segment_fixture test-cloudflare-tunnel.sh +make_segment_fixture test-push-worker-registration.sh check_subset() { - local expected=$1 set=$2 + local suite=$1 segs=$2 set_value=$3 expected=$4 rc=0 actual : > "$CAPTURE" - rc=0 - TEST_SEGMENTS=4 TEST_SEGMENT_SET="$set" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 || rc=$? - [[ "$rc" == 0 ]] || { cat "$TEMP_DIR/output"; exit 1; } - [[ "$(sort "$CAPTURE")" == "$(printf '%s\n' "$expected" | sort)" ]] || { cat "$TEMP_DIR/output"; exit 1; } + TEST_SEGMENTS="$segs" TEST_SEGMENT_SET="$set_value" bash "$TEMP_DIR/tests/run.sh" "$suite" \ + > "$TEMP_DIR/output" 2>&1 || rc=$? + [[ "$rc" == 0 ]] || { cat "$TEMP_DIR/output"; fail "subset '$set_value' for $suite (TEST_SEGMENTS=$segs) exited $rc"; } + actual=$(sort "$CAPTURE" | tr '\n' ' ') + [[ "$actual" == "$(printf '%s\n' "$expected" | sort | tr '\n' ' ')" ]] || { + cat "$TEMP_DIR/output" + fail "subset '$set_value' for $suite ran: $actual" + } } -check_subset 'test-motd.sh-0 -test-motd.sh-3' '0 3' -check_subset 'test-motd.sh-2' '2' -check_subset 'test-motd.sh-0 +check_subset motd 8 '0 1 2 3' 'test-motd.sh-0 +test-motd.sh-1 +test-motd.sh-2 +test-motd.sh-3' +check_subset motd 8 '' 'test-motd.sh-0 test-motd.sh-1 test-motd.sh-2 -test-motd.sh-3' '' -# Anything but a distinct in-range index aborts before the suite starts. -for bad in '0 4' x '-1' '1 1' '4' '1,2'; do +test-motd.sh-3 +test-motd.sh-4 +test-motd.sh-5 +test-motd.sh-6 +test-motd.sh-7' +check_subset motd 1 '' 'test-motd.sh' +check_subset cloudflare 8 '4 5 6 7' 'test-cloudflare-tunnel.sh-4 +test-cloudflare-tunnel.sh-5 +test-cloudflare-tunnel.sh-6 +test-cloudflare-tunnel.sh-7' +check_subset cloudflare 1 '' 'test-cloudflare-tunnel.sh' +check_subset push-worker 4 '2' 'test-push-worker-registration.sh-2' +check_subset push-worker 4 '' 'test-push-worker-registration.sh-0 +test-push-worker-registration.sh-1 +test-push-worker-registration.sh-2 +test-push-worker-registration.sh-3' +check_subset push-worker 1 '' 'test-push-worker-registration.sh' + +check_reject() { + local label=$1 suite=$2 segs=$3 set_value=$4 rc=0 + : > "$CAPTURE" + TEST_SEGMENTS="$segs" TEST_SEGMENT_SET="$set_value" bash "$TEMP_DIR/tests/run.sh" "$suite" \ + > "$TEMP_DIR/output" 2>&1 || rc=$? + [[ "$rc" == 2 ]] || { cat "$TEMP_DIR/output"; fail "$label: exit $rc, expected 2"; } + [[ ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "$label: started a suite"; } +} +# A segment count the suite's source does not have would silently run part of it. +check_reject 'motd cut into two' motd 2 '' +check_reject 'motd cut into four' motd 4 '' +check_reject 'cloudflare cut into four' cloudflare 4 '' +check_reject 'push-worker cut into eight' push-worker 8 '' +check_reject 'push-worker cut into two' push-worker 2 '' +check_reject 'motd segment past its source' motd 8 '8' +check_reject 'subset for an unsegmented suite' push 2 '0' +check_reject 'subset for the other bucket' other 2 '0' +check_reject 'subset without segmented mode' motd 1 '0' +check_reject 'mixed plan with one count' all 8 '' +check_reject 'mixed plan with four segments' all 4 '0 1 2 3' +# Anything but a distinct single-digit in-range index is refused; a file named 1 +# must not turn "[1]" into a real segment. +: > "$TEMP_DIR/1" +for bad in '1 01' '9223372036854775808' '18446744073709551616' '[1]' '1,2' '-1' '1 1' '+1' '0x1' 'x'; do + : > "$CAPTURE" + rc=0 + ( cd "$TEMP_DIR" && TEST_SEGMENTS=8 TEST_SEGMENT_SET="$bad" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 ) || rc=$? + [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "subset '$bad' accepted (exit $rc)"; } +done +for bad in 0 9 08 '4x' '1 4' 9223372036854775808 18446744073709551616; do : > "$CAPTURE" rc=0 - TEST_SEGMENTS=4 TEST_SEGMENT_SET="$bad" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 || rc=$? - [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { printf 'subset %s accepted\n' "$bad"; exit 1; } + TEST_SEGMENTS="$bad" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 || rc=$? + [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "TEST_SEGMENTS=$bad accepted (exit $rc)"; } done printf 'All shard fixture tests passed.\n'