diff --git a/.github/workflows/shell-tests.yml b/.github/workflows/shell-tests.yml index 9a0299b..be5a514 100644 --- a/.github/workflows/shell-tests.yml +++ b/.github/workflows/shell-tests.yml @@ -20,7 +20,7 @@ jobs: steps: - uses: actions/checkout@v7 with: - fetch-depth: 0 + fetch-depth: 1 persist-credentials: false - name: Install pinned ShellCheck @@ -58,7 +58,34 @@ jobs: strategy: fail-fast: false matrix: - shard: [motd, push, other] + include: + # motd and cloudflare are cut into eight segments; each of their two + # jobs runs four of them, so a single suite spans two jobs. + - shard: motd-a + suite: motd + segments: 8 + segment_set: "0 1 2 3" + - shard: motd-b + suite: motd + segments: 8 + segment_set: "4 5 6 7" + - shard: push-worker + suite: push-worker + segments: 4 + - shard: push + suite: push + segments: 1 + - shard: cloudflare-a + suite: cloudflare + segments: 8 + segment_set: "0 1 2 3" + - shard: cloudflare-b + suite: cloudflare + segments: 8 + segment_set: "4 5 6 7" + - shard: other + suite: other + segments: 1 steps: - uses: actions/checkout@v7 with: @@ -84,8 +111,10 @@ jobs: env: TEST_BASE_SHA: ${{ github.event.pull_request.base.sha }} TEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - TEST_SHARD: ${{ matrix.shard }} - run: bash tests/run.sh "$TEST_SHARD" + TEST_SUITE: ${{ matrix.suite }} + TEST_SEGMENTS: ${{ matrix.segments }} + TEST_SEGMENT_SET: ${{ matrix.segment_set }} + run: bash tests/run.sh "$TEST_SUITE" debian-tests: name: Debian 13 / ${{ matrix.shard }} @@ -95,7 +124,34 @@ jobs: strategy: fail-fast: false matrix: - shard: [motd, push, other] + include: + # motd and cloudflare are cut into eight segments; each of their two + # jobs runs four of them, so a single suite spans two jobs. + - shard: motd-a + suite: motd + segments: 8 + segment_set: "0 1 2 3" + - shard: motd-b + suite: motd + segments: 8 + segment_set: "4 5 6 7" + - shard: push-worker + suite: push-worker + segments: 4 + - shard: push + suite: push + segments: 1 + - shard: cloudflare-a + suite: cloudflare + segments: 8 + segment_set: "0 1 2 3" + - shard: cloudflare-b + suite: cloudflare + segments: 8 + segment_set: "4 5 6 7" + - shard: other + suite: other + segments: 1 steps: - name: Install test dependencies run: | @@ -118,14 +174,17 @@ jobs: bash tests/test-selection-git.sh - name: Check Bash syntax + if: matrix.shard == 'other' run: find . -type f -name '*.sh' -print0 | xargs -0 -n1 bash -n - name: Run tests env: TEST_BASE_SHA: ${{ github.event.pull_request.base.sha }} TEST_HEAD_SHA: ${{ github.event.pull_request.head.sha }} - TEST_SHARD: ${{ matrix.shard }} - run: bash tests/run.sh "$TEST_SHARD" + TEST_SUITE: ${{ matrix.suite }} + TEST_SEGMENTS: ${{ matrix.segments }} + TEST_SEGMENT_SET: ${{ matrix.segment_set }} + run: bash tests/run.sh "$TEST_SUITE" # Preserve existing required-check names; skipped/cancelled dependencies fail closed. ubuntu: diff --git a/README.md b/README.md index 078d5e5..76d9173 100644 --- a/README.md +++ b/README.md @@ -23,7 +23,14 @@ 生产宿主机(包括 Netcup)仅执行差异检查及改动脚本的 `bash -n`,不运行全量 ShellCheck 或完整测试集; 分析和测试交由 CI:PR 按完整差异及依赖映射选择套件,README/许可证修改无需业务测试; 公共设施或未知依赖改动回退全量,main 推送仍执行全量。两个必需 Job 始终运行,输出选择和跳过记录。 -Ubuntu 使用固定 ShellCheck v0.10.0:所有受 Git 跟踪、位于 tests/ 之外的 .sh 文件执行 warning 门槛,tests/ 内的 .sh 文件执行 error 门槛;新目录中的生产 .sh 文件自动纳入,不维护生产文件白名单。Zsh 配置 p10k-config.zsh 不交给 Bash ShellCheck。两组 lint 都会执行并累计失败,Debian 不重复 lint。 +每个 OS 的测试作业按七个 shard 并行:`motd-a`/`motd-b`(`test-motd.sh`,各跑四段)、 +`cloudflare-a`/`cloudflare-b`(`test-cloudflare-tunnel.sh`,各跑四段)、`push-worker` +(`test-push-worker-registration.sh`)、`push`(仅 `test-push.sh`)、`other`(其余全部套件); +本机全量用 `tests/run.sh all`。 +重套件(motd、Cloudflare Tunnel、push worker 注册)按语义分段,import 守卫 `seg_is`:`TEST_SEGMENTS` 个进程各跑互不重叠的一段并同时执行(本机默认 1,即按原顺序串行)。 +motd 与 Cloudflare Tunnel 各切成八段:`-a` 作业用 `TEST_SEGMENT_SET="0 1 2 3"`、`-b` 作业用 `"4 5 6 7"`,两个作业的并集恰好覆盖全部段且互不重叠;push worker 注册仍为四段、单作业。 +`TEST_SEGMENT_SET` 只接受互不重复、且在 `TEST_SEGMENTS` 范围内的段下标,非法值时在任何套件启动前以状态 2 退出;未设置则照旧跑全部段。 +Ubuntu 使用固定 ShellCheck v0.10.0:所有受 Git 跟踪、位于 tests/ 之外的 .sh 文件执行 warning 门槛,tests/ 与 tests/lib/ 内的 .sh 文件执行 error 门槛;新目录中的生产 .sh 文件自动纳入,不维护生产文件白名单。Zsh 配置 p10k-config.zsh 不交给 Bash ShellCheck。两组 lint 都会执行并累计失败,Debian 不重复 lint。 例外仅允许局部明确的 SC 规则编号和邻近理由;禁止文件级或全局 warning 抑制。测试 warning 不阻断不等于被 disable,禁止用抑制隐藏行为缺陷。CI 工作流改动继续触发全量测试回退;warning 门槛失败时后续 Ubuntu 行为套件不会执行,不能以 Debian 通过代替双环境验收。 语法检查仍覆盖全部 .sh 脚本;文档修改无需本地全套测试,但仍须通过必需 CI。 本机重测试必须使用原生资源限制及权限隔离;无法隔离或超限时停止,不自动提高额度或转宿主机执行。 diff --git a/tests/run.sh b/tests/run.sh index dbb2aa4..b787d72 100644 --- a/tests/run.sh +++ b/tests/run.sh @@ -6,11 +6,11 @@ readonly ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) # Positional, not exported: nested selection fixtures must still run all suites. shard=${1:-all} if (( $# > 1 )); then - printf 'Usage: tests/run.sh [all|motd|push|other]\n' >&2 + printf 'Usage: tests/run.sh [all|motd|push-worker|push|cloudflare|other]\n' >&2 exit 2 fi case "$shard" in - all|motd|push|other) ;; + all|motd|push-worker|push|cloudflare|other) ;; *) printf 'Invalid test shard: %s\n' "$shard" >&2; exit 2 ;; esac @@ -29,6 +29,7 @@ if [[ -n "${TEST_BASE_SHA:-}" && -n "${TEST_HEAD_SHA:-}" ]]; then tools/push.sh) selected[test-push.sh]=1; selected[test-push-worker-registration.sh]=1 ;; tools/cloudflare_tunnel.sh) selected[test-cloudflare-tunnel.sh]=1 ;; modules/ssh-security.sh) selected[test-ssh-security.sh]=1 ;; + modules/system-optimize.sh) selected[test-system-optimize.sh]=1 ;; linux_setup.sh) selected[test-linux-setup.sh]=1; selected[test-setup-exec.sh]=1 ;; modules/zsh-setup.sh|p10k-config.zsh) selected[test-linux-setup.sh]=1 ;; tests/test-*.sh) @@ -43,35 +44,151 @@ if [[ -n "${TEST_BASE_SHA:-}" && -n "${TEST_HEAD_SHA:-}" ]]; then fi printf 'Test selection: full=%s\n' "$full" +plan=() for test_file in "$ROOT_DIR"/tests/test-*.sh; do - if [[ "$full" != true && -z "${selected[${test_file##*/}]:-}" ]]; then - printf 'SKIP: %s (unaffected by PR diff)\n' "${test_file##*/}" + test_name=$(basename "$test_file") + if [[ "$full" != true && -z "${selected[${test_name}]:-}" ]]; then + printf 'SKIP: %s (unaffected by PR diff)\n' "$test_name" continue fi - test_name=$(basename "$test_file") + # One suite per heavy family so CI can schedule them independently; every + # other suite shares `other`. case "$test_name" in test-motd.sh) suite_shard=motd ;; - test-push.sh|test-push-worker-registration.sh) suite_shard=push ;; + test-push-worker-registration.sh) suite_shard=push-worker ;; + test-push.sh) suite_shard=push ;; + test-cloudflare-tunnel.sh) suite_shard=cloudflare ;; *) suite_shard=other ;; esac if [[ "$shard" != all && "$shard" != "$suite_shard" ]]; then printf 'SKIP: %s (assigned to %s shard)\n' "$test_name" "$suite_shard" continue fi - printf '\n==> %s\n' "$test_name" + plan+=("$test_file") +done + +# Suites carrying CI segment guards, with the number of segments their source +# defines. TEST_SEGMENTS sets how many processes share the work; with +# TEST_SEGMENTS=1 every suite runs in one process, all segments in order. +declare -rA SEGMENTED_SUITES=( + [test-motd.sh]=8 + [test-cloudflare-tunnel.sh]=8 + [test-push-worker-registration.sh]=4 +) +segments=${TEST_SEGMENTS:-1} +# One digit, at most the eight segments a suite actually has: no accepted value +# can overflow shell arithmetic and every index below stays a single character. +[[ "$segments" =~ ^[1-8]$ ]] || { printf 'Invalid TEST_SEGMENTS: %s\n' "$segments" >&2; exit 2; } + +# TEST_SEGMENT_SET narrows one job to a subset of the segments ("0 1 2 3"), so a +# single suite can be spread over several jobs. Unset or blank keeps every +# segment; anything that is not a distinct in-range index aborts before a suite +# starts. `read -a` splits on IFS without pathname expansion, so an entry such +# as "[1]" is compared literally instead of being matched against file names. +segment_set=() +seen_segments=" " +if [[ -n "${TEST_SEGMENT_SET:-}" && -n "${TEST_SEGMENT_SET//[[:space:]]/}" ]]; then + # Flatten embedded newlines first: `read -a` stops at the first line, which + # would drop the rest of the set without a word. + segment_value=${TEST_SEGMENT_SET//$'\t'/ } + segment_value=${segment_value//$'\n'/ } + read -r -a segment_entries <<< "$segment_value" + for segment_index in "${segment_entries[@]}"; do + [[ "$segment_index" =~ ^[0-7]$ ]] \ + || { printf 'Invalid TEST_SEGMENT_SET entry: %s\n' "$segment_index" >&2; exit 2; } + (( segment_index < segments )) \ + || { printf 'Segment %s out of range for TEST_SEGMENTS=%s\n' "$segment_index" "$segments" >&2; exit 2; } + [[ "$seen_segments" != *" $segment_index "* ]] \ + || { printf 'Duplicate segment in TEST_SEGMENT_SET: %s\n' "$segment_index" >&2; exit 2; } + seen_segments+="$segment_index " + segment_set+=("$segment_index") + done + (( segments > 1 )) \ + || { printf 'TEST_SEGMENT_SET requires TEST_SEGMENTS above 1\n' >&2; exit 2; } +fi + +# The whole plan must agree with the segment counts above before the first suite +# starts: a count that matches no suite (or a subset aimed at an unsegmented one) +# would otherwise silently run part of the work and still exit 0. +for test_file in "${plan[@]:-}"; do + [[ -n "$test_file" ]] || continue + test_name=$(basename "$test_file") + declared_segments=${SEGMENTED_SUITES[${test_name}]:-1} + if (( ${#segment_set[@]} )) && [[ -z "${SEGMENTED_SUITES[${test_name}]:-}" ]]; then + printf 'TEST_SEGMENT_SET does not apply to unsegmented suite: %s\n' "$test_name" >&2 + exit 2 + fi + (( segments == 1 || segments == declared_segments )) \ + || { printf 'TEST_SEGMENTS=%s but %s has %s segments\n' "$segments" "$test_name" "$declared_segments" >&2; exit 2; } +done + +is_segmented() { + local name=$1 + (( segments > 1 )) || return 1 + [[ -n "${SEGMENTED_SUITES[${name}]:-}" ]] +} + +run_suite() { + local test_file=$1 seg=${2:-} test_name started status + test_name=$(basename "$test_file") + if [[ -n "$seg" ]]; then + test_name+=" [segment $((seg + 1))/$segments]" + fi started=$SECONDS + printf '\n==> %s\n' "$test_name" # Keep the child shell standalone: an if/|| wrapper can change errexit behavior. + # Nested fixtures re-enter this script, so neither setting may leak to them. set +e - bash "$test_file" + if [[ -n "$seg" ]]; then + env -u TEST_JOBS -u TEST_SEGMENTS -u TEST_SEGMENT_SET TEST_SEGMENT="$seg" bash "$test_file" + else + env -u TEST_JOBS -u TEST_SEGMENTS -u TEST_SEGMENT -u TEST_SEGMENT_SET bash "$test_file" + fi status=$? set -e - elapsed=$((SECONDS - started)) + local elapsed=$((SECONDS - started)) printf 'TIMING: %s elapsed=%ss exit=%s\n' "$test_name" "$elapsed" "$status" if [[ -n "${GITHUB_STEP_SUMMARY:-}" ]]; then printf -- '- `%s`: %ss, exit %s\n' "$test_name" "$elapsed" "$status" >> "$GITHUB_STEP_SUMMARY" fi - if (( status != 0 )); then - exit "$status" + return "$status" +} + +# Segments of one suite share no state, so they run together and fill the +# runner; the suites of a shard still run one after another. +run_segments() { + local test_file=$1 dir k rc failed=0 pids=() + local -a queue=() + dir=$(mktemp -d) + if (( ${#segment_set[@]} )); then + queue=("${segment_set[@]}") + else + for (( k = 0; k < segments; k++ )); do queue+=("$k"); done + fi + for k in "${queue[@]}"; do + ( + rc=0 + run_suite "$test_file" "$k" > "$dir/$k.log" 2>&1 || rc=$? + printf '%s' "$rc" > "$dir/$k.rc" + ) & + pids+=("$!") + done + for k in "${pids[@]}"; do wait "$k" || true; done + for k in "${queue[@]}"; do + cat "$dir/$k.log" + rc=$(cat "$dir/$k.rc" 2>/dev/null || printf 1) + (( rc == 0 )) || failed=1 + done + rm -rf "$dir" + return "$failed" +} + +for test_file in "${plan[@]:-}"; do + [[ -n "$test_file" ]] || continue + if is_segmented "$(basename "$test_file")"; then + run_segments "$test_file" || exit 1 + else + run_suite "$test_file" || exit "$?" fi done diff --git a/tests/test-cloudflare-tunnel.sh b/tests/test-cloudflare-tunnel.sh index c23689a..2e48726 100644 --- a/tests/test-cloudflare-tunnel.sh +++ b/tests/test-cloudflare-tunnel.sh @@ -1,8 +1,17 @@ +#!/usr/bin/env bash +set -euo pipefail +# --- CI segments (tools/segmentize_ranges.py) --- +# TEST_SEGMENT unset: every segment runs in order, exactly like the original. +seg_is() { + [[ -z ${TEST_SEGMENT:-} ]] || (( $1 == TEST_SEGMENT )) +} +# --- end CI segments --- + #!/usr/bin/env bash set -euo pipefail if (( EUID != 0 )); then - exec sudo --preserve-env=PATH bash "$0" "$@" + exec sudo --preserve-env=PATH --preserve-env=TEST_SEGMENT bash "$0" "$@" fi ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) @@ -172,6 +181,15 @@ run_failure_case() { ORIGINAL_PATH=$PATH +# replay the state that a later segment inherited from an earlier one +if [[ -n ${TEST_SEGMENT:-} ]] && (( TEST_SEGMENT != 0 )); then +new_case +require_root() { :; } +check_platform() { :; } +uninstall_transaction_hook() { [[ "$1" != after-disable-auto-update ]] || exit 0; } +fi +if seg_is 0; then + new_case before=$(find "$CASE_DIR/root" -mindepth 1 -printf '%P %y %m\n' | sort) repository_source_content > "$CASE_DIR/source-output" @@ -402,7 +420,9 @@ pass "source remains old when key commit fails" run_failure_case "APT probe failure rollback" 'export FAKE_APT_UPDATE_FAIL=1' 'run_repository_apt_transaction install' run_failure_case "APT install failure rollback" 'export FAKE_APT_INSTALL_FAIL=1' 'run_repository_apt_transaction install' +fi +if seg_is 1; then for signal in HUP INT TERM; do case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac for phase in download validate stage key-commit source-commit apt-probe apt-install; do @@ -454,6 +474,9 @@ if configure_repository >/dev/null 2>&1; then fail "rollback failure unexpectedl find "$CLOUDFLARED_REPOSITORY_STATE_DIR" -maxdepth 2 -type f -name rollback.log -print -quit | grep -q . || fail "rollback failure did not preserve evidence" ) +fi + +if seg_is 2; then pass "rollback failure returns nonzero and preserves evidence" new_case @@ -482,7 +505,9 @@ find "$CLOUDFLARED_STATE_DIR" -maxdepth 1 -type d -name 'uninstall-*' -print -qu fail "managed source backup missing" [[ ! -d "$CLOUDFLARED_STATE_DIR.lock" ]] || fail "managed repository removal left lock" pass "configure then real repository uninstall removes source and preserves keyring" +fi +if seg_is 3; then for signal in HUP INT TERM; do case "$signal" in HUP) expected=129 ;; INT) expected=130 ;; TERM) expected=143 ;; esac for phase in before-snapshot-create capture-auto_update_script capture-auto_update_timer capture-legacy_marker lock-acquired before-disable-auto-update after-disable-auto-update before-apt-remove after-apt-remove before-source-remove after-source-remove final-apt-update before-lock-release; do @@ -513,7 +538,9 @@ for signal in HUP INT TERM; do pass "uninstall $signal=$expected cleans $phase" done done +fi +if seg_is 4; then new_case set_old_generation configure_repository @@ -596,6 +623,7 @@ grep -Fq "ExecStart=$CLOUDFLARED_APT_BIN --no-autoupdate --token-file /etc/cloud pass "legacy binary and service migration behavior remains" + # Each malformed snapshot runs in a separate process: transaction traps and # intentional corrupt fixtures must not leak into subsequent cases. for schema_case in version generation duplicate unknown order state mode uid gid digest created payload-name payload-missing payload-extra payload-fifo payload-symlink manifest-fifo ancestor; do @@ -637,8 +665,9 @@ for schema_case in version generation duplicate unknown order state mode uid gid ) > "$TEST_DIR/schema-$schema_case.log" 2>&1 || fail "schema $schema_case (see $TEST_DIR/schema-$schema_case.log)" pass "uninstall manifest rejects $schema_case" done +fi - +if seg_is 5; then for ownership_case in owned foreign inode mode owner gid symlink directory fifo; do ( new_case @@ -935,7 +964,9 @@ for restore_id in auto_update_script auto_update_service auto_update_timer sourc } pass "restore failure $restore_id continues remaining five targets and preserves original evidence" done +fi +if seg_is 6; then for target_id in auto_update_script auto_update_service auto_update_timer source current legacy_marker; do for bad_metadata in mode owner gid symlink directory fifo; do ( @@ -1016,7 +1047,9 @@ for creation_phase in before-create-marker before-create-manifest-stage before-c pass "creation $creation_phase $creation_fault preserves ownership across publication" done done +fi +if seg_is 7; then for bookkeeping_fault in marker stage rename; do ( new_case @@ -1542,3 +1575,4 @@ grep -Fq 'cloudflare_tunnel.sh install' <<< "$entrypoint_output" || fail "bash - pass "bash -c entrypoint remains compatible" printf 'All cloudflare wrapper tests passed. PASS=%d\n' "$pass_count" +fi diff --git a/tests/test-motd.sh b/tests/test-motd.sh index cccdae2..a7ea717 100644 --- a/tests/test-motd.sh +++ b/tests/test-motd.sh @@ -1,3 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +# --- CI segments (tools/segmentize_ranges.py) --- +# TEST_SEGMENT unset: every segment runs in order, exactly like the original. +seg_is() { + [[ -z ${TEST_SEGMENT:-} ]] || (( $1 == TEST_SEGMENT )) +} +# --- end CI segments --- + #!/usr/bin/env bash set -euo pipefail ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.."&&pwd); @@ -57,6 +66,132 @@ else echo "O|$p|$(stat -c %F "$p")"; fi; done; } +create_legacy(){ printf legacy>"$MOTD_ETC_ROOT/motd.initial-backup"; +: >"$MOTD_ETC_ROOT/issue.initial-absent"; +printf net>"$MOTD_ETC_ROOT/issue.net.initial-backup"; +: >"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.initial-unknown"; +printf uname>"$MOTD_ETC_ROOT/update-motd.d/10-uname.initial-backup"; +: >"$MOTD_ETC_ROOT/update-motd.d/50-motd-news.initial-absent"; +} +cat >"$TEST_DIR/sig.sh"<<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1"; +hit=false +send(){ [[ $hit == false ]]||return; +hit=true; +kill -"$SIGNAL_NAME" "$BASHPID"; +} +motd_snapshot_capture_hook(){ [[ $PHASE == snapshot-building&&$1 == issue ]]&&send; +return 0; +} +motd_transaction_phase_hook(){ [[ $PHASE == snapshot-commit&&$1 == snapshot-commit ]]&&send; +return 0; +} +motd_snapshot_commit_hook(){ [[ $PHASE == snapshot-publish&&$1 == initial&&$3 == pointer ]]&&send; +return 0; +} +motd_target_commit_hook(){ case "$PHASE:$1:$2" in install-first:install:motd|install-middle:install:issue_net|restore-first:restore:motd|restore-middle:restore:issue_net)send;; +esac; +return 0; +} +case $PHASE in restore-*)motd_run_locked_operation restore previous;; +*)motd_run_locked_operation install;; +esac +CHILD +chmod 700 "$TEST_DIR/sig.sh" +state_fingerprint() { + local root="$1" path + [[ -d "$root" ]] || { printf 'absent\n'; return; } + while IFS= read -r path; do + if [[ -L "$path" ]]; then printf 'L|%s|%s\n' "${path#$root/}" "$(readlink "$path")" + elif [[ -f "$path" ]]; then printf 'F|%s|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" "$(sha256sum "$path" | awk '{print $1}')" + elif [[ -d "$path" ]]; then printf 'D|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" + fi + done < <(find "$root" -mindepth 1 -print | sort) +} + +generation_count() { + local directory="$1" + [[ -d "$directory" ]] || { printf '0\n'; return; } + find "$directory" -mindepth 1 -maxdepth 1 -type d ! -name '.*.stage' | wc -l +} + +assert_no_motd_residue() { + local root="$1" residue + residue=$(find "$root" -name '.linux-setup-motd.*' -o -name '.*.pointer.*' -o -name '.pending.*' -o -name '.*.stage' 2>/dev/null) + [[ -z "$residue" ]] || fail "unexpected MOTD residue: $residue" + if [[ -d "$root/state/transactions" ]]; then [[ -z "$(find "$root/state/transactions" -mindepth 1 -print -quit)" ]] || fail "transaction residue remains"; fi +} + +create_complete_legacy_scope() { + local scope="$1" prefix + cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_ETC_ROOT/motd.${scope}-backup" + cp -a "$MOTD_ETC_ROOT/issue" "$MOTD_ETC_ROOT/issue.${scope}-backup" + : > "$MOTD_ETC_ROOT/issue.net.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/issue.net.${scope}-absent" + cp -a "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.${scope}-backup" + cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_ETC_ROOT/update-motd.d/10-uname.${scope}-backup" + : > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent" +} +cat > "$TEST_DIR/pointer-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1" +motd_snapshot_commit_hook() { + if [[ "$1" == previous && "$3" == pointer ]]; then kill -"$SIGNAL_NAME" "$BASHPID"; fi +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/pointer-signal-child.sh" +cat > "$TEST_DIR/kill-pending-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_transaction_phase_hook() { [[ "$1" == pending-active ]] && kill -KILL "$BASHPID"; return 0; } +motd_run_locked_operation install +CHILD +cat > "$TEST_DIR/kill-previous-pointer-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_snapshot_commit_hook() { [[ "$1" == previous && "$3" == pointer ]] && kill -KILL "$BASHPID"; return 0; } +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/kill-pending-child.sh" "$TEST_DIR/kill-previous-pointer-child.sh" +cat > "$TEST_DIR/sigkill-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +. "$1" +motd_transaction_phase_hook() { + [[ "$1" == "$KILL_PHASE" ]] && kill -KILL "$BASHPID" + return 0 +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/sigkill-child.sh" +cat > "$TEST_DIR/building-signal-child.sh" <<'CHILD' +#!/usr/bin/env bash +set -euo pipefail +trap - HUP INT TERM +. "$1" +send_building_signal() { kill -"$SIGNAL_NAME" "$BASHPID"; } +motd_transaction_phase_hook() { + case "$FAIL_PHASE:$1" in + transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) send_building_signal ;; + esac + return 0 +} +motd_snapshot_capture_hook() { + [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0 + case "$FAIL_PHASE:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) send_building_signal ;; esac + return 0 +} +motd_run_locked_operation install +CHILD +chmod 0700 "$TEST_DIR/building-signal-child.sh" +if seg_is 0; then extract_payload "$ROOT_DIR/tools/setup-motd.sh">"$TEST_DIR/t"; extract_payload "$ROOT_DIR/modules/system-customize.sh">"$TEST_DIR/m"; cmp -s "$TEST_DIR/t" "$TEST_DIR/m"||fail payload; @@ -231,13 +366,6 @@ assert_fail "state rejects $k" motd_prepare_state_layout; motd_release_lock ); done -create_legacy(){ printf legacy>"$MOTD_ETC_ROOT/motd.initial-backup"; -: >"$MOTD_ETC_ROOT/issue.initial-absent"; -printf net>"$MOTD_ETC_ROOT/issue.net.initial-backup"; -: >"$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.initial-unknown"; -printf uname>"$MOTD_ETC_ROOT/update-motd.d/10-uname.initial-backup"; -: >"$MOTD_ETC_ROOT/update-motd.d/50-motd-news.initial-absent"; -} ( r="$TEST_DIR/legacy"; create_fixture "$r"; @@ -299,34 +427,9 @@ assert_fail "rollback failure nonzero" motd_run_locked_operation install; assert_eq 6 "$(wc -l<"$r/a")" "rollback attempts six"; [[ -e "$MOTD_STATE_DIR/pending" ]]||fail journal ) -cat >"$TEST_DIR/sig.sh"<<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1"; -hit=false -send(){ [[ $hit == false ]]||return; -hit=true; -kill -"$SIGNAL_NAME" "$BASHPID"; -} -motd_snapshot_capture_hook(){ [[ $PHASE == snapshot-building&&$1 == issue ]]&&send; -return 0; -} -motd_transaction_phase_hook(){ [[ $PHASE == snapshot-commit&&$1 == snapshot-commit ]]&&send; -return 0; -} -motd_snapshot_commit_hook(){ [[ $PHASE == snapshot-publish&&$1 == initial&&$3 == pointer ]]&&send; -return 0; -} -motd_target_commit_hook(){ case "$PHASE:$1:$2" in install-first:install:motd|install-middle:install:issue_net|restore-first:restore:motd|restore-middle:restore:issue_net)send;; -esac; -return 0; -} -case $PHASE in restore-*)motd_run_locked_operation restore previous;; -*)motd_run_locked_operation install;; -esac -CHILD -chmod 700 "$TEST_DIR/sig.sh" +fi + +if seg_is 1; then for ph in snapshot-building snapshot-commit snapshot-publish install-first install-middle restore-first restore-middle; do for sig in HUP INT TERM; do( @@ -472,40 +575,6 @@ motd_run_locked_operation restore previous; assert_eq legacy "$(cat "$MOTD_ETC_ROOT/motd")" "state legacy restore" ) -state_fingerprint() { - local root="$1" path - [[ -d "$root" ]] || { printf 'absent\n'; return; } - while IFS= read -r path; do - if [[ -L "$path" ]]; then printf 'L|%s|%s\n' "${path#$root/}" "$(readlink "$path")" - elif [[ -f "$path" ]]; then printf 'F|%s|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" "$(sha256sum "$path" | awk '{print $1}')" - elif [[ -d "$path" ]]; then printf 'D|%s|%s\n' "${path#$root/}" "$(stat -c %u:%g:%a "$path")" - fi - done < <(find "$root" -mindepth 1 -print | sort) -} - -generation_count() { - local directory="$1" - [[ -d "$directory" ]] || { printf '0\n'; return; } - find "$directory" -mindepth 1 -maxdepth 1 -type d ! -name '.*.stage' | wc -l -} - -assert_no_motd_residue() { - local root="$1" residue - residue=$(find "$root" -name '.linux-setup-motd.*' -o -name '.*.pointer.*' -o -name '.pending.*' -o -name '.*.stage' 2>/dev/null) - [[ -z "$residue" ]] || fail "unexpected MOTD residue: $residue" - if [[ -d "$root/state/transactions" ]]; then [[ -z "$(find "$root/state/transactions" -mindepth 1 -print -quit)" ]] || fail "transaction residue remains"; fi -} - -create_complete_legacy_scope() { - local scope="$1" prefix - cp -a "$MOTD_ETC_ROOT/motd" "$MOTD_ETC_ROOT/motd.${scope}-backup" - cp -a "$MOTD_ETC_ROOT/issue" "$MOTD_ETC_ROOT/issue.${scope}-backup" - : > "$MOTD_ETC_ROOT/issue.net.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/issue.net.${scope}-absent" - cp -a "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome" "$MOTD_ETC_ROOT/update-motd.d/00-custom-welcome.${scope}-backup" - cp -a "$MOTD_ETC_ROOT/update-motd.d/10-uname" "$MOTD_ETC_ROOT/update-motd.d/10-uname.${scope}-backup" - : > "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent"; chmod 0600 "$MOTD_ETC_ROOT/update-motd.d/50-motd-news.${scope}-absent" -} - for legacy_phase in random-id stat open create cat chown chmod sha256 manifest mv; do ( root="$TEST_DIR/legacy-io-$legacy_phase"; create_fixture "$root"; export_paths "$root" @@ -522,19 +591,9 @@ for legacy_phase in random-id stat open create cat chown chmod sha256 manifest m ) done pass "legacy I/O failure matrix preserves data and state" +fi -cat > "$TEST_DIR/pointer-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1" -motd_snapshot_commit_hook() { - if [[ "$1" == previous && "$3" == pointer ]]; then kill -"$SIGNAL_NAME" "$BASHPID"; fi -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/pointer-signal-child.sh" - +if seg_is 2; then for signal_name in HUP INT TERM; do ( root="$TEST_DIR/pointer-signal-$signal_name"; create_fixture "$root"; export_paths "$root" @@ -596,21 +655,6 @@ for pending_kind in broken-symlink symlink directory fifo mode owner gid; do ) done -cat > "$TEST_DIR/kill-pending-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_transaction_phase_hook() { [[ "$1" == pending-active ]] && kill -KILL "$BASHPID"; return 0; } -motd_run_locked_operation install -CHILD -cat > "$TEST_DIR/kill-previous-pointer-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_snapshot_commit_hook() { [[ "$1" == previous && "$3" == pointer ]] && kill -KILL "$BASHPID"; return 0; } -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/kill-pending-child.sh" "$TEST_DIR/kill-previous-pointer-child.sh" for journal_case in pending-directory journal-transaction generation-scope generation-missing old-new-mix; do ( @@ -695,19 +739,9 @@ for native_id in uname motd_news; do [[ -L "$native_path" && "$(readlink "$native_path")" == "$root/external-$native_id" ]] || fail "$native_id previous symlink not restored" ) done +fi -cat > "$TEST_DIR/sigkill-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -. "$1" -motd_transaction_phase_hook() { - [[ "$1" == "$KILL_PHASE" ]] && kill -KILL "$BASHPID" - return 0 -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/sigkill-child.sh" - +if seg_is 3; then for kill_phase in transaction-snapshot-building transaction-final-moved target-committed-motd target-committed-issue_net target-committed-motd_news committed-cleanup committed-old-generation-cleaned committed-pending-removed; do ( root="$TEST_DIR/sigkill-$kill_phase"; create_fixture "$root"; export_paths "$root" @@ -805,8 +839,9 @@ done [[ ! -e "$MOTD_STATE_DIR/pending" ]] || fail "rolledback cleanup retry left pending" assert_eq 2 "$(generation_count "$MOTD_STATE_DIR/generations")" "rolledback cleanup retry restores bounded generations" ) +fi - +if seg_is 4; then for hook_case in content-motd mode-issue_net inode-motd_news type-issue_net; do ( root="$TEST_DIR/hook-target-$hook_case"; create_fixture "$root"; export_paths "$root" @@ -900,26 +935,6 @@ for pending_hook_case in id extra-line inode; do ) done -cat > "$TEST_DIR/building-signal-child.sh" <<'CHILD' -#!/usr/bin/env bash -set -euo pipefail -trap - HUP INT TERM -. "$1" -send_building_signal() { kill -"$SIGNAL_NAME" "$BASHPID"; } -motd_transaction_phase_hook() { - case "$FAIL_PHASE:$1" in - transaction-snapshot-building:transaction-snapshot-building|snapshot-commit:snapshot-commit|transaction-final-move:transaction-final-move) send_building_signal ;; - esac - return 0 -} -motd_snapshot_capture_hook() { - [[ "$MOTD_TRANSACTION_LIFECYCLE" == building ]] || return 0 - case "$FAIL_PHASE:$1" in capture-motd:motd|capture-issue_net:issue_net|capture-motd_news:motd_news) send_building_signal ;; esac - return 0 -} -motd_run_locked_operation install -CHILD -chmod 0700 "$TEST_DIR/building-signal-child.sh" for building_phase in transaction-snapshot-building capture-motd capture-issue_net capture-motd_news snapshot-commit transaction-final-move; do ( @@ -952,7 +967,9 @@ for building_phase in transaction-snapshot-building capture-motd capture-issue_n ) done done +fi +if seg_is 5; then for previous_impl in tool module; do ( root="$TEST_DIR/previous-only-$previous_impl"; create_fixture "$root"; export_paths "$root" @@ -1046,7 +1063,9 @@ done assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "localized stat wrapper preserves regular legacy import" LANG=zh_CN.UTF-8 bash -c '. "$1"; motd_run_locked_operation install' _ "$ROOT_DIR/modules/system-customize.sh" ) +fi +if seg_is 6; then for missing_command in bash chown flock ln od readlink sha256sum touch wc; do ( root="$TEST_DIR/module-missing-$missing_command"; create_fixture "$root"; export_paths "$root" @@ -1179,8 +1198,9 @@ done assert_eq original "$(cat "$MOTD_ETC_ROOT/motd")" "legal state root regular backup imported"; [[ -e "$MOTD_STATE_DIR/initial.current" ]] || fail "legal state root import did not publish initial pointer" assert_eq "$legacy_times" "$(stat -c %X:%Y "$MOTD_STATE_DIR/motd.initial-backup")" "legacy root metadata validation/import preserves timestamps"; pass "legal state root regular, symlink, and markers import" ) +fi - +if seg_is 7; then for rollback_impl in tool module; do for rollback_damage in motd-content issue-mode custom-inode absent-regular absent-symlink absent-fifo symlink-target; do ( @@ -1257,3 +1277,4 @@ for rollback_impl in tool module; do done printf 'All MOTD rollback final validation tests passed.\n' +fi diff --git a/tests/test-push-worker-registration.sh b/tests/test-push-worker-registration.sh index 3ef0fcf..acb194e 100755 --- a/tests/test-push-worker-registration.sh +++ b/tests/test-push-worker-registration.sh @@ -1,3 +1,12 @@ +#!/usr/bin/env bash +set -euo pipefail +# --- CI segments (tools/segmentize_ranges.py) --- +# TEST_SEGMENT unset: every segment runs in order, exactly like the original. +seg_is() { + [[ -z ${TEST_SEGMENT:-} ]] || (( $1 == TEST_SEGMENT )) +} +# --- end CI segments --- + #!/usr/bin/env bash set -euo pipefail # Safe runtime identity only: no environment dump, credentials, or xtrace. @@ -73,6 +82,139 @@ wait_test_process_start() { return 1 } +source "$SCRIPT" +trap 'rm -rf "$TEST_DIR"' EXIT +setup_fixture() { + local root="$1" + CURRENT_FIXTURE_ROOT=$root + mkdir -m 0700 "$root" "$root/runtime" "$root/capture" + TMPDIR="$root/runtime"; initialize_runtime + MAX_PARALLEL=2 + MAX_RETRIES=1 + BATCH_WORKER_FAILED=false + BATCH_WORKER_ERROR=false +} +assert_no_registration_residue() { + local root="$1" + [[ -z "$(find "$root/runtime" \( -name 'worker-registration.*' -o -name '.worker-registration.*' -o -name 'worker-session.*.state' \) -print -quit)" ]] || fail "registration/session residue remains" + (( ${#REGISTERING_WORKERS[@]} == 0 && ${#ACTIVE_WORKERS[@]} == 0 )) || fail "worker arrays remain" + (( ${#WORKER_REGISTRATION_READY_FILES[@]} == 0 && ${#WORKER_REGISTRATION_RELEASE_FILES[@]} == 0 && ${#WORKER_REGISTRATION_STARTS[@]} == 0 && ${#WORKER_REGISTRATION_STAGE_FILES[@]} == 0 )) || fail "registration arrays remain" +} +test_watchdog_process() { + local seconds="$1" marker="$2" target_pid="$3" timer="" + trap '[[ -z "$timer" ]] || kill "$timer" 2>/dev/null || true; [[ -z "$timer" ]] || wait "$timer" 2>/dev/null || true; exit 0' HUP INT TERM + sleep "$seconds" & timer=$! + wait "$timer" 2>/dev/null || exit 0 + printf fired > "$marker" + kill -KILL "$target_pid" 2>/dev/null || true +} + +run_with_watchdog() { + local name="$1" seconds="$2"; shift 2 + local marker="$TEST_DIR/watchdog-$name" pid watchdog rc=0 + "$@" & pid=$! + test_watchdog_process "$seconds" "$marker" "$pid" & watchdog=$! + wait "$pid" || rc=$? + kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true + [[ ! -e "$marker" ]] || fail "$name exceeded watchdog deadline" + return "$rc" +} +write_test_worker_session_manifest() { + local file="$1" state="$2" worker_pid="$3" worker_start="$4" leader_pid="$5" leader_start="$6" pgid="$7" sid="$8" + printf 'version=1\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\n' \ + "$state" "$worker_pid" "$worker_start" "$leader_pid" "$leader_start" "$pgid" "$sid" > "$file" + chmod 0600 "$file" +} + + +assert_retained_runtime_trusted() { + local path="$1" expected_dev="$2" expected_inode="$3" metadata="" dev inode owner gid mode trusted_gid + [[ -d "$path" && ! -L "$path" ]] || fail "retained runtime is not a regular directory: $path" + metadata=$(stat -Lc '%d:%i:%u:%g:%a' -- "$path") || fail "cannot stat retained runtime: $path" + IFS=: read -r dev inode owner gid mode <<< "$metadata" + trusted_gid=$(current_gid) + [[ "$dev" == "$expected_dev" && "$inode" == "$expected_inode" && + "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 700 ]] || + fail "retained runtime trust mismatch: $metadata" +} + +assert_retained_state_trusted() { + local file="$1" metadata="" owner gid mode trusted_gid + [[ -f "$file" && ! -L "$file" ]] || fail "retained worker state is not a regular file: $file" + metadata=$(stat -Lc '%u:%g:%a' -- "$file") || fail "cannot stat retained worker state: $file" + IFS=: read -r owner gid mode <<< "$metadata" + trusted_gid=$(current_gid) + [[ "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 600 ]] || + fail "retained worker state trust mismatch: $metadata" +} +write_active_grace_fixture() { + local root="$1" + mkdir -m 0700 "$root" "$root/bin" "$root/runtime" "$root/ssh" "$root/pids" + printf source > "$root/source" + cat > "$root/config.conf" < "$root/bin/timeout" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/timeout.pid" +[[ ${1:-} == --kill-after=* ]] && shift +shift +"$@" & +wait "$!" +EOF + cat > "$root/bin/sshpass" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/sshpass.pid" +[[ ${1:-} == -e ]] && shift +"$@" & +wait "$!" +EOF + cat > "$root/bin/rsync" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +set -m +printf '%s\n' "$$" > "$PID_DIR/rsync.pid" +ssh fake-target & +wait "$!" +EOF + cat > "$root/bin/ssh" <<'EOF' +#!/usr/bin/env bash +trap '' HUP INT TERM +printf '%s\n' "$$" > "$PID_DIR/ssh.pid" +if [[ "${STATE_PUBLISH_MODE:-active}" == cleanup ]]; then + while [[ ! -e "$STATE_CLEANUP_RELEASE" ]]; do sleep 0.01; done + exit 0 +fi +sleep 300 & +printf '%s\n' "$!" > "$PID_DIR/leaf.pid" +wait "$!" +EOF + chmod 0700 "$root/bin"/* +} + +if seg_is 0; then source_root="$TEST_DIR/source"; mkdir -m 0700 "$source_root" source_result=$(env TMPDIR="$source_root" SCRIPT="$SCRIPT" bash -c ' set -euo pipefail @@ -84,7 +226,6 @@ printf source-safe ') assert_eq source-safe "$source_result" "source keeps zero runtime side effects" # shellcheck source=../tools/push.sh -source "$SCRIPT" ( # Inert direct-child job table: no SSH or production services. sleep 10 & snapshot_child=$! @@ -98,7 +239,6 @@ source "$SCRIPT" trap - EXIT pass "job snapshot preserves direct-child matching and reap semantics" ) -trap 'rm -rf "$TEST_DIR"' EXIT # Deterministic coordinator contract: a live worker can republish after cleanup. ( @@ -121,22 +261,6 @@ trap 'rm -rf "$TEST_DIR"' EXIT pass "live publisher tracking survives successful active cleanup" ) -setup_fixture() { - local root="$1" - CURRENT_FIXTURE_ROOT=$root - mkdir -m 0700 "$root" "$root/runtime" "$root/capture" - TMPDIR="$root/runtime"; initialize_runtime - MAX_PARALLEL=2 - MAX_RETRIES=1 - BATCH_WORKER_FAILED=false - BATCH_WORKER_ERROR=false -} -assert_no_registration_residue() { - local root="$1" - [[ -z "$(find "$root/runtime" \( -name 'worker-registration.*' -o -name '.worker-registration.*' -o -name 'worker-session.*.state' \) -print -quit)" ]] || fail "registration/session residue remains" - (( ${#REGISTERING_WORKERS[@]} == 0 && ${#ACTIVE_WORKERS[@]} == 0 )) || fail "worker arrays remain" - (( ${#WORKER_REGISTRATION_READY_FILES[@]} == 0 && ${#WORKER_REGISTRATION_RELEASE_FILES[@]} == 0 && ${#WORKER_REGISTRATION_STARTS[@]} == 0 && ${#WORKER_REGISTRATION_STAGE_FILES[@]} == 0 )) || fail "registration arrays remain" -} run_delayed_parent_reap_case() ( local depth="$1" root leader leader_start leader_pgid leader_sid @@ -233,25 +357,6 @@ CHILD if [[ "${PUSH_REAP_ONLY:-false}" == true ]]; then exit 0 fi -test_watchdog_process() { - local seconds="$1" marker="$2" target_pid="$3" timer="" - trap '[[ -z "$timer" ]] || kill "$timer" 2>/dev/null || true; [[ -z "$timer" ]] || wait "$timer" 2>/dev/null || true; exit 0' HUP INT TERM - sleep "$seconds" & timer=$! - wait "$timer" 2>/dev/null || exit 0 - printf fired > "$marker" - kill -KILL "$target_pid" 2>/dev/null || true -} - -run_with_watchdog() { - local name="$1" seconds="$2"; shift 2 - local marker="$TEST_DIR/watchdog-$name" pid watchdog rc=0 - "$@" & pid=$! - test_watchdog_process "$seconds" "$marker" "$pid" & watchdog=$! - wait "$pid" || rc=$? - kill "$watchdog" 2>/dev/null || true; wait "$watchdog" 2>/dev/null || true - [[ ! -e "$marker" ]] || fail "$name exceeded watchdog deadline" - return "$rc" -} root_cause_regression_case() ( root="$TEST_DIR/root-cause"; setup_fixture "$root" @@ -585,7 +690,9 @@ pass "failed managed-session reap remains bounded for 20 deterministic runs" [[ ! -e "$state_file" && -e "$root/capture/reaped" ]] || fail "parent fallback evidence incomplete" pass "parent fallback cleans a live failed session without PPID 1 zombie" ) +fi +if seg_is 1; then ( trap - EXIT HUP INT TERM sleep 60 & replacement=$! @@ -673,35 +780,6 @@ chmod 0700 "$TEST_DIR/hard-timeout-cleanup-failure-child.sh" pass "hard timeout cleanup failure is bounded and blocks subsequent command" ) -write_test_worker_session_manifest() { - local file="$1" state="$2" worker_pid="$3" worker_start="$4" leader_pid="$5" leader_start="$6" pgid="$7" sid="$8" - printf 'version=1\nstate=%s\nworker_pid=%s\nworker_start=%s\nleader_pid=%s\nleader_start=%s\npgid=%s\nsid=%s\n' \ - "$state" "$worker_pid" "$worker_start" "$leader_pid" "$leader_start" "$pgid" "$sid" > "$file" - chmod 0600 "$file" -} - - -assert_retained_runtime_trusted() { - local path="$1" expected_dev="$2" expected_inode="$3" metadata="" dev inode owner gid mode trusted_gid - [[ -d "$path" && ! -L "$path" ]] || fail "retained runtime is not a regular directory: $path" - metadata=$(stat -Lc '%d:%i:%u:%g:%a' -- "$path") || fail "cannot stat retained runtime: $path" - IFS=: read -r dev inode owner gid mode <<< "$metadata" - trusted_gid=$(current_gid) - [[ "$dev" == "$expected_dev" && "$inode" == "$expected_inode" && - "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 700 ]] || - fail "retained runtime trust mismatch: $metadata" -} - -assert_retained_state_trusted() { - local file="$1" metadata="" owner gid mode trusted_gid - [[ -f "$file" && ! -L "$file" ]] || fail "retained worker state is not a regular file: $file" - metadata=$(stat -Lc '%u:%g:%a' -- "$file") || fail "cannot stat retained worker state: $file" - IFS=: read -r owner gid mode <<< "$metadata" - trusted_gid=$(current_gid) - [[ "$owner" == "$EUID" && "$gid" == "$trusted_gid" && "$mode" == 600 ]] || - fail "retained worker state trust mismatch: $metadata" -} - run_preserved_cleanup_retry_case() ( local run="$1" root runtime runtime_dev runtime_inode credential leader leader_start leader_pgid leader_sid state_file local owner watchdog rc=0 metadata="" state_status=0 @@ -774,7 +852,9 @@ for runtime_evidence_run in $(seq 1 "$runtime_evidence_stress"); do run_preserved_cleanup_retry_case "$runtime_evidence_run" done pass "failed cleanup evidence retry completed $runtime_evidence_stress deterministic run(s)" +fi +if seg_is 2; then ( local_cleanup_root="$TEST_DIR/active-cleanup-evidence"; setup_fixture "$local_cleanup_root" runtime=$TEMP_DIR; runtime_dev=$TEMP_DIR_DEV; runtime_inode=$TEMP_DIR_INODE @@ -892,7 +972,6 @@ done pass "clean runtime final proof preserves normal success path" ) - cat > "$TEST_DIR/runtime-retention-signal-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1199,72 +1278,6 @@ parallel_fallback_runs=${PUSH_WORKER_REGISTRATION_PARALLEL_STRESS:-1} for parallel_run in $(seq 1 "$parallel_fallback_runs"); do run_parallel_fallback_iteration "$parallel_run"; done pass "parallel fallback completed $parallel_fallback_runs deterministic run(s)" -write_active_grace_fixture() { - local root="$1" - mkdir -m 0700 "$root" "$root/bin" "$root/runtime" "$root/ssh" "$root/pids" - printf source > "$root/source" - cat > "$root/config.conf" < "$root/bin/timeout" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/timeout.pid" -[[ ${1:-} == --kill-after=* ]] && shift -shift -"$@" & -wait "$!" -EOF - cat > "$root/bin/sshpass" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/sshpass.pid" -[[ ${1:-} == -e ]] && shift -"$@" & -wait "$!" -EOF - cat > "$root/bin/rsync" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -set -m -printf '%s\n' "$$" > "$PID_DIR/rsync.pid" -ssh fake-target & -wait "$!" -EOF - cat > "$root/bin/ssh" <<'EOF' -#!/usr/bin/env bash -trap '' HUP INT TERM -printf '%s\n' "$$" > "$PID_DIR/ssh.pid" -if [[ "${STATE_PUBLISH_MODE:-active}" == cleanup ]]; then - while [[ ! -e "$STATE_CLEANUP_RELEASE" ]]; do sleep 0.01; done - exit 0 -fi -sleep 300 & -printf '%s\n' "$!" > "$PID_DIR/leaf.pid" -wait "$!" -EOF - chmod 0700 "$root/bin"/* -} cat > "$TEST_DIR/active-grace-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1323,7 +1336,9 @@ run_active_grace_signal_case() { run_active_grace_signal_case HUP 129 run_active_grace_signal_case INT 130 run_active_grace_signal_case TERM 143 +fi +if seg_is 3; then cat > "$TEST_DIR/live-cleanup-failure-child.sh" <<'CHILD' #!/usr/bin/env bash set -euo pipefail @@ -1550,4 +1565,7 @@ done assert_no_registration_residue "$root"; cleanup_runtime; assert_no_registration_residue "$root" pass "normal registration/session lifecycle cleans all state" ) + printf 'All push worker registration tests passed.\n' +fi + diff --git a/tests/test-selection-git.sh b/tests/test-selection-git.sh index aacb939..c442c16 100644 --- a/tests/test-selection-git.sh +++ b/tests/test-selection-git.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +# These fixtures re-enter run.sh and require its serial, whole-shard behaviour. +unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT TEST_SEGMENT_SET ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) TEMP_DIR=$(mktemp -d) trap 'rm -rf "$TEMP_DIR"' EXIT diff --git a/tests/test-selection.sh b/tests/test-selection.sh index 591b5cc..e220b38 100644 --- a/tests/test-selection.sh +++ b/tests/test-selection.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +# These fixtures re-enter run.sh and require its serial, whole-shard behaviour. +unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT TEST_SEGMENT_SET ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) TEMP_DIR=$(mktemp -d) trap 'rm -rf "$TEMP_DIR"' EXIT diff --git a/tests/test-shards.sh b/tests/test-shards.sh index 598fd82..c8bde8d 100644 --- a/tests/test-shards.sh +++ b/tests/test-shards.sh @@ -1,5 +1,7 @@ #!/usr/bin/env bash set -euo pipefail +# These fixtures re-enter run.sh and require its serial, whole-shard behaviour. +unset TEST_JOBS TEST_SEGMENTS TEST_SEGMENT TEST_SEGMENT_SET # Inert fixtures only: verify partitioning without running production suites. ROOT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) TEMP_DIR=$(mktemp -d) @@ -8,7 +10,7 @@ mkdir -p "$TEMP_DIR/tests" "$TEMP_DIR/bin" cp "$ROOT_DIR/tests/run.sh" "$TEMP_DIR/tests/run.sh" export CAPTURE="$TEMP_DIR/capture" GITHUB_STEP_SUMMARY="$TEMP_DIR/summary" unset TEST_BASE_SHA TEST_HEAD_SHA -for name in motd push push-worker-registration new-suite; do +for name in cloudflare-tunnel motd push push-worker-registration new-suite; do printf '#!/usr/bin/env bash\nprintf "%%s\\n" "%s" >> "$CAPTURE"\n' "test-$name.sh" > "$TEMP_DIR/tests/test-$name.sh" done check() { @@ -18,15 +20,17 @@ check() { bash "$TEMP_DIR/tests/run.sh" "$@" > "$TEMP_DIR/output" 2>&1 [[ "$(cat "$CAPTURE")" == "$expected" ]] || { cat "$TEMP_DIR/output"; exit 1; } } -all=$'test-motd.sh\ntest-new-suite.sh\ntest-push-worker-registration.sh\ntest-push.sh' +all=$'test-cloudflare-tunnel.sh\ntest-motd.sh\ntest-new-suite.sh\ntest-push-worker-registration.sh\ntest-push.sh' check "$all" check "$all" all check test-motd.sh motd -check $'test-push-worker-registration.sh\ntest-push.sh' push +check test-push-worker-registration.sh push-worker +check test-push.sh push +check test-cloudflare-tunnel.sh cloudflare check test-new-suite.sh other # Every current and future suite belongs to exactly one shard. : > "$TEMP_DIR/union" -for shard in motd push other; do +for shard in motd push-worker push cloudflare other; do : > "$CAPTURE" bash "$TEMP_DIR/tests/run.sh" "$shard" > "$TEMP_DIR/output" 2>&1 cat "$CAPTURE" >> "$TEMP_DIR/union" @@ -49,17 +53,274 @@ cat > "$TEMP_DIR/bin/git" <<'STUB' printf 'tools/push.sh\0' STUB chmod +x "$TEMP_DIR/bin/git" +ORIGINAL_PATH=$PATH export PATH="$TEMP_DIR/bin:$PATH" TEST_BASE_SHA=base TEST_HEAD_SHA=head check '' motd +check '' cloudflare check '' other -check $'test-push-worker-registration.sh\ntest-push.sh' push +check test-push.sh push +check test-push-worker-registration.sh push-worker DIFF_FAIL=1 check test-motd.sh motd DIFF_FAIL=1 check test-new-suite.sh other # Child errexit and failure propagation must survive sharding. printf '#!/usr/bin/env bash\nset -e\nfalse\nprintf unexpected >> "$CAPTURE"\n' > "$TEMP_DIR/tests/test-push-worker-registration.sh" : > "$CAPTURE" rc=0 -bash "$TEMP_DIR/tests/run.sh" push > "$TEMP_DIR/output" 2>&1 || rc=$? +bash "$TEMP_DIR/tests/run.sh" push-worker > "$TEMP_DIR/output" 2>&1 || rc=$? [[ "$rc" == 1 && ! -s "$CAPTURE" ]] grep -q 'exit=1' "$TEMP_DIR/output" +PATH=$ORIGINAL_PATH +unset TEST_BASE_SHA TEST_HEAD_SHA +# Segment coverage: one suite can be spread over several jobs, so every segment +# of a suite must be covered by exactly one job in each OS matrix, and every job +# must name real segments of the suite's source. Nothing is deduplicated before +# the checks: a repeated shard name or a repeated segment is a failure. +guards() { + awk '/^if seg_is [0-9]+; then$/ { n = $3; sub(";", "", n); print n }' "$1" +} +# run.sh is the single source of truth for which suites are segmented and how +# many segments each one has. +segmented_suites() { + awk ' + /^declare -rA SEGMENTED_SUITES=\(/ { inside = 1; next } + inside && /^\)/ { inside = 0; next } + inside { + entry = $1 + if (entry !~ /^\[test-[^]]*\.sh\]=[0-9]+$/) next + name = entry; sub(/^\[/, "", name); sub(/\]=.*/, "", name) + count = entry; sub(/^.*\]=/, "", count) + print name, count + } + ' "$ROOT_DIR/tests/run.sh" +} +file_of_suite() { + case $1 in + motd) printf 'test-motd.sh' ;; + cloudflare) printf 'test-cloudflare-tunnel.sh' ;; + push-worker) printf 'test-push-worker-registration.sh' ;; + push) printf 'test-push.sh' ;; + *) return 1 ;; + esac +} +suite_of_file() { + case $1 in + test-motd.sh) printf 'motd' ;; + test-cloudflare-tunnel.sh) printf 'cloudflare' ;; + test-push-worker-registration.sh) printf 'push-worker' ;; + test-push.sh) printf 'push' ;; + *) return 1 ;; + esac +} +fail() { printf '%s\n' "$*" >&2; exit 1; } +workflow="$ROOT_DIR/.github/workflows/shell-tests.yml" +[[ -f "$workflow" ]] +# job|shard|suite|segments|segment_set, one line per matrix entry, duplicates kept. +jobs=$(awk ' + /^ [a-z][a-z0-9-]*:$/ { + if (suite != "") print job "|" shard "|" suite "|" segments "|" set + job = $1; sub(/:$/, "", job); shard = ""; suite = ""; segments = ""; set = "" + next + } + /^[[:space:]]*- shard:/ { + if (suite != "") print job "|" shard "|" suite "|" segments "|" set + shard = $3; suite = ""; segments = ""; set = "" + next + } + /^[[:space:]]+suite:/ { suite = $2; next } + /^[[:space:]]+segments:/ { segments = $2; next } + /^[[:space:]]+segment_set:/ { + set = $0 + sub(/^[[:space:]]*segment_set:[[:space:]]*/, "", set) + gsub(/"/, "", set) + next + } + END { if (suite != "") print job "|" shard "|" suite "|" segments "|" set } +' "$workflow") +[[ -n "$jobs" ]] || fail 'no shard matrix in the workflow' + +# One OS matrix on its own: unique shard names, every segment of every suite +# covered exactly once, union 0..N-1. +check_matrix() { + local job=$1 rows suite row_suite suite_segments segment_set_value + local declared covered index count total expected actual + rows=$(printf '%s\n' "$jobs" | awk -F'|' -v job="$job" '$1 == job') + [[ -n "$rows" ]] || fail "no matrix rows for $job" + if printf '%s\n' "$rows" | cut -d'|' -f2 | sort | uniq -d | grep -q .; then + fail "$job repeats a shard name" + fi + for suite in motd cloudflare push-worker push other; do + declared="" + covered=" " + while IFS='|' read -r _job _shard row_suite suite_segments segment_set_value; do + [[ "$row_suite" == "$suite" ]] || continue + case "$suite_segments" in + ''|*[!0-9]*) fail "$job $suite has an unsupported segment count: '$suite_segments'" ;; + esac + if [[ -z "$declared" ]]; then + declared=$suite_segments + elif [[ "$declared" != "$suite_segments" ]]; then + fail "$job $suite mixes segment counts: $declared and $suite_segments" + fi + if [[ -z "$segment_set_value" ]]; then + for (( index = 0; index < suite_segments; index++ )); do + covered+=" $index" + done + else + covered+=" $segment_set_value" + fi + done <<< "$rows" + [[ -n "$declared" ]] || fail "$job has no job for $suite" + for (( index = 0; index < declared; index++ )); do + count=$(printf '%s' "$covered" | tr ' ' '\n' | grep -cx -- "$index" || true) + [[ "$count" == 1 ]] || fail "$job $suite segment $index covered $count times" + done + total=$(printf '%s' "$covered" | tr ' ' '\n' | grep -cE '^[0-9]+$' || true) + [[ "$total" == "$declared" ]] || fail "$job $suite covers $total segments, declared $declared" + expected=$(seq -s ' ' 0 "$(( declared - 1 ))") + actual=$(printf '%s' "$covered" | tr ' ' '\n' | grep -E '^[0-9]+$' | sort -n -u | tr '\n' ' ') + actual=${actual% } + [[ "$actual" == "$expected" ]] || fail "$job $suite covers '$actual', expected '$expected'" + done +} + +# The declared counts must be the ones the sources actually implement, and both +# OS matrices must agree on them. +declared_segments() { + local job=$1 suite=$2 rows first + rows=$(printf '%s\n' "$jobs" | awk -F'|' -v job="$job" -v suite="$suite" \ + '$1 == job && $3 == suite { print $4 }') + [[ -n "$rows" ]] || fail "$job has no job for $suite" + first=$(printf '%s\n' "$rows" | head -1) + [[ "$(printf '%s\n' "$rows" | grep -cx -- "$first" || true)" == \ + "$(printf '%s\n' "$rows" | grep -c . || true)" ]] || fail "$job $suite mixes segment counts" + printf '%s' "$first" +} + +check_suites() { + local suite file declared ubuntu_declared debian_declared guards_list guard_count expected name + local -A families=() + while read -r name count; do + [[ -n "$name" ]] || continue + families[$name]=$count + done <<< "$(segmented_suites)" + (( ${#families[@]} > 0 )) || fail 'run.sh declares no segmented suite' + for name in "${!families[@]}"; do + suite_of_file "$name" > /dev/null || fail "run.sh segments $name, which is not in the matrix" + done + for suite in motd cloudflare push-worker push other; do + ubuntu_declared=$(declared_segments ubuntu-tests "$suite") + debian_declared=$(declared_segments debian-tests "$suite") + [[ "$ubuntu_declared" == "$debian_declared" ]] \ + || fail "$suite has $ubuntu_declared segments on Ubuntu but $debian_declared on Debian" + declared=$ubuntu_declared + if file=$(file_of_suite "$suite"); then + guards_list=$(guards "$ROOT_DIR/tests/$file") + guard_count=$(printf '%s' "$guards_list" | grep -c . || true) + if (( declared > 1 )); then + [[ -n "${families[$file]:-}" ]] || fail "$suite runs $declared segments but run.sh does not segment $file" + [[ "${families[$file]}" == "$declared" ]] \ + || fail "run.sh cuts $file into ${families[$file]} segments, the matrix says $declared" + [[ "$guard_count" == "$declared" ]] \ + || fail "$file has $guard_count segment guards, the matrix says $declared" + expected=$(seq -s ' ' 0 "$(( declared - 1 ))") + [[ "$(printf '%s\n' "$guards_list" | tr '\n' ' ')" == "$expected " ]] \ + || fail "$file guard numbers are not 0..$(( declared - 1 ))" + else + [[ -z "${families[$file]:-}" ]] || fail "run.sh segments $file but its jobs run it whole" + [[ "$guard_count" == 0 ]] || fail "$file has segment guards but its jobs run it whole" + fi + else + # `other` is a bucket of suites, not a suite that can be cut up. + (( declared == 1 )) || fail "$suite is a bucket and cannot be cut into $declared segments" + fi + done +} + +check_matrix ubuntu-tests +check_matrix debian-tests +check_suites + +# Subset execution: every segmented family must run only the segments its job +# names, and a segment count or subset the suite cannot honour must abort before +# the first suite starts. TEST_SEGMENTS=1 still runs a suite whole in one +# process. +make_segment_fixture() { + printf '#!/usr/bin/env bash\nprintf "%%s\\n" "%s${TEST_SEGMENT:+-$TEST_SEGMENT}" >> "$CAPTURE"\n' "$1" \ + > "$TEMP_DIR/tests/$1" +} +make_segment_fixture test-motd.sh +make_segment_fixture test-cloudflare-tunnel.sh +make_segment_fixture test-push-worker-registration.sh +check_subset() { + local suite=$1 segs=$2 set_value=$3 expected=$4 rc=0 actual + : > "$CAPTURE" + TEST_SEGMENTS="$segs" TEST_SEGMENT_SET="$set_value" bash "$TEMP_DIR/tests/run.sh" "$suite" \ + > "$TEMP_DIR/output" 2>&1 || rc=$? + [[ "$rc" == 0 ]] || { cat "$TEMP_DIR/output"; fail "subset '$set_value' for $suite (TEST_SEGMENTS=$segs) exited $rc"; } + actual=$(sort "$CAPTURE" | tr '\n' ' ') + [[ "$actual" == "$(printf '%s\n' "$expected" | sort | tr '\n' ' ')" ]] || { + cat "$TEMP_DIR/output" + fail "subset '$set_value' for $suite ran: $actual" + } +} +check_subset motd 8 '0 1 2 3' 'test-motd.sh-0 +test-motd.sh-1 +test-motd.sh-2 +test-motd.sh-3' +check_subset motd 8 '' 'test-motd.sh-0 +test-motd.sh-1 +test-motd.sh-2 +test-motd.sh-3 +test-motd.sh-4 +test-motd.sh-5 +test-motd.sh-6 +test-motd.sh-7' +check_subset motd 1 '' 'test-motd.sh' +check_subset cloudflare 8 '4 5 6 7' 'test-cloudflare-tunnel.sh-4 +test-cloudflare-tunnel.sh-5 +test-cloudflare-tunnel.sh-6 +test-cloudflare-tunnel.sh-7' +check_subset cloudflare 1 '' 'test-cloudflare-tunnel.sh' +check_subset push-worker 4 '2' 'test-push-worker-registration.sh-2' +check_subset push-worker 4 '' 'test-push-worker-registration.sh-0 +test-push-worker-registration.sh-1 +test-push-worker-registration.sh-2 +test-push-worker-registration.sh-3' +check_subset push-worker 1 '' 'test-push-worker-registration.sh' + +check_reject() { + local label=$1 suite=$2 segs=$3 set_value=$4 rc=0 + : > "$CAPTURE" + TEST_SEGMENTS="$segs" TEST_SEGMENT_SET="$set_value" bash "$TEMP_DIR/tests/run.sh" "$suite" \ + > "$TEMP_DIR/output" 2>&1 || rc=$? + [[ "$rc" == 2 ]] || { cat "$TEMP_DIR/output"; fail "$label: exit $rc, expected 2"; } + [[ ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "$label: started a suite"; } +} +# A segment count the suite's source does not have would silently run part of it. +check_reject 'motd cut into two' motd 2 '' +check_reject 'motd cut into four' motd 4 '' +check_reject 'cloudflare cut into four' cloudflare 4 '' +check_reject 'push-worker cut into eight' push-worker 8 '' +check_reject 'push-worker cut into two' push-worker 2 '' +check_reject 'motd segment past its source' motd 8 '8' +check_reject 'subset for an unsegmented suite' push 2 '0' +check_reject 'subset for the other bucket' other 2 '0' +check_reject 'subset without segmented mode' motd 1 '0' +check_reject 'mixed plan with one count' all 8 '' +check_reject 'mixed plan with four segments' all 4 '0 1 2 3' +# Anything but a distinct single-digit in-range index is refused; a file named 1 +# must not turn "[1]" into a real segment. +: > "$TEMP_DIR/1" +for bad in '1 01' '9223372036854775808' '18446744073709551616' '[1]' '1,2' '-1' '1 1' '+1' '0x1' 'x'; do + : > "$CAPTURE" + rc=0 + ( cd "$TEMP_DIR" && TEST_SEGMENTS=8 TEST_SEGMENT_SET="$bad" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 ) || rc=$? + [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "subset '$bad' accepted (exit $rc)"; } +done +for bad in 0 9 08 '4x' '1 4' 9223372036854775808 18446744073709551616; do + : > "$CAPTURE" + rc=0 + TEST_SEGMENTS="$bad" bash "$TEMP_DIR/tests/run.sh" motd > "$TEMP_DIR/output" 2>&1 || rc=$? + [[ "$rc" == 2 && ! -s "$CAPTURE" ]] || { cat "$TEMP_DIR/output"; fail "TEST_SEGMENTS=$bad accepted (exit $rc)"; } +done printf 'All shard fixture tests passed.\n'