From 578a1ad98158cb177801e694a97549d62e97d4fa Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 20 Aug 2026 10:41:35 +0200 Subject: [PATCH 01/17] add debian release tests in CI --- .github/workflows/test.yml | 3 + iso/debian_release.conf | 3 + iso/preseeds/preseed.cfg | 2 +- iso/preseeds/preseed_unattended.cfg | 2 +- iso/scripts/check_debian_release_alignment.sh | 58 +++++++++++++++++++ iso/scripts/generate_dappnode_iso_debian.sh | 11 ++++ 6 files changed, 77 insertions(+), 2 deletions(-) create mode 100644 iso/debian_release.conf create mode 100755 iso/scripts/check_debian_release_alignment.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index deb5465a..622f65d3 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -39,6 +39,9 @@ jobs: - name: Checkout uses: actions/checkout@v6 + - name: Verify Debian release alignment + run: bash iso/scripts/check_debian_release_alignment.sh + # Create Debian ISO - name: create Debian ISO run: | diff --git a/iso/debian_release.conf b/iso/debian_release.conf new file mode 100644 index 00000000..54e6b562 --- /dev/null +++ b/iso/debian_release.conf @@ -0,0 +1,3 @@ +# Debian release used by the ISO and its package repositories. +DEBIAN_SUITE="trixie" +DEBIAN_MAJOR_VERSION="13" diff --git a/iso/preseeds/preseed.cfg b/iso/preseeds/preseed.cfg index 1bd38005..fd55caea 100644 --- a/iso/preseeds/preseed.cfg +++ b/iso/preseeds/preseed.cfg @@ -25,7 +25,7 @@ d-i mirror/country string manual d-i mirror/http/hostname string deb.debian.org d-i mirror/http/directory string /debian d-i mirror/http/proxy string -d-i apt-setup/local0/repository string http://deb.debian.org/debian/ trixie main contrib non-free-firmware +d-i apt-setup/local0/repository string http://deb.debian.org/debian/ @DEBIAN_SUITE@ main contrib non-free-firmware d-i apt-setup/cdrom/set-first boolean false d-i apt-setup/cdrom/set-next boolean false d-i apt-setup/cdrom/set-failed boolean false diff --git a/iso/preseeds/preseed_unattended.cfg b/iso/preseeds/preseed_unattended.cfg index 8316fd7c..0521173e 100644 --- a/iso/preseeds/preseed_unattended.cfg +++ b/iso/preseeds/preseed_unattended.cfg @@ -62,7 +62,7 @@ d-i mirror/country string manual d-i mirror/http/hostname string deb.debian.org d-i mirror/http/directory string /debian d-i mirror/http/proxy string -d-i apt-setup/local0/repository string http://deb.debian.org/debian/ trixie main contrib non-free-firmware +d-i apt-setup/local0/repository string http://deb.debian.org/debian/ @DEBIAN_SUITE@ main contrib non-free-firmware d-i apt-setup/cdrom/set-first boolean false d-i apt-setup/cdrom/set-next boolean false d-i apt-setup/cdrom/set-failed boolean false diff --git a/iso/scripts/check_debian_release_alignment.sh b/iso/scripts/check_debian_release_alignment.sh new file mode 100755 index 00000000..2f237584 --- /dev/null +++ b/iso/scripts/check_debian_release_alignment.sh @@ -0,0 +1,58 @@ +#!/bin/bash +set -euo pipefail + +SCRIPTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +ISO_DIR=$(cd "${SCRIPTS_DIR}/.." && pwd) + +source "${ISO_DIR}/debian_release.conf" + +case "${DEBIAN_SUITE}" in + bullseye) suite_major_version="11" ;; + bookworm) suite_major_version="12" ;; + trixie) suite_major_version="13" ;; + forky) suite_major_version="14" ;; + *) + echo "[ERROR] Unknown Debian suite: ${DEBIAN_SUITE}" + exit 1 + ;; +esac + +if [ "${suite_major_version}" != "${DEBIAN_MAJOR_VERSION}" ]; then + echo "[ERROR] Debian suite ${DEBIAN_SUITE} is Debian ${suite_major_version}, not Debian ${DEBIAN_MAJOR_VERSION}" + exit 1 +fi + +generator="${SCRIPTS_DIR}/generate_dappnode_iso_debian.sh" +base_iso_name=$(sed -n 's/^BASE_ISO_NAME="\([^"]*\)"$/\1/p' "${generator}") + +if [[ ! "${base_iso_name}" =~ ^debian-([0-9]+)\. ]]; then + echo "[ERROR] Could not determine the Debian major version from BASE_ISO_NAME=${base_iso_name}" + exit 1 +fi + +iso_major_version="${BASH_REMATCH[1]}" +if [ "${iso_major_version}" != "${DEBIAN_MAJOR_VERSION}" ]; then + echo "[ERROR] Base ISO ${base_iso_name} is Debian ${iso_major_version}, but debian_release.conf expects Debian ${DEBIAN_MAJOR_VERSION} (${DEBIAN_SUITE})" + exit 1 +fi + +expected_repository="d-i apt-setup/local0/repository string http://deb.debian.org/debian/ @DEBIAN_SUITE@ main contrib non-free-firmware" +preseed_files=( + "${ISO_DIR}/preseeds/preseed.cfg" + "${ISO_DIR}/preseeds/preseed_unattended.cfg" +) + +for preseed_file in "${preseed_files[@]}"; do + repository_count=$(grep -Ec '^d-i apt-setup/local[0-9]+/repository string ' "${preseed_file}" || true) + if [ "${repository_count}" -ne 1 ]; then + echo "[ERROR] Expected exactly one additional Debian repository in ${preseed_file}, found ${repository_count}" + exit 1 + fi + + if ! grep -Fqx "${expected_repository}" "${preseed_file}"; then + echo "[ERROR] Debian repository in ${preseed_file} must use the @DEBIAN_SUITE@ placeholder" + exit 1 + fi +done + +echo "[INFO] Debian release alignment is valid: Debian ${DEBIAN_MAJOR_VERSION} (${DEBIAN_SUITE}), ${base_iso_name}" diff --git a/iso/scripts/generate_dappnode_iso_debian.sh b/iso/scripts/generate_dappnode_iso_debian.sh index 4ec4a93d..349aca07 100755 --- a/iso/scripts/generate_dappnode_iso_debian.sh +++ b/iso/scripts/generate_dappnode_iso_debian.sh @@ -4,6 +4,7 @@ set -e SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") source ${SCRIPTS_DIR}/common_iso_generation.sh +source ${SCRIPTS_DIR}/../debian_release.conf BASE_ISO_NAME="debian-13.5.0-amd64-netinst.iso" BASE_ISO_VERSION="${BASE_ISO_NAME#debian-}" @@ -15,6 +16,9 @@ BASE_ISO_SHASUM="95838884f5ea6c82421dfe6baaa5a639dbbe6756c1e380f9fe7a7cb0c1949d2 DAPPNODE_ISO_NAME="${DAPPNODE_ISO_PREFIX}${BASE_ISO_NAME}" DAPPNODE_ISO_PATH="/images/${DAPPNODE_ISO_NAME}" +# Fail before downloading or building if the ISO and repositories target different releases. +bash "${SCRIPTS_DIR}/check_debian_release_alignment.sh" + customize_debian_preseed() { local iso_build_path=$1 local workdir=$2 @@ -50,6 +54,13 @@ customize_debian_preseed() { exit 1 } + # Keep package repositories aligned with the Debian release used by the ISO. + sed -i "s/@DEBIAN_SUITE@/${DEBIAN_SUITE}/g" "${tmp_initrd}/preseed.cfg" + if grep -q "@DEBIAN_SUITE@" "${tmp_initrd}/preseed.cfg"; then + echo "[ERROR] Could not render the Debian suite in the preseed file" + exit 1 + fi + # Recreate (and recompress) the initrd (cd "${tmp_initrd}" && find . -print0 | cpio -0 -ov -H newc | gzip >"${install_dir}/initrd.gz") || { echo "[ERROR] Could not create new initrd" From 0bbc0eb332be31d472cbcb1f37b4f6f2045cc30e Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 20 Aug 2026 10:58:24 +0200 Subject: [PATCH 02/17] debian e2e test with QEMU --- .github/workflows/debian-iso-e2e.yml | 55 +++++++ .github/workflows/test.yml | 18 +- .gitignore | 3 +- test/e2e_debian_iso.sh | 235 +++++++++++++++++++++++++++ 4 files changed, 293 insertions(+), 18 deletions(-) create mode 100644 .github/workflows/debian-iso-e2e.yml create mode 100755 test/e2e_debian_iso.sh diff --git a/.github/workflows/debian-iso-e2e.yml b/.github/workflows/debian-iso-e2e.yml new file mode 100644 index 00000000..d234ee08 --- /dev/null +++ b/.github/workflows/debian-iso-e2e.yml @@ -0,0 +1,55 @@ +name: Debian ISO end-to-end + +on: + workflow_dispatch: + push: + branches-ignore: [master] + pull_request: + branches: [master] + +concurrency: + group: debian-iso-e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + install: + name: Build and install Debian ISO + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Verify Debian release alignment + run: bash iso/scripts/check_debian_release_alignment.sh + + - name: Build Debian unattended ISO + run: | + export BASE_OS=debian + export UNATTENDED=true + docker compose up --build + ls -lrt images/Dappnode-debian-*.iso + + - name: Install QEMU test dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils xorriso sshpass + if [ -e /dev/kvm ]; then + sudo chmod a+rw /dev/kvm + fi + + - name: Install Debian ISO end to end + run: | + debian_iso=$(find images -maxdepth 1 -type f -name 'Dappnode-debian-*-amd64-netinst.iso' -print -quit) + test -n "${debian_iso}" + bash test/e2e_debian_iso.sh "${debian_iso}" + + - name: Upload installation logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: debian-installation-e2e-logs + path: test-output/debian-e2e/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 622f65d3..9f99a6a6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -32,29 +32,13 @@ jobs: sudo /bin/bash ./scripts/dappnode_uninstall.sh y iso: - name: test Debian and Ubuntu ISO + name: test Ubuntu ISO runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v6 - - name: Verify Debian release alignment - run: bash iso/scripts/check_debian_release_alignment.sh - - # Create Debian ISO - - name: create Debian ISO - run: | - export BASE_OS=debian - export UNATTENDED=true - docker compose up --build - ls images/ - - # Verify Debian ISO creation - - name: verify Debian ISO - run: | - ls -lrt images/Dappnode-debian-*.iso - # Create Ubuntu ISO - name: create Ubuntu ISO run: | diff --git a/.gitignore b/.gitignore index ed3c1e18..15ec4cc3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ docs/_build -images \ No newline at end of file +images +test-output/ diff --git a/test/e2e_debian_iso.sh b/test/e2e_debian_iso.sh new file mode 100755 index 00000000..6165dbaf --- /dev/null +++ b/test/e2e_debian_iso.sh @@ -0,0 +1,235 @@ +#!/bin/bash +set -Eeuo pipefail + +usage() { + echo "Usage: $0 " +} + +if [ "$#" -ne 1 ]; then + usage + exit 2 +fi + +ISO_PATH=$(realpath "$1") +if [ ! -f "${ISO_PATH}" ]; then + echo "[ERROR] ISO not found: ${ISO_PATH}" + exit 1 +fi + +required_commands=(qemu-img qemu-system-x86_64 xorriso ssh sshpass) +for command_name in "${required_commands[@]}"; do + if ! command -v "${command_name}" >/dev/null 2>&1; then + echo "[ERROR] Missing required command: ${command_name}" + exit 1 + fi +done + +OUTPUT_DIR=${E2E_OUTPUT_DIR:-"${PWD}/test-output/debian-e2e"} +mkdir -p "${OUTPUT_DIR}" +OUTPUT_DIR=$(realpath "${OUTPUT_DIR}") + +TEMP_ROOT=${RUNNER_TEMP:-/tmp} +VM_DIR=$(mktemp -d "${TEMP_ROOT%/}/dappnode-debian-e2e.XXXXXX") +DISK_PATH="${VM_DIR}/debian.qcow2" +KERNEL_PATH="${VM_DIR}/vmlinuz" +INITRD_PATH="${VM_DIR}/initrd.gz" +INSTALLER_SERIAL_LOG="${OUTPUT_DIR}/installer-serial.log" +INSTALLER_PROCESS_LOG="${OUTPUT_DIR}/installer-qemu.log" +SYSTEM_SERIAL_LOG="${OUTPUT_DIR}/installed-system-serial.log" +SYSTEM_PROCESS_LOG="${OUTPUT_DIR}/installed-system-qemu.log" +INSTALLED_SYSTEM_REPORT="${OUTPUT_DIR}/installed-system-report.log" + +VM_DISK_SIZE=${E2E_VM_DISK_SIZE:-32G} +VM_MEMORY_MB=${E2E_VM_MEMORY_MB:-4096} +VM_CPUS=${E2E_VM_CPUS:-2} +INSTALL_TIMEOUT_SECONDS=${E2E_INSTALL_TIMEOUT_SECONDS:-3600} +BOOT_TIMEOUT_SECONDS=${E2E_BOOT_TIMEOUT_SECONDS:-600} +SSH_PORT=${E2E_SSH_PORT:-2222} +SSH_PASSWORD=${E2E_SSH_PASSWORD:-dappnode.s0} +qemu_pid="" + +cleanup() { + exit_code=$? + trap - EXIT INT TERM + + if [ -n "${qemu_pid}" ] && kill -0 "${qemu_pid}" 2>/dev/null; then + kill "${qemu_pid}" 2>/dev/null || true + wait "${qemu_pid}" 2>/dev/null || true + fi + + if [ "${KEEP_E2E_VM:-false}" = "true" ]; then + echo "[INFO] Keeping VM files in ${VM_DIR}" + else + case "${VM_DIR}" in + "${TEMP_ROOT%/}"/dappnode-debian-e2e.*) rm -rf -- "${VM_DIR}" ;; + *) echo "[WARN] Refusing to remove unexpected VM directory: ${VM_DIR}" ;; + esac + fi + + exit "${exit_code}" +} +trap cleanup EXIT INT TERM + +show_failure_logs() { + echo "[INFO] Last installer serial output:" + tail -n 200 "${INSTALLER_SERIAL_LOG}" 2>/dev/null || true + echo "[INFO] Last installed-system serial output:" + tail -n 200 "${SYSTEM_SERIAL_LOG}" 2>/dev/null || true + echo "[INFO] QEMU process output:" + tail -n 100 "${INSTALLER_PROCESS_LOG}" 2>/dev/null || true + tail -n 100 "${SYSTEM_PROCESS_LOG}" 2>/dev/null || true +} + +wait_for_process_exit() { + process_id=$1 + timeout_seconds=$2 + description=$3 + deadline=$((SECONDS + timeout_seconds)) + + while kill -0 "${process_id}" 2>/dev/null; do + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "[ERROR] Timed out waiting for ${description}" + return 1 + fi + sleep 10 + done + + wait "${process_id}" +} + +qemu_acceleration=(-accel "tcg,thread=multi") +if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + qemu_acceleration=(-accel kvm -cpu host) + echo "[INFO] Using KVM acceleration" +else + echo "[WARN] /dev/kvm is unavailable; using slower TCG emulation" +fi + +echo "[INFO] Extracting the Debian installer kernel and initrd from ${ISO_PATH}" +xorriso -osirrox on -indev "${ISO_PATH}" -extract /install.amd/vmlinuz "${KERNEL_PATH}" >/dev/null 2>&1 +xorriso -osirrox on -indev "${ISO_PATH}" -extract /install.amd/initrd.gz "${INITRD_PATH}" >/dev/null 2>&1 + +echo "[INFO] Creating ${VM_DISK_SIZE} virtual installation disk" +qemu-img create -q -f qcow2 "${DISK_PATH}" "${VM_DISK_SIZE}" + +echo "[INFO] Booting the unattended installer" +qemu-system-x86_64 \ + "${qemu_acceleration[@]}" \ + -m "${VM_MEMORY_MB}" \ + -smp "${VM_CPUS}" \ + -drive "file=${DISK_PATH},format=qcow2,if=virtio" \ + -cdrom "${ISO_PATH}" \ + -kernel "${KERNEL_PATH}" \ + -initrd "${INITRD_PATH}" \ + -append "auto=true priority=critical console=ttyS0,115200n8 --- quiet" \ + -nic user,model=virtio-net-pci \ + -display none \ + -monitor none \ + -serial "file:${INSTALLER_SERIAL_LOG}" \ + -no-reboot \ + >"${INSTALLER_PROCESS_LOG}" 2>&1 & +qemu_pid=$! + +if ! wait_for_process_exit "${qemu_pid}" "${INSTALL_TIMEOUT_SECONDS}" "the Debian installer to complete"; then + show_failure_logs + exit 1 +fi +qemu_pid="" + +echo "[INFO] Installer completed; booting the installed virtual disk" +qemu-system-x86_64 \ + "${qemu_acceleration[@]}" \ + -m "${VM_MEMORY_MB}" \ + -smp "${VM_CPUS}" \ + -drive "file=${DISK_PATH},format=qcow2,if=virtio" \ + -boot order=c \ + -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${SSH_PORT}-:22" \ + -display none \ + -monitor none \ + -serial "file:${SYSTEM_SERIAL_LOG}" \ + >"${SYSTEM_PROCESS_LOG}" 2>&1 & +qemu_pid=$! + +ssh_options=( + -p "${SSH_PORT}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o ConnectTimeout=5 + -o ConnectionAttempts=1 + -o LogLevel=ERROR +) + +ssh_guest() { + SSHPASS="${SSH_PASSWORD}" sshpass -e ssh "${ssh_options[@]}" dappnode@127.0.0.1 "$@" +} + +echo "[INFO] Waiting for SSH on the installed system" +boot_deadline=$((SECONDS + BOOT_TIMEOUT_SECONDS)) +while ! ssh_guest true >/dev/null 2>&1; do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + echo "[ERROR] Installed-system VM exited before SSH became available" + show_failure_logs + exit 1 + fi + if [ "${SECONDS}" -ge "${boot_deadline}" ]; then + echo "[ERROR] Timed out waiting for SSH on the installed system" + show_failure_logs + exit 1 + fi + sleep 10 +done + +echo "[INFO] Validating the installed Debian and DAppNode system" +ssh_guest bash -s <<'REMOTE_CHECKS' | tee "${INSTALLED_SYSTEM_REPORT}" +set -euo pipefail + +source /etc/os-release +if [ "${ID}" != "debian" ] || [ "${VERSION_ID}" != "13" ] || [ "${VERSION_CODENAME}" != "trixie" ]; then + echo "[ERROR] Expected Debian 13 (trixie), found ${PRETTY_NAME}" + exit 1 +fi + +if grep -R -H -E --include='*.list' --include='*.sources' \ + '(^|[[:space:]])bookworm([[:space:]]|$)' /etc/apt 2>/dev/null; then + echo "[ERROR] Found a Bookworm APT source on the installed system" + exit 1 +fi + +dpkg_audit=$(dpkg --audit) +if [ -n "${dpkg_audit}" ]; then + echo "[ERROR] dpkg reports incomplete or broken packages:" + echo "${dpkg_audit}" + exit 1 +fi + +package_version() { + dpkg-query -W -f='${Version}' "$1" +} + +tasksel_version=$(package_version tasksel) +task_english_version=$(package_version task-english) +debconf_version=$(package_version debconf) +python_debconf_version=$(package_version python3-debconf) + +if [ "${tasksel_version}" != "${task_english_version}" ]; then + echo "[ERROR] tasksel (${tasksel_version}) and task-english (${task_english_version}) do not match" + exit 1 +fi +if [ "${debconf_version}" != "${python_debconf_version}" ]; then + echo "[ERROR] debconf (${debconf_version}) and python3-debconf (${python_debconf_version}) do not match" + exit 1 +fi + +test -x /usr/src/dappnode/scripts/dappnode_install.sh +test -s /usr/src/dappnode/logs/iso_install.log +docker --version +docker compose version +docker info >/dev/null + +echo "Installed OS: ${PRETTY_NAME}" +echo "tasksel family: ${tasksel_version}" +echo "debconf family: ${debconf_version}" +echo "[INFO] End-to-end Debian ISO installation checks passed" +REMOTE_CHECKS + +echo "[INFO] Complete Debian ISO installation succeeded" From bd6ca2f3c89ead9972adcb1aa06ff6b1bdd43b02 Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 20 Aug 2026 11:10:18 +0200 Subject: [PATCH 03/17] organize tests --- .github/workflows/test.yml | 21 -- .github/workflows/ubuntu-iso-e2e.yml | 52 +++++ test/e2e_debian_iso.sh | 243 ++--------------------- test/e2e_iso_install.sh | 282 +++++++++++++++++++++++++++ test/e2e_ubuntu_iso.sh | 25 +++ 5 files changed, 373 insertions(+), 250 deletions(-) create mode 100644 .github/workflows/ubuntu-iso-e2e.yml create mode 100755 test/e2e_iso_install.sh create mode 100755 test/e2e_ubuntu_iso.sh diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 9f99a6a6..45a09ec9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,24 +30,3 @@ jobs: - name: Uninstall DAppNode run: | sudo /bin/bash ./scripts/dappnode_uninstall.sh y - - iso: - name: test Ubuntu ISO - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v6 - - # Create Ubuntu ISO - - name: create Ubuntu ISO - run: | - export BASE_OS=ubuntu - export UNATTENDED=true - docker compose up --build - ls images/ - - # Verify Ubuntu ISO creation - - name: verify Ubuntu ISO - run: | - ls -lrt images/Dappnode-ubuntu-*.iso diff --git a/.github/workflows/ubuntu-iso-e2e.yml b/.github/workflows/ubuntu-iso-e2e.yml new file mode 100644 index 00000000..a78bd604 --- /dev/null +++ b/.github/workflows/ubuntu-iso-e2e.yml @@ -0,0 +1,52 @@ +name: Ubuntu ISO end-to-end + +on: + workflow_dispatch: + push: + branches-ignore: [master] + pull_request: + branches: [master] + +concurrency: + group: ubuntu-iso-e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + install: + name: Build and install Ubuntu ISO + runs-on: ubuntu-latest + timeout-minutes: 120 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Build Ubuntu unattended ISO + run: | + export BASE_OS=ubuntu + export UNATTENDED=true + docker compose up --build + ls -lrt images/Dappnode-ubuntu-*.iso + + - name: Install QEMU test dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils xorriso sshpass + if [ -e /dev/kvm ]; then + sudo chmod a+rw /dev/kvm + fi + + - name: Install Ubuntu ISO end to end + run: | + ubuntu_iso=$(find images -maxdepth 1 -type f -name 'Dappnode-ubuntu-*-live-server-amd64.iso' -print -quit) + test -n "${ubuntu_iso}" + bash test/e2e_ubuntu_iso.sh "${ubuntu_iso}" + + - name: Upload installation logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ubuntu-installation-e2e-logs + path: test-output/ubuntu-e2e/ + if-no-files-found: ignore + retention-days: 7 diff --git a/test/e2e_debian_iso.sh b/test/e2e_debian_iso.sh index 6165dbaf..d7cb28b4 100755 --- a/test/e2e_debian_iso.sh +++ b/test/e2e_debian_iso.sh @@ -1,235 +1,20 @@ #!/bin/bash +# Configures the shared ISO installation harness for Debian's installer layout. +# The Debian end-to-end workflow calls this wrapper with its unattended ISO. set -Eeuo pipefail -usage() { - echo "Usage: $0 " -} +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) -if [ "$#" -ne 1 ]; then - usage - exit 2 -fi +source "${REPO_ROOT}/iso/debian_release.conf" -ISO_PATH=$(realpath "$1") -if [ ! -f "${ISO_PATH}" ]; then - echo "[ERROR] ISO not found: ${ISO_PATH}" - exit 1 -fi +E2E_DISTRO="debian" +E2E_EXPECTED_ID="debian" +E2E_EXPECTED_VERSION="${DEBIAN_MAJOR_VERSION}" +E2E_EXPECTED_CODENAME="${DEBIAN_SUITE}" +E2E_KERNEL_ISO_PATH="/install.amd/vmlinuz" +E2E_INITRD_ISO_PATH="/install.amd/initrd.gz" +E2E_KERNEL_ARGS="auto=true priority=critical console=ttyS0,115200n8 --- quiet" -required_commands=(qemu-img qemu-system-x86_64 xorriso ssh sshpass) -for command_name in "${required_commands[@]}"; do - if ! command -v "${command_name}" >/dev/null 2>&1; then - echo "[ERROR] Missing required command: ${command_name}" - exit 1 - fi -done - -OUTPUT_DIR=${E2E_OUTPUT_DIR:-"${PWD}/test-output/debian-e2e"} -mkdir -p "${OUTPUT_DIR}" -OUTPUT_DIR=$(realpath "${OUTPUT_DIR}") - -TEMP_ROOT=${RUNNER_TEMP:-/tmp} -VM_DIR=$(mktemp -d "${TEMP_ROOT%/}/dappnode-debian-e2e.XXXXXX") -DISK_PATH="${VM_DIR}/debian.qcow2" -KERNEL_PATH="${VM_DIR}/vmlinuz" -INITRD_PATH="${VM_DIR}/initrd.gz" -INSTALLER_SERIAL_LOG="${OUTPUT_DIR}/installer-serial.log" -INSTALLER_PROCESS_LOG="${OUTPUT_DIR}/installer-qemu.log" -SYSTEM_SERIAL_LOG="${OUTPUT_DIR}/installed-system-serial.log" -SYSTEM_PROCESS_LOG="${OUTPUT_DIR}/installed-system-qemu.log" -INSTALLED_SYSTEM_REPORT="${OUTPUT_DIR}/installed-system-report.log" - -VM_DISK_SIZE=${E2E_VM_DISK_SIZE:-32G} -VM_MEMORY_MB=${E2E_VM_MEMORY_MB:-4096} -VM_CPUS=${E2E_VM_CPUS:-2} -INSTALL_TIMEOUT_SECONDS=${E2E_INSTALL_TIMEOUT_SECONDS:-3600} -BOOT_TIMEOUT_SECONDS=${E2E_BOOT_TIMEOUT_SECONDS:-600} -SSH_PORT=${E2E_SSH_PORT:-2222} -SSH_PASSWORD=${E2E_SSH_PASSWORD:-dappnode.s0} -qemu_pid="" - -cleanup() { - exit_code=$? - trap - EXIT INT TERM - - if [ -n "${qemu_pid}" ] && kill -0 "${qemu_pid}" 2>/dev/null; then - kill "${qemu_pid}" 2>/dev/null || true - wait "${qemu_pid}" 2>/dev/null || true - fi - - if [ "${KEEP_E2E_VM:-false}" = "true" ]; then - echo "[INFO] Keeping VM files in ${VM_DIR}" - else - case "${VM_DIR}" in - "${TEMP_ROOT%/}"/dappnode-debian-e2e.*) rm -rf -- "${VM_DIR}" ;; - *) echo "[WARN] Refusing to remove unexpected VM directory: ${VM_DIR}" ;; - esac - fi - - exit "${exit_code}" -} -trap cleanup EXIT INT TERM - -show_failure_logs() { - echo "[INFO] Last installer serial output:" - tail -n 200 "${INSTALLER_SERIAL_LOG}" 2>/dev/null || true - echo "[INFO] Last installed-system serial output:" - tail -n 200 "${SYSTEM_SERIAL_LOG}" 2>/dev/null || true - echo "[INFO] QEMU process output:" - tail -n 100 "${INSTALLER_PROCESS_LOG}" 2>/dev/null || true - tail -n 100 "${SYSTEM_PROCESS_LOG}" 2>/dev/null || true -} - -wait_for_process_exit() { - process_id=$1 - timeout_seconds=$2 - description=$3 - deadline=$((SECONDS + timeout_seconds)) - - while kill -0 "${process_id}" 2>/dev/null; do - if [ "${SECONDS}" -ge "${deadline}" ]; then - echo "[ERROR] Timed out waiting for ${description}" - return 1 - fi - sleep 10 - done - - wait "${process_id}" -} - -qemu_acceleration=(-accel "tcg,thread=multi") -if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then - qemu_acceleration=(-accel kvm -cpu host) - echo "[INFO] Using KVM acceleration" -else - echo "[WARN] /dev/kvm is unavailable; using slower TCG emulation" -fi - -echo "[INFO] Extracting the Debian installer kernel and initrd from ${ISO_PATH}" -xorriso -osirrox on -indev "${ISO_PATH}" -extract /install.amd/vmlinuz "${KERNEL_PATH}" >/dev/null 2>&1 -xorriso -osirrox on -indev "${ISO_PATH}" -extract /install.amd/initrd.gz "${INITRD_PATH}" >/dev/null 2>&1 - -echo "[INFO] Creating ${VM_DISK_SIZE} virtual installation disk" -qemu-img create -q -f qcow2 "${DISK_PATH}" "${VM_DISK_SIZE}" - -echo "[INFO] Booting the unattended installer" -qemu-system-x86_64 \ - "${qemu_acceleration[@]}" \ - -m "${VM_MEMORY_MB}" \ - -smp "${VM_CPUS}" \ - -drive "file=${DISK_PATH},format=qcow2,if=virtio" \ - -cdrom "${ISO_PATH}" \ - -kernel "${KERNEL_PATH}" \ - -initrd "${INITRD_PATH}" \ - -append "auto=true priority=critical console=ttyS0,115200n8 --- quiet" \ - -nic user,model=virtio-net-pci \ - -display none \ - -monitor none \ - -serial "file:${INSTALLER_SERIAL_LOG}" \ - -no-reboot \ - >"${INSTALLER_PROCESS_LOG}" 2>&1 & -qemu_pid=$! - -if ! wait_for_process_exit "${qemu_pid}" "${INSTALL_TIMEOUT_SECONDS}" "the Debian installer to complete"; then - show_failure_logs - exit 1 -fi -qemu_pid="" - -echo "[INFO] Installer completed; booting the installed virtual disk" -qemu-system-x86_64 \ - "${qemu_acceleration[@]}" \ - -m "${VM_MEMORY_MB}" \ - -smp "${VM_CPUS}" \ - -drive "file=${DISK_PATH},format=qcow2,if=virtio" \ - -boot order=c \ - -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${SSH_PORT}-:22" \ - -display none \ - -monitor none \ - -serial "file:${SYSTEM_SERIAL_LOG}" \ - >"${SYSTEM_PROCESS_LOG}" 2>&1 & -qemu_pid=$! - -ssh_options=( - -p "${SSH_PORT}" - -o StrictHostKeyChecking=no - -o UserKnownHostsFile=/dev/null - -o ConnectTimeout=5 - -o ConnectionAttempts=1 - -o LogLevel=ERROR -) - -ssh_guest() { - SSHPASS="${SSH_PASSWORD}" sshpass -e ssh "${ssh_options[@]}" dappnode@127.0.0.1 "$@" -} - -echo "[INFO] Waiting for SSH on the installed system" -boot_deadline=$((SECONDS + BOOT_TIMEOUT_SECONDS)) -while ! ssh_guest true >/dev/null 2>&1; do - if ! kill -0 "${qemu_pid}" 2>/dev/null; then - echo "[ERROR] Installed-system VM exited before SSH became available" - show_failure_logs - exit 1 - fi - if [ "${SECONDS}" -ge "${boot_deadline}" ]; then - echo "[ERROR] Timed out waiting for SSH on the installed system" - show_failure_logs - exit 1 - fi - sleep 10 -done - -echo "[INFO] Validating the installed Debian and DAppNode system" -ssh_guest bash -s <<'REMOTE_CHECKS' | tee "${INSTALLED_SYSTEM_REPORT}" -set -euo pipefail - -source /etc/os-release -if [ "${ID}" != "debian" ] || [ "${VERSION_ID}" != "13" ] || [ "${VERSION_CODENAME}" != "trixie" ]; then - echo "[ERROR] Expected Debian 13 (trixie), found ${PRETTY_NAME}" - exit 1 -fi - -if grep -R -H -E --include='*.list' --include='*.sources' \ - '(^|[[:space:]])bookworm([[:space:]]|$)' /etc/apt 2>/dev/null; then - echo "[ERROR] Found a Bookworm APT source on the installed system" - exit 1 -fi - -dpkg_audit=$(dpkg --audit) -if [ -n "${dpkg_audit}" ]; then - echo "[ERROR] dpkg reports incomplete or broken packages:" - echo "${dpkg_audit}" - exit 1 -fi - -package_version() { - dpkg-query -W -f='${Version}' "$1" -} - -tasksel_version=$(package_version tasksel) -task_english_version=$(package_version task-english) -debconf_version=$(package_version debconf) -python_debconf_version=$(package_version python3-debconf) - -if [ "${tasksel_version}" != "${task_english_version}" ]; then - echo "[ERROR] tasksel (${tasksel_version}) and task-english (${task_english_version}) do not match" - exit 1 -fi -if [ "${debconf_version}" != "${python_debconf_version}" ]; then - echo "[ERROR] debconf (${debconf_version}) and python3-debconf (${python_debconf_version}) do not match" - exit 1 -fi - -test -x /usr/src/dappnode/scripts/dappnode_install.sh -test -s /usr/src/dappnode/logs/iso_install.log -docker --version -docker compose version -docker info >/dev/null - -echo "Installed OS: ${PRETTY_NAME}" -echo "tasksel family: ${tasksel_version}" -echo "debconf family: ${debconf_version}" -echo "[INFO] End-to-end Debian ISO installation checks passed" -REMOTE_CHECKS - -echo "[INFO] Complete Debian ISO installation succeeded" +source "${SCRIPT_DIR}/e2e_iso_install.sh" +run_e2e_iso_install "$@" diff --git a/test/e2e_iso_install.sh b/test/e2e_iso_install.sh new file mode 100755 index 00000000..7aff7532 --- /dev/null +++ b/test/e2e_iso_install.sh @@ -0,0 +1,282 @@ +#!/bin/bash +# Shared QEMU harness that installs a DAppNode ISO onto a virtual disk. +# It boots the result and validates the OS, packages, Docker, and DAppNode files. +set -Eeuo pipefail + +run_e2e_iso_install() { + if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + return 2 + fi + + local iso_path + iso_path=$(realpath "$1") + if [ ! -f "${iso_path}" ]; then + echo "[ERROR] ISO not found: ${iso_path}" + return 1 + fi + + local required_commands=(qemu-img qemu-system-x86_64 xorriso ssh sshpass) + local command_name + for command_name in "${required_commands[@]}"; do + if ! command -v "${command_name}" >/dev/null 2>&1; then + echo "[ERROR] Missing required command: ${command_name}" + return 1 + fi + done + + local output_dir=${E2E_OUTPUT_DIR:-"${PWD}/test-output/${E2E_DISTRO}-e2e"} + mkdir -p "${output_dir}" + output_dir=$(realpath "${output_dir}") + + temp_root=${RUNNER_TEMP:-/tmp} + vm_dir=$(mktemp -d "${temp_root%/}/dappnode-${E2E_DISTRO}-e2e.XXXXXX") + local disk_path="${vm_dir}/${E2E_DISTRO}.qcow2" + local kernel_path="${vm_dir}/vmlinuz" + local initrd_path="${vm_dir}/initrd" + local installer_serial_log="${output_dir}/installer-serial.log" + local installer_process_log="${output_dir}/installer-qemu.log" + local system_serial_log="${output_dir}/installed-system-serial.log" + local system_process_log="${output_dir}/installed-system-qemu.log" + local installed_system_report="${output_dir}/installed-system-report.log" + + local vm_disk_size=${E2E_VM_DISK_SIZE:-32G} + local vm_memory_mb=${E2E_VM_MEMORY_MB:-4096} + local vm_cpus=${E2E_VM_CPUS:-2} + local install_timeout_seconds=${E2E_INSTALL_TIMEOUT_SECONDS:-3600} + local boot_timeout_seconds=${E2E_BOOT_TIMEOUT_SECONDS:-600} + local ssh_port=${E2E_SSH_PORT:-2222} + local ssh_password=${E2E_SSH_PASSWORD:-dappnode.s0} + qemu_pid="" + + cleanup_e2e_vm() { + local exit_code=$? + trap - EXIT INT TERM + + if [ -n "${qemu_pid}" ] && kill -0 "${qemu_pid}" 2>/dev/null; then + kill "${qemu_pid}" 2>/dev/null || true + wait "${qemu_pid}" 2>/dev/null || true + fi + + if [ "${KEEP_E2E_VM:-false}" = "true" ]; then + echo "[INFO] Keeping VM files in ${vm_dir}" + else + case "${vm_dir}" in + "${temp_root%/}"/dappnode-"${E2E_DISTRO}"-e2e.*) rm -rf -- "${vm_dir}" ;; + *) echo "[WARN] Refusing to remove unexpected VM directory: ${vm_dir}" ;; + esac + fi + + exit "${exit_code}" + } + trap cleanup_e2e_vm EXIT INT TERM + + show_failure_logs() { + echo "[INFO] Last installer serial output:" + tail -n 200 "${installer_serial_log}" 2>/dev/null || true + echo "[INFO] Last installed-system serial output:" + tail -n 200 "${system_serial_log}" 2>/dev/null || true + echo "[INFO] QEMU process output:" + tail -n 100 "${installer_process_log}" 2>/dev/null || true + tail -n 100 "${system_process_log}" 2>/dev/null || true + } + + wait_for_process_exit() { + local process_id=$1 + local timeout_seconds=$2 + local description=$3 + local deadline=$((SECONDS + timeout_seconds)) + + while kill -0 "${process_id}" 2>/dev/null; do + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "[ERROR] Timed out waiting for ${description}" + return 1 + fi + sleep 10 + done + + wait "${process_id}" + } + + local qemu_acceleration=(-accel "tcg,thread=multi") + if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + qemu_acceleration=(-accel kvm -cpu host) + echo "[INFO] Using KVM acceleration" + else + echo "[WARN] /dev/kvm is unavailable; using slower TCG emulation" + fi + + echo "[INFO] Extracting the ${E2E_DISTRO} installer kernel and initrd from ${iso_path}" + xorriso -osirrox on -indev "${iso_path}" -extract "${E2E_KERNEL_ISO_PATH}" "${kernel_path}" >/dev/null 2>&1 + xorriso -osirrox on -indev "${iso_path}" -extract "${E2E_INITRD_ISO_PATH}" "${initrd_path}" >/dev/null 2>&1 + + echo "[INFO] Creating ${vm_disk_size} virtual installation disk" + qemu-img create -q -f qcow2 "${disk_path}" "${vm_disk_size}" + + echo "[INFO] Booting the unattended ${E2E_DISTRO} installer" + qemu-system-x86_64 \ + "${qemu_acceleration[@]}" \ + -m "${vm_memory_mb}" \ + -smp "${vm_cpus}" \ + -drive "file=${disk_path},format=qcow2,if=virtio" \ + -cdrom "${iso_path}" \ + -kernel "${kernel_path}" \ + -initrd "${initrd_path}" \ + -append "${E2E_KERNEL_ARGS}" \ + -nic user,model=virtio-net-pci \ + -display none \ + -monitor none \ + -serial "file:${installer_serial_log}" \ + -no-reboot \ + >"${installer_process_log}" 2>&1 & + qemu_pid=$! + + if ! wait_for_process_exit "${qemu_pid}" "${install_timeout_seconds}" "the ${E2E_DISTRO} installer to complete"; then + show_failure_logs + return 1 + fi + qemu_pid="" + + echo "[INFO] Installer completed; booting the installed virtual disk" + qemu-system-x86_64 \ + "${qemu_acceleration[@]}" \ + -m "${vm_memory_mb}" \ + -smp "${vm_cpus}" \ + -drive "file=${disk_path},format=qcow2,if=virtio" \ + -boot order=c \ + -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ + -display none \ + -monitor none \ + -serial "file:${system_serial_log}" \ + >"${system_process_log}" 2>&1 & + qemu_pid=$! + + local ssh_options=( + -p "${ssh_port}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o ConnectTimeout=5 + -o ConnectionAttempts=1 + -o LogLevel=ERROR + ) + + ssh_guest() { + SSHPASS="${ssh_password}" sshpass -e ssh "${ssh_options[@]}" dappnode@127.0.0.1 "$@" + } + + echo "[INFO] Waiting for SSH on the installed system" + local boot_deadline=$((SECONDS + boot_timeout_seconds)) + while ! ssh_guest true >/dev/null 2>&1; do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + echo "[ERROR] Installed-system VM exited before SSH became available" + show_failure_logs + return 1 + fi + if [ "${SECONDS}" -ge "${boot_deadline}" ]; then + echo "[ERROR] Timed out waiting for SSH on the installed system" + show_failure_logs + return 1 + fi + sleep 10 + done + + echo "[INFO] Validating the installed ${E2E_DISTRO} and DAppNode system" + ssh_guest bash -s -- \ + "${E2E_DISTRO}" \ + "${E2E_EXPECTED_ID}" \ + "${E2E_EXPECTED_VERSION}" \ + "${E2E_EXPECTED_CODENAME}" <<'REMOTE_CHECKS' | tee "${installed_system_report}" +set -euo pipefail + +expected_distro=$1 +expected_id=$2 +expected_version=$3 +expected_codename=$4 + +source /etc/os-release +if [ "${expected_codename}" = "AUTO" ]; then + expected_codename=${VERSION_CODENAME} +fi +if [ "${ID}" != "${expected_id}" ] || [ "${VERSION_ID}" != "${expected_version}" ] || [ "${VERSION_CODENAME}" != "${expected_codename}" ]; then + echo "[ERROR] Expected ${expected_id} ${expected_version} (${expected_codename}), found ${PRETTY_NAME}" + exit 1 +fi + +apt_sources=$(grep -R -h -E --include='*.list' --include='*.sources' \ + '^(deb |Suites:)' /etc/apt 2>/dev/null || true) +if ! grep -Eq "(^|[[:space:]])${expected_codename}([[:space:]-]|$)" <<<"${apt_sources}"; then + echo "[ERROR] No ${expected_codename} APT source found on the installed system" + exit 1 +fi + +dpkg_audit=$(dpkg --audit) +if [ -n "${dpkg_audit}" ]; then + echo "[ERROR] dpkg reports incomplete or broken packages:" + echo "${dpkg_audit}" + exit 1 +fi + +package_version() { + dpkg-query -W -f='${Version}' "$1" +} + +case "${expected_distro}" in + debian) + for debian_suite in bullseye bookworm trixie forky; do + if [ "${debian_suite}" != "${expected_codename}" ] && \ + grep -R -q -E --include='*.list' --include='*.sources' \ + "(^|[[:space:]])${debian_suite}([[:space:]-]|$)" /etc/apt 2>/dev/null; then + echo "[ERROR] Found stale Debian suite ${debian_suite} in APT sources" + exit 1 + fi + done + + tasksel_version=$(package_version tasksel) + task_english_version=$(package_version task-english) + debconf_version=$(package_version debconf) + python_debconf_version=$(package_version python3-debconf) + + if [ "${tasksel_version}" != "${task_english_version}" ]; then + echo "[ERROR] tasksel (${tasksel_version}) and task-english (${task_english_version}) do not match" + exit 1 + fi + if [ "${debconf_version}" != "${python_debconf_version}" ]; then + echo "[ERROR] debconf (${debconf_version}) and python3-debconf (${python_debconf_version}) do not match" + exit 1 + fi + + echo "tasksel family: ${tasksel_version}" + echo "debconf family: ${debconf_version}" + ;; + ubuntu) + ubuntu_minimal_version=$(package_version ubuntu-minimal) + if ! grep -R -q -E --include='*.list' --include='*.sources' \ + 'https?://[^[:space:]]*ubuntu\.com/ubuntu' /etc/apt 2>/dev/null; then + echo "[ERROR] No official Ubuntu archive found in APT sources" + exit 1 + fi + if grep -R -H -E --include='*.list' --include='*.sources' \ + 'https?://deb\.debian\.org/debian' /etc/apt 2>/dev/null; then + echo "[ERROR] Found a Debian APT source on the installed Ubuntu system" + exit 1 + fi + echo "ubuntu-minimal: ${ubuntu_minimal_version}" + ;; + *) + echo "[ERROR] Unsupported installed-system check: ${expected_distro}" + exit 1 + ;; +esac + +test -x /usr/src/dappnode/scripts/dappnode_install.sh +test -s /usr/src/dappnode/logs/iso_install.log +docker --version +docker compose version +docker info >/dev/null + +echo "Installed OS: ${PRETTY_NAME}" +echo "[INFO] End-to-end ${expected_distro} ISO installation checks passed" +REMOTE_CHECKS + + echo "[INFO] Complete ${E2E_DISTRO} ISO installation succeeded" +} diff --git a/test/e2e_ubuntu_iso.sh b/test/e2e_ubuntu_iso.sh new file mode 100755 index 00000000..a58ac46e --- /dev/null +++ b/test/e2e_ubuntu_iso.sh @@ -0,0 +1,25 @@ +#!/bin/bash +# Configures the shared ISO installation harness for Ubuntu's autoinstall layout. +# The Ubuntu end-to-end workflow calls this wrapper with its unattended ISO. +set -Eeuo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) +GENERATOR="${REPO_ROOT}/iso/scripts/generate_dappnode_iso_ubuntu.sh" + +base_iso_name=$(sed -n 's/^BASE_ISO_NAME="\{0,1\}\([^"[:space:]]*\)"\{0,1\}$/\1/p' "${GENERATOR}") +if [[ ! "${base_iso_name}" =~ ^ubuntu-([0-9]+\.[0-9]+)(\.[0-9]+)?-live-server-amd64\.iso$ ]]; then + echo "[ERROR] Could not determine the Ubuntu release from BASE_ISO_NAME=${base_iso_name}" + exit 1 +fi + +E2E_DISTRO="ubuntu" +E2E_EXPECTED_ID="ubuntu" +E2E_EXPECTED_VERSION="${BASH_REMATCH[1]}" +E2E_EXPECTED_CODENAME="AUTO" +E2E_KERNEL_ISO_PATH="/casper/vmlinuz" +E2E_INITRD_ISO_PATH="/casper/initrd" +E2E_KERNEL_ARGS="autoinstall console=ttyS0,115200n8 --- quiet" + +source "${SCRIPT_DIR}/e2e_iso_install.sh" +run_e2e_iso_install "$@" From 3484461691e0f8c88b7532128238f8fa21325286 Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 20 Aug 2026 11:11:34 +0200 Subject: [PATCH 04/17] limit token to read only --- .github/workflows/debian-iso-e2e.yml | 3 +++ .github/workflows/ubuntu-iso-e2e.yml | 3 +++ 2 files changed, 6 insertions(+) diff --git a/.github/workflows/debian-iso-e2e.yml b/.github/workflows/debian-iso-e2e.yml index d234ee08..ce64b217 100644 --- a/.github/workflows/debian-iso-e2e.yml +++ b/.github/workflows/debian-iso-e2e.yml @@ -7,6 +7,9 @@ on: pull_request: branches: [master] +permissions: + contents: read + concurrency: group: debian-iso-e2e-${{ github.ref }} cancel-in-progress: true diff --git a/.github/workflows/ubuntu-iso-e2e.yml b/.github/workflows/ubuntu-iso-e2e.yml index a78bd604..60af70eb 100644 --- a/.github/workflows/ubuntu-iso-e2e.yml +++ b/.github/workflows/ubuntu-iso-e2e.yml @@ -7,6 +7,9 @@ on: pull_request: branches: [master] +permissions: + contents: read + concurrency: group: ubuntu-iso-e2e-${{ github.ref }} cancel-in-progress: true From 36948c88f129604570621a7aca24b259109b01ab Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 20 Aug 2026 11:34:45 +0200 Subject: [PATCH 05/17] unattended usb tests --- .github/workflows/debian-iso-e2e.yml | 20 +-- .github/workflows/ubuntu-iso-e2e.yml | 20 +-- test/e2e_debian_iso.sh | 7 +- test/e2e_iso_install.sh | 231 ++++++++++++++++++++++----- test/e2e_ubuntu_iso.sh | 7 +- 5 files changed, 215 insertions(+), 70 deletions(-) diff --git a/.github/workflows/debian-iso-e2e.yml b/.github/workflows/debian-iso-e2e.yml index ce64b217..699a7f43 100644 --- a/.github/workflows/debian-iso-e2e.yml +++ b/.github/workflows/debian-iso-e2e.yml @@ -1,11 +1,10 @@ -name: Debian ISO end-to-end +# Builds the unattended Debian ISO and installs it from UEFI USB media in QEMU. +# It completes the required reboots and verifies the installed DAppNode system. +name: Debian unattended USB ISO end-to-end on: workflow_dispatch: - push: - branches-ignore: [master] pull_request: - branches: [master] permissions: contents: read @@ -16,9 +15,9 @@ concurrency: jobs: install: - name: Build and install Debian ISO + name: Install Debian unattended ISO via UEFI USB runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 180 steps: - name: Checkout @@ -37,12 +36,15 @@ jobs: - name: Install QEMU test dependencies run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils xorriso sshpass + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf socat sshpass if [ -e /dev/kvm ]; then sudo chmod a+rw /dev/kvm fi - - name: Install Debian ISO end to end + - name: Complete Debian unattended UEFI USB installation + env: + E2E_BOOT_MODE: usb + E2E_FIRMWARE: uefi run: | debian_iso=$(find images -maxdepth 1 -type f -name 'Dappnode-debian-*-amd64-netinst.iso' -print -quit) test -n "${debian_iso}" @@ -52,7 +54,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: debian-installation-e2e-logs + name: debian-unattended-usb-installation-logs path: test-output/debian-e2e/ if-no-files-found: ignore retention-days: 7 diff --git a/.github/workflows/ubuntu-iso-e2e.yml b/.github/workflows/ubuntu-iso-e2e.yml index 60af70eb..e3e2658b 100644 --- a/.github/workflows/ubuntu-iso-e2e.yml +++ b/.github/workflows/ubuntu-iso-e2e.yml @@ -1,11 +1,10 @@ -name: Ubuntu ISO end-to-end +# Builds the unattended Ubuntu ISO and installs it from UEFI USB media in QEMU. +# It completes the required reboots and verifies the installed DAppNode system. +name: Ubuntu unattended USB ISO end-to-end on: workflow_dispatch: - push: - branches-ignore: [master] pull_request: - branches: [master] permissions: contents: read @@ -16,9 +15,9 @@ concurrency: jobs: install: - name: Build and install Ubuntu ISO + name: Install Ubuntu unattended ISO via UEFI USB runs-on: ubuntu-latest - timeout-minutes: 120 + timeout-minutes: 180 steps: - name: Checkout @@ -34,12 +33,15 @@ jobs: - name: Install QEMU test dependencies run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils xorriso sshpass + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf socat sshpass if [ -e /dev/kvm ]; then sudo chmod a+rw /dev/kvm fi - - name: Install Ubuntu ISO end to end + - name: Complete Ubuntu unattended UEFI USB installation + env: + E2E_BOOT_MODE: usb + E2E_FIRMWARE: uefi run: | ubuntu_iso=$(find images -maxdepth 1 -type f -name 'Dappnode-ubuntu-*-live-server-amd64.iso' -print -quit) test -n "${ubuntu_iso}" @@ -49,7 +51,7 @@ jobs: if: always() uses: actions/upload-artifact@v4 with: - name: ubuntu-installation-e2e-logs + name: ubuntu-unattended-usb-installation-logs path: test-output/ubuntu-e2e/ if-no-files-found: ignore retention-days: 7 diff --git a/test/e2e_debian_iso.sh b/test/e2e_debian_iso.sh index d7cb28b4..9f9c2393 100755 --- a/test/e2e_debian_iso.sh +++ b/test/e2e_debian_iso.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Configures the shared ISO installation harness for Debian's installer layout. -# The Debian end-to-end workflow calls this wrapper with its unattended ISO. +# Runs the generated unattended Debian ISO through the complete UEFI USB test. +# It adds Debian release and package-alignment checks to the shared harness. set -Eeuo pipefail SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -12,9 +12,6 @@ E2E_DISTRO="debian" E2E_EXPECTED_ID="debian" E2E_EXPECTED_VERSION="${DEBIAN_MAJOR_VERSION}" E2E_EXPECTED_CODENAME="${DEBIAN_SUITE}" -E2E_KERNEL_ISO_PATH="/install.amd/vmlinuz" -E2E_INITRD_ISO_PATH="/install.amd/initrd.gz" -E2E_KERNEL_ARGS="auto=true priority=critical console=ttyS0,115200n8 --- quiet" source "${SCRIPT_DIR}/e2e_iso_install.sh" run_e2e_iso_install "$@" diff --git a/test/e2e_iso_install.sh b/test/e2e_iso_install.sh index 7aff7532..a0bd493d 100755 --- a/test/e2e_iso_install.sh +++ b/test/e2e_iso_install.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Shared QEMU harness that installs a DAppNode ISO onto a virtual disk. -# It boots the result and validates the OS, packages, Docker, and DAppNode files. +# Boots an unattended DAppNode ISO as USB media and installs it onto a virtual disk. +# It completes the first-boot flow, reboots, and validates the running DAppNode system. set -Eeuo pipefail run_e2e_iso_install() { @@ -16,7 +16,7 @@ run_e2e_iso_install() { return 1 fi - local required_commands=(qemu-img qemu-system-x86_64 xorriso ssh sshpass) + local required_commands=(qemu-img qemu-system-x86_64 ssh sshpass socat) local command_name for command_name in "${required_commands[@]}"; do if ! command -v "${command_name}" >/dev/null 2>&1; then @@ -32,12 +32,16 @@ run_e2e_iso_install() { temp_root=${RUNNER_TEMP:-/tmp} vm_dir=$(mktemp -d "${temp_root%/}/dappnode-${E2E_DISTRO}-e2e.XXXXXX") local disk_path="${vm_dir}/${E2E_DISTRO}.qcow2" - local kernel_path="${vm_dir}/vmlinuz" - local initrd_path="${vm_dir}/initrd" local installer_serial_log="${output_dir}/installer-serial.log" local installer_process_log="${output_dir}/installer-qemu.log" - local system_serial_log="${output_dir}/installed-system-serial.log" - local system_process_log="${output_dir}/installed-system-qemu.log" + local installer_monitor_socket="${vm_dir}/installer-monitor.sock" + local installer_screenshot="${output_dir}/installer-screen.ppm" + local first_boot_monitor_socket="${vm_dir}/first-boot-monitor.sock" + local first_boot_screenshot="${output_dir}/first-boot-screen.ppm" + local first_boot_serial_log="${output_dir}/first-boot-serial.log" + local first_boot_process_log="${output_dir}/first-boot-qemu.log" + local system_serial_log="${output_dir}/final-system-serial.log" + local system_process_log="${output_dir}/final-system-qemu.log" local installed_system_report="${output_dir}/installed-system-report.log" local vm_disk_size=${E2E_VM_DISK_SIZE:-32G} @@ -45,10 +49,19 @@ run_e2e_iso_install() { local vm_cpus=${E2E_VM_CPUS:-2} local install_timeout_seconds=${E2E_INSTALL_TIMEOUT_SECONDS:-3600} local boot_timeout_seconds=${E2E_BOOT_TIMEOUT_SECONDS:-600} + local first_boot_timeout_seconds=${E2E_FIRST_BOOT_TIMEOUT_SECONDS:-1800} + local postinstall_timeout_seconds=${E2E_POSTINSTALL_TIMEOUT_SECONDS:-1200} local ssh_port=${E2E_SSH_PORT:-2222} local ssh_password=${E2E_SSH_PASSWORD:-dappnode.s0} + local boot_mode=${E2E_BOOT_MODE:-usb} + local firmware=${E2E_FIRMWARE:-uefi} qemu_pid="" + if [ "${boot_mode}" != "usb" ] || [ "${firmware}" != "uefi" ]; then + echo "[ERROR] This unattended ISO test requires E2E_BOOT_MODE=usb and E2E_FIRMWARE=uefi" + return 1 + fi + cleanup_e2e_vm() { local exit_code=$? trap - EXIT INT TERM @@ -72,12 +85,23 @@ run_e2e_iso_install() { trap cleanup_e2e_vm EXIT INT TERM show_failure_logs() { + if [ -S "${installer_monitor_socket}" ]; then + printf 'screendump %s\n' "${installer_screenshot}" | \ + socat - "UNIX-CONNECT:${installer_monitor_socket}" >/dev/null 2>&1 || true + fi + if [ -S "${first_boot_monitor_socket}" ]; then + printf 'screendump %s\n' "${first_boot_screenshot}" | \ + socat - "UNIX-CONNECT:${first_boot_monitor_socket}" >/dev/null 2>&1 || true + fi echo "[INFO] Last installer serial output:" tail -n 200 "${installer_serial_log}" 2>/dev/null || true - echo "[INFO] Last installed-system serial output:" + echo "[INFO] Last first-boot serial output:" + tail -n 200 "${first_boot_serial_log}" 2>/dev/null || true + echo "[INFO] Last final-system serial output:" tail -n 200 "${system_serial_log}" 2>/dev/null || true echo "[INFO] QEMU process output:" tail -n 100 "${installer_process_log}" 2>/dev/null || true + tail -n 100 "${first_boot_process_log}" 2>/dev/null || true tail -n 100 "${system_process_log}" 2>/dev/null || true } @@ -106,26 +130,57 @@ run_e2e_iso_install() { echo "[WARN] /dev/kvm is unavailable; using slower TCG emulation" fi - echo "[INFO] Extracting the ${E2E_DISTRO} installer kernel and initrd from ${iso_path}" - xorriso -osirrox on -indev "${iso_path}" -extract "${E2E_KERNEL_ISO_PATH}" "${kernel_path}" >/dev/null 2>&1 - xorriso -osirrox on -indev "${iso_path}" -extract "${E2E_INITRD_ISO_PATH}" "${initrd_path}" >/dev/null 2>&1 + local ovmf_code="" + local ovmf_vars="" + local ovmf_candidate + for ovmf_candidate in /usr/share/OVMF/OVMF_CODE_4M.fd /usr/share/OVMF/OVMF_CODE.fd; do + if [ -f "${ovmf_candidate}" ]; then + ovmf_code=${ovmf_candidate} + break + fi + done + for ovmf_candidate in /usr/share/OVMF/OVMF_VARS_4M.fd /usr/share/OVMF/OVMF_VARS.fd; do + if [ -f "${ovmf_candidate}" ]; then + ovmf_vars=${ovmf_candidate} + break + fi + done + if [ -z "${ovmf_code}" ] || [ -z "${ovmf_vars}" ]; then + echo "[ERROR] UEFI firmware files were not found; install the ovmf package" + return 1 + fi + local writable_ovmf_vars="${vm_dir}/OVMF_VARS.fd" + cp "${ovmf_vars}" "${writable_ovmf_vars}" + local firmware_args=( + -drive "if=pflash,format=raw,readonly=on,file=${ovmf_code}" + -drive "if=pflash,format=raw,file=${writable_ovmf_vars}" + ) echo "[INFO] Creating ${vm_disk_size} virtual installation disk" qemu-img create -q -f qcow2 "${disk_path}" "${vm_disk_size}" - echo "[INFO] Booting the unattended ${E2E_DISTRO} installer" + local target_disk_install_args=( + -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" + -device "virtio-blk-pci,drive=target_disk,bootindex=2" + ) + local installer_media_args=( + -device "qemu-xhci,id=installer_xhci" + -drive "file=${iso_path},format=raw,if=none,readonly=on,id=installer_media" + -device "usb-storage,drive=installer_media,bootindex=1" + ) + + echo "[INFO] Booting the unattended ${E2E_DISTRO} ISO as ${firmware}/${boot_mode}" qemu-system-x86_64 \ "${qemu_acceleration[@]}" \ + "${firmware_args[@]}" \ -m "${vm_memory_mb}" \ -smp "${vm_cpus}" \ - -drive "file=${disk_path},format=qcow2,if=virtio" \ - -cdrom "${iso_path}" \ - -kernel "${kernel_path}" \ - -initrd "${initrd_path}" \ - -append "${E2E_KERNEL_ARGS}" \ + "${target_disk_install_args[@]}" \ + "${installer_media_args[@]}" \ + -boot menu=off \ -nic user,model=virtio-net-pci \ -display none \ - -monitor none \ + -monitor "unix:${installer_monitor_socket},server=on,wait=off" \ -serial "file:${installer_serial_log}" \ -no-reboot \ >"${installer_process_log}" 2>&1 & @@ -137,18 +192,21 @@ run_e2e_iso_install() { fi qemu_pid="" - echo "[INFO] Installer completed; booting the installed virtual disk" + echo "[INFO] Installer completed and virtual USB removed; starting the first disk boot" qemu-system-x86_64 \ "${qemu_acceleration[@]}" \ + "${firmware_args[@]}" \ -m "${vm_memory_mb}" \ -smp "${vm_cpus}" \ - -drive "file=${disk_path},format=qcow2,if=virtio" \ - -boot order=c \ + -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" \ + -device "virtio-blk-pci,drive=target_disk,bootindex=1" \ + -boot menu=off \ -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ -display none \ - -monitor none \ - -serial "file:${system_serial_log}" \ - >"${system_process_log}" 2>&1 & + -monitor "unix:${first_boot_monitor_socket},server=on,wait=off" \ + -serial "file:${first_boot_serial_log}" \ + -no-reboot \ + >"${first_boot_process_log}" 2>&1 & qemu_pid=$! local ssh_options=( @@ -164,23 +222,108 @@ run_e2e_iso_install() { SSHPASS="${ssh_password}" sshpass -e ssh "${ssh_options[@]}" dappnode@127.0.0.1 "$@" } - echo "[INFO] Waiting for SSH on the installed system" - local boot_deadline=$((SECONDS + boot_timeout_seconds)) - while ! ssh_guest true >/dev/null 2>&1; do - if ! kill -0 "${qemu_pid}" 2>/dev/null; then - echo "[ERROR] Installed-system VM exited before SSH became available" - show_failure_logs - return 1 - fi - if [ "${SECONDS}" -ge "${boot_deadline}" ]; then - echo "[ERROR] Timed out waiting for SSH on the installed system" - show_failure_logs - return 1 - fi - sleep 10 - done + wait_for_ssh() { + local description=$1 + local timeout_seconds=$2 + local deadline=$((SECONDS + timeout_seconds)) + + echo "[INFO] Waiting for SSH on ${description}" + while ! ssh_guest true >/dev/null 2>&1; do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + echo "[ERROR] ${description} VM exited before SSH became available" + return 1 + fi + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "[ERROR] Timed out waiting for SSH on ${description}" + return 1 + fi + sleep 10 + done + } + + wait_for_guest_command() { + local timeout_seconds=$1 + local description=$2 + local guest_command=$3 + local deadline=$((SECONDS + timeout_seconds)) + + echo "[INFO] Waiting for ${description}" + while ! ssh_guest "${guest_command}" >/dev/null 2>&1; do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + echo "[ERROR] VM exited while waiting for ${description}" + return 1 + fi + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "[ERROR] Timed out waiting for ${description}" + return 1 + fi + sleep 10 + done + } + + if ! wait_for_ssh "the first installed-system boot" "${boot_timeout_seconds}"; then + show_failure_logs + return 1 + fi + + if ! wait_for_guest_command \ + "${first_boot_timeout_seconds}" \ + "DAppNode's first-boot installation test" \ + "test ! -e /usr/src/dappnode/.firstboot"; then + show_failure_logs + return 1 + fi + + echo "[INFO] Acknowledging the completed first-boot test" + printf 'sendkey ret\n' | \ + socat - "UNIX-CONNECT:${first_boot_monitor_socket}" >/dev/null + + if ! wait_for_guest_command \ + 120 \ + "the first-boot installer process to exit" \ + "! pgrep -f '[d]appnode_test_install.sh|[/]usr/src/dappnode/scripts/dappnode_install.sh' >/dev/null"; then + show_failure_logs + return 1 + fi + + echo "[INFO] Rebooting after the completed first-boot test" + ssh_guest "sudo -S -p '' systemctl reboot" <<<"${ssh_password}" >/dev/null 2>&1 || true + if ! wait_for_process_exit "${qemu_pid}" 120 "the first-boot VM to request its reboot"; then + show_failure_logs + return 1 + fi + qemu_pid="" + + echo "[INFO] Starting the final installed system without the virtual USB" + qemu-system-x86_64 \ + "${qemu_acceleration[@]}" \ + "${firmware_args[@]}" \ + -m "${vm_memory_mb}" \ + -smp "${vm_cpus}" \ + -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" \ + -device "virtio-blk-pci,drive=target_disk,bootindex=1" \ + -boot menu=off \ + -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ + -display none \ + -monitor none \ + -serial "file:${system_serial_log}" \ + >"${system_process_log}" 2>&1 & + qemu_pid=$! + + if ! wait_for_ssh "the final installed system" "${boot_timeout_seconds}"; then + show_failure_logs + return 1 + fi + + if ! wait_for_guest_command \ + "${postinstall_timeout_seconds}" \ + "DAppNode core services to start" \ + "test ! -e /usr/src/dappnode/.firstboot && ! grep -Fq '/usr/src/dappnode/scripts/dappnode_install.sh' /etc/rc.local && docker ps --format '{{.Names}}' | grep -Fxq 'DAppNodeCore-dappmanager.dnp.dappnode.eth'"; then + show_failure_logs + return 1 + fi - echo "[INFO] Validating the installed ${E2E_DISTRO} and DAppNode system" + echo "[INFO] Validating the completed unattended ${E2E_DISTRO} USB installation" ssh_guest bash -s -- \ "${E2E_DISTRO}" \ "${E2E_EXPECTED_ID}" \ @@ -270,13 +413,17 @@ esac test -x /usr/src/dappnode/scripts/dappnode_install.sh test -s /usr/src/dappnode/logs/iso_install.log +test -s /usr/src/dappnode/logs/dappnode_install.log +test ! -e /usr/src/dappnode/.firstboot +! grep -Fq '/usr/src/dappnode/scripts/dappnode_install.sh' /etc/rc.local docker --version docker compose version docker info >/dev/null +docker ps --format '{{.Names}}' | grep -Fxq 'DAppNodeCore-dappmanager.dnp.dappnode.eth' echo "Installed OS: ${PRETTY_NAME}" -echo "[INFO] End-to-end ${expected_distro} ISO installation checks passed" +echo "[INFO] End-to-end unattended ${expected_distro} UEFI USB installation checks passed" REMOTE_CHECKS - echo "[INFO] Complete ${E2E_DISTRO} ISO installation succeeded" + echo "[INFO] Complete unattended ${E2E_DISTRO} UEFI USB installation succeeded" } diff --git a/test/e2e_ubuntu_iso.sh b/test/e2e_ubuntu_iso.sh index a58ac46e..2c00f16c 100755 --- a/test/e2e_ubuntu_iso.sh +++ b/test/e2e_ubuntu_iso.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Configures the shared ISO installation harness for Ubuntu's autoinstall layout. -# The Ubuntu end-to-end workflow calls this wrapper with its unattended ISO. +# Runs the generated unattended Ubuntu ISO through the complete UEFI USB test. +# It adds Ubuntu release and repository checks to the shared harness. set -Eeuo pipefail SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) @@ -17,9 +17,6 @@ E2E_DISTRO="ubuntu" E2E_EXPECTED_ID="ubuntu" E2E_EXPECTED_VERSION="${BASH_REMATCH[1]}" E2E_EXPECTED_CODENAME="AUTO" -E2E_KERNEL_ISO_PATH="/casper/vmlinuz" -E2E_INITRD_ISO_PATH="/casper/initrd" -E2E_KERNEL_ARGS="autoinstall console=ttyS0,115200n8 --- quiet" source "${SCRIPT_DIR}/e2e_iso_install.sh" run_e2e_iso_install "$@" From f0718d003db63866909ec269084f233877141a69 Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 20 Aug 2026 13:03:23 +0200 Subject: [PATCH 06/17] fix unattended usb tests --- .github/workflows/debian-iso-e2e.yml | 3 +++ .github/workflows/ubuntu-iso-e2e.yml | 3 +++ scripts/check-disks.sh | 7 ++++--- scripts/dappnode_install.sh | 11 ++++++++++- 4 files changed, 20 insertions(+), 4 deletions(-) diff --git a/.github/workflows/debian-iso-e2e.yml b/.github/workflows/debian-iso-e2e.yml index 699a7f43..1f7d9655 100644 --- a/.github/workflows/debian-iso-e2e.yml +++ b/.github/workflows/debian-iso-e2e.yml @@ -41,6 +41,9 @@ jobs: sudo chmod a+rw /dev/kvm fi + - name: Enable ICMP for QEMU user networking + run: sudo sysctl -w net.ipv4.ping_group_range="0 2147483647" + - name: Complete Debian unattended UEFI USB installation env: E2E_BOOT_MODE: usb diff --git a/.github/workflows/ubuntu-iso-e2e.yml b/.github/workflows/ubuntu-iso-e2e.yml index e3e2658b..49a105b7 100644 --- a/.github/workflows/ubuntu-iso-e2e.yml +++ b/.github/workflows/ubuntu-iso-e2e.yml @@ -38,6 +38,9 @@ jobs: sudo chmod a+rw /dev/kvm fi + - name: Enable ICMP for QEMU user networking + run: sudo sysctl -w net.ipv4.ping_group_range="0 2147483647" + - name: Complete Ubuntu unattended UEFI USB installation env: E2E_BOOT_MODE: usb diff --git a/scripts/check-disks.sh b/scripts/check-disks.sh index d7385c41..8aa71677 100755 --- a/scripts/check-disks.sh +++ b/scripts/check-disks.sh @@ -1,6 +1,6 @@ #!/bin/sh -# -# This is run by d-i before the partman step (e.g. d-i partman/early_command) +# Selects non-USB disks for Debian partitioning before the partman step. +# It gives GRUB the same targets so the installer USB is never selected. USBDEV=$(list-devices usb-partition | sed "s/\(.*\)./\1/" | sort -u | head -1 ); if [ -z "${USBDEV}" ]; then @@ -12,4 +12,5 @@ for DISK in ${DEVICE}; do DISKS="${DISKS} ${DISK}"; done; DISKS=$(echo "${DISKS}" | sed "s/^ //g"); -debconf-set partman-auto/disk "$DISKS"; \ No newline at end of file +debconf-set partman-auto/disk "$DISKS"; +debconf-set grub-installer/bootdev "$DISKS"; diff --git a/scripts/dappnode_install.sh b/scripts/dappnode_install.sh index 49ecf3f2..a25c8447 100755 --- a/scripts/dappnode_install.sh +++ b/scripts/dappnode_install.sh @@ -428,7 +428,16 @@ normalize_ipfs_version_ref() { local cid_path="$ref" local manifest_url="${IPFS_ENDPOINT%/}${cid_path}/dappnode_package.json" local manifest - manifest="$(download_stdout "$manifest_url" 2>/dev/null || true)" + local attempt + for attempt in 1 2 3 4 5; do + manifest="$(download_stdout "$manifest_url" 2>/dev/null || true)" + if [[ -n "$manifest" ]]; then + break + fi + if [[ "$attempt" -lt 5 ]]; then + sleep 2 + fi + done if [[ -z "$manifest" ]]; then error "Could not fetch IPFS manifest for ${comp} from: $manifest_url" error "Provide ${comp}_VERSION as /ipfs/: (example: /ipfs/Qm...:0.2.11)" From b5b6d122cc8ae62b14be2949248e5c66cea6312a Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 20 Aug 2026 13:09:22 +0200 Subject: [PATCH 07/17] Revert check-disks.sh to master This PR should be scoped to the e2e test workflows. The grub-installer/bootdev override added in f0718d0 is a change to the shipped Debian installer, and it is very likely a no-op for the test it was added for: the e2e harness is UEFI-only, where GRUB installs to the ESP and bootdev is the BIOS/i386-pc question. Co-Authored-By: Claude Opus 5 --- scripts/check-disks.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/scripts/check-disks.sh b/scripts/check-disks.sh index 8aa71677..d7385c41 100755 --- a/scripts/check-disks.sh +++ b/scripts/check-disks.sh @@ -1,6 +1,6 @@ #!/bin/sh -# Selects non-USB disks for Debian partitioning before the partman step. -# It gives GRUB the same targets so the installer USB is never selected. +# +# This is run by d-i before the partman step (e.g. d-i partman/early_command) USBDEV=$(list-devices usb-partition | sed "s/\(.*\)./\1/" | sort -u | head -1 ); if [ -z "${USBDEV}" ]; then @@ -12,5 +12,4 @@ for DISK in ${DEVICE}; do DISKS="${DISKS} ${DISK}"; done; DISKS=$(echo "${DISKS}" | sed "s/^ //g"); -debconf-set partman-auto/disk "$DISKS"; -debconf-set grub-installer/bootdev "$DISKS"; +debconf-set partman-auto/disk "$DISKS"; \ No newline at end of file From 68cea4220c0d13051390570dad16cab7290f1be5 Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 20 Aug 2026 14:52:39 +0200 Subject: [PATCH 08/17] Point grub-installer at the target disk, not the installer USB The preseed sets grub-installer/bootdev to "default", which also marks the question as seen. grub-installer then takes this path: db_fget grub-installer/bootdev seen if [ "$RET" = true ] && db_get grub-installer/bootdev && [ "$RET" ]; then if [ "$RET" = "default" ]; then bootdev=$default_bootdev $default_bootdev is the first grub-mkdevicemap entry -- (hd0), the installer USB itself. The safeguard against installing onto the installation media only runs in the *:grub-pc branch, so nothing protects a UEFI install. The EFI install still succeeds, because grub-install writes to the ESP on the real disk, but the step then exits 1 reading the ISO9660 partition table: grub-installer: info: Installing grub on '/dev/sda' grub-installer: info: grub-install ran successfully main-menu: (process:10646): Can't read partition table from /dev/sda main-menu: WARNING **: Configuring 'grub-installer' failed with error code 1 Reuse the non-USB disk list already computed for partman. Verified by a local QEMU UEFI USB install of the Debian 13.5.0 unattended ISO, which now completes the installer in ~4 minutes instead of stalling on the failure dialog. Co-Authored-By: Claude Opus 5 --- scripts/check-disks.sh | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/scripts/check-disks.sh b/scripts/check-disks.sh index d7385c41..72070d31 100755 --- a/scripts/check-disks.sh +++ b/scripts/check-disks.sh @@ -12,4 +12,14 @@ for DISK in ${DEVICE}; do DISKS="${DISKS} ${DISK}"; done; DISKS=$(echo "${DISKS}" | sed "s/^ //g"); -debconf-set partman-auto/disk "$DISKS"; \ No newline at end of file +debconf-set partman-auto/disk "$DISKS"; + +# grub-installer/bootdev is preseeded to "default", which marks the question as +# already seen. grub-installer then resolves "default" to the first entry of +# grub-mkdevicemap, i.e. (hd0) -- the installer USB itself. Its safeguard against +# installing onto the installation media only runs for grub-pc, so on UEFI it +# happily targets the stick and then dies reading its partition table: +# Can't read partition table from /dev/sda +# Point it at the same non-USB disks partman is using. grub-installer accepts a +# space separated list and installs to each entry in turn. +debconf-set grub-installer/bootdev "$DISKS"; From 9fbd7d4c02016ba27d7f11c3ec62f5c6524eeb16 Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 20 Aug 2026 14:52:50 +0200 Subject: [PATCH 09/17] Capture d-i consoles and guest state when an e2e run fails A failed run only tailed the serial logs, which hold three lines of EFI stub output because d-i draws its UI on VT1 and streams its syslog to VT4, and the first-boot test runs on a virtual terminal via openvt. Both real failures found so far were invisible in the artifacts. Screendump every console instead, and dump the guest's install logs, docker state and connectivity over SSH, which is always up by the time a first-boot wait times out. Co-Authored-By: Claude Opus 5 --- test/e2e_iso_install.sh | 54 +++++++++++++++++++++++++++++++++++------ 1 file changed, 46 insertions(+), 8 deletions(-) diff --git a/test/e2e_iso_install.sh b/test/e2e_iso_install.sh index a0bd493d..c0f80fea 100755 --- a/test/e2e_iso_install.sh +++ b/test/e2e_iso_install.sh @@ -84,15 +84,53 @@ run_e2e_iso_install() { } trap cleanup_e2e_vm EXIT INT TERM + monitor_command() { + local socket=$1 + local command=$2 + [ -S "${socket}" ] || return 0 + printf '%s\n' "${command}" | \ + socat - "UNIX-CONNECT:${socket}" >/dev/null 2>&1 || true + } + + capture_vt_screens() { + # d-i draws its UI on VT1 and streams its syslog to VT4, so a failed + # installer step is only diagnosable if every console is captured. + local socket=$1 + local prefix=$2 + local vt + [ -S "${socket}" ] || return 0 + for vt in 1 2 3 4; do + monitor_command "${socket}" "sendkey alt-f${vt}" + sleep 1 + monitor_command "${socket}" "screendump ${prefix}-vt${vt}.ppm" + done + monitor_command "${socket}" "sendkey alt-f1" + } + + dump_guest_diagnostics() { + # SSH is already up whenever a first-boot wait times out, and the + # installer logs there explain far more than the serial console can. + declare -F ssh_guest >/dev/null 2>&1 || return 0 + echo "[INFO] Collecting guest diagnostics over SSH:" + ssh_guest " + echo '--- .firstboot ---'; ls -la /usr/src/dappnode/.firstboot 2>&1 + echo '--- dappnode processes ---'; ps aux | grep -E '[d]appnode_(test_)?install' 2>&1 + echo '--- docker ps ---'; docker ps --format '{{.Names}}\t{{.Status}}' 2>&1 + echo '--- docker images ---'; docker images 2>&1 + echo '--- connectivity ---' + ping -c 1 -W 5 google.com 2>&1 | tail -3 + curl -sS -o /dev/null -w 'curl https: %{http_code}\n' --max-time 10 https://www.google.com/generate_204 2>&1 + echo '--- iso_install.log ---'; tail -n 60 /usr/src/dappnode/logs/iso_install.log 2>&1 + echo '--- dappnode_install.log ---'; tail -n 80 /usr/src/dappnode/logs/dappnode_install.log 2>&1 + " 2>&1 | sed 's/^/ /' || echo " [WARN] Could not collect guest diagnostics" + } + show_failure_logs() { - if [ -S "${installer_monitor_socket}" ]; then - printf 'screendump %s\n' "${installer_screenshot}" | \ - socat - "UNIX-CONNECT:${installer_monitor_socket}" >/dev/null 2>&1 || true - fi - if [ -S "${first_boot_monitor_socket}" ]; then - printf 'screendump %s\n' "${first_boot_screenshot}" | \ - socat - "UNIX-CONNECT:${first_boot_monitor_socket}" >/dev/null 2>&1 || true - fi + capture_vt_screens "${installer_monitor_socket}" "${output_dir}/installer-screen" + capture_vt_screens "${first_boot_monitor_socket}" "${output_dir}/first-boot-screen" + monitor_command "${installer_monitor_socket}" "screendump ${installer_screenshot}" + monitor_command "${first_boot_monitor_socket}" "screendump ${first_boot_screenshot}" + dump_guest_diagnostics echo "[INFO] Last installer serial output:" tail -n 200 "${installer_serial_log}" 2>/dev/null || true echo "[INFO] Last first-boot serial output:" From ccbf0f48755d14e61d42a2bb7cbc01d93842ed66 Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 20 Aug 2026 15:07:16 +0200 Subject: [PATCH 10/17] Stop the first-boot test hanging when ICMP is filtered The connectivity check was ping-only, so any network that filters ICMP failed it. error_exit then blocks on `read` forever, .firstboot is never removed, and the machine sits on "Check installation source. Press enter to continue". That is not just CI: corporate, hotel and cloud networks filter ICMP too, and a real DAppNode behind one dies the same way on its first boot. GitHub runners hit it because Azure drops ICMP egress -- the ping_group_range sysctl in the e2e workflows lets QEMU open the socket but cannot make the echo come back. Fall back to an HTTPS probe, then to a bare TCP connect for the case where neither curl nor wget is installed yet. Verified on a local QEMU UEFI USB install of the Ubuntu 24.04.3 unattended ISO with `iptables -A OUTPUT -p icmp --icmp-type echo-request -j DROP` applied in the guest before the test ran: ping fails, the check passes, and the run completes instead of stalling for the full 30 minute timeout. Co-Authored-By: Claude Opus 5 --- scripts/dappnode_test_install.sh | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/scripts/dappnode_test_install.sh b/scripts/dappnode_test_install.sh index 4d5d704d..21f193d8 100755 --- a/scripts/dappnode_test_install.sh +++ b/scripts/dappnode_test_install.sh @@ -22,9 +22,33 @@ detect_installation_type() { fi } +# ICMP is filtered on many corporate, hotel and cloud networks, so a failed +# ping alone does not mean the machine is offline. Confirm over HTTPS before +# declaring the installation broken. +check_connectivity() { + ping -c 1 -q google.com >/dev/null 2>&1 && return 0 + + if command -v curl >/dev/null 2>&1; then + curl -fsS --max-time 10 -o /dev/null https://www.google.com/generate_204 && return 0 + fi + + if command -v wget >/dev/null 2>&1; then + wget -q --timeout=10 -O /dev/null https://www.google.com/generate_204 && return 0 + fi + + # Neither downloader is guaranteed to be present this early, so fall back to + # a plain TCP connect, which bash can do on its own. + if (exec 3<>/dev/tcp/www.google.com/443) 2>/dev/null; then + exec 3<&- 3>&- + return 0 + fi + + return 1 +} + components=(BIND IPFS VPN DAPPMANAGER WIFI) detect_installation_type -if ping -c 1 -q google.com >&/dev/null; then +if check_connectivity; then echo -e "\e[32m Connectivity OK\n \e[0m" else error_exit From 133128fb6dcb068c60c374bb554baa6fdfe8f21b Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 8 Oct 2026 12:35:52 +0200 Subject: [PATCH 11/17] Download the newest Debian point release from debian-cd Debian only moves a point release into cdimage/archive once it has been superseded, so the archive URL 404s for the current release. That broke every automated base ISO bump (e.g. #727, debian-13.7.0). Try debian-cd first and fall back to the archive, and drop partial downloads so a failed attempt is not reused on the next build. Co-Authored-By: Claude Opus 5.5 --- iso/scripts/common_iso_generation.sh | 24 +++++++++++++++++---- iso/scripts/generate_dappnode_iso_debian.sh | 7 ++++-- 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/iso/scripts/common_iso_generation.sh b/iso/scripts/common_iso_generation.sh index 1cd65a0c..5306f958 100644 --- a/iso/scripts/common_iso_generation.sh +++ b/iso/scripts/common_iso_generation.sh @@ -4,16 +4,32 @@ WORKDIR="/usr/src/app" ISO_BUILD_PATH="${WORKDIR}/dappnode-iso" DAPPNODE_ISO_PREFIX="Dappnode-" +# Usage: download_iso [fallback_url...] +# URLs are tried in order, which covers mirrors that move older releases elsewhere. download_iso() { local iso_path=$1 local iso_name=$2 - local iso_url=$3 + shift 2 echo "[INFO] Downloading base ISO image: ${iso_name}..." - if [ ! -f "${iso_path}" ]; then - wget "${iso_url}" -O "${iso_path}" + if [ -f "${iso_path}" ]; then + echo "[INFO] Download complete!" + return 0 fi - echo "[INFO] Download complete!" + + local iso_url + for iso_url in "$@"; do + if wget "${iso_url}" -O "${iso_path}"; then + echo "[INFO] Download complete!" + return 0 + fi + # wget leaves a partial file behind, which would be reused on the next run + rm -f "${iso_path}" + echo "[WARN] Could not download ${iso_url}" + done + + echo "[ERROR] Could not download ${iso_name} from any known location" + exit 1 } verify_download() { diff --git a/iso/scripts/generate_dappnode_iso_debian.sh b/iso/scripts/generate_dappnode_iso_debian.sh index 349aca07..e81e2a77 100755 --- a/iso/scripts/generate_dappnode_iso_debian.sh +++ b/iso/scripts/generate_dappnode_iso_debian.sh @@ -10,7 +10,10 @@ BASE_ISO_NAME="debian-13.5.0-amd64-netinst.iso" BASE_ISO_VERSION="${BASE_ISO_NAME#debian-}" BASE_ISO_VERSION="${BASE_ISO_VERSION%-amd64-netinst.iso}" BASE_ISO_PATH="/images/${BASE_ISO_NAME}" -BASE_ISO_URL="https://cdimage.debian.org/mirror/cdimage/archive/${BASE_ISO_VERSION}/amd64/iso-cd/${BASE_ISO_NAME}" +# Debian serves the newest point release from debian-cd/ and only moves it to the +# archive once it is superseded, so try both. +BASE_ISO_URL="https://cdimage.debian.org/debian-cd/${BASE_ISO_VERSION}/amd64/iso-cd/${BASE_ISO_NAME}" +BASE_ISO_ARCHIVE_URL="https://cdimage.debian.org/mirror/cdimage/archive/${BASE_ISO_VERSION}/amd64/iso-cd/${BASE_ISO_NAME}" BASE_ISO_SHASUM="95838884f5ea6c82421dfe6baaa5a639dbbe6756c1e380f9fe7a7cb0c1949d2a ${BASE_ISO_PATH}" DAPPNODE_ISO_NAME="${DAPPNODE_ISO_PREFIX}${BASE_ISO_NAME}" @@ -98,7 +101,7 @@ generate_debian_iso() { -isohybrid-gpt-basdat -o "${iso_output_path}" ${iso_build_path} } -download_iso "${BASE_ISO_PATH}" "${BASE_ISO_NAME}" "${BASE_ISO_URL}" +download_iso "${BASE_ISO_PATH}" "${BASE_ISO_NAME}" "${BASE_ISO_URL}" "${BASE_ISO_ARCHIVE_URL}" verify_download "${BASE_ISO_PATH}" "${BASE_ISO_SHASUM}" clean_old_files "${ISO_BUILD_PATH}" "${DAPPNODE_ISO_PREFIX}" extract_iso "${BASE_ISO_PATH}" "${ISO_BUILD_PATH}" From 705cf9fdb1c25143dba44bd92e4b0d65e6043f13 Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 8 Oct 2026 13:44:47 +0200 Subject: [PATCH 12/17] Keep the ISO E2E tests from changing installer logic Restore every installer, preseed and ISO-generation file to master so this PR only adds tests. The harness now reproduces real hardware instead of patching the installer around QEMU: - Install onto an NVMe disk rather than virtio-blk. - Boot the guest on a TAP network with DHCP, DNS and NAT. GitHub runners drop outbound ICMP, so the host answers the guest's echo requests and the first-boot `ping google.com` check behaves as on a normal LAN. - Read the expected Debian release from the generator and preseed instead of a new config file. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/debian-iso-e2e.yml | 12 +-- .github/workflows/ubuntu-iso-e2e.yml | 9 +- iso/debian_release.conf | 3 - iso/preseeds/preseed.cfg | 2 +- iso/preseeds/preseed_unattended.cfg | 2 +- iso/scripts/check_debian_release_alignment.sh | 58 ------------- iso/scripts/common_iso_generation.sh | 24 +----- iso/scripts/generate_dappnode_iso_debian.sh | 18 +--- scripts/check-disks.sh | 12 +-- scripts/dappnode_install.sh | 11 +-- scripts/dappnode_test_install.sh | 26 +----- test/e2e_debian_iso.sh | 20 ++++- test/e2e_iso_install.sh | 41 +++++++-- test/e2e_network.sh | 84 +++++++++++++++++++ 14 files changed, 156 insertions(+), 166 deletions(-) delete mode 100644 iso/debian_release.conf delete mode 100755 iso/scripts/check_debian_release_alignment.sh create mode 100755 test/e2e_network.sh diff --git a/.github/workflows/debian-iso-e2e.yml b/.github/workflows/debian-iso-e2e.yml index 1f7d9655..423291d2 100644 --- a/.github/workflows/debian-iso-e2e.yml +++ b/.github/workflows/debian-iso-e2e.yml @@ -23,9 +23,6 @@ jobs: - name: Checkout uses: actions/checkout@v6 - - name: Verify Debian release alignment - run: bash iso/scripts/check_debian_release_alignment.sh - - name: Build Debian unattended ISO run: | export BASE_OS=debian @@ -36,18 +33,21 @@ jobs: - name: Install QEMU test dependencies run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf socat sshpass + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf socat sshpass dnsmasq-base if [ -e /dev/kvm ]; then sudo chmod a+rw /dev/kvm fi - - name: Enable ICMP for QEMU user networking - run: sudo sysctl -w net.ipv4.ping_group_range="0 2147483647" + - name: Create the guest network + run: | + mkdir -p test-output/debian-e2e + sudo E2E_NETWORK_STATE_DIR="${PWD}/test-output/debian-e2e/network" bash test/e2e_network.sh up - name: Complete Debian unattended UEFI USB installation env: E2E_BOOT_MODE: usb E2E_FIRMWARE: uefi + E2E_TAP_IFACE: dne2e0 run: | debian_iso=$(find images -maxdepth 1 -type f -name 'Dappnode-debian-*-amd64-netinst.iso' -print -quit) test -n "${debian_iso}" diff --git a/.github/workflows/ubuntu-iso-e2e.yml b/.github/workflows/ubuntu-iso-e2e.yml index 49a105b7..5d038b61 100644 --- a/.github/workflows/ubuntu-iso-e2e.yml +++ b/.github/workflows/ubuntu-iso-e2e.yml @@ -33,18 +33,21 @@ jobs: - name: Install QEMU test dependencies run: | sudo apt-get update - sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf socat sshpass + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf socat sshpass dnsmasq-base if [ -e /dev/kvm ]; then sudo chmod a+rw /dev/kvm fi - - name: Enable ICMP for QEMU user networking - run: sudo sysctl -w net.ipv4.ping_group_range="0 2147483647" + - name: Create the guest network + run: | + mkdir -p test-output/ubuntu-e2e + sudo E2E_NETWORK_STATE_DIR="${PWD}/test-output/ubuntu-e2e/network" bash test/e2e_network.sh up - name: Complete Ubuntu unattended UEFI USB installation env: E2E_BOOT_MODE: usb E2E_FIRMWARE: uefi + E2E_TAP_IFACE: dne2e0 run: | ubuntu_iso=$(find images -maxdepth 1 -type f -name 'Dappnode-ubuntu-*-live-server-amd64.iso' -print -quit) test -n "${ubuntu_iso}" diff --git a/iso/debian_release.conf b/iso/debian_release.conf deleted file mode 100644 index 54e6b562..00000000 --- a/iso/debian_release.conf +++ /dev/null @@ -1,3 +0,0 @@ -# Debian release used by the ISO and its package repositories. -DEBIAN_SUITE="trixie" -DEBIAN_MAJOR_VERSION="13" diff --git a/iso/preseeds/preseed.cfg b/iso/preseeds/preseed.cfg index fd55caea..1bd38005 100644 --- a/iso/preseeds/preseed.cfg +++ b/iso/preseeds/preseed.cfg @@ -25,7 +25,7 @@ d-i mirror/country string manual d-i mirror/http/hostname string deb.debian.org d-i mirror/http/directory string /debian d-i mirror/http/proxy string -d-i apt-setup/local0/repository string http://deb.debian.org/debian/ @DEBIAN_SUITE@ main contrib non-free-firmware +d-i apt-setup/local0/repository string http://deb.debian.org/debian/ trixie main contrib non-free-firmware d-i apt-setup/cdrom/set-first boolean false d-i apt-setup/cdrom/set-next boolean false d-i apt-setup/cdrom/set-failed boolean false diff --git a/iso/preseeds/preseed_unattended.cfg b/iso/preseeds/preseed_unattended.cfg index 0521173e..8316fd7c 100644 --- a/iso/preseeds/preseed_unattended.cfg +++ b/iso/preseeds/preseed_unattended.cfg @@ -62,7 +62,7 @@ d-i mirror/country string manual d-i mirror/http/hostname string deb.debian.org d-i mirror/http/directory string /debian d-i mirror/http/proxy string -d-i apt-setup/local0/repository string http://deb.debian.org/debian/ @DEBIAN_SUITE@ main contrib non-free-firmware +d-i apt-setup/local0/repository string http://deb.debian.org/debian/ trixie main contrib non-free-firmware d-i apt-setup/cdrom/set-first boolean false d-i apt-setup/cdrom/set-next boolean false d-i apt-setup/cdrom/set-failed boolean false diff --git a/iso/scripts/check_debian_release_alignment.sh b/iso/scripts/check_debian_release_alignment.sh deleted file mode 100755 index 2f237584..00000000 --- a/iso/scripts/check_debian_release_alignment.sh +++ /dev/null @@ -1,58 +0,0 @@ -#!/bin/bash -set -euo pipefail - -SCRIPTS_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) -ISO_DIR=$(cd "${SCRIPTS_DIR}/.." && pwd) - -source "${ISO_DIR}/debian_release.conf" - -case "${DEBIAN_SUITE}" in - bullseye) suite_major_version="11" ;; - bookworm) suite_major_version="12" ;; - trixie) suite_major_version="13" ;; - forky) suite_major_version="14" ;; - *) - echo "[ERROR] Unknown Debian suite: ${DEBIAN_SUITE}" - exit 1 - ;; -esac - -if [ "${suite_major_version}" != "${DEBIAN_MAJOR_VERSION}" ]; then - echo "[ERROR] Debian suite ${DEBIAN_SUITE} is Debian ${suite_major_version}, not Debian ${DEBIAN_MAJOR_VERSION}" - exit 1 -fi - -generator="${SCRIPTS_DIR}/generate_dappnode_iso_debian.sh" -base_iso_name=$(sed -n 's/^BASE_ISO_NAME="\([^"]*\)"$/\1/p' "${generator}") - -if [[ ! "${base_iso_name}" =~ ^debian-([0-9]+)\. ]]; then - echo "[ERROR] Could not determine the Debian major version from BASE_ISO_NAME=${base_iso_name}" - exit 1 -fi - -iso_major_version="${BASH_REMATCH[1]}" -if [ "${iso_major_version}" != "${DEBIAN_MAJOR_VERSION}" ]; then - echo "[ERROR] Base ISO ${base_iso_name} is Debian ${iso_major_version}, but debian_release.conf expects Debian ${DEBIAN_MAJOR_VERSION} (${DEBIAN_SUITE})" - exit 1 -fi - -expected_repository="d-i apt-setup/local0/repository string http://deb.debian.org/debian/ @DEBIAN_SUITE@ main contrib non-free-firmware" -preseed_files=( - "${ISO_DIR}/preseeds/preseed.cfg" - "${ISO_DIR}/preseeds/preseed_unattended.cfg" -) - -for preseed_file in "${preseed_files[@]}"; do - repository_count=$(grep -Ec '^d-i apt-setup/local[0-9]+/repository string ' "${preseed_file}" || true) - if [ "${repository_count}" -ne 1 ]; then - echo "[ERROR] Expected exactly one additional Debian repository in ${preseed_file}, found ${repository_count}" - exit 1 - fi - - if ! grep -Fqx "${expected_repository}" "${preseed_file}"; then - echo "[ERROR] Debian repository in ${preseed_file} must use the @DEBIAN_SUITE@ placeholder" - exit 1 - fi -done - -echo "[INFO] Debian release alignment is valid: Debian ${DEBIAN_MAJOR_VERSION} (${DEBIAN_SUITE}), ${base_iso_name}" diff --git a/iso/scripts/common_iso_generation.sh b/iso/scripts/common_iso_generation.sh index 5306f958..1cd65a0c 100644 --- a/iso/scripts/common_iso_generation.sh +++ b/iso/scripts/common_iso_generation.sh @@ -4,32 +4,16 @@ WORKDIR="/usr/src/app" ISO_BUILD_PATH="${WORKDIR}/dappnode-iso" DAPPNODE_ISO_PREFIX="Dappnode-" -# Usage: download_iso [fallback_url...] -# URLs are tried in order, which covers mirrors that move older releases elsewhere. download_iso() { local iso_path=$1 local iso_name=$2 - shift 2 + local iso_url=$3 echo "[INFO] Downloading base ISO image: ${iso_name}..." - if [ -f "${iso_path}" ]; then - echo "[INFO] Download complete!" - return 0 + if [ ! -f "${iso_path}" ]; then + wget "${iso_url}" -O "${iso_path}" fi - - local iso_url - for iso_url in "$@"; do - if wget "${iso_url}" -O "${iso_path}"; then - echo "[INFO] Download complete!" - return 0 - fi - # wget leaves a partial file behind, which would be reused on the next run - rm -f "${iso_path}" - echo "[WARN] Could not download ${iso_url}" - done - - echo "[ERROR] Could not download ${iso_name} from any known location" - exit 1 + echo "[INFO] Download complete!" } verify_download() { diff --git a/iso/scripts/generate_dappnode_iso_debian.sh b/iso/scripts/generate_dappnode_iso_debian.sh index e81e2a77..4ec4a93d 100755 --- a/iso/scripts/generate_dappnode_iso_debian.sh +++ b/iso/scripts/generate_dappnode_iso_debian.sh @@ -4,24 +4,17 @@ set -e SCRIPTS_DIR=$(dirname "${BASH_SOURCE[0]}") source ${SCRIPTS_DIR}/common_iso_generation.sh -source ${SCRIPTS_DIR}/../debian_release.conf BASE_ISO_NAME="debian-13.5.0-amd64-netinst.iso" BASE_ISO_VERSION="${BASE_ISO_NAME#debian-}" BASE_ISO_VERSION="${BASE_ISO_VERSION%-amd64-netinst.iso}" BASE_ISO_PATH="/images/${BASE_ISO_NAME}" -# Debian serves the newest point release from debian-cd/ and only moves it to the -# archive once it is superseded, so try both. -BASE_ISO_URL="https://cdimage.debian.org/debian-cd/${BASE_ISO_VERSION}/amd64/iso-cd/${BASE_ISO_NAME}" -BASE_ISO_ARCHIVE_URL="https://cdimage.debian.org/mirror/cdimage/archive/${BASE_ISO_VERSION}/amd64/iso-cd/${BASE_ISO_NAME}" +BASE_ISO_URL="https://cdimage.debian.org/mirror/cdimage/archive/${BASE_ISO_VERSION}/amd64/iso-cd/${BASE_ISO_NAME}" BASE_ISO_SHASUM="95838884f5ea6c82421dfe6baaa5a639dbbe6756c1e380f9fe7a7cb0c1949d2a ${BASE_ISO_PATH}" DAPPNODE_ISO_NAME="${DAPPNODE_ISO_PREFIX}${BASE_ISO_NAME}" DAPPNODE_ISO_PATH="/images/${DAPPNODE_ISO_NAME}" -# Fail before downloading or building if the ISO and repositories target different releases. -bash "${SCRIPTS_DIR}/check_debian_release_alignment.sh" - customize_debian_preseed() { local iso_build_path=$1 local workdir=$2 @@ -57,13 +50,6 @@ customize_debian_preseed() { exit 1 } - # Keep package repositories aligned with the Debian release used by the ISO. - sed -i "s/@DEBIAN_SUITE@/${DEBIAN_SUITE}/g" "${tmp_initrd}/preseed.cfg" - if grep -q "@DEBIAN_SUITE@" "${tmp_initrd}/preseed.cfg"; then - echo "[ERROR] Could not render the Debian suite in the preseed file" - exit 1 - fi - # Recreate (and recompress) the initrd (cd "${tmp_initrd}" && find . -print0 | cpio -0 -ov -H newc | gzip >"${install_dir}/initrd.gz") || { echo "[ERROR] Could not create new initrd" @@ -101,7 +87,7 @@ generate_debian_iso() { -isohybrid-gpt-basdat -o "${iso_output_path}" ${iso_build_path} } -download_iso "${BASE_ISO_PATH}" "${BASE_ISO_NAME}" "${BASE_ISO_URL}" "${BASE_ISO_ARCHIVE_URL}" +download_iso "${BASE_ISO_PATH}" "${BASE_ISO_NAME}" "${BASE_ISO_URL}" verify_download "${BASE_ISO_PATH}" "${BASE_ISO_SHASUM}" clean_old_files "${ISO_BUILD_PATH}" "${DAPPNODE_ISO_PREFIX}" extract_iso "${BASE_ISO_PATH}" "${ISO_BUILD_PATH}" diff --git a/scripts/check-disks.sh b/scripts/check-disks.sh index 72070d31..d7385c41 100755 --- a/scripts/check-disks.sh +++ b/scripts/check-disks.sh @@ -12,14 +12,4 @@ for DISK in ${DEVICE}; do DISKS="${DISKS} ${DISK}"; done; DISKS=$(echo "${DISKS}" | sed "s/^ //g"); -debconf-set partman-auto/disk "$DISKS"; - -# grub-installer/bootdev is preseeded to "default", which marks the question as -# already seen. grub-installer then resolves "default" to the first entry of -# grub-mkdevicemap, i.e. (hd0) -- the installer USB itself. Its safeguard against -# installing onto the installation media only runs for grub-pc, so on UEFI it -# happily targets the stick and then dies reading its partition table: -# Can't read partition table from /dev/sda -# Point it at the same non-USB disks partman is using. grub-installer accepts a -# space separated list and installs to each entry in turn. -debconf-set grub-installer/bootdev "$DISKS"; +debconf-set partman-auto/disk "$DISKS"; \ No newline at end of file diff --git a/scripts/dappnode_install.sh b/scripts/dappnode_install.sh index ed97b853..b9519bc1 100755 --- a/scripts/dappnode_install.sh +++ b/scripts/dappnode_install.sh @@ -428,16 +428,7 @@ normalize_ipfs_version_ref() { local cid_path="$ref" local manifest_url="${IPFS_ENDPOINT%/}${cid_path}/dappnode_package.json" local manifest - local attempt - for attempt in 1 2 3 4 5; do - manifest="$(download_stdout "$manifest_url" 2>/dev/null || true)" - if [[ -n "$manifest" ]]; then - break - fi - if [[ "$attempt" -lt 5 ]]; then - sleep 2 - fi - done + manifest="$(download_stdout "$manifest_url" 2>/dev/null || true)" if [[ -z "$manifest" ]]; then error "Could not fetch IPFS manifest for ${comp} from: $manifest_url" error "Provide ${comp}_VERSION as /ipfs/: (example: /ipfs/Qm...:0.2.11)" diff --git a/scripts/dappnode_test_install.sh b/scripts/dappnode_test_install.sh index 21f193d8..4d5d704d 100755 --- a/scripts/dappnode_test_install.sh +++ b/scripts/dappnode_test_install.sh @@ -22,33 +22,9 @@ detect_installation_type() { fi } -# ICMP is filtered on many corporate, hotel and cloud networks, so a failed -# ping alone does not mean the machine is offline. Confirm over HTTPS before -# declaring the installation broken. -check_connectivity() { - ping -c 1 -q google.com >/dev/null 2>&1 && return 0 - - if command -v curl >/dev/null 2>&1; then - curl -fsS --max-time 10 -o /dev/null https://www.google.com/generate_204 && return 0 - fi - - if command -v wget >/dev/null 2>&1; then - wget -q --timeout=10 -O /dev/null https://www.google.com/generate_204 && return 0 - fi - - # Neither downloader is guaranteed to be present this early, so fall back to - # a plain TCP connect, which bash can do on its own. - if (exec 3<>/dev/tcp/www.google.com/443) 2>/dev/null; then - exec 3<&- 3>&- - return 0 - fi - - return 1 -} - components=(BIND IPFS VPN DAPPMANAGER WIFI) detect_installation_type -if check_connectivity; then +if ping -c 1 -q google.com >&/dev/null; then echo -e "\e[32m Connectivity OK\n \e[0m" else error_exit diff --git a/test/e2e_debian_iso.sh b/test/e2e_debian_iso.sh index 9f9c2393..b1388dd8 100755 --- a/test/e2e_debian_iso.sh +++ b/test/e2e_debian_iso.sh @@ -5,13 +5,27 @@ set -Eeuo pipefail SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) REPO_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) +GENERATOR="${REPO_ROOT}/iso/scripts/generate_dappnode_iso_debian.sh" +PRESEED="${REPO_ROOT}/iso/preseeds/preseed_unattended.cfg" -source "${REPO_ROOT}/iso/debian_release.conf" +base_iso_name=$(sed -n 's/^BASE_ISO_NAME="\{0,1\}\([^"[:space:]]*\)"\{0,1\}$/\1/p' "${GENERATOR}") +if [[ ! "${base_iso_name}" =~ ^debian-([0-9]+)\. ]]; then + echo "[ERROR] Could not determine the Debian release from BASE_ISO_NAME=${base_iso_name}" + exit 1 +fi +debian_major_version=${BASH_REMATCH[1]} + +# The installed system must track the suite the preseed configures for APT. +debian_suite=$(sed -n 's|^d-i apt-setup/local0/repository string http://deb\.debian\.org/debian/ \([a-z]*\) .*|\1|p' "${PRESEED}") +if [ -z "${debian_suite}" ]; then + echo "[ERROR] Could not determine the Debian suite from ${PRESEED}" + exit 1 +fi E2E_DISTRO="debian" E2E_EXPECTED_ID="debian" -E2E_EXPECTED_VERSION="${DEBIAN_MAJOR_VERSION}" -E2E_EXPECTED_CODENAME="${DEBIAN_SUITE}" +E2E_EXPECTED_VERSION="${debian_major_version}" +E2E_EXPECTED_CODENAME="${debian_suite}" source "${SCRIPT_DIR}/e2e_iso_install.sh" run_e2e_iso_install "$@" diff --git a/test/e2e_iso_install.sh b/test/e2e_iso_install.sh index c0f80fea..98df0d66 100755 --- a/test/e2e_iso_install.sh +++ b/test/e2e_iso_install.sh @@ -197,10 +197,35 @@ run_e2e_iso_install() { echo "[INFO] Creating ${vm_disk_size} virtual installation disk" qemu-img create -q -f qcow2 "${disk_path}" "${vm_disk_size}" + # NVMe matches the disks DAppNode hardware ships with, including how the + # installer enumerates it next to the USB stick. local target_disk_install_args=( -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" - -device "virtio-blk-pci,drive=target_disk,bootindex=2" + -device "nvme,serial=dappnode-e2e,drive=target_disk,bootindex=2" ) + local target_disk_boot_args=( + -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" + -device "nvme,serial=dappnode-e2e,drive=target_disk,bootindex=1" + ) + + # On a TAP network (see test/e2e_network.sh) the guest sees a regular LAN, + # including replies to ping. User networking needs no root but drops ICMP. + local installer_network_args guest_network_args ssh_host + if [ -n "${tap_iface}" ]; then + installer_network_args=( + -netdev "tap,id=net0,ifname=${tap_iface},script=no,downscript=no" + -device "virtio-net-pci,netdev=net0,mac=${guest_mac}" + ) + guest_network_args=("${installer_network_args[@]}") + ssh_host=${guest_ip} + ssh_port=22 + echo "[INFO] Using TAP network ${tap_iface}; guest address ${guest_ip}" + else + installer_network_args=(-nic user,model=virtio-net-pci) + guest_network_args=(-nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${ssh_port}-:22") + ssh_host=127.0.0.1 + echo "[WARN] E2E_TAP_IFACE is unset; using user networking, where the guest cannot ping" + fi local installer_media_args=( -device "qemu-xhci,id=installer_xhci" -drive "file=${iso_path},format=raw,if=none,readonly=on,id=installer_media" @@ -216,7 +241,7 @@ run_e2e_iso_install() { "${target_disk_install_args[@]}" \ "${installer_media_args[@]}" \ -boot menu=off \ - -nic user,model=virtio-net-pci \ + "${installer_network_args[@]}" \ -display none \ -monitor "unix:${installer_monitor_socket},server=on,wait=off" \ -serial "file:${installer_serial_log}" \ @@ -236,10 +261,9 @@ run_e2e_iso_install() { "${firmware_args[@]}" \ -m "${vm_memory_mb}" \ -smp "${vm_cpus}" \ - -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" \ - -device "virtio-blk-pci,drive=target_disk,bootindex=1" \ + "${target_disk_boot_args[@]}" \ -boot menu=off \ - -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ + "${guest_network_args[@]}" \ -display none \ -monitor "unix:${first_boot_monitor_socket},server=on,wait=off" \ -serial "file:${first_boot_serial_log}" \ @@ -257,7 +281,7 @@ run_e2e_iso_install() { ) ssh_guest() { - SSHPASS="${ssh_password}" sshpass -e ssh "${ssh_options[@]}" dappnode@127.0.0.1 "$@" + SSHPASS="${ssh_password}" sshpass -e ssh "${ssh_options[@]}" "dappnode@${ssh_host}" "$@" } wait_for_ssh() { @@ -338,10 +362,9 @@ run_e2e_iso_install() { "${firmware_args[@]}" \ -m "${vm_memory_mb}" \ -smp "${vm_cpus}" \ - -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" \ - -device "virtio-blk-pci,drive=target_disk,bootindex=1" \ + "${target_disk_boot_args[@]}" \ -boot menu=off \ - -nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${ssh_port}-:22" \ + "${guest_network_args[@]}" \ -display none \ -monitor none \ -serial "file:${system_serial_log}" \ diff --git a/test/e2e_network.sh b/test/e2e_network.sh new file mode 100755 index 00000000..5e3c5beb --- /dev/null +++ b/test/e2e_network.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Creates the TAP network the E2E guests boot on. Run as root: up | down +# The guest gets DHCP, DNS and NAT like on a home LAN. CI providers drop outbound +# ICMP, so echo requests from the guest are answered by this host instead: the +# DAppNode first-boot check pings google.com and must see a reply. +set -Eeuo pipefail + +E2E_TAP_IFACE=${E2E_TAP_IFACE:-dne2e0} +E2E_HOST_IP=${E2E_HOST_IP:-192.168.77.1} +E2E_GUEST_IP=${E2E_GUEST_IP:-192.168.77.10} +E2E_GUEST_MAC=${E2E_GUEST_MAC:-52:54:00:d4:00:10} +SUBNET="${E2E_HOST_IP%.*}.0/24" +STATE_DIR=${E2E_NETWORK_STATE_DIR:-/run/dappnode-e2e} + +if [ "$(id -u)" -ne 0 ]; then + echo "[ERROR] $0 must run as root" + exit 1 +fi + +tap_owner=${SUDO_USER:-root} + +network_up() { + mkdir -p "${STATE_DIR}" + + ip tuntap add dev "${E2E_TAP_IFACE}" mode tap user "${tap_owner}" + ip addr add "${E2E_HOST_IP}/24" dev "${E2E_TAP_IFACE}" + ip link set "${E2E_TAP_IFACE}" up + + sysctl -q -w net.ipv4.ip_forward=1 + iptables -t nat -A POSTROUTING -s "${SUBNET}" ! -o "${E2E_TAP_IFACE}" -j MASQUERADE + iptables -t nat -A PREROUTING -i "${E2E_TAP_IFACE}" -p icmp --icmp-type echo-request \ + -j DNAT --to-destination "${E2E_HOST_IP}" + # Docker sets the FORWARD policy to DROP, so allow the guest explicitly. + iptables -I FORWARD 1 -i "${E2E_TAP_IFACE}" -j ACCEPT + iptables -I FORWARD 1 -o "${E2E_TAP_IFACE}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + + local upstream=(--resolv-file=/etc/resolv.conf) + if [ -f /run/systemd/resolve/resolv.conf ]; then + # The stub at 127.0.0.53 is fine for the host, but read the real upstreams. + upstream=(--resolv-file=/run/systemd/resolve/resolv.conf) + fi + + # Stay root: the state directory may live in a workspace other users cannot enter. + dnsmasq \ + --interface="${E2E_TAP_IFACE}" \ + --bind-interfaces \ + --except-interface=lo \ + --listen-address="${E2E_HOST_IP}" \ + --dhcp-range="${E2E_GUEST_IP},${E2E_GUEST_IP},255.255.255.0,12h" \ + --dhcp-host="${E2E_GUEST_MAC},${E2E_GUEST_IP}" \ + --dhcp-option=option:router,"${E2E_HOST_IP}" \ + --dhcp-option=option:dns-server,"${E2E_HOST_IP}" \ + --dhcp-leasefile="${STATE_DIR}/dnsmasq.leases" \ + --pid-file="${STATE_DIR}/dnsmasq.pid" \ + --log-facility="${STATE_DIR}/dnsmasq.log" \ + --log-dhcp \ + --no-hosts \ + --user=root \ + "${upstream[@]}" + + echo "[INFO] Guest network ${E2E_TAP_IFACE} is up: host ${E2E_HOST_IP}, guest ${E2E_GUEST_IP} (${E2E_GUEST_MAC})" +} + +network_down() { + if [ -f "${STATE_DIR}/dnsmasq.pid" ]; then + kill "$(cat "${STATE_DIR}/dnsmasq.pid")" 2>/dev/null || true + fi + iptables -D FORWARD -o "${E2E_TAP_IFACE}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true + iptables -D FORWARD -i "${E2E_TAP_IFACE}" -j ACCEPT 2>/dev/null || true + iptables -t nat -D PREROUTING -i "${E2E_TAP_IFACE}" -p icmp --icmp-type echo-request \ + -j DNAT --to-destination "${E2E_HOST_IP}" 2>/dev/null || true + iptables -t nat -D POSTROUTING -s "${SUBNET}" ! -o "${E2E_TAP_IFACE}" -j MASQUERADE 2>/dev/null || true + ip link delete "${E2E_TAP_IFACE}" 2>/dev/null || true + rm -rf "${STATE_DIR}" +} + +case "${1:-}" in + up) network_up ;; + down) network_down ;; + *) + echo "Usage: $0 up|down" + exit 2 + ;; +esac From a7db0a555575fa87c6a5d7eff4e02e35962f7398 Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 8 Oct 2026 13:48:20 +0200 Subject: [PATCH 13/17] Declare the TAP settings in the E2E harness and upload network logs Co-Authored-By: Claude Opus 5.5 --- .github/workflows/debian-iso-e2e.yml | 4 ++++ .github/workflows/ubuntu-iso-e2e.yml | 4 ++++ test/e2e_iso_install.sh | 3 +++ 3 files changed, 11 insertions(+) diff --git a/.github/workflows/debian-iso-e2e.yml b/.github/workflows/debian-iso-e2e.yml index 423291d2..6464e82b 100644 --- a/.github/workflows/debian-iso-e2e.yml +++ b/.github/workflows/debian-iso-e2e.yml @@ -53,6 +53,10 @@ jobs: test -n "${debian_iso}" bash test/e2e_debian_iso.sh "${debian_iso}" + - name: Make network logs readable + if: always() + run: sudo chmod -R a+rX test-output || true + - name: Upload installation logs if: always() uses: actions/upload-artifact@v4 diff --git a/.github/workflows/ubuntu-iso-e2e.yml b/.github/workflows/ubuntu-iso-e2e.yml index 5d038b61..f257928a 100644 --- a/.github/workflows/ubuntu-iso-e2e.yml +++ b/.github/workflows/ubuntu-iso-e2e.yml @@ -53,6 +53,10 @@ jobs: test -n "${ubuntu_iso}" bash test/e2e_ubuntu_iso.sh "${ubuntu_iso}" + - name: Make network logs readable + if: always() + run: sudo chmod -R a+rX test-output || true + - name: Upload installation logs if: always() uses: actions/upload-artifact@v4 diff --git a/test/e2e_iso_install.sh b/test/e2e_iso_install.sh index 98df0d66..02973926 100755 --- a/test/e2e_iso_install.sh +++ b/test/e2e_iso_install.sh @@ -53,6 +53,9 @@ run_e2e_iso_install() { local postinstall_timeout_seconds=${E2E_POSTINSTALL_TIMEOUT_SECONDS:-1200} local ssh_port=${E2E_SSH_PORT:-2222} local ssh_password=${E2E_SSH_PASSWORD:-dappnode.s0} + local tap_iface=${E2E_TAP_IFACE:-} + local guest_ip=${E2E_GUEST_IP:-192.168.77.10} + local guest_mac=${E2E_GUEST_MAC:-52:54:00:d4:00:10} local boot_mode=${E2E_BOOT_MODE:-usb} local firmware=${E2E_FIRMWARE:-uefi} qemu_pid="" From b2198d774d1223c703373dea937c9b15bf150d07 Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 8 Oct 2026 13:55:04 +0200 Subject: [PATCH 14/17] Retry the ISO build in the E2E workflows cdimage.debian.org currently resolves to a dead mirror address part of the time, and one bad DNS answer should not fail a 15 minute E2E run. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/debian-iso-e2e.yml | 8 +++++++- .github/workflows/ubuntu-iso-e2e.yml | 8 +++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/.github/workflows/debian-iso-e2e.yml b/.github/workflows/debian-iso-e2e.yml index 6464e82b..5be039c2 100644 --- a/.github/workflows/debian-iso-e2e.yml +++ b/.github/workflows/debian-iso-e2e.yml @@ -27,7 +27,13 @@ jobs: run: | export BASE_OS=debian export UNATTENDED=true - docker compose up --build + # Retry so a transient mirror outage does not fail the whole E2E run. + for attempt in 1 2 3; do + docker compose up --build --abort-on-container-exit --exit-code-from dappnode_installer && break + [ "${attempt}" -lt 3 ] || exit 1 + echo "::warning::ISO build attempt ${attempt} failed; retrying in 30s" + sleep 30 + done ls -lrt images/Dappnode-debian-*.iso - name: Install QEMU test dependencies diff --git a/.github/workflows/ubuntu-iso-e2e.yml b/.github/workflows/ubuntu-iso-e2e.yml index f257928a..2e4eb70f 100644 --- a/.github/workflows/ubuntu-iso-e2e.yml +++ b/.github/workflows/ubuntu-iso-e2e.yml @@ -27,7 +27,13 @@ jobs: run: | export BASE_OS=ubuntu export UNATTENDED=true - docker compose up --build + # Retry so a transient mirror outage does not fail the whole E2E run. + for attempt in 1 2 3; do + docker compose up --build --abort-on-container-exit --exit-code-from dappnode_installer && break + [ "${attempt}" -lt 3 ] || exit 1 + echo "::warning::ISO build attempt ${attempt} failed; retrying in 30s" + sleep 30 + done ls -lrt images/Dappnode-ubuntu-*.iso - name: Install QEMU test dependencies From b2e5c215a1c76f4639934f0b986d8abbab5acd4a Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 8 Oct 2026 14:05:51 +0200 Subject: [PATCH 15/17] Keep the guest NIC name stable across E2E boots The installer VM has a USB controller that later boots do not, which shifted the NIC to another PCI slot and renamed it. Debian's /etc/network/interfaces names the interface, so the installed system never brought up its network. Pin the NIC's PCI address like real hardware, where removing the USB stick renames nothing. Co-Authored-By: Claude Opus 5.5 --- test/e2e_iso_install.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/test/e2e_iso_install.sh b/test/e2e_iso_install.sh index 02973926..b0f28716 100755 --- a/test/e2e_iso_install.sh +++ b/test/e2e_iso_install.sh @@ -215,9 +215,12 @@ run_e2e_iso_install() { # including replies to ping. User networking needs no root but drops ICMP. local installer_network_args guest_network_args ssh_host if [ -n "${tap_iface}" ]; then + # Pin the PCI slot: the USB controller only exists while installing, and + # without a fixed address the NIC name would change after the install + # (enp0s5 -> enp0s4), unlike on real hardware. installer_network_args=( -netdev "tap,id=net0,ifname=${tap_iface},script=no,downscript=no" - -device "virtio-net-pci,netdev=net0,mac=${guest_mac}" + -device "virtio-net-pci,netdev=net0,mac=${guest_mac},addr=0x10" ) guest_network_args=("${installer_network_args[@]}") ssh_host=${guest_ip} From cf48fa80c172a8b7cc460308300bc56fb73d2f0a Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 8 Oct 2026 15:57:45 +0200 Subject: [PATCH 16/17] Fail stuck ISO E2E runs sooner A hung installer used to burn up to 60 minutes, and a stuck first boot another 30, before failing. Cut the limits to a few times a healthy run (install 30 min, first-boot test 15 min, core services 10 min) and the job timeout from 180 to 90 minutes, which still covers the worst case of every phase plus the retried ISO build. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/debian-iso-e2e.yml | 2 +- .github/workflows/ubuntu-iso-e2e.yml | 2 +- test/e2e_iso_install.sh | 9 ++++++--- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.github/workflows/debian-iso-e2e.yml b/.github/workflows/debian-iso-e2e.yml index 5be039c2..09abe855 100644 --- a/.github/workflows/debian-iso-e2e.yml +++ b/.github/workflows/debian-iso-e2e.yml @@ -17,7 +17,7 @@ jobs: install: name: Install Debian unattended ISO via UEFI USB runs-on: ubuntu-latest - timeout-minutes: 180 + timeout-minutes: 90 steps: - name: Checkout diff --git a/.github/workflows/ubuntu-iso-e2e.yml b/.github/workflows/ubuntu-iso-e2e.yml index 2e4eb70f..65bc9b66 100644 --- a/.github/workflows/ubuntu-iso-e2e.yml +++ b/.github/workflows/ubuntu-iso-e2e.yml @@ -17,7 +17,7 @@ jobs: install: name: Install Ubuntu unattended ISO via UEFI USB runs-on: ubuntu-latest - timeout-minutes: 180 + timeout-minutes: 90 steps: - name: Checkout diff --git a/test/e2e_iso_install.sh b/test/e2e_iso_install.sh index b0f28716..f993ba6c 100755 --- a/test/e2e_iso_install.sh +++ b/test/e2e_iso_install.sh @@ -47,10 +47,13 @@ run_e2e_iso_install() { local vm_disk_size=${E2E_VM_DISK_SIZE:-32G} local vm_memory_mb=${E2E_VM_MEMORY_MB:-4096} local vm_cpus=${E2E_VM_CPUS:-2} - local install_timeout_seconds=${E2E_INSTALL_TIMEOUT_SECONDS:-3600} + # Healthy runs take 4-8 min to install, 1-3 min for the first-boot test and + # well under a minute to start core services. A stuck installer waits on a + # dialog forever, so keep the limits a few times the normal duration. + local install_timeout_seconds=${E2E_INSTALL_TIMEOUT_SECONDS:-1800} local boot_timeout_seconds=${E2E_BOOT_TIMEOUT_SECONDS:-600} - local first_boot_timeout_seconds=${E2E_FIRST_BOOT_TIMEOUT_SECONDS:-1800} - local postinstall_timeout_seconds=${E2E_POSTINSTALL_TIMEOUT_SECONDS:-1200} + local first_boot_timeout_seconds=${E2E_FIRST_BOOT_TIMEOUT_SECONDS:-900} + local postinstall_timeout_seconds=${E2E_POSTINSTALL_TIMEOUT_SECONDS:-600} local ssh_port=${E2E_SSH_PORT:-2222} local ssh_password=${E2E_SSH_PASSWORD:-dappnode.s0} local tap_iface=${E2E_TAP_IFACE:-} From 4744bdd403ce6425db4d20cb75e528311f6e1d57 Mon Sep 17 00:00:00 2001 From: Marketen Date: Thu, 8 Oct 2026 15:59:28 +0200 Subject: [PATCH 17/17] Run the ISO E2E tests every Monday at 07:00 Madrid time Most of an ISO install comes from live upstream repositories, so master can break without any PR. A weekly run against master catches that. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/debian-iso-e2e.yml | 4 ++++ .github/workflows/ubuntu-iso-e2e.yml | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/.github/workflows/debian-iso-e2e.yml b/.github/workflows/debian-iso-e2e.yml index 09abe855..70ad716d 100644 --- a/.github/workflows/debian-iso-e2e.yml +++ b/.github/workflows/debian-iso-e2e.yml @@ -5,6 +5,10 @@ name: Debian unattended USB ISO end-to-end on: workflow_dispatch: pull_request: + # Weekly run against master to catch breakage from upstream package changes. + schedule: + - cron: "0 7 * * 1" + timezone: "Europe/Madrid" permissions: contents: read diff --git a/.github/workflows/ubuntu-iso-e2e.yml b/.github/workflows/ubuntu-iso-e2e.yml index 65bc9b66..5b034da1 100644 --- a/.github/workflows/ubuntu-iso-e2e.yml +++ b/.github/workflows/ubuntu-iso-e2e.yml @@ -5,6 +5,10 @@ name: Ubuntu unattended USB ISO end-to-end on: workflow_dispatch: pull_request: + # Weekly run against master to catch breakage from upstream package changes. + schedule: + - cron: "0 7 * * 1" + timezone: "Europe/Madrid" permissions: contents: read