diff --git a/.github/workflows/debian-iso-e2e.yml b/.github/workflows/debian-iso-e2e.yml new file mode 100644 index 00000000..70ad716d --- /dev/null +++ b/.github/workflows/debian-iso-e2e.yml @@ -0,0 +1,77 @@ +# Builds the unattended Debian ISO and installs it from UEFI USB media in QEMU. +# It completes the required reboots and verifies the installed DAppNode system. +name: Debian unattended USB ISO end-to-end + +on: + workflow_dispatch: + pull_request: + # Weekly run against master to catch breakage from upstream package changes. + schedule: + - cron: "0 7 * * 1" + timezone: "Europe/Madrid" + +permissions: + contents: read + +concurrency: + group: debian-iso-e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + install: + name: Install Debian unattended ISO via UEFI USB + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Build Debian unattended ISO + run: | + export BASE_OS=debian + export UNATTENDED=true + # Retry so a transient mirror outage does not fail the whole E2E run. + for attempt in 1 2 3; do + docker compose up --build --abort-on-container-exit --exit-code-from dappnode_installer && break + [ "${attempt}" -lt 3 ] || exit 1 + echo "::warning::ISO build attempt ${attempt} failed; retrying in 30s" + sleep 30 + done + ls -lrt images/Dappnode-debian-*.iso + + - name: Install QEMU test dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf socat sshpass dnsmasq-base + if [ -e /dev/kvm ]; then + sudo chmod a+rw /dev/kvm + fi + + - name: Create the guest network + run: | + mkdir -p test-output/debian-e2e + sudo E2E_NETWORK_STATE_DIR="${PWD}/test-output/debian-e2e/network" bash test/e2e_network.sh up + + - name: Complete Debian unattended UEFI USB installation + env: + E2E_BOOT_MODE: usb + E2E_FIRMWARE: uefi + E2E_TAP_IFACE: dne2e0 + run: | + debian_iso=$(find images -maxdepth 1 -type f -name 'Dappnode-debian-*-amd64-netinst.iso' -print -quit) + test -n "${debian_iso}" + bash test/e2e_debian_iso.sh "${debian_iso}" + + - name: Make network logs readable + if: always() + run: sudo chmod -R a+rX test-output || true + + - name: Upload installation logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: debian-unattended-usb-installation-logs + path: test-output/debian-e2e/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index deb5465a..45a09ec9 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,37 +30,3 @@ jobs: - name: Uninstall DAppNode run: | sudo /bin/bash ./scripts/dappnode_uninstall.sh y - - iso: - name: test Debian and Ubuntu ISO - runs-on: ubuntu-latest - - steps: - - name: Checkout - uses: actions/checkout@v6 - - # Create Debian ISO - - name: create Debian ISO - run: | - export BASE_OS=debian - export UNATTENDED=true - docker compose up --build - ls images/ - - # Verify Debian ISO creation - - name: verify Debian ISO - run: | - ls -lrt images/Dappnode-debian-*.iso - - # Create Ubuntu ISO - - name: create Ubuntu ISO - run: | - export BASE_OS=ubuntu - export UNATTENDED=true - docker compose up --build - ls images/ - - # Verify Ubuntu ISO creation - - name: verify Ubuntu ISO - run: | - ls -lrt images/Dappnode-ubuntu-*.iso diff --git a/.github/workflows/ubuntu-iso-e2e.yml b/.github/workflows/ubuntu-iso-e2e.yml new file mode 100644 index 00000000..5b034da1 --- /dev/null +++ b/.github/workflows/ubuntu-iso-e2e.yml @@ -0,0 +1,77 @@ +# Builds the unattended Ubuntu ISO and installs it from UEFI USB media in QEMU. +# It completes the required reboots and verifies the installed DAppNode system. +name: Ubuntu unattended USB ISO end-to-end + +on: + workflow_dispatch: + pull_request: + # Weekly run against master to catch breakage from upstream package changes. + schedule: + - cron: "0 7 * * 1" + timezone: "Europe/Madrid" + +permissions: + contents: read + +concurrency: + group: ubuntu-iso-e2e-${{ github.ref }} + cancel-in-progress: true + +jobs: + install: + name: Install Ubuntu unattended ISO via UEFI USB + runs-on: ubuntu-latest + timeout-minutes: 90 + + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Build Ubuntu unattended ISO + run: | + export BASE_OS=ubuntu + export UNATTENDED=true + # Retry so a transient mirror outage does not fail the whole E2E run. + for attempt in 1 2 3; do + docker compose up --build --abort-on-container-exit --exit-code-from dappnode_installer && break + [ "${attempt}" -lt 3 ] || exit 1 + echo "::warning::ISO build attempt ${attempt} failed; retrying in 30s" + sleep 30 + done + ls -lrt images/Dappnode-ubuntu-*.iso + + - name: Install QEMU test dependencies + run: | + sudo apt-get update + sudo apt-get install -y --no-install-recommends qemu-system-x86 qemu-utils ovmf socat sshpass dnsmasq-base + if [ -e /dev/kvm ]; then + sudo chmod a+rw /dev/kvm + fi + + - name: Create the guest network + run: | + mkdir -p test-output/ubuntu-e2e + sudo E2E_NETWORK_STATE_DIR="${PWD}/test-output/ubuntu-e2e/network" bash test/e2e_network.sh up + + - name: Complete Ubuntu unattended UEFI USB installation + env: + E2E_BOOT_MODE: usb + E2E_FIRMWARE: uefi + E2E_TAP_IFACE: dne2e0 + run: | + ubuntu_iso=$(find images -maxdepth 1 -type f -name 'Dappnode-ubuntu-*-live-server-amd64.iso' -print -quit) + test -n "${ubuntu_iso}" + bash test/e2e_ubuntu_iso.sh "${ubuntu_iso}" + + - name: Make network logs readable + if: always() + run: sudo chmod -R a+rX test-output || true + + - name: Upload installation logs + if: always() + uses: actions/upload-artifact@v4 + with: + name: ubuntu-unattended-usb-installation-logs + path: test-output/ubuntu-e2e/ + if-no-files-found: ignore + retention-days: 7 diff --git a/.gitignore b/.gitignore index ed3c1e18..15ec4cc3 100644 --- a/.gitignore +++ b/.gitignore @@ -1,2 +1,3 @@ docs/_build -images \ No newline at end of file +images +test-output/ diff --git a/test/e2e_debian_iso.sh b/test/e2e_debian_iso.sh new file mode 100755 index 00000000..b1388dd8 --- /dev/null +++ b/test/e2e_debian_iso.sh @@ -0,0 +1,31 @@ +#!/bin/bash +# Runs the generated unattended Debian ISO through the complete UEFI USB test. +# It adds Debian release and package-alignment checks to the shared harness. +set -Eeuo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) +GENERATOR="${REPO_ROOT}/iso/scripts/generate_dappnode_iso_debian.sh" +PRESEED="${REPO_ROOT}/iso/preseeds/preseed_unattended.cfg" + +base_iso_name=$(sed -n 's/^BASE_ISO_NAME="\{0,1\}\([^"[:space:]]*\)"\{0,1\}$/\1/p' "${GENERATOR}") +if [[ ! "${base_iso_name}" =~ ^debian-([0-9]+)\. ]]; then + echo "[ERROR] Could not determine the Debian release from BASE_ISO_NAME=${base_iso_name}" + exit 1 +fi +debian_major_version=${BASH_REMATCH[1]} + +# The installed system must track the suite the preseed configures for APT. +debian_suite=$(sed -n 's|^d-i apt-setup/local0/repository string http://deb\.debian\.org/debian/ \([a-z]*\) .*|\1|p' "${PRESEED}") +if [ -z "${debian_suite}" ]; then + echo "[ERROR] Could not determine the Debian suite from ${PRESEED}" + exit 1 +fi + +E2E_DISTRO="debian" +E2E_EXPECTED_ID="debian" +E2E_EXPECTED_VERSION="${debian_major_version}" +E2E_EXPECTED_CODENAME="${debian_suite}" + +source "${SCRIPT_DIR}/e2e_iso_install.sh" +run_e2e_iso_install "$@" diff --git a/test/e2e_iso_install.sh b/test/e2e_iso_install.sh new file mode 100755 index 00000000..f993ba6c --- /dev/null +++ b/test/e2e_iso_install.sh @@ -0,0 +1,499 @@ +#!/bin/bash +# Boots an unattended DAppNode ISO as USB media and installs it onto a virtual disk. +# It completes the first-boot flow, reboots, and validates the running DAppNode system. +set -Eeuo pipefail + +run_e2e_iso_install() { + if [ "$#" -ne 1 ]; then + echo "Usage: $0 " + return 2 + fi + + local iso_path + iso_path=$(realpath "$1") + if [ ! -f "${iso_path}" ]; then + echo "[ERROR] ISO not found: ${iso_path}" + return 1 + fi + + local required_commands=(qemu-img qemu-system-x86_64 ssh sshpass socat) + local command_name + for command_name in "${required_commands[@]}"; do + if ! command -v "${command_name}" >/dev/null 2>&1; then + echo "[ERROR] Missing required command: ${command_name}" + return 1 + fi + done + + local output_dir=${E2E_OUTPUT_DIR:-"${PWD}/test-output/${E2E_DISTRO}-e2e"} + mkdir -p "${output_dir}" + output_dir=$(realpath "${output_dir}") + + temp_root=${RUNNER_TEMP:-/tmp} + vm_dir=$(mktemp -d "${temp_root%/}/dappnode-${E2E_DISTRO}-e2e.XXXXXX") + local disk_path="${vm_dir}/${E2E_DISTRO}.qcow2" + local installer_serial_log="${output_dir}/installer-serial.log" + local installer_process_log="${output_dir}/installer-qemu.log" + local installer_monitor_socket="${vm_dir}/installer-monitor.sock" + local installer_screenshot="${output_dir}/installer-screen.ppm" + local first_boot_monitor_socket="${vm_dir}/first-boot-monitor.sock" + local first_boot_screenshot="${output_dir}/first-boot-screen.ppm" + local first_boot_serial_log="${output_dir}/first-boot-serial.log" + local first_boot_process_log="${output_dir}/first-boot-qemu.log" + local system_serial_log="${output_dir}/final-system-serial.log" + local system_process_log="${output_dir}/final-system-qemu.log" + local installed_system_report="${output_dir}/installed-system-report.log" + + local vm_disk_size=${E2E_VM_DISK_SIZE:-32G} + local vm_memory_mb=${E2E_VM_MEMORY_MB:-4096} + local vm_cpus=${E2E_VM_CPUS:-2} + # Healthy runs take 4-8 min to install, 1-3 min for the first-boot test and + # well under a minute to start core services. A stuck installer waits on a + # dialog forever, so keep the limits a few times the normal duration. + local install_timeout_seconds=${E2E_INSTALL_TIMEOUT_SECONDS:-1800} + local boot_timeout_seconds=${E2E_BOOT_TIMEOUT_SECONDS:-600} + local first_boot_timeout_seconds=${E2E_FIRST_BOOT_TIMEOUT_SECONDS:-900} + local postinstall_timeout_seconds=${E2E_POSTINSTALL_TIMEOUT_SECONDS:-600} + local ssh_port=${E2E_SSH_PORT:-2222} + local ssh_password=${E2E_SSH_PASSWORD:-dappnode.s0} + local tap_iface=${E2E_TAP_IFACE:-} + local guest_ip=${E2E_GUEST_IP:-192.168.77.10} + local guest_mac=${E2E_GUEST_MAC:-52:54:00:d4:00:10} + local boot_mode=${E2E_BOOT_MODE:-usb} + local firmware=${E2E_FIRMWARE:-uefi} + qemu_pid="" + + if [ "${boot_mode}" != "usb" ] || [ "${firmware}" != "uefi" ]; then + echo "[ERROR] This unattended ISO test requires E2E_BOOT_MODE=usb and E2E_FIRMWARE=uefi" + return 1 + fi + + cleanup_e2e_vm() { + local exit_code=$? + trap - EXIT INT TERM + + if [ -n "${qemu_pid}" ] && kill -0 "${qemu_pid}" 2>/dev/null; then + kill "${qemu_pid}" 2>/dev/null || true + wait "${qemu_pid}" 2>/dev/null || true + fi + + if [ "${KEEP_E2E_VM:-false}" = "true" ]; then + echo "[INFO] Keeping VM files in ${vm_dir}" + else + case "${vm_dir}" in + "${temp_root%/}"/dappnode-"${E2E_DISTRO}"-e2e.*) rm -rf -- "${vm_dir}" ;; + *) echo "[WARN] Refusing to remove unexpected VM directory: ${vm_dir}" ;; + esac + fi + + exit "${exit_code}" + } + trap cleanup_e2e_vm EXIT INT TERM + + monitor_command() { + local socket=$1 + local command=$2 + [ -S "${socket}" ] || return 0 + printf '%s\n' "${command}" | \ + socat - "UNIX-CONNECT:${socket}" >/dev/null 2>&1 || true + } + + capture_vt_screens() { + # d-i draws its UI on VT1 and streams its syslog to VT4, so a failed + # installer step is only diagnosable if every console is captured. + local socket=$1 + local prefix=$2 + local vt + [ -S "${socket}" ] || return 0 + for vt in 1 2 3 4; do + monitor_command "${socket}" "sendkey alt-f${vt}" + sleep 1 + monitor_command "${socket}" "screendump ${prefix}-vt${vt}.ppm" + done + monitor_command "${socket}" "sendkey alt-f1" + } + + dump_guest_diagnostics() { + # SSH is already up whenever a first-boot wait times out, and the + # installer logs there explain far more than the serial console can. + declare -F ssh_guest >/dev/null 2>&1 || return 0 + echo "[INFO] Collecting guest diagnostics over SSH:" + ssh_guest " + echo '--- .firstboot ---'; ls -la /usr/src/dappnode/.firstboot 2>&1 + echo '--- dappnode processes ---'; ps aux | grep -E '[d]appnode_(test_)?install' 2>&1 + echo '--- docker ps ---'; docker ps --format '{{.Names}}\t{{.Status}}' 2>&1 + echo '--- docker images ---'; docker images 2>&1 + echo '--- connectivity ---' + ping -c 1 -W 5 google.com 2>&1 | tail -3 + curl -sS -o /dev/null -w 'curl https: %{http_code}\n' --max-time 10 https://www.google.com/generate_204 2>&1 + echo '--- iso_install.log ---'; tail -n 60 /usr/src/dappnode/logs/iso_install.log 2>&1 + echo '--- dappnode_install.log ---'; tail -n 80 /usr/src/dappnode/logs/dappnode_install.log 2>&1 + " 2>&1 | sed 's/^/ /' || echo " [WARN] Could not collect guest diagnostics" + } + + show_failure_logs() { + capture_vt_screens "${installer_monitor_socket}" "${output_dir}/installer-screen" + capture_vt_screens "${first_boot_monitor_socket}" "${output_dir}/first-boot-screen" + monitor_command "${installer_monitor_socket}" "screendump ${installer_screenshot}" + monitor_command "${first_boot_monitor_socket}" "screendump ${first_boot_screenshot}" + dump_guest_diagnostics + echo "[INFO] Last installer serial output:" + tail -n 200 "${installer_serial_log}" 2>/dev/null || true + echo "[INFO] Last first-boot serial output:" + tail -n 200 "${first_boot_serial_log}" 2>/dev/null || true + echo "[INFO] Last final-system serial output:" + tail -n 200 "${system_serial_log}" 2>/dev/null || true + echo "[INFO] QEMU process output:" + tail -n 100 "${installer_process_log}" 2>/dev/null || true + tail -n 100 "${first_boot_process_log}" 2>/dev/null || true + tail -n 100 "${system_process_log}" 2>/dev/null || true + } + + wait_for_process_exit() { + local process_id=$1 + local timeout_seconds=$2 + local description=$3 + local deadline=$((SECONDS + timeout_seconds)) + + while kill -0 "${process_id}" 2>/dev/null; do + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "[ERROR] Timed out waiting for ${description}" + return 1 + fi + sleep 10 + done + + wait "${process_id}" + } + + local qemu_acceleration=(-accel "tcg,thread=multi") + if [ -c /dev/kvm ] && [ -r /dev/kvm ] && [ -w /dev/kvm ]; then + qemu_acceleration=(-accel kvm -cpu host) + echo "[INFO] Using KVM acceleration" + else + echo "[WARN] /dev/kvm is unavailable; using slower TCG emulation" + fi + + local ovmf_code="" + local ovmf_vars="" + local ovmf_candidate + for ovmf_candidate in /usr/share/OVMF/OVMF_CODE_4M.fd /usr/share/OVMF/OVMF_CODE.fd; do + if [ -f "${ovmf_candidate}" ]; then + ovmf_code=${ovmf_candidate} + break + fi + done + for ovmf_candidate in /usr/share/OVMF/OVMF_VARS_4M.fd /usr/share/OVMF/OVMF_VARS.fd; do + if [ -f "${ovmf_candidate}" ]; then + ovmf_vars=${ovmf_candidate} + break + fi + done + if [ -z "${ovmf_code}" ] || [ -z "${ovmf_vars}" ]; then + echo "[ERROR] UEFI firmware files were not found; install the ovmf package" + return 1 + fi + local writable_ovmf_vars="${vm_dir}/OVMF_VARS.fd" + cp "${ovmf_vars}" "${writable_ovmf_vars}" + local firmware_args=( + -drive "if=pflash,format=raw,readonly=on,file=${ovmf_code}" + -drive "if=pflash,format=raw,file=${writable_ovmf_vars}" + ) + + echo "[INFO] Creating ${vm_disk_size} virtual installation disk" + qemu-img create -q -f qcow2 "${disk_path}" "${vm_disk_size}" + + # NVMe matches the disks DAppNode hardware ships with, including how the + # installer enumerates it next to the USB stick. + local target_disk_install_args=( + -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" + -device "nvme,serial=dappnode-e2e,drive=target_disk,bootindex=2" + ) + local target_disk_boot_args=( + -drive "file=${disk_path},format=qcow2,if=none,id=target_disk" + -device "nvme,serial=dappnode-e2e,drive=target_disk,bootindex=1" + ) + + # On a TAP network (see test/e2e_network.sh) the guest sees a regular LAN, + # including replies to ping. User networking needs no root but drops ICMP. + local installer_network_args guest_network_args ssh_host + if [ -n "${tap_iface}" ]; then + # Pin the PCI slot: the USB controller only exists while installing, and + # without a fixed address the NIC name would change after the install + # (enp0s5 -> enp0s4), unlike on real hardware. + installer_network_args=( + -netdev "tap,id=net0,ifname=${tap_iface},script=no,downscript=no" + -device "virtio-net-pci,netdev=net0,mac=${guest_mac},addr=0x10" + ) + guest_network_args=("${installer_network_args[@]}") + ssh_host=${guest_ip} + ssh_port=22 + echo "[INFO] Using TAP network ${tap_iface}; guest address ${guest_ip}" + else + installer_network_args=(-nic user,model=virtio-net-pci) + guest_network_args=(-nic "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:${ssh_port}-:22") + ssh_host=127.0.0.1 + echo "[WARN] E2E_TAP_IFACE is unset; using user networking, where the guest cannot ping" + fi + local installer_media_args=( + -device "qemu-xhci,id=installer_xhci" + -drive "file=${iso_path},format=raw,if=none,readonly=on,id=installer_media" + -device "usb-storage,drive=installer_media,bootindex=1" + ) + + echo "[INFO] Booting the unattended ${E2E_DISTRO} ISO as ${firmware}/${boot_mode}" + qemu-system-x86_64 \ + "${qemu_acceleration[@]}" \ + "${firmware_args[@]}" \ + -m "${vm_memory_mb}" \ + -smp "${vm_cpus}" \ + "${target_disk_install_args[@]}" \ + "${installer_media_args[@]}" \ + -boot menu=off \ + "${installer_network_args[@]}" \ + -display none \ + -monitor "unix:${installer_monitor_socket},server=on,wait=off" \ + -serial "file:${installer_serial_log}" \ + -no-reboot \ + >"${installer_process_log}" 2>&1 & + qemu_pid=$! + + if ! wait_for_process_exit "${qemu_pid}" "${install_timeout_seconds}" "the ${E2E_DISTRO} installer to complete"; then + show_failure_logs + return 1 + fi + qemu_pid="" + + echo "[INFO] Installer completed and virtual USB removed; starting the first disk boot" + qemu-system-x86_64 \ + "${qemu_acceleration[@]}" \ + "${firmware_args[@]}" \ + -m "${vm_memory_mb}" \ + -smp "${vm_cpus}" \ + "${target_disk_boot_args[@]}" \ + -boot menu=off \ + "${guest_network_args[@]}" \ + -display none \ + -monitor "unix:${first_boot_monitor_socket},server=on,wait=off" \ + -serial "file:${first_boot_serial_log}" \ + -no-reboot \ + >"${first_boot_process_log}" 2>&1 & + qemu_pid=$! + + local ssh_options=( + -p "${ssh_port}" + -o StrictHostKeyChecking=no + -o UserKnownHostsFile=/dev/null + -o ConnectTimeout=5 + -o ConnectionAttempts=1 + -o LogLevel=ERROR + ) + + ssh_guest() { + SSHPASS="${ssh_password}" sshpass -e ssh "${ssh_options[@]}" "dappnode@${ssh_host}" "$@" + } + + wait_for_ssh() { + local description=$1 + local timeout_seconds=$2 + local deadline=$((SECONDS + timeout_seconds)) + + echo "[INFO] Waiting for SSH on ${description}" + while ! ssh_guest true >/dev/null 2>&1; do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + echo "[ERROR] ${description} VM exited before SSH became available" + return 1 + fi + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "[ERROR] Timed out waiting for SSH on ${description}" + return 1 + fi + sleep 10 + done + } + + wait_for_guest_command() { + local timeout_seconds=$1 + local description=$2 + local guest_command=$3 + local deadline=$((SECONDS + timeout_seconds)) + + echo "[INFO] Waiting for ${description}" + while ! ssh_guest "${guest_command}" >/dev/null 2>&1; do + if ! kill -0 "${qemu_pid}" 2>/dev/null; then + echo "[ERROR] VM exited while waiting for ${description}" + return 1 + fi + if [ "${SECONDS}" -ge "${deadline}" ]; then + echo "[ERROR] Timed out waiting for ${description}" + return 1 + fi + sleep 10 + done + } + + if ! wait_for_ssh "the first installed-system boot" "${boot_timeout_seconds}"; then + show_failure_logs + return 1 + fi + + if ! wait_for_guest_command \ + "${first_boot_timeout_seconds}" \ + "DAppNode's first-boot installation test" \ + "test ! -e /usr/src/dappnode/.firstboot"; then + show_failure_logs + return 1 + fi + + echo "[INFO] Acknowledging the completed first-boot test" + printf 'sendkey ret\n' | \ + socat - "UNIX-CONNECT:${first_boot_monitor_socket}" >/dev/null + + if ! wait_for_guest_command \ + 120 \ + "the first-boot installer process to exit" \ + "! pgrep -f '[d]appnode_test_install.sh|[/]usr/src/dappnode/scripts/dappnode_install.sh' >/dev/null"; then + show_failure_logs + return 1 + fi + + echo "[INFO] Rebooting after the completed first-boot test" + ssh_guest "sudo -S -p '' systemctl reboot" <<<"${ssh_password}" >/dev/null 2>&1 || true + if ! wait_for_process_exit "${qemu_pid}" 120 "the first-boot VM to request its reboot"; then + show_failure_logs + return 1 + fi + qemu_pid="" + + echo "[INFO] Starting the final installed system without the virtual USB" + qemu-system-x86_64 \ + "${qemu_acceleration[@]}" \ + "${firmware_args[@]}" \ + -m "${vm_memory_mb}" \ + -smp "${vm_cpus}" \ + "${target_disk_boot_args[@]}" \ + -boot menu=off \ + "${guest_network_args[@]}" \ + -display none \ + -monitor none \ + -serial "file:${system_serial_log}" \ + >"${system_process_log}" 2>&1 & + qemu_pid=$! + + if ! wait_for_ssh "the final installed system" "${boot_timeout_seconds}"; then + show_failure_logs + return 1 + fi + + if ! wait_for_guest_command \ + "${postinstall_timeout_seconds}" \ + "DAppNode core services to start" \ + "test ! -e /usr/src/dappnode/.firstboot && ! grep -Fq '/usr/src/dappnode/scripts/dappnode_install.sh' /etc/rc.local && docker ps --format '{{.Names}}' | grep -Fxq 'DAppNodeCore-dappmanager.dnp.dappnode.eth'"; then + show_failure_logs + return 1 + fi + + echo "[INFO] Validating the completed unattended ${E2E_DISTRO} USB installation" + ssh_guest bash -s -- \ + "${E2E_DISTRO}" \ + "${E2E_EXPECTED_ID}" \ + "${E2E_EXPECTED_VERSION}" \ + "${E2E_EXPECTED_CODENAME}" <<'REMOTE_CHECKS' | tee "${installed_system_report}" +set -euo pipefail + +expected_distro=$1 +expected_id=$2 +expected_version=$3 +expected_codename=$4 + +source /etc/os-release +if [ "${expected_codename}" = "AUTO" ]; then + expected_codename=${VERSION_CODENAME} +fi +if [ "${ID}" != "${expected_id}" ] || [ "${VERSION_ID}" != "${expected_version}" ] || [ "${VERSION_CODENAME}" != "${expected_codename}" ]; then + echo "[ERROR] Expected ${expected_id} ${expected_version} (${expected_codename}), found ${PRETTY_NAME}" + exit 1 +fi + +apt_sources=$(grep -R -h -E --include='*.list' --include='*.sources' \ + '^(deb |Suites:)' /etc/apt 2>/dev/null || true) +if ! grep -Eq "(^|[[:space:]])${expected_codename}([[:space:]-]|$)" <<<"${apt_sources}"; then + echo "[ERROR] No ${expected_codename} APT source found on the installed system" + exit 1 +fi + +dpkg_audit=$(dpkg --audit) +if [ -n "${dpkg_audit}" ]; then + echo "[ERROR] dpkg reports incomplete or broken packages:" + echo "${dpkg_audit}" + exit 1 +fi + +package_version() { + dpkg-query -W -f='${Version}' "$1" +} + +case "${expected_distro}" in + debian) + for debian_suite in bullseye bookworm trixie forky; do + if [ "${debian_suite}" != "${expected_codename}" ] && \ + grep -R -q -E --include='*.list' --include='*.sources' \ + "(^|[[:space:]])${debian_suite}([[:space:]-]|$)" /etc/apt 2>/dev/null; then + echo "[ERROR] Found stale Debian suite ${debian_suite} in APT sources" + exit 1 + fi + done + + tasksel_version=$(package_version tasksel) + task_english_version=$(package_version task-english) + debconf_version=$(package_version debconf) + python_debconf_version=$(package_version python3-debconf) + + if [ "${tasksel_version}" != "${task_english_version}" ]; then + echo "[ERROR] tasksel (${tasksel_version}) and task-english (${task_english_version}) do not match" + exit 1 + fi + if [ "${debconf_version}" != "${python_debconf_version}" ]; then + echo "[ERROR] debconf (${debconf_version}) and python3-debconf (${python_debconf_version}) do not match" + exit 1 + fi + + echo "tasksel family: ${tasksel_version}" + echo "debconf family: ${debconf_version}" + ;; + ubuntu) + ubuntu_minimal_version=$(package_version ubuntu-minimal) + if ! grep -R -q -E --include='*.list' --include='*.sources' \ + 'https?://[^[:space:]]*ubuntu\.com/ubuntu' /etc/apt 2>/dev/null; then + echo "[ERROR] No official Ubuntu archive found in APT sources" + exit 1 + fi + if grep -R -H -E --include='*.list' --include='*.sources' \ + 'https?://deb\.debian\.org/debian' /etc/apt 2>/dev/null; then + echo "[ERROR] Found a Debian APT source on the installed Ubuntu system" + exit 1 + fi + echo "ubuntu-minimal: ${ubuntu_minimal_version}" + ;; + *) + echo "[ERROR] Unsupported installed-system check: ${expected_distro}" + exit 1 + ;; +esac + +test -x /usr/src/dappnode/scripts/dappnode_install.sh +test -s /usr/src/dappnode/logs/iso_install.log +test -s /usr/src/dappnode/logs/dappnode_install.log +test ! -e /usr/src/dappnode/.firstboot +! grep -Fq '/usr/src/dappnode/scripts/dappnode_install.sh' /etc/rc.local +docker --version +docker compose version +docker info >/dev/null +docker ps --format '{{.Names}}' | grep -Fxq 'DAppNodeCore-dappmanager.dnp.dappnode.eth' + +echo "Installed OS: ${PRETTY_NAME}" +echo "[INFO] End-to-end unattended ${expected_distro} UEFI USB installation checks passed" +REMOTE_CHECKS + + echo "[INFO] Complete unattended ${E2E_DISTRO} UEFI USB installation succeeded" +} diff --git a/test/e2e_network.sh b/test/e2e_network.sh new file mode 100755 index 00000000..5e3c5beb --- /dev/null +++ b/test/e2e_network.sh @@ -0,0 +1,84 @@ +#!/bin/bash +# Creates the TAP network the E2E guests boot on. Run as root: up | down +# The guest gets DHCP, DNS and NAT like on a home LAN. CI providers drop outbound +# ICMP, so echo requests from the guest are answered by this host instead: the +# DAppNode first-boot check pings google.com and must see a reply. +set -Eeuo pipefail + +E2E_TAP_IFACE=${E2E_TAP_IFACE:-dne2e0} +E2E_HOST_IP=${E2E_HOST_IP:-192.168.77.1} +E2E_GUEST_IP=${E2E_GUEST_IP:-192.168.77.10} +E2E_GUEST_MAC=${E2E_GUEST_MAC:-52:54:00:d4:00:10} +SUBNET="${E2E_HOST_IP%.*}.0/24" +STATE_DIR=${E2E_NETWORK_STATE_DIR:-/run/dappnode-e2e} + +if [ "$(id -u)" -ne 0 ]; then + echo "[ERROR] $0 must run as root" + exit 1 +fi + +tap_owner=${SUDO_USER:-root} + +network_up() { + mkdir -p "${STATE_DIR}" + + ip tuntap add dev "${E2E_TAP_IFACE}" mode tap user "${tap_owner}" + ip addr add "${E2E_HOST_IP}/24" dev "${E2E_TAP_IFACE}" + ip link set "${E2E_TAP_IFACE}" up + + sysctl -q -w net.ipv4.ip_forward=1 + iptables -t nat -A POSTROUTING -s "${SUBNET}" ! -o "${E2E_TAP_IFACE}" -j MASQUERADE + iptables -t nat -A PREROUTING -i "${E2E_TAP_IFACE}" -p icmp --icmp-type echo-request \ + -j DNAT --to-destination "${E2E_HOST_IP}" + # Docker sets the FORWARD policy to DROP, so allow the guest explicitly. + iptables -I FORWARD 1 -i "${E2E_TAP_IFACE}" -j ACCEPT + iptables -I FORWARD 1 -o "${E2E_TAP_IFACE}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT + + local upstream=(--resolv-file=/etc/resolv.conf) + if [ -f /run/systemd/resolve/resolv.conf ]; then + # The stub at 127.0.0.53 is fine for the host, but read the real upstreams. + upstream=(--resolv-file=/run/systemd/resolve/resolv.conf) + fi + + # Stay root: the state directory may live in a workspace other users cannot enter. + dnsmasq \ + --interface="${E2E_TAP_IFACE}" \ + --bind-interfaces \ + --except-interface=lo \ + --listen-address="${E2E_HOST_IP}" \ + --dhcp-range="${E2E_GUEST_IP},${E2E_GUEST_IP},255.255.255.0,12h" \ + --dhcp-host="${E2E_GUEST_MAC},${E2E_GUEST_IP}" \ + --dhcp-option=option:router,"${E2E_HOST_IP}" \ + --dhcp-option=option:dns-server,"${E2E_HOST_IP}" \ + --dhcp-leasefile="${STATE_DIR}/dnsmasq.leases" \ + --pid-file="${STATE_DIR}/dnsmasq.pid" \ + --log-facility="${STATE_DIR}/dnsmasq.log" \ + --log-dhcp \ + --no-hosts \ + --user=root \ + "${upstream[@]}" + + echo "[INFO] Guest network ${E2E_TAP_IFACE} is up: host ${E2E_HOST_IP}, guest ${E2E_GUEST_IP} (${E2E_GUEST_MAC})" +} + +network_down() { + if [ -f "${STATE_DIR}/dnsmasq.pid" ]; then + kill "$(cat "${STATE_DIR}/dnsmasq.pid")" 2>/dev/null || true + fi + iptables -D FORWARD -o "${E2E_TAP_IFACE}" -m conntrack --ctstate RELATED,ESTABLISHED -j ACCEPT 2>/dev/null || true + iptables -D FORWARD -i "${E2E_TAP_IFACE}" -j ACCEPT 2>/dev/null || true + iptables -t nat -D PREROUTING -i "${E2E_TAP_IFACE}" -p icmp --icmp-type echo-request \ + -j DNAT --to-destination "${E2E_HOST_IP}" 2>/dev/null || true + iptables -t nat -D POSTROUTING -s "${SUBNET}" ! -o "${E2E_TAP_IFACE}" -j MASQUERADE 2>/dev/null || true + ip link delete "${E2E_TAP_IFACE}" 2>/dev/null || true + rm -rf "${STATE_DIR}" +} + +case "${1:-}" in + up) network_up ;; + down) network_down ;; + *) + echo "Usage: $0 up|down" + exit 2 + ;; +esac diff --git a/test/e2e_ubuntu_iso.sh b/test/e2e_ubuntu_iso.sh new file mode 100755 index 00000000..2c00f16c --- /dev/null +++ b/test/e2e_ubuntu_iso.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Runs the generated unattended Ubuntu ISO through the complete UEFI USB test. +# It adds Ubuntu release and repository checks to the shared harness. +set -Eeuo pipefail + +SCRIPT_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +REPO_ROOT=$(cd "${SCRIPT_DIR}/.." && pwd) +GENERATOR="${REPO_ROOT}/iso/scripts/generate_dappnode_iso_ubuntu.sh" + +base_iso_name=$(sed -n 's/^BASE_ISO_NAME="\{0,1\}\([^"[:space:]]*\)"\{0,1\}$/\1/p' "${GENERATOR}") +if [[ ! "${base_iso_name}" =~ ^ubuntu-([0-9]+\.[0-9]+)(\.[0-9]+)?-live-server-amd64\.iso$ ]]; then + echo "[ERROR] Could not determine the Ubuntu release from BASE_ISO_NAME=${base_iso_name}" + exit 1 +fi + +E2E_DISTRO="ubuntu" +E2E_EXPECTED_ID="ubuntu" +E2E_EXPECTED_VERSION="${BASH_REMATCH[1]}" +E2E_EXPECTED_CODENAME="AUTO" + +source "${SCRIPT_DIR}/e2e_iso_install.sh" +run_e2e_iso_install "$@"