From e25a8babd526b7a1ccc721979fe5fa2f12d66e3d Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 20:50:15 +0530 Subject: [PATCH 01/20] fix(migration): roll back when a migration step exits through EE::error() A helper that calls EE::error() inside a migration step exited the process, so none of the completed steps were reverted, and a PHP Error was fatal for the same reason. DB and container migration failures also called exit(1) directly, bypassing the step processor and its error message. RevertableStepProcessor can now capture exits: with it on, EE::error()/EE::halt() inside an up or down step throw EE\ExitException, which fails that step and runs the rollback. The migration pipeline and the image migration turn it on. Both processors now catch Throwable, the DB and container migration executors rethrow instead of exiting, and EE::runcommand() restores the previous capture setting instead of clearing it. --- php/EE/Migration/Containers.php | 2 +- .../Migration/CustomContainerMigrations.php | 3 +- php/EE/Migration/Executor.php | 3 +- php/EE/RevertableStepProcessor.php | 57 +++++++++++++++++-- php/EE/Runner.php | 3 +- php/class-ee.php | 20 ++++++- 6 files changed, 75 insertions(+), 13 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 8c078e60d..9965db1c1 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -22,7 +22,7 @@ class Containers { public static function start_container_migration() { EE\Utils\delem_log( 'Starting container migration' ); - self::$rsp = new RevertableStepProcessor(); + self::$rsp = new RevertableStepProcessor( true ); $img_versions = EE\Utils\get_image_versions(); $current_versions = self::get_current_docker_images_versions(); diff --git a/php/EE/Migration/CustomContainerMigrations.php b/php/EE/Migration/CustomContainerMigrations.php index 1fcb1f8bd..984ec0e9e 100644 --- a/php/EE/Migration/CustomContainerMigrations.php +++ b/php/EE/Migration/CustomContainerMigrations.php @@ -34,7 +34,8 @@ public static function execute_migrations() { self::execute_migration_stack( $migrations ); } catch ( \Throwable $e ) { Utils\delem_log( 'ee migration ended abruptly' ); - exit( 1 ); + // Rethrown, not exit(1): the migration step processor reports it and rolls back. + throw $e; } EE::debug( 'Successfully migrated EasyEngine' ); diff --git a/php/EE/Migration/Executor.php b/php/EE/Migration/Executor.php index 249e00df9..176158665 100644 --- a/php/EE/Migration/Executor.php +++ b/php/EE/Migration/Executor.php @@ -30,7 +30,8 @@ public static function execute_migrations() { self::execute_migration_stack( $migrations ); } catch ( \Throwable $e ) { Utils\delem_log( 'ee migration ended abruptly' ); - exit( 1 ); + // Rethrown, not exit(1): the migration step processor reports it and rolls back. + throw $e; } EE::debug( 'Successfully migrated EasyEngine' ); diff --git a/php/EE/RevertableStepProcessor.php b/php/EE/RevertableStepProcessor.php index fde925787..8e806b57c 100644 --- a/php/EE/RevertableStepProcessor.php +++ b/php/EE/RevertableStepProcessor.php @@ -18,6 +18,16 @@ class RevertableStepProcessor { /** @var int Keeps track of steps executed. All items in $steps till this index have been executed */ private $execution_index = 0; + /** @var bool Whether an `EE::error()`/`EE::halt()` inside a step throws, so the rollback still runs, instead of exiting. */ + private $capture_exit; + + /** + * @param bool $capture_exit Turn exits inside up and down steps into failures of that step. + */ + public function __construct( $capture_exit = false ) { + $this->capture_exit = (bool) $capture_exit; + } + /** * Adds a new step. * @@ -53,11 +63,11 @@ public function execute( $show_error = true ) { $context = $step['context']; try { EE::debug( "Executing $context... " ); - call_user_func_array( $step['up'], $step['up_params'] ?? [] ); + $this->call( $step['up'], $step['up_params'] ); $this->execution_index ++; EE::debug( "Executed $context." ); - } catch ( \Exception $e ) { - $exception_message = $e->getMessage(); + } catch ( \Throwable $e ) { + $exception_message = self::describe( $e ); $callable = EE\Utils\get_callable_name( $step['up'] ); if ( $show_error ) { EE::error( "Error while executing $context. $callable: $exception_message", false ); @@ -86,15 +96,50 @@ public function rollback() { try { EE::debug( "Reverting $context... " ); if ( null !== $step['down'] ) { - call_user_func_array( $step['down'], $step['down_params'] ?? [] ); + $this->call( $step['down'], $step['down_params'] ); } EE::debug( "Reverted $context" ); - } catch ( \Exception $e ) { - $exception_message = $e->getMessage(); + } catch ( \Throwable $e ) { + $exception_message = self::describe( $e ); EE::debug( "Encountered error while reverting $context: $exception_message. If possible, do it manually" ); } finally { $this->execution_index --; } } } + + /** + * Calls a step callable, capturing exits if this processor does. + * + * @param callable $callable Step callable. + * @param array|null $params Its parameters. + */ + private function call( $callable, $params ) { + if ( ! $this->capture_exit ) { + call_user_func_array( $callable, $params ?? [] ); + + return; + } + + $previous = EE::set_capture_exit( true ); + try { + call_user_func_array( $callable, $params ?? [] ); + } finally { + EE::set_capture_exit( $previous ); + } + } + + /** + * @param \Throwable $e Failure of a step. + * + * @return string + */ + private static function describe( \Throwable $e ) { + // EE::error() has already printed its message. + if ( $e instanceof ExitException ) { + return sprintf( 'exited with code %d', $e->getCode() ); + } + + return $e->getMessage(); + } } diff --git a/php/EE/Runner.php b/php/EE/Runner.php index 2b96b1c73..03f73a65d 100644 --- a/php/EE/Runner.php +++ b/php/EE/Runner.php @@ -167,7 +167,8 @@ private function migrate() { EE::error( 'EasyEngine update requires minimum 5GB disk space to run. Please free up some space and try again.' ); } - $rsp = new \EE\RevertableStepProcessor(); + // Capture EE::error() exits inside the steps so a failure still reverts what ran. + $rsp = new \EE\RevertableStepProcessor( true ); $rsp->add_step( 'ee-db-migrations', 'EE\Migration\Executor::execute_migrations' ); $rsp->add_step( 'ee-custom-container-migrations', 'EE\Migration\CustomContainerMigrations::execute_migrations', 'EE\Migration\CustomContainerMigrations::revert_executed_migrations' ); diff --git a/php/class-ee.php b/php/class-ee.php index 72b455764..390f2c263 100644 --- a/php/class-ee.php +++ b/php/class-ee.php @@ -790,6 +790,20 @@ public static function halt( $return_code ) { exit( $return_code ); } + /** + * Make `EE::error()` and `EE::halt()` throw `EE\ExitException` instead of exiting. + * + * @param bool $capture Whether to capture exits. + * + * @return bool The previous setting, to restore afterwards. + */ + public static function set_capture_exit( $capture ) { + $previous = self::$capture_exit; + self::$capture_exit = (bool) $capture; + + return $previous; + } + /** * Display a multi-line error message in a red box. Doesn't exit script. * @@ -1248,6 +1262,7 @@ public static function runcommand( $command, $options = array() ) { self::$logger = new EE\Loggers\Execution; self::$logger->ob_start(); } + $previous_capture_exit = self::$capture_exit; if ( ! $exit_error ) { self::$capture_exit = true; } @@ -1281,9 +1296,8 @@ public static function runcommand( $command, $options = array() ) { ); } } - if ( ! $exit_error ) { - self::$capture_exit = false; - } + // Restore rather than clear: a caller (e.g. a migration step) may be capturing exits itself. + self::$capture_exit = $previous_capture_exit; } if ( ( true === $return || 'stdout' === $return ) && 'json' === $parse ) { From 67464422b992670bc88c0e76e4a4a81b7fba335f Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 20:53:10 +0530 Subject: [PATCH 02/20] fix(migration): recreate global services one at a time behind a readiness gate The image migration stopped and removed every updated global service in one step, regenerated the compose file and only then brought them back, so all sites were down for the whole window, and an exit in between left the proxy and the DB removed. Nothing checked that a recreated service came back: a failed `up -d` or a crash-looping container counted as success, the new tags were saved and the old images pruned. Now the new compose file is written while the old containers keep running, and each updated service is recreated on its own with `up -d ` in the order cron, newrelic-daemon, redis, db, nginx-proxy. The proxy goes last so a failing backend is rolled back before every site loses its proxy; the DB goes just before it so its datadir is only reverted onto the old image when the proxy fails. Each recreate waits until the container runs without restarting, is healthy if the image has a healthcheck, passes its probe (nginx-proxy: `nginx -t` and an HTTP response on the published port 80; db: `mariadb-admin`/`mysqladmin ping` over TCP, which the entrypoint's temporary server doesn't answer; redis: `redis-cli ping`), and still does so 2 s later without a restart. cron and newrelic-daemon have no readiness signal and only need to have started: cron restart-loops while it has no jobs, which must not roll back every upgrade, so that only warns. The timeout is 60 s (300 s for the DB), overridable with `global-service-ready-timeout` in config.yml. A failure throws, so the existing revert restores the old compose file and recreates the services from it; that revert now checks their readiness too and warns. --- php/EE/Migration/Containers.php | 22 +-- php/EE/Migration/GlobalContainers.php | 231 ++++++++++++++++++++++++++ 2 files changed, 238 insertions(+), 15 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 9965db1c1..5c63523ae 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -258,14 +258,7 @@ private static function migrate_global_containers( $updated_images ) { [ $global_compose_file_backup_path, $global_compose_file_path, $updated_global_images ] ); - self::$rsp->add_step( - 'stop-global-containers', - 'EE\Migration\GlobalContainers::down_global_containers', - null, - [ $updated_global_images ], - null - ); - + // The services keep running until each is recreated in turn: no step stops them all at once. self::$rsp->add_step( 'generate-global-docker-compose-file', 'EE\Service\Utils\generate_global_docker_compose_yml', @@ -274,17 +267,16 @@ private static function migrate_global_containers( $updated_images ) { null ); + // No down steps: the backup step's down restores the old compose file and recreates these services from it. $all_global_images = GlobalContainers::get_all_global_images_with_service_name(); - foreach ( $updated_global_images as $image_name ) { + foreach ( GlobalContainers::sort_for_upgrade( $updated_global_images ) as $image_name ) { $global_container_name = $all_global_images[ $image_name ]; - $global_service_name = ltrim( $global_container_name, 'services_' ); - $global_service_name = rtrim( $global_service_name, '_1' ); self::$rsp->add_step( "upgrade-$global_container_name-container", - "EE\Migration\GlobalContainers::global_service_up", - "EE\Migration\GlobalContainers::global_service_down", - [ $global_service_name ], - [ $global_service_name ] + 'EE\Migration\GlobalContainers::upgrade_global_service', + null, + [ GlobalContainers::get_service_name( $global_container_name ), $global_container_name ], + null ); } } diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index ca762bf54..5c9ba6b97 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -10,6 +10,229 @@ */ class GlobalContainers { + /** + * Upgrade order. The proxy goes last so that a failing backend is rolled back before every site loses its proxy, + * and the database just before it, so its datadir is only reverted onto the old image if the proxy fails. + */ + const UPGRADE_ORDER = [ + 'easyengine/cron', + 'easyengine/newrelic-daemon', + 'easyengine/redis', + 'easyengine/mariadb', + 'easyengine/nginx-proxy', + ]; + + /** Seconds a service may take to become ready (`global-service-ready-timeout` in config.yml overrides it). */ + const READY_TIMEOUT = 60; + + /** MariaDB may run crash recovery or a datadir upgrade first. */ + const DB_READY_TIMEOUT = 300; + + /** Seconds a service must stay up without restarting after its first successful probe. */ + const SETTLE_SECONDS = 2; + + /** Services with a readiness probe. The others (cron, newrelic-daemon) only need to have started. */ + const PROBED_SERVICES = [ 'global-nginx-proxy', 'global-db', 'global-redis' ]; + + /** @var callable|null Test seam: replaces sleep( $seconds ) while waiting for a service. */ + public static $sleeper; + + /** + * Sort updated global images into the order their services are upgraded in. + * + * @param array $images Updated global images. + * + * @return array + */ + public static function sort_for_upgrade( $images ) { + return array_values( array_merge( array_diff( $images, self::UPGRADE_ORDER ), array_intersect( self::UPGRADE_ORDER, $images ) ) ); + } + + /** + * Compose service name of a global container, e.g. `global-db` for `services_global-db_1`. + * + * @param string $container Container name. + * + * @return string + */ + public static function get_service_name( $container ) { + return preg_replace( '/^services_(.+)_1$/', '$1', $container ); + } + + /** + * Recreate one global service with the image in the (already regenerated) global compose file, + * then wait until it is ready. Throws on failure, so the image migration rolls back. + * + * @param string $service Compose service, e.g. `global-db`. + * @param string $container Its container name. + * + * @throws \Exception + */ + public static function upgrade_global_service( $service, $container ) { + EE::debug( "Start upgrading $container" ); + + if ( ! chdir( EE_SERVICE_DIR ) || ! EE::exec( \EE_DOCKER::docker_compose_with_custom() . ' up -d ' . $service ) ) { + throw new \Exception( "Unable to recreate $container with its new image. Please check logs for more details." ); + } + + self::wait_until_ready( $service, $container ); + EE::debug( "Complete upgrading $container" ); + } + + /** + * Wait until a global container runs, isn't restarting, is healthy if it has a healthcheck, + * and passes its service's readiness probe, then stays so for a moment. Unprobed services only need to have started. + * + * @param string $service Compose service, e.g. `global-db`. + * @param string $container Its container name. + * @param int|null $timeout Seconds; defaults to the service's timeout. + * + * @throws \Exception When it isn't ready in time or its healthcheck reports unhealthy. + */ + public static function wait_until_ready( $service, $container, $timeout = null ) { + if ( null === $timeout ) { + $timeout = self::get_ready_timeout( $service ); + } + $start = microtime( true ); + $slept = 0; + $reason = ''; + + while ( true ) { + $state = self::get_container_state( $container ); + if ( 'unhealthy' === $state['health'] ) { + throw new \Exception( "$container is unhealthy according to its healthcheck." ); + } + + // No readiness signal, and cron restart-loops while it has no jobs (CRON-1): having started is enough. + if ( ! in_array( $service, self::PROBED_SERVICES, true ) && in_array( $state['status'], [ 'running', 'restarting' ], true ) && in_array( $state['health'], [ 'healthy', 'none' ], true ) ) { + if ( 'running' !== $state['status'] || 'false' !== $state['restarting'] ) { + EE::warning( "$container keeps restarting. Please check `docker logs $container`." ); + } + + return; + } + + $reason = self::not_ready_reason( $service, $container, $state ); + if ( '' === $reason ) { + self::pause( self::SETTLE_SECONDS ); + $slept += self::SETTLE_SECONDS; + $settled = self::get_container_state( $container ); + $reason = $settled['restart_count'] !== $state['restart_count'] ? 'it restarted' : self::not_ready_reason( $service, $container, $settled ); + if ( '' === $reason ) { + EE::debug( sprintf( '%s is ready after %.1f s', $container, microtime( true ) - $start ) ); + + return; + } + } + + // Time slept counts too, so a test sleeper that returns at once still ends the wait. + if ( max( microtime( true ) - $start, $slept ) >= $timeout ) { + throw new \Exception( sprintf( '%s is not ready after %d s (%s). Please check `docker logs %s`.', $container, $timeout, $reason, $container ) ); + } + self::pause( 1 ); + ++$slept; + } + } + + /** + * @param string $service Compose service. + * + * @return int Seconds. + */ + private static function get_ready_timeout( $service ) { + $configured = \EE\Utils\get_config_value( 'global-service-ready-timeout' ); + if ( is_numeric( $configured ) && $configured > 0 ) { + return (int) $configured; + } + + return 'global-db' === $service ? self::DB_READY_TIMEOUT : self::READY_TIMEOUT; + } + + /** + * @param string $container Container name. + * + * @return array status, restarting, restart_count and health (`none` without a healthcheck); empty strings if it doesn't exist. + */ + private static function get_container_state( $container ) { + $format = '{{.State.Status}} {{.State.Restarting}} {{.RestartCount}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}'; + $result = EE::launch( sprintf( "docker inspect --format '%s' %s", $format, escapeshellarg( $container ) ) ); + $parts = 0 === $result->return_code ? preg_split( '/\s+/', trim( $result->stdout ) ) : []; + + return array_combine( [ 'status', 'restarting', 'restart_count', 'health' ], array_pad( array_slice( $parts, 0, 4 ), 4, '' ) ); + } + + /** + * @param string $service Compose service. + * @param string $container Container name. + * @param array $state From get_container_state(). + * + * @return string Why it isn't ready, or '' when it is. + */ + private static function not_ready_reason( $service, $container, $state ) { + if ( '' === $state['status'] ) { + return 'no such container'; + } + if ( 'running' !== $state['status'] || 'false' !== $state['restarting'] ) { + return "status {$state['status']}"; + } + if ( ! in_array( $state['health'], [ 'healthy', 'none' ], true ) ) { + return "health {$state['health']}"; + } + + return self::probe( $service, $container ); + } + + /** + * Service-specific readiness probe. + * + * @param string $service Compose service. + * @param string $container Container name. + * + * @return string Why it failed, or '' when it passed. + */ + private static function probe( $service, $container ) { + $exec = 'docker exec ' . escapeshellarg( $container ) . ' '; + + switch ( $service ) { + case 'global-nginx-proxy': + if ( ! EE::exec( $exec . 'nginx -t' ) ) { + return 'nginx -t failed'; + } + // Any HTTP status proves nginx answers through the published port; 443 is the same process. + $port = (string) \EE\Utils\get_config_value( 'proxy_80_port', '80' ); + $host = false === strpos( $port, ':' ) ? '127.0.0.1' : substr( $port, 0, strrpos( $port, ':' ) ); + $port = false === strpos( $port, ':' ) ? $port : substr( $port, strrpos( $port, ':' ) + 1 ); + $status = trim( EE::launch( sprintf( "curl -s -o /dev/null -w '%%{http_code}' --max-time 5 %s", escapeshellarg( "http://$host:$port/" ) ) )->stdout ); + + return preg_match( '/^[1-5]\d\d$/', $status ) ? '' : "no HTTP response on port $port"; + + case 'global-db': + // Over TCP: the entrypoint's temporary init/upgrade server listens on the socket only. Access denied still means alive. + $ping = 'for c in mariadb-admin mysqladmin; do command -v $c >/dev/null 2>&1 && exec $c ping -h 127.0.0.1 --protocol=tcp --connect-timeout=2 --silent; done; exit 1'; + + return EE::exec( $exec . 'sh -c ' . escapeshellarg( $ping ) ) ? '' : 'mysqladmin ping failed'; + + case 'global-redis': + $pong = trim( EE::launch( $exec . 'redis-cli ping' )->stdout ); + + return 'PONG' === $pong ? '' : 'redis-cli ping failed'; + } + + return ''; + } + + /** + * @param int|float $seconds Seconds to sleep. + */ + private static function pause( $seconds ) { + if ( self::$sleeper ) { + call_user_func( self::$sleeper, $seconds ); + + return; + } + usleep( (int) ( $seconds * 1000000 ) ); + } + /** * Get global containers which has new image update. * @@ -58,6 +281,14 @@ public static function revert_global_containers( $source_path, $dest_path, $upda if ( ! EE::exec( \EE_DOCKER::docker_compose_with_custom() . ' up -d ' . $services_to_regenerate ) ) { throw new \Exception( 'Unable to downgrade global containers. Please check logs for more details.' ); } + foreach ( $updated_images as $image_name ) { + $container = $all_global_images[ $image_name ]; + try { + self::wait_until_ready( self::get_service_name( $container ), $container ); + } catch ( \Exception $e ) { + EE::warning( 'After restoring the previous image: ' . $e->getMessage() ); + } + } EE::debug( 'Complete restoring global docker-compose.yml file from backup' ); } From 5b935df28efa606c40e09faf24aec90318001e82 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 20:55:59 +0530 Subject: [PATCH 03/20] fix(migration): pull only images that something uses The image migration pulled every image in img-versions.json that no running container used and the DB didn't track, on every run, without ever recording it. That is easyengine/mailhog and the amd64-only easyengine/php (stable), which no site type uses: the pull costs time and Docker Hub quota everywhere, fails without registry access, and aborts the whole migration on arm64 hosts. Such images have nothing to upgrade, so they are no longer pulled; Compose pulls them when a container first needs them. easyengine/php also joins the images that are left to Compose when they do change. --- php/EE/Migration/Containers.php | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 5c63523ae..d52aae148 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -40,20 +40,16 @@ public static function start_container_migration() { 'easyengine/php8.3', 'easyengine/php8.4', 'easyengine/php8.5', + 'easyengine/php', 'easyengine/newrelic-daemon', ]; + // An image no container runs and the DB doesn't track (e.g. mailhog, the amd64-only easyengine/php) has nothing + // to upgrade: pulling it only fails on hosts it isn't built for. Compose pulls it when something first uses it. foreach ( $img_versions as $img => $version ) { - - if ( array_key_exists( $img, $current_versions ) ) { - if ( $current_versions[ $img ] !== $version ) { - $updated_images[] = $img; - if ( ! in_array( $img, $skip_download ) ) { - self::pull_or_error( $img, $version ); - } - } - } else { - if ( ! in_array( $img, $skip_download ) ) { + if ( array_key_exists( $img, $current_versions ) && $current_versions[ $img ] !== $version ) { + $updated_images[] = $img; + if ( ! in_array( $img, $skip_download, true ) ) { self::pull_or_error( $img, $version ); } } From b204545d7a1bc85d73cbc02021164796ad237682 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 20:58:18 +0530 Subject: [PATCH 04/20] fix(migration): install docker-compose only when missing or too old, and verify it CheckAndUpdateDockerOne never called the parent constructor, so its fresh-install skip was dead code: every first run replaced the host's docker-compose, whatever its version, with a downloaded 1.29.2 (a URL that 404s on aarch64), and the image migration then replaced that with 2.27.0. A host with compose v5 ended up on 2.27.0. The download had no --fail and no checksum and ignored its exit codes, so an HTTP error page could become the binary, and the revert never restored anything because it copied without overwriting. update_docker_compose() now installs 2.27.0 only when docker-compose is missing or older than 2.26.1, the minimum EasyEngine checks for, so it never downgrades. It resolves the release asset from the OS and architecture (x86_64, aarch64/arm64, armv6/7, ppc64le, s390x, riscv64), downloads with --fail, checks the published sha256 and that the new binary runs and reports at least 2.26.1, and only then moves it into place. A failure warns and keeps the installed binary. The revert restores the backup only if this run replaced the binary, never a stale one from an earlier run. The migration now calls the parent constructor, so it is skipped on fresh installs, and on upgrades it uses the same helper instead of forcing 1.29.2. --- ...easyengine_check_and_update_docker_one.php | 24 +--- php/EE/Migration/Containers.php | 130 ++++++++++++++++-- 2 files changed, 120 insertions(+), 34 deletions(-) diff --git a/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php b/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php index 9a25e20e4..d16b04f69 100644 --- a/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php +++ b/migrations/db/20240502102550_easyengine_check_and_update_docker_one.php @@ -4,12 +4,12 @@ use EE; use EE\Migration\Base; -use Symfony\Component\Filesystem\Filesystem; class CheckAndUpdateDockerOne extends Base { public function __construct() { + parent::__construct(); if ( $this->is_first_execution ) { $this->skip_this_migration = true; } @@ -48,32 +48,14 @@ public function up() { } EE::debug( 'Checking docker-compose version' ); - $docker_compose_version = trim( EE::launch( 'docker-compose version --short' )->stdout ); - $docker_compose_path = EE::launch( 'command -v docker-compose' )->stdout; - $docker_compose_path = trim( $docker_compose_path ); - $docker_compose_backup_path = EE_BACKUP_DIR . '/docker-compose.backup'; - $docker_compose_new_path = EE_BACKUP_DIR . '/docker-compose'; - $fs = new Filesystem(); - if ( ! $fs->exists( EE_BACKUP_DIR ) ) { - $fs->mkdir( EE_BACKUP_DIR ); - } - $fs->copy( $docker_compose_path, $docker_compose_backup_path ); - - if ( version_compare( '1.29.2', $docker_compose_version, '!=' ) ) { - EE::exec( "curl -L https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m) -o $docker_compose_new_path && chmod +x $docker_compose_new_path" ); - EE::exec( "mv $docker_compose_new_path $docker_compose_path" ); - } + // Installs only when docker-compose is missing or too old; it never downgrades. + Containers::update_docker_compose(); // Check the version again post update. $docker_version = trim( EE::launch( 'docker version --format "{{.Server.Version}}"' )->stdout ); if ( version_compare( $docker_version, '20.10.10', '<' ) ) { EE::error( 'Docker version should be 20.10.10 or above. Please update Docker and try again.' ); } - - $docker_compose_version = trim( EE::launch( 'docker-compose version --short' )->stdout ); - if ( version_compare( '1.29.2', $docker_compose_version, '!=' ) ) { - EE::error( 'Docker-compose version should be 1.29.2. Please update Docker-compose and try again.' ); - } } /** diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index d52aae148..a02d76ab2 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -11,9 +11,18 @@ */ class Containers { + /** EasyEngine needs at least this docker-compose (see Runner::check_requirements()). */ + const DOCKER_COMPOSE_MIN_VERSION = '2.26.1'; + + /** Installed when docker-compose is missing or older than the minimum. */ + const DOCKER_COMPOSE_INSTALL_VERSION = '2.27.0'; + /** @var RevertableStepProcessor $rsp Keeps track of migration state. Reverts on error */ private static $rsp; + /** @var bool Whether update_docker_compose() replaced the binary in this process. */ + private static $replaced_docker_compose = false; + /** * Main entry point for the script. Migrates all containers * @@ -123,38 +132,133 @@ public static function docker_cleanup() { } /** - * Update docker-compose to v2.27.0 if lower version is installed. + * Install docker-compose DOCKER_COMPOSE_INSTALL_VERSION when it is missing or older than DOCKER_COMPOSE_MIN_VERSION. + * A newer one is never replaced. A failed download or check only warns and keeps the installed binary. */ public static function update_docker_compose() { - $docker_compose_version = trim( EE::launch( 'docker-compose version --short' )->stdout ); + $installed = self::get_docker_compose_version(); + if ( '' !== $installed && version_compare( $installed, self::DOCKER_COMPOSE_MIN_VERSION, '>=' ) ) { + EE::debug( "docker-compose $installed is recent enough, not replacing it." ); + + return; + } + + $version = self::DOCKER_COMPOSE_INSTALL_VERSION; + $url = self::get_docker_compose_download_url( $version, php_uname( 's' ), php_uname( 'm' ) ); + if ( '' === $url ) { + EE::warning( sprintf( 'EasyEngine requires docker-compose %s or newer, and there is no docker-compose %s build for %s %s. Please install it manually.', self::DOCKER_COMPOSE_MIN_VERSION, $version, php_uname( 's' ), php_uname( 'm' ) ) ); + + return; + } + $docker_compose_path = trim( EE::launch( 'command -v docker-compose' )->stdout ); + $docker_compose_path = '' === $docker_compose_path ? '/usr/local/bin/docker-compose' : $docker_compose_path; $docker_compose_backup_path = EE_BACKUP_DIR . '/docker-compose.backup'; $docker_compose_new_path = EE_BACKUP_DIR . '/docker-compose'; - if ( version_compare( '2.27.0', $docker_compose_version, '>' ) ) { + $fs = new Filesystem(); + $fs->mkdir( EE_BACKUP_DIR ); + $fs->remove( $docker_compose_new_path ); + + $failure = ''; + if ( ! EE::exec( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 -o %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $url ) ) ) ) { + $failure = "the download of $url failed"; + } else { + $checksum = strtok( trim( EE::launch( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 %s', escapeshellarg( "$url.sha256" ) ) )->stdout ), " \t" ); + $actual = is_file( $docker_compose_new_path ) ? hash_file( 'sha256', $docker_compose_new_path ) : ''; + if ( ! preg_match( '/^[0-9a-f]{64}$/', (string) $checksum ) || ! hash_equals( (string) $checksum, (string) $actual ) ) { + $failure = "the checksum of $url doesn't match"; + } else { + $fs->chmod( $docker_compose_new_path, 0755 ); + $new_version = self::normalize_version( EE::launch( escapeshellarg( $docker_compose_new_path ) . ' version --short' )->stdout ); + if ( ! version_compare( $new_version, self::DOCKER_COMPOSE_MIN_VERSION, '>=' ) ) { + $failure = 'the downloaded binary does not run'; + } + } + } - $fs = new Filesystem(); - if ( ! $fs->exists( EE_BACKUP_DIR ) ) { - $fs->mkdir( EE_BACKUP_DIR ); + if ( '' === $failure ) { + if ( $fs->exists( $docker_compose_path ) ) { + $fs->copy( $docker_compose_path, $docker_compose_backup_path, true ); } - $fs->copy( $docker_compose_path, $docker_compose_backup_path ); + if ( EE::exec( sprintf( 'mv %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $docker_compose_path ) ) ) ) { + self::$replaced_docker_compose = true; + EE::debug( "Installed docker-compose $version at $docker_compose_path" ); - EE::exec( "curl -L https://github.com/docker/compose/releases/download/v2.27.0/docker-compose-$(uname -s)-$(uname -m) -o $docker_compose_new_path && chmod +x $docker_compose_new_path" ); - EE::exec( "mv $docker_compose_new_path $docker_compose_path" ); + return; + } + $failure = "it could not be moved to $docker_compose_path"; } + + $fs->remove( $docker_compose_new_path ); + EE::warning( sprintf( 'Could not install docker-compose %s (%s). EasyEngine requires docker-compose %s or newer, found %s.', $version, $failure, self::DOCKER_COMPOSE_MIN_VERSION, '' === $installed ? 'none' : $installed ) ); } /** - * Revert docker-compose to previous version. + * Revert docker-compose to the binary update_docker_compose() replaced in this run, if it did. + * An older backup from an earlier run is never restored. */ public static function revert_docker_compose() { - $docker_compose_path = EE::launch( 'command -v docker-compose' )->stdout; - $docker_compose_path = trim( $docker_compose_path ); + if ( ! self::$replaced_docker_compose ) { + return; + } + + $docker_compose_path = trim( EE::launch( 'command -v docker-compose' )->stdout ); $docker_compose_backup_path = EE_BACKUP_DIR . '/docker-compose.backup'; $fs = new Filesystem(); - $fs->copy( $docker_compose_backup_path, $docker_compose_path ); + if ( '' !== $docker_compose_path && $fs->exists( $docker_compose_backup_path ) ) { + $fs->copy( $docker_compose_backup_path, $docker_compose_path, true ); + } + self::$replaced_docker_compose = false; + } + + /** + * @return string Installed docker-compose version as x.y.z, or '' if there is none. + */ + public static function get_docker_compose_version() { + return self::normalize_version( EE::launch( 'docker-compose version --short' )->stdout ); + } + + /** + * Release asset URL of a docker-compose v2 build. + * + * @param string $version Version without `v`. + * @param string $os `uname -s`, e.g. Linux or Darwin. + * @param string $arch `uname -m`. + * + * @return string '' when there is no build for this platform. + */ + public static function get_docker_compose_download_url( $version, $os, $arch ) { + $os = strtolower( $os ); + $arches = [ + 'x86_64' => 'x86_64', + 'amd64' => 'x86_64', + 'aarch64' => 'aarch64', + 'arm64' => 'aarch64', + 'armv7l' => 'armv7', + 'armv6l' => 'armv6', + 'ppc64le' => 'ppc64le', + 's390x' => 's390x', + 'riscv64' => 'riscv64', + ]; + $arch = $arches[ strtolower( $arch ) ] ?? ''; + + if ( '' === $arch || ! ( 'linux' === $os || ( 'darwin' === $os && in_array( $arch, [ 'x86_64', 'aarch64' ], true ) ) ) ) { + return ''; + } + + return sprintf( 'https://github.com/docker/compose/releases/download/v%s/docker-compose-%s-%s', $version, $os, $arch ); + } + + /** + * @param string $output Output of `docker-compose version --short`, e.g. `2.27.0`, `v2.40.3` or `5.5.1-desktop.1`. + * + * @return string x.y.z, or ''. + */ + private static function normalize_version( $output ) { + return preg_match( '/(\d+\.\d+\.\d+)/', (string) $output, $matches ) ? $matches[1] : ''; } /** From 026461fed746024b418ff8fa457264f814223ca8 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:00:41 +0530 Subject: [PATCH 05/20] fix(db): create the schema of an empty EasyEngine database EE_DB decided whether the schema existed by whether the file existed, so a 0-byte ee.sqlite (left by running the sqlite3 CLI on the not-yet-created DB, or by a first run that died after PDO created the file) made every command that reaches the migration check die with an uncaught 'no such table: options' until the file was deleted. A missing, empty or table-less database now gets the base tables, created with IF NOT EXISTS in one transaction. A database that has tables but lacks migrations or options, or isn't a SQLite file, is left untouched and ee stops with a message naming the file and how to recover, instead of an uncaught exception. --- php/class-ee-db.php | 44 ++++++++++++++++++++++++++------------------ 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/php/class-ee-db.php b/php/class-ee-db.php index 3d361198a..4c67656b1 100644 --- a/php/class-ee-db.php +++ b/php/class-ee-db.php @@ -32,17 +32,23 @@ public function __construct() { */ private static function init_db() { - if ( ! ( file_exists( DB ) ) ) { - self::create_required_tables(); - - return; - } + // PDO creates a missing file; an empty one (e.g. left by the sqlite3 CLI or an interrupted first run) has no schema either. + clearstatcache( true, DB ); + $is_new = ! file_exists( DB ) || 0 === filesize( DB ); try { self::$pdo = new PDO( 'sqlite:' . DB ); self::$pdo->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); + $tables = $is_new ? [] : self::$pdo->query( "SELECT name FROM sqlite_master WHERE type = 'table'" )->fetchAll( PDO::FETCH_COLUMN ); } catch ( PDOException $exception ) { - EE::error( 'Unable to initialize connection to EE sqlite database: ' . $exception->getMessage() ); + EE::error( sprintf( 'Unable to open the EasyEngine database %s: %s. If it is damaged, restore it from a backup, or move it away to start with an empty database.', DB, $exception->getMessage() ) ); + } + + if ( empty( $tables ) ) { + self::create_required_tables(); + } elseif ( array_diff( [ 'migrations', 'options' ], $tables ) ) { + // Tables but no base schema: not ours to initialise over. + EE::error( sprintf( 'The EasyEngine database %s lacks the %s table(s). Restore it from a backup, or move it away to start with an empty database.', DB, implode( ', ', array_diff( [ 'migrations', 'options' ], $tables ) ) ) ); } } @@ -50,30 +56,32 @@ private static function init_db() { * Sqlite database creation. */ private static function create_required_tables() { - try { - self::$pdo = new PDO( 'sqlite:' . DB ); - self::$pdo->setAttribute( PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION ); - } catch ( PDOException $exception ) { - EE::error( $exception->getMessage() ); - } - $query = 'CREATE TABLE migrations ( + $queries = [ + 'CREATE TABLE IF NOT EXISTS migrations ( id INTEGER, migration VARCHAR, type VARCHAR, timestamp DATETIME, PRIMARY KEY (id) - );'; - - $query .= 'CREATE TABLE options ( + );', + 'CREATE TABLE IF NOT EXISTS options ( key VARCHAR NOT NULL, value VARCHAR NOT NULL, PRIMARY KEY (key) - );'; + );', + ]; try { - self::$pdo->exec( $query ); + self::$pdo->beginTransaction(); + foreach ( $queries as $query ) { + self::$pdo->exec( $query ); + } + self::$pdo->commit(); } catch ( PDOException $exception ) { + if ( self::$pdo->inTransaction() ) { + self::$pdo->rollBack(); + } EE::error( 'Encountered Error while creating table: ' . $exception->getMessage() ); } } From ee371241f878f2082cbb6a9ab8a6a52857465aeb Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:03:10 +0530 Subject: [PATCH 06/20] fix(migration): never restore a docker-compose backup from an earlier run When no docker-compose was installed, update_docker_compose() made no backup but still marked the binary as replaced, so a failed image migration restored whatever docker-compose.backup an earlier run had left over the new install. The old backup is now removed before a new one is taken. --- php/EE/Migration/Containers.php | 2 ++ 1 file changed, 2 insertions(+) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index a02d76ab2..757ffc3a5 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -179,6 +179,8 @@ public static function update_docker_compose() { } if ( '' === $failure ) { + // A backup from an earlier run must never be restored over this install. + $fs->remove( $docker_compose_backup_path ); if ( $fs->exists( $docker_compose_path ) ) { $fs->copy( $docker_compose_path, $docker_compose_backup_path, true ); } From a935f0ac933a4b41d7aefcc909b62ff3cd283342 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:05:25 +0530 Subject: [PATCH 07/20] fix(migration): accept a password-protected redis and fractional timeouts in the readiness gate redis-cli ping answers NOAUTH when redis.conf sets requirepass, so a working redis failed the gate and every upgrade rolled back; NOAUTH now counts as alive, as 'Access denied' already does for the db. A fractional global-service-ready-timeout (e.g. 0.5) was truncated to 0 and failed the first probe at once; it is now rounded up. The comment on the upgrade order no longer claims the db is only reverted when the proxy fails: a failing site step after the globals reverts all of them, as before. --- php/EE/Migration/GlobalContainers.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 5c9ba6b97..ad605aa05 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -12,7 +12,8 @@ class GlobalContainers { /** * Upgrade order. The proxy goes last so that a failing backend is rolled back before every site loses its proxy, - * and the database just before it, so its datadir is only reverted onto the old image if the proxy fails. + * and the database just before it, so of the globals only a failing proxy reverts its datadir onto the old image + * (a failure in a later site step still reverts every global). */ const UPGRADE_ORDER = [ 'easyengine/cron', @@ -142,7 +143,7 @@ public static function wait_until_ready( $service, $container, $timeout = null ) private static function get_ready_timeout( $service ) { $configured = \EE\Utils\get_config_value( 'global-service-ready-timeout' ); if ( is_numeric( $configured ) && $configured > 0 ) { - return (int) $configured; + return (int) ceil( $configured ); } return 'global-db' === $service ? self::DB_READY_TIMEOUT : self::READY_TIMEOUT; @@ -215,7 +216,8 @@ private static function probe( $service, $container ) { case 'global-redis': $pong = trim( EE::launch( $exec . 'redis-cli ping' )->stdout ); - return 'PONG' === $pong ? '' : 'redis-cli ping failed'; + // NOAUTH (a requirepass in redis.conf) still means redis answers. + return 'PONG' === $pong || 0 === strpos( $pong, 'NOAUTH' ) ? '' : 'redis-cli ping failed'; } return ''; From 498d20e45252aeaec45f28f174940bc61611d808 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:07:49 +0530 Subject: [PATCH 08/20] fix(migration): report a missing migration file instead of a class-not-found error Both executors threw `new Exception()` inside the EE\Migration namespace, which is an Error for the missing class EE\Migration\Exception. That used to end in exit(1); now that the executors rethrow, the step processor would report 'Class "EE\Migration\Exception" not found' as the reason. They throw \Exception with the missing path instead. --- php/EE/Migration/CustomContainerMigrations.php | 2 +- php/EE/Migration/Executor.php | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/php/EE/Migration/CustomContainerMigrations.php b/php/EE/Migration/CustomContainerMigrations.php index 984ec0e9e..1f9b50981 100644 --- a/php/EE/Migration/CustomContainerMigrations.php +++ b/php/EE/Migration/CustomContainerMigrations.php @@ -161,7 +161,7 @@ private static function execute_migration_stack( $migrations ) { if ( ! file_exists( $migration_path ) ) { EE::error( "Unable to find migration file at $migration_path", false ); - throw new Exception(); + throw new \Exception( "Unable to find migration file at $migration_path" ); } require( $migration_path ); diff --git a/php/EE/Migration/Executor.php b/php/EE/Migration/Executor.php index 176158665..5749e3cf5 100644 --- a/php/EE/Migration/Executor.php +++ b/php/EE/Migration/Executor.php @@ -101,7 +101,7 @@ private static function execute_migration_stack( $migrations ) { if ( ! file_exists( $migration_path ) ) { EE::error( "Unable to find migration file at $migration_path", false ); - throw new Exception(); + throw new \Exception( "Unable to find migration file at $migration_path" ); } require( $migration_path ); From 40e64642618ca0b200a6f8fa57fa9bb06afd5d97 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:10:38 +0530 Subject: [PATCH 09/20] fix(migration): revert only the docker-compose replacement of the image migration's own step The replaced flag was process-wide, so when the CheckAndUpdateDockerOne DB migration (which has no undo) installed 2.27.0 and the image migration's update-compose step then had nothing to do, a later failure in the image migration still restored the old binary, possibly compose v1, before the site and global reverts ran against containers the v2 binary had created. Each update_docker_compose() call now starts with the flag cleared, so the step's revert undoes only its own replacement. --- php/EE/Migration/Containers.php | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 757ffc3a5..1a2ef731e 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -20,7 +20,7 @@ class Containers { /** @var RevertableStepProcessor $rsp Keeps track of migration state. Reverts on error */ private static $rsp; - /** @var bool Whether update_docker_compose() replaced the binary in this process. */ + /** @var bool Whether the last update_docker_compose() call replaced the binary. */ private static $replaced_docker_compose = false; /** @@ -137,7 +137,9 @@ public static function docker_cleanup() { */ public static function update_docker_compose() { - $installed = self::get_docker_compose_version(); + // Only this call's replacement may be reverted, not one made earlier in the run (e.g. by a DB migration, which has no undo). + self::$replaced_docker_compose = false; + $installed = self::get_docker_compose_version(); if ( '' !== $installed && version_compare( $installed, self::DOCKER_COMPOSE_MIN_VERSION, '>=' ) ) { EE::debug( "docker-compose $installed is recent enough, not replacing it." ); @@ -198,7 +200,7 @@ public static function update_docker_compose() { } /** - * Revert docker-compose to the binary update_docker_compose() replaced in this run, if it did. + * Revert docker-compose to the binary the last update_docker_compose() call replaced, if it did. * An older backup from an earlier run is never restored. */ public static function revert_docker_compose() { From 8634458fbf4d60151a7b1b405f2a0261240d7abe Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:13:05 +0530 Subject: [PATCH 10/20] fix(ee): let a migration step capture exits from EE::runcommand() With exit_error on (the default), runcommand() ended the process with a raw exit() when a launched command failed, and an in-process command's EE::error() was swallowed into a return code whenever the caller was already capturing exits. Inside a migration step (e.g. auth-command's global auth update reached from a site's admin-tools enable) the first skipped the rollback and the second hid the failure. A failed launched command now goes through EE::halt(), which exits as before unless exits are captured, and an in-process exit with a non-zero code is rethrown to a capturing caller. --- php/class-ee.php | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/php/class-ee.php b/php/class-ee.php index 390f2c263..6156f9dde 100644 --- a/php/class-ee.php +++ b/php/class-ee.php @@ -1238,7 +1238,8 @@ public static function runcommand( $command, $options = array() ) { if ( -1 == $return_code ) { self::warning( 'Spawned process returned exit code -1, which could be caused by a custom compiled version of PHP that uses the --enable-sigchild option.' ); } elseif ( $return_code && $exit_error ) { - exit( $return_code ); + // halt(), not exit(): a migration step capturing exits must still roll back. + self::halt( $return_code ); } if ( true === $return || 'stdout' === $return ) { $retval = trim( $stdout ); @@ -1263,6 +1264,7 @@ public static function runcommand( $command, $options = array() ) { self::$logger->ob_start(); } $previous_capture_exit = self::$capture_exit; + $exit_exception = null; if ( ! $exit_error ) { self::$capture_exit = true; } @@ -1275,6 +1277,10 @@ public static function runcommand( $command, $options = array() ) { $return_code = 0; } catch ( ExitException $e ) { $return_code = $e->getCode(); + // With exit_error the caller would have exited; a caller that captures exits gets the exit instead. + if ( $exit_error && $return_code ) { + $exit_exception = $e; + } } if ( $return ) { $execution_logger = self::$logger; @@ -1298,6 +1304,9 @@ public static function runcommand( $command, $options = array() ) { } // Restore rather than clear: a caller (e.g. a migration step) may be capturing exits itself. self::$capture_exit = $previous_capture_exit; + if ( $exit_exception ) { + throw $exit_exception; + } } if ( ( true === $return || 'stdout' === $return ) && 'json' === $parse ) { From 019901b70aeec6d1d11446834cf97275fffae244 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:15:46 +0530 Subject: [PATCH 11/20] fix(migration): update docker-compose before recreating global services Recreating a service in place with `up -d ` fails with docker-compose v1 on Docker 25 and newer: v1 stops and renames the old container, then crashes with KeyError: 'ContainerConfig', and a retry or the rollback's `up -d` fails the same way, so the service stays down. The old flow stopped and removed the services first, which v1 still handles, and only updated docker-compose at the end, so a v1 host would lose its first recreated global and never reach the update. The update-compose step now runs first in the image migration (and so is reverted last), and when docker-compose is still older than 2.26.1 (the download failed) the global recreates and the rollback stop and remove the service before `up -d`. --- php/EE/Migration/Containers.php | 7 ++++--- php/EE/Migration/GlobalContainers.php | 23 +++++++++++++++++++++-- 2 files changed, 25 insertions(+), 5 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 1a2ef731e..334806e15 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -69,9 +69,10 @@ public static function start_container_migration() { return; } + // First: compose v1 can't recreate containers on Docker 25+, and on a rollback it is restored last. + self::maybe_update_docker_compose(); self::migrate_global_containers( $updated_images ); self::migrate_site_containers( $updated_images ); - self::maybe_update_docker_compose(); self::save_upgraded_image_versions( $current_versions, $img_versions, $updated_images ); if ( ! self::$rsp->execute() ) { @@ -82,8 +83,8 @@ public static function start_container_migration() { } /** - * Maybe update docker-compose at the end of migration. - * Need to update to latest docker-compose version for new template changes. + * Maybe update docker-compose before any container is recreated, for the new templates and because v1 fails to + * recreate containers on current Docker. */ public static function maybe_update_docker_compose() { diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index ad605aa05..0f9cfa133 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -72,7 +72,7 @@ public static function get_service_name( $container ) { public static function upgrade_global_service( $service, $container ) { EE::debug( "Start upgrading $container" ); - if ( ! chdir( EE_SERVICE_DIR ) || ! EE::exec( \EE_DOCKER::docker_compose_with_custom() . ' up -d ' . $service ) ) { + if ( ! chdir( EE_SERVICE_DIR ) || ! self::compose_up( $service ) ) { throw new \Exception( "Unable to recreate $container with its new image. Please check logs for more details." ); } @@ -80,6 +80,25 @@ public static function upgrade_global_service( $service, $container ) { EE::debug( "Complete upgrading $container" ); } + /** + * `up -d` global services in the current directory. docker-compose v1 can't recreate a container on Docker 25+ + * (KeyError: 'ContainerConfig', leaving it stopped), so below the minimum version they are stopped and removed first. + * + * @param string $services Space-separated compose services. + * + * @return bool + */ + private static function compose_up( $services ) { + $compose = \EE_DOCKER::docker_compose_with_custom(); + $up = "$compose up -d $services"; + $version = Containers::get_docker_compose_version(); + if ( '' !== $version && version_compare( $version, Containers::DOCKER_COMPOSE_MIN_VERSION, '<' ) ) { + return EE::exec( "$compose stop $services && $compose rm -f $services && $up" ); + } + + return EE::exec( $up ); + } + /** * Wait until a global container runs, isn't restarting, is healthy if it has a healthcheck, * and passes its service's readiness probe, then stays so for a moment. Unprobed services only need to have started. @@ -280,7 +299,7 @@ public static function revert_global_containers( $source_path, $dest_path, $upda chdir( EE_ROOT_DIR . '/services' ); - if ( ! EE::exec( \EE_DOCKER::docker_compose_with_custom() . ' up -d ' . $services_to_regenerate ) ) { + if ( ! self::compose_up( trim( $services_to_regenerate ) ) ) { throw new \Exception( 'Unable to downgrade global containers. Please check logs for more details.' ); } foreach ( $updated_images as $image_name ) { From 7b7e265c3a683e9dd77729de27727c2b454344ba Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:18:36 +0530 Subject: [PATCH 12/20] fix(migration): check the running proxy config first, bound the compose download, bypass http_proxy in the probe A vhost.d/conf.d/htpasswd file broken after the running proxy loaded it made the recreated proxy fail, and the rollback's proxy on the old image failed the same way, taking every site down; the proxy step now runs nginx -t in the running container first and fails before touching it. The docker-compose download and its .sha256 had only a connect timeout, so a stalled transfer hung the upgrade; they now have --max-time. The proxy probe's curl followed an exported http_proxy, so a corporate proxy could time the probe out or answer for it; it now uses --noproxy '*'. --- php/EE/Migration/Containers.php | 5 +++-- php/EE/Migration/GlobalContainers.php | 9 ++++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 334806e15..ad7dce0ea 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -165,10 +165,11 @@ public static function update_docker_compose() { $fs->remove( $docker_compose_new_path ); $failure = ''; - if ( ! EE::exec( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 -o %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $url ) ) ) ) { + // --max-time: a stalled download must not hang the upgrade. + if ( ! EE::exec( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 --max-time 300 -o %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $url ) ) ) ) { $failure = "the download of $url failed"; } else { - $checksum = strtok( trim( EE::launch( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 %s', escapeshellarg( "$url.sha256" ) ) )->stdout ), " \t" ); + $checksum = strtok( trim( EE::launch( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 --max-time 60 %s', escapeshellarg( "$url.sha256" ) ) )->stdout ), " \t" ); $actual = is_file( $docker_compose_new_path ) ? hash_file( 'sha256', $docker_compose_new_path ) : ''; if ( ! preg_match( '/^[0-9a-f]{64}$/', (string) $checksum ) || ! hash_equals( (string) $checksum, (string) $actual ) ) { $failure = "the checksum of $url doesn't match"; diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 0f9cfa133..742826d8b 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -72,6 +72,12 @@ public static function get_service_name( $container ) { public static function upgrade_global_service( $service, $container ) { EE::debug( "Start upgrading $container" ); + // A config already broken on disk would stop the new proxy and the restored one alike: refuse before touching it. + if ( 'global-nginx-proxy' === $service && 'running' === self::get_container_state( $container )['status'] + && ! EE::exec( 'docker exec ' . escapeshellarg( $container ) . ' nginx -t' ) ) { + throw new \Exception( "The configuration of the running $container fails `nginx -t`, so a recreated proxy would not start. Fix it (see `docker exec $container nginx -t`) and run the upgrade again." ); + } + if ( ! chdir( EE_SERVICE_DIR ) || ! self::compose_up( $service ) ) { throw new \Exception( "Unable to recreate $container with its new image. Please check logs for more details." ); } @@ -222,7 +228,8 @@ private static function probe( $service, $container ) { $port = (string) \EE\Utils\get_config_value( 'proxy_80_port', '80' ); $host = false === strpos( $port, ':' ) ? '127.0.0.1' : substr( $port, 0, strrpos( $port, ':' ) ); $port = false === strpos( $port, ':' ) ? $port : substr( $port, strrpos( $port, ':' ) + 1 ); - $status = trim( EE::launch( sprintf( "curl -s -o /dev/null -w '%%{http_code}' --max-time 5 %s", escapeshellarg( "http://$host:$port/" ) ) )->stdout ); + // --noproxy: an http_proxy exported for pulls must not answer for the local proxy. + $status = trim( EE::launch( sprintf( "curl -s --noproxy '*' -o /dev/null -w '%%{http_code}' --max-time 5 %s", escapeshellarg( "http://$host:$port/" ) ) )->stdout ); return preg_match( '/^[1-5]\d\d$/', $status ) ? '' : "no HTTP response on port $port"; From e495dcf46f730206dd1151f6efaaed4ad0b0961f Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:21:21 +0530 Subject: [PATCH 13/20] fix(migration): check the running proxy config before any global service is recreated The nginx -t check of the running proxy ran in the proxy's own upgrade step, which comes last, so a config already broken on disk only stopped the upgrade after cron, redis and the database were recreated on their new images, and the rollback then started the old database image on a datadir the new one may have upgraded. The check now runs as its own step before the global compose file is backed up and regenerated, so a broken proxy config stops the upgrade before any service is touched. --- php/EE/Migration/Containers.php | 13 ++++++++++++- php/EE/Migration/GlobalContainers.php | 20 ++++++++++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index ad7dce0ea..102466456 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -355,6 +355,18 @@ private static function migrate_global_containers( $updated_images ) { $global_compose_file_path = EE_ROOT_DIR . '/services/docker-compose.yml'; $global_compose_file_backup_path = EE_BACKUP_DIR . '/services/docker-compose.yml.backup'; + $all_global_images = GlobalContainers::get_all_global_images_with_service_name(); + + if ( in_array( 'easyengine/nginx-proxy', $updated_global_images, true ) ) { + // Before anything is recreated: a rollback after the DB moved to a new image may not get its datadir back. + self::$rsp->add_step( + 'check-running-proxy-config', + 'EE\Migration\GlobalContainers::check_running_proxy_config', + null, + [ $all_global_images['easyengine/nginx-proxy'] ], + null + ); + } self::$rsp->add_step( 'backup-global-docker-compose-file', @@ -374,7 +386,6 @@ private static function migrate_global_containers( $updated_images ) { ); // No down steps: the backup step's down restores the old compose file and recreates these services from it. - $all_global_images = GlobalContainers::get_all_global_images_with_service_name(); foreach ( GlobalContainers::sort_for_upgrade( $updated_global_images ) as $image_name ) { $global_container_name = $all_global_images[ $image_name ]; self::$rsp->add_step( diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 742826d8b..715da1f65 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -60,6 +60,20 @@ public static function get_service_name( $container ) { return preg_replace( '/^services_(.+)_1$/', '$1', $container ); } + /** + * Refuse to upgrade when the running proxy's config fails `nginx -t`: it lives on volumes, so the recreated proxy and + * the restored one would both fail to start. Runs before any global service is recreated. + * + * @param string $container Proxy container name. + * + * @throws \Exception + */ + public static function check_running_proxy_config( $container ) { + if ( 'running' === self::get_container_state( $container )['status'] && ! EE::exec( 'docker exec ' . escapeshellarg( $container ) . ' nginx -t' ) ) { + throw new \Exception( "The configuration of the running $container fails `nginx -t`, so a recreated proxy would not start. Fix it (see `docker exec $container nginx -t`) and run the upgrade again." ); + } + } + /** * Recreate one global service with the image in the (already regenerated) global compose file, * then wait until it is ready. Throws on failure, so the image migration rolls back. @@ -72,12 +86,6 @@ public static function get_service_name( $container ) { public static function upgrade_global_service( $service, $container ) { EE::debug( "Start upgrading $container" ); - // A config already broken on disk would stop the new proxy and the restored one alike: refuse before touching it. - if ( 'global-nginx-proxy' === $service && 'running' === self::get_container_state( $container )['status'] - && ! EE::exec( 'docker exec ' . escapeshellarg( $container ) . ' nginx -t' ) ) { - throw new \Exception( "The configuration of the running $container fails `nginx -t`, so a recreated proxy would not start. Fix it (see `docker exec $container nginx -t`) and run the upgrade again." ); - } - if ( ! chdir( EE_SERVICE_DIR ) || ! self::compose_up( $service ) ) { throw new \Exception( "Unable to recreate $container with its new image. Please check logs for more details." ); } From 853379a2e85c658c3b341916ae3473784b9bd13d Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:23:45 +0530 Subject: [PATCH 14/20] fix(migration): stop and remove first only the global services docker-compose v1 must recreate Every recreate and the rollback stopped and removed the listed services before up -d whenever docker-compose was older than 2.26.1. Only v1 needs that (it fails with KeyError: 'ContainerConfig' when it recreates a container on Docker 25+); v2 builds below 2.26.1 recreate in place. And because the rollback listed every updated service, a failure early in the upgrade also stopped and removed the database and the proxy that were still running untouched on their old images, taking every site down while the rollback ran. Now only v1 takes the stop and remove path, and only for the services this run has actually recreated; up -d leaves the others alone on both versions. The rollback also derives the service names with the same helper as the upgrade steps. --- php/EE/Migration/Containers.php | 1 + php/EE/Migration/GlobalContainers.php | 33 +++++++++++++++++++-------- 2 files changed, 25 insertions(+), 9 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 102466456..0c75cca2f 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -356,6 +356,7 @@ private static function migrate_global_containers( $updated_images ) { $global_compose_file_path = EE_ROOT_DIR . '/services/docker-compose.yml'; $global_compose_file_backup_path = EE_BACKUP_DIR . '/services/docker-compose.yml.backup'; $all_global_images = GlobalContainers::get_all_global_images_with_service_name(); + GlobalContainers::reset_recreated(); if ( in_array( 'easyengine/nginx-proxy', $updated_global_images, true ) ) { // Before anything is recreated: a rollback after the DB moved to a new image may not get its datadir back. diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 715da1f65..4d0f61b86 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -38,6 +38,9 @@ class GlobalContainers { /** @var callable|null Test seam: replaces sleep( $seconds ) while waiting for a service. */ public static $sleeper; + /** @var string[] Services this run has recreated, the only ones a rollback must recreate again. */ + private static $recreated = []; + /** * Sort updated global images into the order their services are upgraded in. * @@ -60,6 +63,13 @@ public static function get_service_name( $container ) { return preg_replace( '/^services_(.+)_1$/', '$1', $container ); } + /** + * Start a new image migration with no service recorded as recreated. + */ + public static function reset_recreated() { + self::$recreated = []; + } + /** * Refuse to upgrade when the running proxy's config fails `nginx -t`: it lives on volumes, so the recreated proxy and * the restored one would both fail to start. Runs before any global service is recreated. @@ -86,7 +96,8 @@ public static function check_running_proxy_config( $container ) { public static function upgrade_global_service( $service, $container ) { EE::debug( "Start upgrading $container" ); - if ( ! chdir( EE_SERVICE_DIR ) || ! self::compose_up( $service ) ) { + self::$recreated[] = $service; + if ( ! chdir( EE_SERVICE_DIR ) || ! self::compose_up( $service, [ $service ] ) ) { throw new \Exception( "Unable to recreate $container with its new image. Please check logs for more details." ); } @@ -96,18 +107,21 @@ public static function upgrade_global_service( $service, $container ) { /** * `up -d` global services in the current directory. docker-compose v1 can't recreate a container on Docker 25+ - * (KeyError: 'ContainerConfig', leaving it stopped), so below the minimum version they are stopped and removed first. + * (KeyError: 'ContainerConfig', leaving it stopped), so with v1 the services to recreate are stopped and removed first. * - * @param string $services Space-separated compose services. + * @param string $services Space-separated compose services. + * @param string[] $recreate Those whose container must be replaced; v1's `up -d` leaves the others alone. * * @return bool */ - private static function compose_up( $services ) { + private static function compose_up( $services, $recreate ) { $compose = \EE_DOCKER::docker_compose_with_custom(); $up = "$compose up -d $services"; $version = Containers::get_docker_compose_version(); - if ( '' !== $version && version_compare( $version, Containers::DOCKER_COMPOSE_MIN_VERSION, '<' ) ) { - return EE::exec( "$compose stop $services && $compose rm -f $services && $up" ); + if ( $recreate && '' !== $version && version_compare( $version, '2.0.0', '<' ) ) { + $recreate = implode( ' ', $recreate ); + + return EE::exec( "$compose stop $recreate && $compose rm -f $recreate && $up" ); } return EE::exec( $up ); @@ -302,9 +316,10 @@ public static function revert_global_containers( $source_path, $dest_path, $upda $services_to_regenerate = ''; $all_global_images = self::get_all_global_images_with_service_name(); foreach ( $updated_images as $image_name ) { - $global_container_name = $all_global_images[ $image_name ]; - $services_to_regenerate .= ltrim( rtrim( $global_container_name, '_1' ), 'services_' ) . ' '; + $services_to_regenerate .= self::get_service_name( $all_global_images[ $image_name ] ) . ' '; } + $recreated = self::$recreated; + self::$recreated = []; if ( empty( trim( $services_to_regenerate ) ) ) { return; } @@ -314,7 +329,7 @@ public static function revert_global_containers( $source_path, $dest_path, $upda chdir( EE_ROOT_DIR . '/services' ); - if ( ! self::compose_up( trim( $services_to_regenerate ) ) ) { + if ( ! self::compose_up( trim( $services_to_regenerate ), array_values( array_intersect( explode( ' ', trim( $services_to_regenerate ) ), $recreated ) ) ) ) { throw new \Exception( 'Unable to downgrade global containers. Please check logs for more details.' ); } foreach ( $updated_images as $image_name ) { From 95b8944b4abad59af89770bdde3c4ab4c7c891fa Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:26:16 +0530 Subject: [PATCH 15/20] fix(migration): bound the retries of the docker-compose download curl retries a transfer that hit --max-time, so with --retry 3 a stalled download could hold the upgrade for about 20 minutes, and the upgrade tries the install up to twice per run when it fails. --retry-max-time stops new attempts after two minutes (one for the checksum). --- php/EE/Migration/Containers.php | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 0c75cca2f..4f7d10586 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -165,11 +165,11 @@ public static function update_docker_compose() { $fs->remove( $docker_compose_new_path ); $failure = ''; - // --max-time: a stalled download must not hang the upgrade. - if ( ! EE::exec( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 --max-time 300 -o %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $url ) ) ) ) { + // --max-time and --retry-max-time: a stalled download must not hang the upgrade (curl retries timeouts too). + if ( ! EE::exec( sprintf( 'curl -fsSL --retry 3 --retry-max-time 120 --connect-timeout 10 --max-time 300 -o %s %s', escapeshellarg( $docker_compose_new_path ), escapeshellarg( $url ) ) ) ) { $failure = "the download of $url failed"; } else { - $checksum = strtok( trim( EE::launch( sprintf( 'curl -fsSL --retry 3 --connect-timeout 10 --max-time 60 %s', escapeshellarg( "$url.sha256" ) ) )->stdout ), " \t" ); + $checksum = strtok( trim( EE::launch( sprintf( 'curl -fsSL --retry 3 --retry-max-time 60 --connect-timeout 10 --max-time 60 %s', escapeshellarg( "$url.sha256" ) ) )->stdout ), " \t" ); $actual = is_file( $docker_compose_new_path ) ? hash_file( 'sha256', $docker_compose_new_path ) : ''; if ( ! preg_match( '/^[0-9a-f]{64}$/', (string) $checksum ) || ! hash_equals( (string) $checksum, (string) $actual ) ) { $failure = "the checksum of $url doesn't match"; From 2b6a4c7c4a18f14be5aa90154fc23f6346b1e8a5 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 21:36:29 +0530 Subject: [PATCH 16/20] fix(migration): keep the global compose snapshot of an upgrade that did not complete An upgrade killed after it recreated some global services leaves their new tags in the global compose file. The next attempt copied that half-upgraded file over the snapshot, so if it then failed, its rollback restored the new tags and recreated the services it had not touched yet, the database and the proxy included, on their new images instead of putting them back. The snapshot is now marked pending until the upgrade completes or is rolled back, and a run that finds a pending snapshot keeps it instead of taking a new one. --- php/EE/Migration/Containers.php | 11 ++++++++++- php/EE/Migration/GlobalContainers.php | 28 +++++++++++++++++++++++++++ 2 files changed, 38 insertions(+), 1 deletion(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 4f7d10586..608482f6f 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -114,6 +114,15 @@ public static function save_upgraded_image_versions( $current_versions, $new_ver [ $current_versions, $updated_images ] ); + // The upgrade is committed: a later run takes a fresh snapshot of the global compose file. + self::$rsp->add_step( + 'clear-pending-global-compose-backup', + 'EE\Migration\GlobalContainers::clear_pending_compose_backup', + null, + [ EE_BACKUP_DIR . '/services/docker-compose.yml.backup' ], + null + ); + self::$rsp->add_step( 'prune-old-docker-images', 'EE\Migration\Containers::docker_cleanup', @@ -371,7 +380,7 @@ private static function migrate_global_containers( $updated_images ) { self::$rsp->add_step( 'backup-global-docker-compose-file', - 'EE\Migration\SiteContainers::backup_restore', + 'EE\Migration\GlobalContainers::backup_compose_file', 'EE\Migration\GlobalContainers::revert_global_containers', [ $global_compose_file_path, $global_compose_file_backup_path ], [ $global_compose_file_backup_path, $global_compose_file_path, $updated_global_images ] diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 4d0f61b86..4f2a25d03 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -63,6 +63,33 @@ public static function get_service_name( $container ) { return preg_replace( '/^services_(.+)_1$/', '$1', $container ); } + /** + * Snapshot the global compose file before it is regenerated. A run that was interrupted after its snapshot left the + * services half upgraded, so its snapshot (marked pending until a run completes) is kept instead of the current file. + * + * @param string $source Global compose file. + * @param string $backup Snapshot path. + */ + public static function backup_compose_file( $source, $backup ) { + $fs = new Filesystem(); + if ( $fs->exists( [ $backup, $backup . '.pending' ] ) ) { + EE::debug( "Keeping $backup from an upgrade that did not complete" ); + + return; + } + $fs->copy( $source, $backup, true ); + $fs->touch( $backup . '.pending' ); + } + + /** + * Mark the compose snapshot as no longer needed by a later run: the upgrade completed or was rolled back. + * + * @param string $backup Snapshot path. + */ + public static function clear_pending_compose_backup( $backup ) { + ( new Filesystem() )->remove( $backup . '.pending' ); + } + /** * Start a new image migration with no service recorded as recreated. */ @@ -340,6 +367,7 @@ public static function revert_global_containers( $source_path, $dest_path, $upda EE::warning( 'After restoring the previous image: ' . $e->getMessage() ); } } + self::clear_pending_compose_backup( $source_path ); EE::debug( 'Complete restoring global docker-compose.yml file from backup' ); } From 8aaa57c5edc73a633845e671acb1e6b5547ba169 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 22:02:59 +0530 Subject: [PATCH 17/20] fix(migration): roll back the global services an interrupted attempt already upgraded A service that a killed upgrade had already recreated runs its new image, so the next attempt doesn't count it as updated. When that attempt failed, the rollback restored the old compose file but left the service on the new image, out of step with its compose configuration and recorded tag. The rollback now also recreates every global service whose running image differs from the one in the restored compose file, and checks its readiness like the others. --- php/EE/Migration/GlobalContainers.php | 47 ++++++++++++++++++++++----- 1 file changed, 39 insertions(+), 8 deletions(-) diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 4f2a25d03..c219489b8 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -4,6 +4,7 @@ use EE; use Symfony\Component\Filesystem\Filesystem; +use Symfony\Component\Yaml\Yaml; /** * Upgrade existing global containers to new docker-image @@ -340,29 +341,34 @@ public static function get_updated_global_images( $updated_images ) { */ public static function revert_global_containers( $source_path, $dest_path, $updated_images ) { - $services_to_regenerate = ''; - $all_global_images = self::get_all_global_images_with_service_name(); + $all_global_images = self::get_all_global_images_with_service_name(); + $services = []; foreach ( $updated_images as $image_name ) { - $services_to_regenerate .= self::get_service_name( $all_global_images[ $image_name ] ) . ' '; + $services[] = self::get_service_name( $all_global_images[ $image_name ] ); } $recreated = self::$recreated; self::$recreated = []; - if ( empty( trim( $services_to_regenerate ) ) ) { + if ( empty( $services ) ) { return; } EE::debug( 'Start restoring global docker-compose.yml file from backup' ); $fs = new Filesystem(); $fs->copy( $source_path, $dest_path, true ); + // Services an earlier, interrupted attempt moved to a new image aren't among this run's updates: put them back too. + $moved = array_diff( self::get_services_off_compose_file( $dest_path ), $services ); + $services = array_merge( $services, $moved ); + $recreate = array_values( array_merge( array_intersect( $services, $recreated ), $moved ) ); + chdir( EE_ROOT_DIR . '/services' ); - if ( ! self::compose_up( trim( $services_to_regenerate ), array_values( array_intersect( explode( ' ', trim( $services_to_regenerate ) ), $recreated ) ) ) ) { + if ( ! self::compose_up( implode( ' ', $services ), $recreate ) ) { throw new \Exception( 'Unable to downgrade global containers. Please check logs for more details.' ); } - foreach ( $updated_images as $image_name ) { - $container = $all_global_images[ $image_name ]; + $containers = array_combine( array_map( [ self::class, 'get_service_name' ], $all_global_images ), $all_global_images ); + foreach ( $services as $service ) { try { - self::wait_until_ready( self::get_service_name( $container ), $container ); + self::wait_until_ready( $service, $containers[ $service ] ); } catch ( \Exception $e ) { EE::warning( 'After restoring the previous image: ' . $e->getMessage() ); } @@ -371,6 +377,31 @@ public static function revert_global_containers( $source_path, $dest_path, $upda EE::debug( 'Complete restoring global docker-compose.yml file from backup' ); } + /** + * @param string $compose_file Global compose file. + * + * @return string[] Global services that run an image other than the one the compose file sets for them. + */ + private static function get_services_off_compose_file( $compose_file ) { + try { + $config = Yaml::parseFile( $compose_file ); + } catch ( \Exception $e ) { + return []; + } + $services = []; + foreach ( self::get_all_global_images_with_service_name() as $container ) { + $service = self::get_service_name( $container ); + $image = is_array( $config ) && isset( $config['services'][ $service ]['image'] ) ? $config['services'][ $service ]['image'] : ''; + $running = EE::launch( sprintf( "docker inspect --format '{{.State.Status}} {{.Config.Image}}' %s", escapeshellarg( $container ) ) ); + $parts = 0 === $running->return_code ? preg_split( '/\s+/', trim( $running->stdout ) ) : []; + if ( '' !== $image && 2 === count( $parts ) && 'running' === $parts[0] && $parts[1] !== $image ) { + $services[] = $service; + } + } + + return $services; + } + /** * Stop global container and remove them. * From 73d60d42d37c2582851253b2aef3e2adf5ac8d28 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 22:14:47 +0530 Subject: [PATCH 18/20] fix(migration): read the global compose snapshot with Spyc, which the phar ships The phar leaves out symfony/yaml, so the rollback stopped with a class-not-found error before it recreated anything. --- php/EE/Migration/GlobalContainers.php | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index c219489b8..70ab407a0 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -4,7 +4,7 @@ use EE; use Symfony\Component\Filesystem\Filesystem; -use Symfony\Component\Yaml\Yaml; +use Mustangostang\Spyc; /** * Upgrade existing global containers to new docker-image @@ -383,11 +383,8 @@ public static function revert_global_containers( $source_path, $dest_path, $upda * @return string[] Global services that run an image other than the one the compose file sets for them. */ private static function get_services_off_compose_file( $compose_file ) { - try { - $config = Yaml::parseFile( $compose_file ); - } catch ( \Exception $e ) { - return []; - } + // Spyc, not symfony/yaml: the phar doesn't ship symfony/yaml. + $config = is_file( $compose_file ) ? Spyc::YAMLLoad( $compose_file ) : []; $services = []; foreach ( self::get_all_global_images_with_service_name() as $container ) { $service = self::get_service_name( $container ); From ef097e193b21557f744f733b8e13a7669a89a3a9 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Fri, 23 Oct 2026 22:35:13 +0530 Subject: [PATCH 19/20] fix(migration): recheck global services an interrupted upgrade recreated A killed upgrade leaves the services it recreated on their new tags without recording them, and running containers take precedence over the recorded tags. The next run therefore skipped them: one stuck in a restart loop on a broken new image was never gated, and with nothing else to update the run reported success and left the pending snapshot behind. While a pending snapshot exists, the global services now count by their recorded tags, so the next run recreates (a no-op when nothing changed) and gates them, and can roll them back to the snapshot. A restart-looping global service counts as present, both when choosing the services to upgrade and when a rollback looks for services off the snapshot. --- php/EE/Migration/Containers.php | 11 +++++++++++ php/EE/Migration/GlobalContainers.php | 17 ++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) diff --git a/php/EE/Migration/Containers.php b/php/EE/Migration/Containers.php index 608482f6f..b2ce883c6 100644 --- a/php/EE/Migration/Containers.php +++ b/php/EE/Migration/Containers.php @@ -37,6 +37,17 @@ public static function start_container_migration() { $current_versions = self::get_current_docker_images_versions(); $updated_images = []; + // An interrupted upgrade recreated some global services without recording their tags. Count those by their + // recorded tags, so this run checks their readiness again and can roll them back to the pending snapshot. + if ( GlobalContainers::has_pending_compose_backup( EE_BACKUP_DIR . '/services/docker-compose.yml.backup' ) ) { + foreach ( array_keys( GlobalContainers::get_all_global_images_with_service_name() ) as $image ) { + $recorded = EE\Model\Option::get( $image ); + if ( $recorded ) { + $current_versions[ $image ] = $recorded; + } + } + } + $skip_download = [ 'easyengine/php5.6', 'easyengine/php7.0', diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 70ab407a0..88d9a1998 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -73,7 +73,7 @@ public static function get_service_name( $container ) { */ public static function backup_compose_file( $source, $backup ) { $fs = new Filesystem(); - if ( $fs->exists( [ $backup, $backup . '.pending' ] ) ) { + if ( self::has_pending_compose_backup( $backup ) ) { EE::debug( "Keeping $backup from an upgrade that did not complete" ); return; @@ -82,6 +82,15 @@ public static function backup_compose_file( $source, $backup ) { $fs->touch( $backup . '.pending' ); } + /** + * @param string $backup Snapshot path. + * + * @return bool Whether an upgrade that did not complete left its snapshot. + */ + public static function has_pending_compose_backup( $backup ) { + return ( new Filesystem() )->exists( [ $backup, $backup . '.pending' ] ); + } + /** * Mark the compose snapshot as no longer needed by a later run: the upgrade completed or was rolled back. * @@ -323,7 +332,8 @@ public static function get_updated_global_images( $updated_images ) { $global_images = self::get_all_global_images_with_service_name(); $running_global_services = []; foreach ( $global_images as $image => $container_name ) { - if ( 'running' === \EE_DOCKER::container_status( $container_name ) ) { + // A restart-looping service is upgraded too (State.Running is false between its restarts). + if ( in_array( self::get_container_state( $container_name )['status'], [ 'running', 'restarting' ], true ) ) { $running_global_services[] = $image; } } @@ -391,7 +401,8 @@ private static function get_services_off_compose_file( $compose_file ) { $image = is_array( $config ) && isset( $config['services'][ $service ]['image'] ) ? $config['services'][ $service ]['image'] : ''; $running = EE::launch( sprintf( "docker inspect --format '{{.State.Status}} {{.Config.Image}}' %s", escapeshellarg( $container ) ) ); $parts = 0 === $running->return_code ? preg_split( '/\s+/', trim( $running->stdout ) ) : []; - if ( '' !== $image && 2 === count( $parts ) && 'running' === $parts[0] && $parts[1] !== $image ) { + // A restart-looping container counts: it is the failed new image to put back. One never created doesn't. + if ( '' !== $image && 2 === count( $parts ) && in_array( $parts[0], [ 'running', 'restarting' ], true ) && $parts[1] !== $image ) { $services[] = $service; } } From 625d170145c71a2e333cc5b65d39fde00f620114 Mon Sep 17 00:00:00 2001 From: Riddhesh Sanghvi Date: Mon, 26 Oct 2026 08:26:28 +0530 Subject: [PATCH 20/20] fix(migration): bound every readiness probe with a timeout inside the container The readiness probes ran `docker exec` with no time limit, so a redis or database that accepts connections but never answers hung the upgrade forever. Each probe (redis-cli ping, the database ping and nginx -t, including the check of the running proxy config) now runs under `timeout 10` inside the container, and a probe killed by it counts as not ready, so the existing wait and rollback handle it. Images whose `timeout` doesn't work (busybox before 1.30) run the probe as before. --- php/EE/Migration/GlobalContainers.php | 62 +++++++++++++++++++++++---- 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/php/EE/Migration/GlobalContainers.php b/php/EE/Migration/GlobalContainers.php index 88d9a1998..6320e387b 100644 --- a/php/EE/Migration/GlobalContainers.php +++ b/php/EE/Migration/GlobalContainers.php @@ -33,6 +33,9 @@ class GlobalContainers { /** Seconds a service must stay up without restarting after its first successful probe. */ const SETTLE_SECONDS = 2; + /** Seconds one readiness probe may run inside the container before it counts as failed. */ + const PROBE_TIMEOUT = 10; + /** Services with a readiness probe. The others (cron, newrelic-daemon) only need to have started. */ const PROBED_SERVICES = [ 'global-nginx-proxy', 'global-db', 'global-redis' ]; @@ -116,7 +119,14 @@ public static function reset_recreated() { * @throws \Exception */ public static function check_running_proxy_config( $container ) { - if ( 'running' === self::get_container_state( $container )['status'] && ! EE::exec( 'docker exec ' . escapeshellarg( $container ) . ' nginx -t' ) ) { + if ( 'running' !== self::get_container_state( $container )['status'] ) { + return; + } + $result = self::probe_exec( $container, 'nginx', '-t' ); + if ( self::timed_out( $result ) ) { + throw new \Exception( sprintf( '`nginx -t` in the running %s did not finish within %d s, so the upgrade cannot check its configuration. Check `docker exec %s nginx -t` and run the upgrade again.', $container, self::PROBE_TIMEOUT, $container ) ); + } + if ( $result->return_code ) { throw new \Exception( "The configuration of the running $container fails `nginx -t`, so a recreated proxy would not start. Fix it (see `docker exec $container nginx -t`) and run the upgrade again." ); } } @@ -276,12 +286,11 @@ private static function not_ready_reason( $service, $container, $state ) { * @return string Why it failed, or '' when it passed. */ private static function probe( $service, $container ) { - $exec = 'docker exec ' . escapeshellarg( $container ) . ' '; - switch ( $service ) { case 'global-nginx-proxy': - if ( ! EE::exec( $exec . 'nginx -t' ) ) { - return 'nginx -t failed'; + $result = self::probe_exec( $container, 'nginx', '-t' ); + if ( $result->return_code ) { + return self::probe_failure( 'nginx -t', $result ); } // Any HTTP status proves nginx answers through the published port; 443 is the same process. $port = (string) \EE\Utils\get_config_value( 'proxy_80_port', '80' ); @@ -296,18 +305,55 @@ private static function probe( $service, $container ) { // Over TCP: the entrypoint's temporary init/upgrade server listens on the socket only. Access denied still means alive. $ping = 'for c in mariadb-admin mysqladmin; do command -v $c >/dev/null 2>&1 && exec $c ping -h 127.0.0.1 --protocol=tcp --connect-timeout=2 --silent; done; exit 1'; - return EE::exec( $exec . 'sh -c ' . escapeshellarg( $ping ) ) ? '' : 'mysqladmin ping failed'; + $result = self::probe_exec( $container, 'sh', '-c', $ping ); + + return $result->return_code ? self::probe_failure( 'mysqladmin ping', $result ) : ''; case 'global-redis': - $pong = trim( EE::launch( $exec . 'redis-cli ping' )->stdout ); + $result = self::probe_exec( $container, 'redis-cli', 'ping' ); + $pong = trim( $result->stdout ); // NOAUTH (a requirepass in redis.conf) still means redis answers. - return 'PONG' === $pong || 0 === strpos( $pong, 'NOAUTH' ) ? '' : 'redis-cli ping failed'; + return 'PONG' === $pong || 0 === strpos( $pong, 'NOAUTH' ) ? '' : self::probe_failure( 'redis-cli ping', $result ); } return ''; } + /** + * Run a probe command in a container under `timeout`, so a service that accepts connections but never answers + * can't hang the upgrade. Where `timeout` doesn't work (busybox before 1.30 wants `-t`), the command runs unbounded. + * + * @param string $container Container name. + * @param string ...$args Command and its arguments. + * + * @return \EE\ProcessRun + */ + private static function probe_exec( $container, ...$args ) { + $bounded = sprintf( 'if timeout 1 true >/dev/null 2>&1; then exec timeout %d "$@"; fi; exec "$@"', self::PROBE_TIMEOUT ); + + return EE::launch( 'docker exec ' . escapeshellarg( $container ) . ' sh -c ' . escapeshellarg( $bounded ) . ' probe ' . implode( ' ', array_map( 'escapeshellarg', $args ) ) ); + } + + /** + * @param \EE\ProcessRun $result From probe_exec(). + * + * @return bool Whether `timeout` killed the probe: 124 from coreutils, 143 (SIGTERM) from busybox. + */ + private static function timed_out( $result ) { + return in_array( (int) $result->return_code, [ 124, 143 ], true ); + } + + /** + * @param string $probe Probe name. + * @param \EE\ProcessRun $result From probe_exec(). + * + * @return string Why the probe failed. + */ + private static function probe_failure( $probe, $result ) { + return self::timed_out( $result ) ? sprintf( '%s timed out after %d s', $probe, self::PROBE_TIMEOUT ) : "$probe failed"; + } + /** * @param int|float $seconds Seconds to sleep. */