From 22a03d8853ceaf9d4125946e55925c32c61448ad Mon Sep 17 00:00:00 2001 From: Mohamed Rasik Farid Date: Thu, 5 Nov 2026 20:53:47 +0530 Subject: [PATCH 1/2] fix: prevent random test runner deadlock with one job --- .github/scripts/run-random-tests.sh | 5 ++- .github/scripts/test-run-random-tests.sh | 44 +++++++++++++++++++++ .github/workflows/test-random-execution.yml | 5 +++ 3 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/test-run-random-tests.sh diff --git a/.github/scripts/run-random-tests.sh b/.github/scripts/run-random-tests.sh index a0a962efbba5..5b51d2a22cde 100755 --- a/.github/scripts/run-random-tests.sh +++ b/.github/scripts/run-random-tests.sh @@ -665,7 +665,10 @@ cleanup_finished_pids() { active+=("$pid") fi done - bg_pids=("${active[@]:-}") + bg_pids=() + if [[ ${#active[@]} -gt 0 ]]; then + bg_pids=("${active[@]}") + fi } spawn_limited_job() { diff --git a/.github/scripts/test-run-random-tests.sh b/.github/scripts/test-run-random-tests.sh new file mode 100644 index 000000000000..3b6a30d4f887 --- /dev/null +++ b/.github/scripts/test-run-random-tests.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash + +# Exercise the scheduler in an isolated project with a lightweight PHPUnit stub. +set -eu + +script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +test_root="$(mktemp -d)" +trap 'rm -rf "$test_root"' EXIT + +mkdir -p "$test_root/.github/scripts" "$test_root/vendor/bin" \ + "$test_root/tests/system/First" "$test_root/tests/system/Second" +cp "$script_dir/run-random-tests.sh" "$test_root/.github/scripts/" +printf '%s\n' First Second > "$test_root/.github/scripts/random-tests-config.txt" + +cat > "$test_root/vendor/bin/phpunit" <<'PHPUNIT' +#!/usr/bin/env bash +printf '%s\n' "$1" >> calls.log +sleep 0.05 +PHPUNIT +chmod +x "$test_root/vendor/bin/phpunit" + +# A deadline makes a scheduler deadlock fail instead of hanging the CI job. +for max_jobs in 1 2; do + : > "$test_root/calls.log" + if ! GITHUB_ACTIONS=true timeout --kill-after=1s 5s \ + bash "$test_root/.github/scripts/run-random-tests.sh" \ + --max-jobs "$max_jobs" --repeat 2 --timeout 0 \ + > "$test_root/output.log" 2>&1; then + cat "$test_root/output.log" + echo "Scheduler failed with --max-jobs $max_jobs" >&2 + exit 1 + fi + + for component in First Second; do + count=$(grep -c "^tests/system/$component$" "$test_root/calls.log" || true) + if [[ "$count" -ne 2 ]]; then + cat "$test_root/output.log" + echo "Expected two runs of $component, got $count" >&2 + exit 1 + fi + done +done + +echo 'Random test scheduler checks passed.' diff --git a/.github/workflows/test-random-execution.yml b/.github/workflows/test-random-execution.yml index a98d50cec3ad..3d0b8b072de7 100644 --- a/.github/workflows/test-random-execution.yml +++ b/.github/workflows/test-random-execution.yml @@ -7,6 +7,7 @@ on: - '4.*' paths: - '.github/scripts/run-random-tests.sh' + - '.github/scripts/test-run-random-tests.sh' - '.github/scripts/random-tests-config.txt' - '.github/workflows/test-random-execution.yml' - 'phpunit.dist.xml' @@ -19,6 +20,7 @@ on: - '4.*' paths: - '.github/scripts/run-random-tests.sh' + - '.github/scripts/test-run-random-tests.sh' - '.github/scripts/random-tests-config.txt' - '.github/workflows/test-random-execution.yml' - 'phpunit.dist.xml' @@ -173,6 +175,9 @@ jobs: persist-credentials: false fetch-depth: 0 + - name: Test random-test scheduler + run: bash .github/scripts/test-run-random-tests.sh + - name: Setup PHP ${{ matrix.php-version }} uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # 2.40.0 with: From 736bcd3916581958530f3f3989b9389487dc02ec Mon Sep 17 00:00:00 2001 From: Mohamed Rasik Farid Date: Thu, 5 Nov 2026 22:11:36 +0530 Subject: [PATCH 2/2] test: move random runner regression coverage to AutoReview --- .github/scripts/test-run-random-tests.sh | 44 ------- .github/workflows/test-random-execution.yml | 5 - .../system/AutoReview/RunRandomTestsTest.php | 109 ++++++++++++++++++ 3 files changed, 109 insertions(+), 49 deletions(-) delete mode 100644 .github/scripts/test-run-random-tests.sh create mode 100644 tests/system/AutoReview/RunRandomTestsTest.php diff --git a/.github/scripts/test-run-random-tests.sh b/.github/scripts/test-run-random-tests.sh deleted file mode 100644 index 3b6a30d4f887..000000000000 --- a/.github/scripts/test-run-random-tests.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# Exercise the scheduler in an isolated project with a lightweight PHPUnit stub. -set -eu - -script_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -test_root="$(mktemp -d)" -trap 'rm -rf "$test_root"' EXIT - -mkdir -p "$test_root/.github/scripts" "$test_root/vendor/bin" \ - "$test_root/tests/system/First" "$test_root/tests/system/Second" -cp "$script_dir/run-random-tests.sh" "$test_root/.github/scripts/" -printf '%s\n' First Second > "$test_root/.github/scripts/random-tests-config.txt" - -cat > "$test_root/vendor/bin/phpunit" <<'PHPUNIT' -#!/usr/bin/env bash -printf '%s\n' "$1" >> calls.log -sleep 0.05 -PHPUNIT -chmod +x "$test_root/vendor/bin/phpunit" - -# A deadline makes a scheduler deadlock fail instead of hanging the CI job. -for max_jobs in 1 2; do - : > "$test_root/calls.log" - if ! GITHUB_ACTIONS=true timeout --kill-after=1s 5s \ - bash "$test_root/.github/scripts/run-random-tests.sh" \ - --max-jobs "$max_jobs" --repeat 2 --timeout 0 \ - > "$test_root/output.log" 2>&1; then - cat "$test_root/output.log" - echo "Scheduler failed with --max-jobs $max_jobs" >&2 - exit 1 - fi - - for component in First Second; do - count=$(grep -c "^tests/system/$component$" "$test_root/calls.log" || true) - if [[ "$count" -ne 2 ]]; then - cat "$test_root/output.log" - echo "Expected two runs of $component, got $count" >&2 - exit 1 - fi - done -done - -echo 'Random test scheduler checks passed.' diff --git a/.github/workflows/test-random-execution.yml b/.github/workflows/test-random-execution.yml index 3d0b8b072de7..a98d50cec3ad 100644 --- a/.github/workflows/test-random-execution.yml +++ b/.github/workflows/test-random-execution.yml @@ -7,7 +7,6 @@ on: - '4.*' paths: - '.github/scripts/run-random-tests.sh' - - '.github/scripts/test-run-random-tests.sh' - '.github/scripts/random-tests-config.txt' - '.github/workflows/test-random-execution.yml' - 'phpunit.dist.xml' @@ -20,7 +19,6 @@ on: - '4.*' paths: - '.github/scripts/run-random-tests.sh' - - '.github/scripts/test-run-random-tests.sh' - '.github/scripts/random-tests-config.txt' - '.github/workflows/test-random-execution.yml' - 'phpunit.dist.xml' @@ -175,9 +173,6 @@ jobs: persist-credentials: false fetch-depth: 0 - - name: Test random-test scheduler - run: bash .github/scripts/test-run-random-tests.sh - - name: Setup PHP ${{ matrix.php-version }} uses: shivammathur/setup-php@eb7c497e18156a6bbabfef1d3a82760b9eda3962 # 2.40.0 with: diff --git a/tests/system/AutoReview/RunRandomTestsTest.php b/tests/system/AutoReview/RunRandomTestsTest.php new file mode 100644 index 000000000000..db84163c8780 --- /dev/null +++ b/tests/system/AutoReview/RunRandomTestsTest.php @@ -0,0 +1,109 @@ + + * + * For the full copyright and license information, please view + * the LICENSE file that was distributed with this source code. + */ + +namespace CodeIgniter\AutoReview; + +use FilesystemIterator; +use PHPUnit\Framework\Attributes\CoversNothing; +use PHPUnit\Framework\Attributes\DataProvider; +use PHPUnit\Framework\Attributes\Group; +use PHPUnit\Framework\TestCase; +use RecursiveDirectoryIterator; +use RecursiveIteratorIterator; + +/** + * @internal + */ +#[CoversNothing] +#[Group('AutoReview')] +final class RunRandomTestsTest extends TestCase +{ + private string $project; + + protected function setUp(): void + { + parent::setUp(); + + if (PHP_OS_FAMILY === 'Windows') { + $this->markTestSkipped('The random test runner requires Bash and GNU coreutils.'); + } + + $this->project = sys_get_temp_dir() . '/ci4-random-tests-' . bin2hex(random_bytes(4)); + + foreach (['.github/scripts', 'vendor/bin', 'tests/system/First', 'tests/system/Second'] as $directory) { + mkdir($this->project . '/' . $directory, 0777, true); + } + + copy(ROOTPATH . '.github/scripts/run-random-tests.sh', $this->project . '/.github/scripts/run-random-tests.sh'); + file_put_contents($this->project . '/.github/scripts/random-tests-config.txt', "First\nSecond\n"); + file_put_contents($this->project . '/vendor/bin/phpunit', <<<'BASH' + #!/usr/bin/env bash + printf '%s\n' "$1" >> calls.log + sleep 0.05 + + BASH); + chmod($this->project . '/vendor/bin/phpunit', 0755); + } + + protected function tearDown(): void + { + if (isset($this->project)) { + $files = new RecursiveIteratorIterator( + new RecursiveDirectoryIterator($this->project, FilesystemIterator::SKIP_DOTS), + RecursiveIteratorIterator::CHILD_FIRST, + ); + + foreach ($files as $file) { + $file->isDir() ? rmdir($file->getPathname()) : unlink($file->getPathname()); + } + + rmdir($this->project); + } + + parent::tearDown(); + } + + #[DataProvider('provideRunsEveryComponentOnEachRepetition')] + public function testRunsEveryComponentOnEachRepetition(int $maxJobs): void + { + // Bound the command so a scheduler regression fails instead of hanging PHPUnit. + exec(sprintf( + 'GITHUB_ACTIONS=true timeout --kill-after=1s 5s bash %s --max-jobs %d --repeat 2 --timeout 0 2>&1', + escapeshellarg($this->project . '/.github/scripts/run-random-tests.sh'), + $maxJobs, + ), $output, $exitCode); + $output = implode("\n", $output); + + $this->assertSame(0, $exitCode, $output); + $this->assertStringContainsString('All components passed random execution tests!', $output); + + $calls = file($this->project . '/calls.log', FILE_IGNORE_NEW_LINES); + + $this->assertIsArray($calls); + $this->assertCount(4, $calls); + + foreach (['First', 'Second'] as $component) { + $this->assertCount(2, array_keys($calls, 'tests/system/' . $component, true)); + } + } + + /** + * @return iterable + */ + public static function provideRunsEveryComponentOnEachRepetition(): iterable + { + yield 'one job' => [1]; + + yield 'two jobs' => [2]; + } +}