Visual suites: data-driven VisualTestsSuite, selector masks, per-run browser options

README.md

@@ -28,6 +28,29 @@ Resolution priority: fluent `onVersion()` → `$psVersion` property → `PRESTAF
 
 `onVersion()` throws `InvalidArgumentException` on malformed input (expected format: `1.7`, `1.7.8`, `1.7.8.11`, `9`, `9.0`, `9.0.1`, etc.).
 
+## Run only some suite folders
+
+`prestaflow run <path>` runs every suite under `<path>`. To run only some of
+its sub-folders, list them, comma-separated, relative to `<path>`:
+
+```bash
+./vendor/bin/prestaflow run tests --suites=BackOffice,FrontOffice/Checkout
+PRESTAFLOW_SUITES=BackOffice,FrontOffice/Checkout ./vendor/bin/prestaflow run tests
+```
+
+- Each name is a sub-folder of `<path>`, scanned recursively; nested paths such
+  as `FrontOffice/Checkout` are allowed. The run takes the union of the folders.
+- `--suites` wins over `PRESTAFLOW_SUITES`. The GitHub Action sets
+  `PRESTAFLOW_SUITES` from its `suites` input.
+- A name that matches no folder fails the run (non-zero exit code), listing the
+  missing names and the available sub-folders. A typo never becomes a green
+  job that ran zero tests.
+- A filtered run that ends up with no suite at all (an empty folder, or
+  nothing left after `--group` / `--draft`) fails too, for the same reason.
+- Absolute paths and `..` are refused: the filter can only narrow `<path>`.
+- `--group` and `--draft` still apply, on the suites of the selected folders.
+- Unset or empty: every suite under `<path>` runs, as before.
+
 ## Run a suite against a throwaway shop
 
 `docker-compose.yml` boots a disposable PrestaShop from the official

bin/prestaflow

@@ -28,48 +28,6 @@ if (!$autoloadLoaded) {
     exit(1);
 }
 
-use PrestaFlow\Library\Command\ExecuteSuite;
-use Symfony\Component\Console\Application;
-use Symfony\Component\Console\Command\Command;
-use Symfony\Component\Console\ConsoleEvents;
-use Symfony\Component\Console\Event\ConsoleErrorEvent;
-use Symfony\Component\Console\Style\SymfonyStyle;
-use Symfony\Component\EventDispatcher\EventDispatcher;
-
-$dispatcher = new EventDispatcher();
-
-$dispatcher->addListener(ConsoleEvents::ERROR, function (ConsoleErrorEvent $event): void {
-
-    $input = $event->getInput();
-    $output = $event->getOutput();
-
-    $io = new SymfonyStyle($input, $output);
-
-    $command = $event->getCommand();
-
-    $error = $event->getError();
-
-    $io->newLine();
-
-    $io->writeln(sprintf('<fg=red;options=bold>ERROR</> <fg=white>%s</>', $error->getMessage()));
-    $io->writeln(sprintf('<fg=gray;options=bold>TRACE</> <fg=white>%s</>', $error->getFile() . ':' . $error->getLine()));
-
-    foreach ($error->getTrace() as $trace) {
-        $io->writeln(sprintf('<fg=gray;options=bold>TRACE</> <fg=white>%s</>', ($trace['file'] ?? '[internal]') . ':' . ($trace['line'] ?? '?')));
-    }
-
-    // gets the current exit code (the exception code)
-    $exitCode = $event->getExitCode();
-
-    $event->setExitCode(Command::SUCCESS);
-
-    // changes the exception to another one
-    $event->setError(new \LogicException('Caught exception', $exitCode, $event->getError()));
-});
-
-$application = new Application();
-
-$application->add(new ExecuteSuite());
-$application->setDispatcher($dispatcher);
-
-$application->run();
+// Toute la configuration vit dans PrestaFlow\Library\Console\Application (testée) :
+// une exception non rattrapée y donne un code de sortie non nul.
+(new PrestaFlow\Library\Console\Application())->run();

composer.json

@@ -3,6 +3,9 @@
     "description": "PrestaFlow is an open-source set of prebuilt tests components, ready-to-use examples made for the PrestaShop E-commerce software.",
     "type": "library",
     "license": "MIT",
+    "bin": [
+        "bin/prestaflow"
+    ],
     "autoload": {
         "psr-4": {
             "PrestaFlow\\Library\\": "src/",
@@ -32,9 +35,9 @@
         "chrome-php/chrome": "^1.10",
         "sapientpro/image-comparator": "^1.0",
         "inceddy/image-compare": "^1.0",
-        "symfony/console": "^6.0",
+        "symfony/console": "^6.0 || ^7.0",
         "vlucas/phpdotenv": "^5.6",
-        "symfony/event-dispatcher": "^6.0",
+        "symfony/event-dispatcher": "^6.0 || ^7.0",
         "smalot/pdfparser": "^2.11"
     },
     "config": {

src/Command/ExecuteSuite.php

@@ -55,6 +55,7 @@ protected function configure(): void
             ->addOption('visual-report', null, InputOption::VALUE_OPTIONAL, 'Écrit un rapport de régression visuelle (HTML)', false)
             ->addOption('visual-report-tz', null, InputOption::VALUE_REQUIRED, 'Fuseau horaire du stamp du rapport visuel (ex. Europe/Brussels). Défaut : env PRESTAFLOW_TZ ou UTC.')
             ->addOption('draft', 'd', InputOption::VALUE_NEGATABLE, 'Draft mode')
+            ->addOption('suites', null, InputOption::VALUE_REQUIRED, 'Comma-separated sub-folders of the suites path to run (e.g. BackOffice,FrontOffice/Checkout). Overrides env PRESTAFLOW_SUITES.')
             ->addArgument('folder', InputArgument::OPTIONAL, 'The folder name', 'tests')
             ->addOption(
                 'group',
@@ -117,6 +118,7 @@ protected function outputTitle()
 
     public function execute(InputInterface $input, OutputInterface $output): int
     {
+        $this->beginRun();
         $this->handleDir(dirname($this->file));
         $this->cli = true;
         $this->output = $output;
@@ -138,7 +140,11 @@ public function execute(InputInterface $input, OutputInterface $output): int
         $junitPath = ($junitOption === false) ? null : ($junitOption ?: 'prestaflow/junit.xml');
 
         try {
-            $testSuites = $this->resolveSuitePaths((string) $input->getArgument('folder'));
+            // --suites wins over PRESTAFLOW_SUITES (set by the GitHub Action).
+            $suitesFilter = $this->parseSuitesFilter(
+                $input->getOption('suites') ?? Env::get('PRESTAFLOW_SUITES')
+            );
+            $testSuites = $this->resolveSuitePaths((string) $input->getArgument('folder'), $suitesFilter);
         } catch (Error $e) {
             $this->sections['progressIndicator']->finish('Finished');
             $this->sections['progressBar']->clear();
@@ -149,6 +155,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
             $this->sections['progressIndicator']->finish('Finished');
             $this->sections['progressBar']->clear();
 
+            $this->failOnEmptyFilteredRun($suitesFilter, (string) $input->getArgument('folder'));
             $this->success('Tests folder is empty', newLine: true);
             return Command::SUCCESS;
         };
@@ -255,6 +262,7 @@ public function execute(InputInterface $input, OutputInterface $output): int
         }
 
         if (!$nbSuites) {
+            $this->failOnEmptyFilteredRun($suitesFilter, (string) $input->getArgument('folder'));
             $this->success('Tests folder is empty', newLine: true);
             return Command::SUCCESS;
         };
@@ -295,14 +303,36 @@ public function execute(InputInterface $input, OutputInterface $output): int
             $this->success('Rapport visuel écrit dans ' . $visualPath, newLine: true, force: true);
         }
 
+        $this->releaseBrowser();
+
+        return $summary->hasFailures() ? Command::FAILURE : Command::SUCCESS;
+    }
+
+    /**
+     * Give this run a browser of its own.
+     *
+     * The run ends by closing its browser (releaseBrowser()), so it must never
+     * pick up one another PrestaFlow process is driving, nor let that process
+     * find and close its own. Scoping the socket file to this run does both.
+     */
+    public function beginRun(): void
+    {
+        \PrestaFlow\Library\Tests\TestsSuite::scopeBrowserFilesTo(
+            'run-' . getmypid() . '-' . bin2hex(random_bytes(4))
+        );
+    }
+
+    /**
+     * Close the browser this run launched and forget its socket file.
+     */
+    public function releaseBrowser(): void
+    {
         try {
             \PrestaFlow\Library\Tests\TestsSuite::getBrowser(force: false)?->close();
         } catch (\Throwable $e) {
         }
         @unlink(\PrestaFlow\Library\Tests\TestsSuite::getFilePath('.browser'));
         @unlink(\PrestaFlow\Library\Tests\TestsSuite::getFilePath('.browser-options'));
-
-        return $summary->hasFailures() ? Command::FAILURE : Command::SUCCESS;
     }
 
     protected function handleDir($path)
@@ -443,14 +473,24 @@ public function candidatePaths(string $argument): array
      *
      * @throws Error when the argument matches neither a directory nor a suite file
      */
-    public function resolveSuitePaths(string $argument): array
+    public function resolveSuitePaths(string $argument, array $subFolders = []): array
     {
         foreach ($this->candidatePaths($argument) as $path) {
             if (is_dir($path)) {
-                return $this->getTestsSuites($path);
+                return $subFolders === []
+                    ? $this->getTestsSuites($path)
+                    : $this->getTestsSuitesIn($path, $subFolders);
             }
 
             if (is_file($path)) {
+                if ($subFolders !== []) {
+                    throw new Error(sprintf(
+                        '[%s] is a single suite file: a suites filter (%s) needs a folder',
+                        $path,
+                        implode(', ', $subFolders)
+                    ));
+                }
+
                 if (!str_ends_with($path, '.php')) {
                     throw new Error(sprintf('[%s] is not a PHP suite file', $path));
                 }
@@ -462,6 +502,97 @@ public function resolveSuitePaths(string $argument): array
         throw new Error(sprintf('The suites path [%s] doesn\'t seem to exist', $argument));
     }
 
+    /**
+     * Parse a suites filter (--suites or PRESTAFLOW_SUITES): comma-separated
+     * sub-folders of the suites path, nested ones allowed (FrontOffice/Checkout).
+     * Names are trimmed, empty ones dropped, duplicates removed.
+     *
+     * @return array<int, string> an empty array means "no filter"
+     *
+     * @throws Error on an absolute path or a `..` segment: the filter may only
+     *               narrow the suites path, never leave it
+     */
+    public function parseSuitesFilter(?string $raw): array
+    {
+        $names = [];
+        foreach (explode(',', (string) $raw) as $name) {
+            $name = trim(str_replace('\\', '/', $name));
+            if ($name === '') {
+                continue;
+            }
+
+            if (str_starts_with($name, '/') || preg_match('#^[A-Za-z]:#', $name)) {
+                throw new Error(sprintf('Suites filter [%s]: absolute paths are not allowed, name a sub-folder of the suites path', $name));
+            }
+
+            $segments = array_values(array_filter(explode('/', $name), fn ($segment) => $segment !== '' && $segment !== '.'));
+            if (in_array('..', $segments, true)) {
+                throw new Error(sprintf('Suites filter [%s]: ".." is not allowed, name a sub-folder of the suites path', $name));
+            }
+            if ($segments === []) {
+                continue;
+            }
+
+            $names[] = implode('/', $segments);
+        }
+
+        return array_values(array_unique($names));
+    }
+
+    /**
+     * A run narrowed by --suites / PRESTAFLOW_SUITES that ends up with no suite
+     * is a mistake in the filter, not an empty project: fail it rather than
+     * report a green run of zero tests. Unfiltered, an empty folder still passes.
+     *
+     * @throws Error when $suitesFilter is not empty
+     */
+    protected function failOnEmptyFilteredRun(array $suitesFilter, string $root): void
+    {
+        if ($suitesFilter !== []) {
+            throw new Error(sprintf(
+                'Suites filter [%s] selected no suite under [%s]',
+                implode(', ', $suitesFilter),
+                $root
+            ));
+        }
+    }
+
+    /**
+     * Suites of the given sub-folders of $root (the union, without duplicates).
+     *
+     * @return array<int, string>
+     *
+     * @throws Error when a name matches no folder, so that a filter typo never
+     *               turns into a green run of zero tests
+     */
+    public function getTestsSuitesIn(string $root, array $subFolders): array
+    {
+        $missing = array_values(array_filter($subFolders, fn ($name) => !is_dir($root . '/' . $name)));
+        if ($missing !== []) {
+            $available = array_values(array_filter(
+                scandir($root) ?: [],
+                fn ($entry) => $entry !== '.' && $entry !== '..' && is_dir($root . '/' . $entry)
+            ));
+            sort($available);
+
+            throw new Error(sprintf(
+                'Suites filter: no folder [%s] under [%s]. Available sub-folders: %s',
+                implode(', ', $missing),
+                $root,
+                $available === [] ? '(none)' : implode(', ', $available)
+            ));
+        }
+
+        $testSuites = [];
+        foreach ($subFolders as $name) {
+            foreach ($this->getTestsSuites($root . '/' . $name) as $suitePath) {
+                $testSuites[] = $suitePath;
+            }
+        }
+
+        return array_values(array_unique($testSuites));
+    }
+
     public function getTestsSuites($folderPath)
     {
         $testSuites = [];

src/Console/Application.php

@@ -0,0 +1,40 @@
+<?php
+
+namespace PrestaFlow\Library\Console;
+
+use PrestaFlow\Library\Command\ExecuteSuite;
+use Symfony\Component\Console\Application as BaseApplication;
+use Symfony\Component\Console\Output\OutputInterface;
+
+/**
+ * The `prestaflow` console application.
+ *
+ * Lives here rather than in bin/prestaflow so its exit codes can be tested.
+ * An uncaught exception or error (Chrome that does not start, a suites path
+ * that does not exist) is rendered as ERROR/TRACE lines and ends the run with
+ * a non-zero exit code; a run whose assertions fail still exits with 1, and a
+ * clean run with 0.
+ */
+class Application extends BaseApplication
+{
+    public function __construct()
+    {
+        parent::__construct('PrestaFlow', \PrestaFlow\Library\Traits\AppVersion::APP_VERSION);
+
+        $this->add(new ExecuteSuite());
+
+        // Most failures inside a run are \Error (resolveSuitePaths() throws one):
+        // without this they would escape run() as a PHP fatal error.
+        $this->setCatchErrors(true);
+    }
+
+    protected function doRenderThrowable(\Throwable $e, OutputInterface $output): void
+    {
+        $output->writeln(sprintf('<fg=red;options=bold>ERROR</> <fg=white>%s</>', $e->getMessage()));
+        $output->writeln(sprintf('<fg=gray;options=bold>TRACE</> <fg=white>%s</>', $e->getFile() . ':' . $e->getLine()));
+
+        foreach ($e->getTrace() as $trace) {
+            $output->writeln(sprintf('<fg=gray;options=bold>TRACE</> <fg=white>%s</>', ($trace['file'] ?? '[internal]') . ':' . ($trace['line'] ?? '?')));
+        }
+    }
+}