From 93fbd61c8eb7509b6ae261198b0374ef4205dea2 Mon Sep 17 00:00:00 2001 From: Jochen Delabie Date: Fri, 4 Sep 2026 09:44:12 +0200 Subject: [PATCH] cli: --app-url and .tar.gz apps for Expo / EAS builds (TB-379) eas build hands back a download URL, and for iOS simulator builds the artifact is a .tar.gz with the .app inside. The CLI now accepts: - --app-url : downloads .apk/.ipa/.zip/.tar.gz to a temp dir (extension from the URL path, Content-Disposition or Content-Type), with a clear message for expired signed links (401/403). - .tar.gz as an app format, local or downloaded: extracted with the system tar and the shallowest .app bundle is used. - testingbot upload . - Inside EAS Build, the build id/profile/platform are attached as metadata and EAS_BUILD_GIT_COMMIT_HASH is the default commit SHA. Temp files are removed after the upload, including on failure. Only one app source may be given (file, --app-url or --app-binary-id). --- README.md | 339 +++++++++++++++++--------------- src/cli.ts | 51 ++++- src/models/maestro_options.ts | 8 + src/providers/maestro.ts | 131 ++++++++++-- src/utils/app_source.ts | 265 +++++++++++++++++++++++++ tests/cli.test.ts | 126 ++++++++++++ tests/providers/maestro.test.ts | 172 +++++++++++++++- tests/utils/app_source.test.ts | 213 ++++++++++++++++++++ 8 files changed, 1122 insertions(+), 183 deletions(-) create mode 100644 src/utils/app_source.ts create mode 100644 tests/utils/app_source.test.ts diff --git a/README.md b/README.md index 718602c..92622d9 100644 --- a/README.md +++ b/README.md @@ -89,92 +89,94 @@ testingbot maestro [options] ``` **Arguments:** -- `app` - Path to your app file (.apk, .ipa, .app, or .zip) + +- `app` - Path to your app file (.apk, .ipa, .app, .zip, or an EAS iOS simulator .tar.gz) - `flows` - One or more paths to flow files (.yaml/.yml), directories, .zip files, or glob patterns **App Options:** -| Option | Description | -|--------|-------------| -| `--app ` | Path to the application under test (alternative to the positional `app` argument) | -| `--other-app ` | Additional companion app to install on the device alongside `--app`. Accepts a local file path (`.apk`, `.ipa`, `.app`, `.zip`) **or** a `tb://` / `http(s)://...` URL — local paths are uploaded; URLs are passed through to the run as-is. Repeatable, **max 4** entries. | -| `--app-binary-id ` | Reuse the app of a project uploaded earlier (`testingbot upload`, or any previous run's Project ID) instead of uploading one. Every positional argument is then a flow. The platform is taken from the stored app unless `--platform` is given | +| Option | Description | +| ----------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--app ` | Path to the application under test (alternative to the positional `app` argument) | +| `--other-app ` | Additional companion app to install on the device alongside `--app`. Accepts a local file path (`.apk`, `.ipa`, `.app`, `.zip`) **or** a `tb://` / `http(s)://...` URL — local paths are uploaded; URLs are passed through to the run as-is. Repeatable, **max 4** entries. | +| `--app-url ` | Download the app from an http(s) URL instead of a local file: `.apk`, `.ipa`, `.zip` or an EAS Build iOS `.tar.gz` (the `.app` inside is extracted automatically). Every positional argument is then a flow. Signed URLs such as EAS links expire after about an hour, so pass a fresh one | +| `--app-binary-id ` | Reuse the app of a project uploaded earlier (`testingbot upload`, or any previous run's Project ID) instead of uploading one. Every positional argument is then a flow. The platform is taken from the stored app unless `--platform` is given | **Device Options:** -| Option | Description | -|--------|-------------| -| `--device ` | Device name (e.g., "Pixel 9", "iPhone 16") | -| `--platform ` | Platform: Android or iOS | -| `--deviceVersion ` | OS version (e.g., "14", "17.2") | -| `--real-device` | Use a real device instead of emulator/simulator | -| `--device-matrix ` | Run every flow on each listed device in one go. Cells are `[:][:real]`, comma-separated or repeatable. Cannot be combined with `--device` or `--deviceVersion`; `--real-device` (or an `.ipa` app) applies to every cell | -| `--orientation ` | Screen orientation: PORTRAIT or LANDSCAPE | -| `--device-locale ` | Device locale (e.g., "en_US", "de_DE") | -| `--timezone ` | Timezone (e.g., "America/New_York", "Europe/London") | +| Option | Description | +| ----------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--device ` | Device name (e.g., "Pixel 9", "iPhone 16") | +| `--platform ` | Platform: Android or iOS | +| `--deviceVersion ` | OS version (e.g., "14", "17.2") | +| `--real-device` | Use a real device instead of emulator/simulator | +| `--device-matrix ` | Run every flow on each listed device in one go. Cells are `[:][:real]`, comma-separated or repeatable. Cannot be combined with `--device` or `--deviceVersion`; `--real-device` (or an `.ipa` app) applies to every cell | +| `--orientation ` | Screen orientation: PORTRAIT or LANDSCAPE | +| `--device-locale ` | Device locale (e.g., "en_US", "de_DE") | +| `--timezone ` | Timezone (e.g., "America/New_York", "Europe/London") | **Test Configuration:** -| Option | Description | -|--------|-------------| -| `--name ` | Test name for dashboard identification | -| `--build ` | Build identifier for grouping test runs | -| `--groups ` | Tag the test session with one or more groups (comma-separated). Groups appear on the test in the TestingBot dashboard | -| `--include-tags ` | Only run flows with these tags (comma-separated) | -| `--exclude-tags ` | Exclude flows with these tags (comma-separated) | -| `--exclude-flows ` | Flow files, directories or glob patterns to leave out of the run (comma-separated, repeatable). An excluded flow that another flow still invokes via `runFlow` is bundled as a subflow but never runs on its own | -| `-e, --env ` | Environment variable for flows (can be repeated) | -| `--config ` | Path to a custom Maestro config file (default: config.yaml in project root) | -| `--maestro-version ` | Maestro version to use (e.g., "2.0.10") | +| Option | Description | +| ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--name ` | Test name for dashboard identification | +| `--build ` | Build identifier for grouping test runs | +| `--groups ` | Tag the test session with one or more groups (comma-separated). Groups appear on the test in the TestingBot dashboard | +| `--include-tags ` | Only run flows with these tags (comma-separated) | +| `--exclude-tags ` | Exclude flows with these tags (comma-separated) | +| `--exclude-flows ` | Flow files, directories or glob patterns to leave out of the run (comma-separated, repeatable). An excluded flow that another flow still invokes via `runFlow` is bundled as a subflow but never runs on its own | +| `-e, --env ` | Environment variable for flows (can be repeated) | +| `--config ` | Path to a custom Maestro config file (default: config.yaml in project root) | +| `--maestro-version ` | Maestro version to use (e.g., "2.0.10") | **Network & Location:** -| Option | Description | -|--------|-------------| -| `--throttle-network ` | Network throttling: 4G, 3G, Edge, airplane, or disable | -| `--geo-country-code ` | Geographic IP location (ISO country code, e.g., "US", "DE") | +| Option | Description | +| ---------------------------- | ----------------------------------------------------------- | +| `--throttle-network ` | Network throttling: 4G, 3G, Edge, airplane, or disable | +| `--geo-country-code ` | Geographic IP location (ISO country code, e.g., "US", "DE") | **Tunnel:** -| Option | Description | -|--------|-------------| -| `-t, --tunnel` | Start a TestingBot tunnel for this test run (cannot be combined with `--async`) | -| `--tunnel-identifier ` | Identifier for the tunnel, allowing multiple tunnels in parallel | +| Option | Description | +| -------------------------- | ------------------------------------------------------------------------------- | +| `-t, --tunnel` | Start a TestingBot tunnel for this test run (cannot be combined with `--async`) | +| `--tunnel-identifier ` | Identifier for the tunnel, allowing multiple tunnels in parallel | **Output Options:** -| Option | Description | -|--------|-------------| -| `--async` | Start tests and exit without waiting for results | -| `-q, --quiet` | Suppress progress output | -| `--json` | Print results as a single JSON document on stdout (logs move to stderr). Implies `--quiet`. Exit code 2 when tests fail | -| `--json-file` | Write results as JSON to a file (default: `_testingbot.json` in the current directory). Implies `--quiet`. Exit code stays 0 when tests fail so the pipeline can gate on the file | -| `--json-file-name ` | Custom path for the JSON results file (requires `--json-file`) | -| `--report ` | Download report after completion: `html`, `html-detailed`, `junit` or `allure` | -| `--report-output-dir ` | Directory to save reports (required with --report) | -| `--download-artifacts [mode]` | Download test artifacts (logs, screenshots, video). Mode: `all` (default) or `failed` | -| `--artifacts-output-dir ` | Directory to save artifacts zip (defaults to current directory) | +| Option | Description | +| ------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--async` | Start tests and exit without waiting for results | +| `-q, --quiet` | Suppress progress output | +| `--json` | Print results as a single JSON document on stdout (logs move to stderr). Implies `--quiet`. Exit code 2 when tests fail | +| `--json-file` | Write results as JSON to a file (default: `_testingbot.json` in the current directory). Implies `--quiet`. Exit code stays 0 when tests fail so the pipeline can gate on the file | +| `--json-file-name ` | Custom path for the JSON results file (requires `--json-file`) | +| `--report ` | Download report after completion: `html`, `html-detailed`, `junit` or `allure` | +| `--report-output-dir ` | Directory to save reports (required with --report) | +| `--download-artifacts [mode]` | Download test artifacts (logs, screenshots, video). Mode: `all` (default) or `failed` | +| `--artifacts-output-dir ` | Directory to save artifacts zip (defaults to current directory) | **Advanced Options:** -| Option | Description | -|--------|-------------| -| `--shard-split ` | Split flows into N parallel sessions for faster execution | -| `--retry ` | Retry failed flows up to N times (0-2, default 0). Re-runs only the flows (or shards) that failed, the moment they fail, while the rest of the run continues. Cannot be combined with `--async`. | -| `--ignore-checksum-check` | Skip checksum verification and always upload the app | +| Option | Description | +| ------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | +| `--shard-split ` | Split flows into N parallel sessions for faster execution | +| `--retry ` | Retry failed flows up to N times (0-2, default 0). Re-runs only the flows (or shards) that failed, the moment they fail, while the rest of the run continues. Cannot be combined with `--async`. | +| `--ignore-checksum-check` | Skip checksum verification and always upload the app | > **Note on `--retry`:** a failed flow/shard is retried immediately — as soon as it fails — without waiting for the other flows in the run to finish. Retry attempts appear live in the flow table marked with a `↻` icon. Each flow is retried independently up to N times, stopping as soon as that flow passes. Pass/fail uses the result of the **last** attempt per flow (last-attempt-wins), consistently across the CLI exit code, the TestingBot dashboard, and reports downloaded via `--report`. **CI/CD Integration:** -| Option | Description | -|--------|-------------| -| `--branch ` | Git branch this test run was built from | -| `--commit-sha ` | Git commit SHA associated with this test run | -| `--pull-request-id ` | Pull request ID this test run originated from | -| `--pr-url ` | Pull request URL this test run originated from | -| `--repo-name ` | Repository name (e.g., GitHub repo slug) | -| `--repo-owner ` | Repository owner (e.g., GitHub organization or username) | +| Option | Description | +| ---------------------------- | -------------------------------------------------------------------------------------------------------------------- | +| `--branch ` | Git branch this test run was built from | +| `--commit-sha ` | Git commit SHA associated with this test run | +| `--pull-request-id ` | Pull request ID this test run originated from | +| `--pr-url ` | Pull request URL this test run originated from | +| `--repo-name ` | Repository name (e.g., GitHub repo slug) | +| `--repo-owner ` | Repository owner (e.g., GitHub organization or username) | | `-m, --metadata ` | Free-form metadata attached to the run and shown in the dashboard (repeatable, e.g. `-m team=mobile -m env=staging`) | **Allure reports:** `--report allure` converts each run's results into Allure result files under `/allure-results/`, one JSON per flow with its steps, status and failure details. Render them with `allure serve /allure-results` (requires the [Allure CLI](https://allurereport.org/docs/install/)). Results from several runs or shards accumulate in the same directory. @@ -338,13 +340,32 @@ Every `maestro` run also prints its Project ID after the app upload, so any prev **`upload `** -| Option | Description | -|--------|-------------| +| Option | Description | +| ------------------------- | ---------------------------------------------------- | | `--ignore-checksum-check` | Skip checksum verification and always upload the app | -| `-q, --quiet` | Suppress upload progress | +| `-q, --quiet` | Suppress upload progress | `--json` returns `{ provider, appId, file, url }`. Fails with exit code `1` if the upload was rejected. +### Expo / EAS Build + +`eas build` produces a download URL rather than a local file, and for iOS simulator builds the artifact is a `.tar.gz` containing the `.app`. Both are handled directly: + +```sh +# iOS simulator build from EAS (tar.gz with the .app inside) +URL=$(eas build --platform ios --profile preview --json --non-interactive | jq -r '.[0].artifacts.buildUrl') +testingbot maestro --app-url "$URL" ./flows --device "iPhone 16" + +# Android build from EAS +URL=$(eas build --platform android --profile preview --json --non-interactive | jq -r '.[0].artifacts.buildUrl') +testingbot maestro --app-url "$URL" ./flows --device "Pixel 9" + +# A local tar.gz works too +testingbot maestro build.tar.gz ./flows +``` + +Inside an EAS Build or EAS Workflows job, the run is tagged automatically with the EAS build id, profile and platform, and `EAS_BUILD_GIT_COMMIT_HASH` is used as the commit SHA unless `--commit-sha` is given. `testingbot upload ` accepts URLs as well. + ### Status, artifacts and list Commands for working with Maestro projects after they were started, typically together with `--async`. Every command accepts `--api-key` / `--api-secret`, `--debug`, and the `--json`, `--json-file`, `--json-file-name` output flags described under [JSON Output](#json-output). @@ -368,30 +389,30 @@ testingbot list --count 25 --offset 25 --json **`status --id `** -| Option | Description | -|--------|-------------| -| `-w, --wait` | Block until every run has finished, showing the same live flow table as a foreground run. Ctrl-C detaches without cancelling the runs | -| `-q, --quiet` | Suppress progress output | +| Option | Description | +| ------------- | ------------------------------------------------------------------------------------------------------------------------------------- | +| `-w, --wait` | Block until every run has finished, showing the same live flow table as a foreground run. Ctrl-C detaches without cancelling the runs | +| `-q, --quiet` | Suppress progress output | Exit code is `0` while the project is still running (JSON `outcome: "running"`), `0`/`2` once it completed, `1` on errors. **`artifacts --id `** -| Option | Description | -|--------|-------------| -| `--report ` | Download report: `html`, `html-detailed`, `junit` or `allure` | -| `--report-output-dir ` | Directory to save reports (required with `--report`) | -| `--download-artifacts [mode]` | Download logs, screenshots and video. Mode: `all` (default) or `failed` | -| `--artifacts-output-dir ` | Directory to save the artifacts zip (defaults to current directory) | +| Option | Description | +| ------------------------------- | ----------------------------------------------------------------------- | +| `--report ` | Download report: `html`, `html-detailed`, `junit` or `allure` | +| `--report-output-dir ` | Directory to save reports (required with `--report`) | +| `--download-artifacts [mode]` | Download logs, screenshots and video. Mode: `all` (default) or `failed` | +| `--artifacts-output-dir ` | Directory to save the artifacts zip (defaults to current directory) | Fails with exit code `1` if the project is still running; use `status --wait` first. **`list`** -| Option | Description | -|--------|-------------| -| `--count ` | Maximum number of projects to return (default 10) | -| `--offset ` | Number of projects to skip, for pagination | +| Option | Description | +| ------------------- | ------------------------------------------------- | +| `--count ` | Maximum number of projects to return (default 10) | +| `--offset ` | Number of projects to skip, for pagination | Projects are listed newest first with id, name, state, run and flow counts. `--json` returns `{ provider, meta: { offset, count, total }, projects: [...] }` with a dashboard `url` per project. @@ -404,69 +425,70 @@ testingbot espresso [appFile] [testAppFile] [options] ``` **Arguments:** + - `appFile` - Path to application APK file - `testAppFile` - Path to test APK file containing Espresso tests **Device Options:** -| Option | Description | -|--------|-------------| -| `--app ` | Path to application APK file | -| `--test-app ` | Path to test APK file | -| `--device ` | Device name (e.g., "Pixel 6", "Samsung.*") | -| `--platform-version ` | Android OS version (e.g., "12", "13", "14") | -| `--real-device` | Use a real device instead of an emulator | -| `--tablet-only` | Only allocate tablet devices | -| `--phone-only` | Only allocate phone devices | -| `--locale ` | Device locale (e.g., "en_US", "de_DE") | -| `--timezone ` | Timezone (e.g., "America/New_York", "Europe/London") | +| Option | Description | +| ------------------------------ | ---------------------------------------------------- | +| `--app ` | Path to application APK file | +| `--test-app ` | Path to test APK file | +| `--device ` | Device name (e.g., "Pixel 6", "Samsung.\*") | +| `--platform-version ` | Android OS version (e.g., "12", "13", "14") | +| `--real-device` | Use a real device instead of an emulator | +| `--tablet-only` | Only allocate tablet devices | +| `--phone-only` | Only allocate phone devices | +| `--locale ` | Device locale (e.g., "en_US", "de_DE") | +| `--timezone ` | Timezone (e.g., "America/New_York", "Europe/London") | **Test Configuration:** -| Option | Description | -|--------|-------------| -| `--name ` | Test name for dashboard identification | -| `--build ` | Build identifier for grouping test runs | -| `--test-runner ` | Custom test instrumentation runner | -| `--language ` | App language (ISO 639-1 code, e.g., "en", "fr", "de") | +| Option | Description | +| ------------------------ | ----------------------------------------------------- | +| `--name ` | Test name for dashboard identification | +| `--build ` | Build identifier for grouping test runs | +| `--test-runner ` | Custom test instrumentation runner | +| `--language ` | App language (ISO 639-1 code, e.g., "en", "fr", "de") | **Test Filtering:** -| Option | Description | -|--------|-------------| -| `--class ` | Run tests in specific classes (comma-separated fully qualified names) | -| `--not-class ` | Exclude tests in specific classes | -| `--package ` | Run tests in specific packages (comma-separated) | -| `--not-package ` | Exclude tests in specific packages | -| `--annotation ` | Run tests with specific annotations (comma-separated) | -| `--not-annotation ` | Exclude tests with specific annotations | -| `--size ` | Run tests by size: small, medium, large (comma-separated) | +| Option | Description | +| -------------------------------- | --------------------------------------------------------------------- | +| `--class ` | Run tests in specific classes (comma-separated fully qualified names) | +| `--not-class ` | Exclude tests in specific classes | +| `--package ` | Run tests in specific packages (comma-separated) | +| `--not-package ` | Exclude tests in specific packages | +| `--annotation ` | Run tests with specific annotations (comma-separated) | +| `--not-annotation ` | Exclude tests with specific annotations | +| `--size ` | Run tests by size: small, medium, large (comma-separated) | **Network & Location:** -| Option | Description | -|--------|-------------| -| `--throttle-network ` | Network throttling: 4G, 3G, Edge, or airplane | -| `--geo-location ` | Geographic IP location (ISO country code, e.g., "US", "DE") | +| Option | Description | +| ---------------------------- | ----------------------------------------------------------- | +| `--throttle-network ` | Network throttling: 4G, 3G, Edge, or airplane | +| `--geo-location ` | Geographic IP location (ISO country code, e.g., "US", "DE") | **Tunnel:** -| Option | Description | -|--------|-------------| -| `-t, --tunnel` | Start a TestingBot tunnel for this test run (cannot be combined with `--async`) | -| `--tunnel-identifier ` | Identifier for the tunnel, allowing multiple tunnels in parallel | +| Option | Description | +| -------------------------- | ------------------------------------------------------------------------------- | +| `-t, --tunnel` | Start a TestingBot tunnel for this test run (cannot be combined with `--async`) | +| `--tunnel-identifier ` | Identifier for the tunnel, allowing multiple tunnels in parallel | **Output Options:** -| Option | Description | -|--------|-------------| -| `--async` | Start tests and exit without waiting for results | -| `-q, --quiet` | Suppress progress output | -| `--json` | Print results as a single JSON document on stdout (logs move to stderr). Implies `--quiet`. Exit code 2 when tests fail | -| `--json-file` | Write results as JSON to a file (default: `_testingbot.json` in the current directory). Implies `--quiet`. Exit code stays 0 when tests fail so the pipeline can gate on the file | -| `--json-file-name ` | Custom path for the JSON results file (requires `--json-file`) | -| `--report ` | Download report after completion: html or junit | -| `--report-output-dir ` | Directory to save reports (required with --report) | +| Option | Description | +| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--async` | Start tests and exit without waiting for results | +| `-q, --quiet` | Suppress progress output | +| `--json` | Print results as a single JSON document on stdout (logs move to stderr). Implies `--quiet`. Exit code 2 when tests fail | +| `--json-file` | Write results as JSON to a file (default: `_testingbot.json` in the current directory). Implies `--quiet`. Exit code stays 0 when tests fail so the pipeline can gate on the file | +| `--json-file-name ` | Custom path for the JSON results file (requires `--json-file`) | +| `--report ` | Download report after completion: html or junit | +| `--report-output-dir ` | Directory to save reports (required with --report) | **Examples:** @@ -519,57 +541,58 @@ testingbot xcuitest [appFile] [testAppFile] [options] ``` **Arguments:** + - `appFile` - Path to application IPA file - `testAppFile` - Path to test ZIP file containing XCUITests **Device Options:** -| Option | Description | -|--------|-------------| -| `--app ` | Path to application IPA file | -| `--test-app ` | Path to test ZIP file | -| `--device ` | Device name (e.g., "iPhone 15", "iPad.*") | -| `--platform-version ` | iOS version (e.g., "17.0", "18.2") | -| `--real-device` | Use a real device instead of a simulator | -| `--tablet-only` | Only allocate tablet devices | -| `--phone-only` | Only allocate phone devices | -| `--orientation ` | Screen orientation: PORTRAIT or LANDSCAPE | -| `--locale ` | Device locale (e.g., "DE", "US") | -| `--timezone ` | Timezone (e.g., "America/New_York", "Europe/London") | +| Option | Description | +| ------------------------------ | ---------------------------------------------------- | +| `--app ` | Path to application IPA file | +| `--test-app ` | Path to test ZIP file | +| `--device ` | Device name (e.g., "iPhone 15", "iPad.\*") | +| `--platform-version ` | iOS version (e.g., "17.0", "18.2") | +| `--real-device` | Use a real device instead of a simulator | +| `--tablet-only` | Only allocate tablet devices | +| `--phone-only` | Only allocate phone devices | +| `--orientation ` | Screen orientation: PORTRAIT or LANDSCAPE | +| `--locale ` | Device locale (e.g., "DE", "US") | +| `--timezone ` | Timezone (e.g., "America/New_York", "Europe/London") | **Test Configuration:** -| Option | Description | -|--------|-------------| -| `--name ` | Test name for dashboard identification | -| `--build ` | Build identifier for grouping test runs | +| Option | Description | +| ------------------- | ----------------------------------------------------- | +| `--name ` | Test name for dashboard identification | +| `--build ` | Build identifier for grouping test runs | | `--language ` | App language (ISO 639-1 code, e.g., "en", "fr", "de") | **Network & Location:** -| Option | Description | -|--------|-------------| -| `--throttle-network ` | Network throttling: 4G, 3G, Edge, or airplane | -| `--geo-location ` | Geographic IP location (ISO country code, e.g., "US", "DE") | +| Option | Description | +| ---------------------------- | ----------------------------------------------------------- | +| `--throttle-network ` | Network throttling: 4G, 3G, Edge, or airplane | +| `--geo-location ` | Geographic IP location (ISO country code, e.g., "US", "DE") | **Tunnel:** -| Option | Description | -|--------|-------------| -| `-t, --tunnel` | Start a TestingBot tunnel for this test run (cannot be combined with `--async`) | -| `--tunnel-identifier ` | Identifier for the tunnel, allowing multiple tunnels in parallel | +| Option | Description | +| -------------------------- | ------------------------------------------------------------------------------- | +| `-t, --tunnel` | Start a TestingBot tunnel for this test run (cannot be combined with `--async`) | +| `--tunnel-identifier ` | Identifier for the tunnel, allowing multiple tunnels in parallel | **Output Options:** -| Option | Description | -|--------|-------------| -| `--async` | Start tests and exit without waiting for results | -| `-q, --quiet` | Suppress progress output | -| `--json` | Print results as a single JSON document on stdout (logs move to stderr). Implies `--quiet`. Exit code 2 when tests fail | -| `--json-file` | Write results as JSON to a file (default: `_testingbot.json` in the current directory). Implies `--quiet`. Exit code stays 0 when tests fail so the pipeline can gate on the file | -| `--json-file-name ` | Custom path for the JSON results file (requires `--json-file`) | -| `--report ` | Download report after completion: html or junit | -| `--report-output-dir ` | Directory to save reports (required with --report) | +| Option | Description | +| ---------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `--async` | Start tests and exit without waiting for results | +| `-q, --quiet` | Suppress progress output | +| `--json` | Print results as a single JSON document on stdout (logs move to stderr). Implies `--quiet`. Exit code 2 when tests fail | +| `--json-file` | Write results as JSON to a file (default: `_testingbot.json` in the current directory). Implies `--quiet`. Exit code stays 0 when tests fail so the pipeline can gate on the file | +| `--json-file-name ` | Custom path for the JSON results file (requires `--json-file`) | +| `--report ` | Download report after completion: html or junit | +| `--report-output-dir ` | Directory to save reports (required with --report) | **Examples:** @@ -624,6 +647,7 @@ testingbot xcuitest app.ipa app-test.zip \ ### Real-time Progress By default, the CLI shows real-time progress updates including: + - Test status updates with actual device names (even when using wildcards) - Device allocation status - Live output from Maestro flows @@ -633,6 +657,7 @@ Use `--quiet` to suppress progress output. ### Graceful Shutdown Press `Ctrl+C` to gracefully stop running tests. The CLI will: + 1. Stop all active test runs on TestingBot 2. Clean up resources 3. Exit with appropriate status code @@ -663,11 +688,11 @@ Artifacts are saved as a zip file named after the `--build` value (or with a tim ## Exit Codes -| Code | Meaning | -|------|---------| -| `0` | All tests passed (also for `--async`, `--dry-run`, and failed tests with `--json-file`) | -| `1` | CLI or infrastructure error: invalid arguments, missing credentials, upload failure, timeout | -| `2` | One or more tests failed | +| Code | Meaning | +| ---- | -------------------------------------------------------------------------------------------- | +| `0` | All tests passed (also for `--async`, `--dry-run`, and failed tests with `--json-file`) | +| `1` | CLI or infrastructure error: invalid arguments, missing credentials, upload failure, timeout | +| `2` | One or more tests failed | Distinguishing `1` from `2` lets CI decide whether to retry the job or fail the build. diff --git a/src/cli.ts b/src/cli.ts index 053c25e..63f7cc2 100644 --- a/src/cli.ts +++ b/src/cli.ts @@ -24,6 +24,7 @@ import Maestro from './providers/maestro'; import Login from './providers/login'; import Credentials from './models/credentials'; import path from 'node:path'; +import { isAppUrl } from './utils/app_source'; import type { DeviceMatrixCell, RunMetadata } from './models/maestro_options'; import TestingBotError from './models/testingbot_error'; import { redirectLogsToStderr } from './logger'; @@ -160,6 +161,26 @@ function parseDeviceMatrix( }); } +/** + * CI metadata available inside an EAS Build job (Expo). Only the commit hash + * is exposed as a first-class field; the build id, profile and platform go + * into the free-form metadata so the run can be traced back to the build. + */ +function easBuildMetadata(env: NodeJS.ProcessEnv): { + commitSha?: string; + custom: Record; +} { + if (env.EAS_BUILD !== 'true' && !env.EAS_BUILD_ID) return { custom: {} }; + const custom: Record = {}; + if (env.EAS_BUILD_ID) custom.easBuildId = env.EAS_BUILD_ID; + if (env.EAS_BUILD_PROFILE) custom.easBuildProfile = env.EAS_BUILD_PROFILE; + if (env.EAS_BUILD_PLATFORM) custom.easBuildPlatform = env.EAS_BUILD_PLATFORM; + return { + commitSha: env.EAS_BUILD_GIT_COMMIT_HASH || undefined, + custom, + }; +} + /** Drops unset fields; returns undefined when nothing is set. */ function buildRunMetadata(fields: RunMetadata): RunMetadata | undefined { const metadata: Record = {}; @@ -543,6 +564,10 @@ program '--app ', 'Path to application under test (.apk, .ipa, .app, or .zip).', ) + .option( + '--app-url ', + 'Download the app from a URL instead of a local file (.apk, .ipa, .zip or an EAS iOS .tar.gz). All positional arguments are then flows.', + ) .option( '--app-binary-id ', 'Reuse the app of a project uploaded earlier (see "testingbot upload") instead of uploading one. All positional arguments are then flows.', @@ -792,8 +817,8 @@ program let app: string; let flows: string[]; - if (args.app || args.appBinaryId != null) { - // With --app or --app-binary-id, every positional argument is a flow + if (args.app || args.appBinaryId != null || args.appUrl) { + // With --app, --app-url or --app-binary-id, every positional is a flow app = args.app ?? ''; flows = appFileArg ? [appFileArg, ...(flowsArgs || [])] @@ -806,8 +831,8 @@ program flows = [...flows, ...aliasFlows]; const missing: string[] = []; - if (!app && args.appBinaryId == null) - missing.push(', --app or --app-binary-id'); + if (!app && args.appBinaryId == null && !args.appUrl) + missing.push(', --app, --app-url or --app-binary-id'); if (flows.length === 0) missing.push( ' (one or more flow files, directories, or globs)', @@ -855,14 +880,19 @@ program } } + const eas = easBuildMetadata(process.env); + const custom = { + ...eas.custom, + ...(parseKeyValues(args.metadata, '--metadata') ?? {}), + }; const metadata = buildRunMetadata({ - commitSha: args.commitSha, + commitSha: args.commitSha ?? eas.commitSha, pullRequestId: args.pullRequestId, pullRequestUrl: args.prUrl, repoName: args.repoName, repoOwner: args.repoOwner, branch: args.branch, - custom: parseKeyValues(args.metadata, '--metadata'), + custom: Object.keys(custom).length > 0 ? custom : undefined, }); const options = new MaestroOptions(app, flows, args.device, { @@ -903,6 +933,7 @@ program metadata, otherApps, appBinaryId: args.appBinaryId, + appUrl: args.appUrl, }); if (args.debug) { enableDebugLogging(); @@ -1127,7 +1158,10 @@ withFlags( .description( 'Upload a Maestro app once and get a project ID to reuse with "testingbot maestro --app-binary-id".', ) - .argument('', 'Path to the app (.apk, .ipa, .app or .zip)') + .argument( + '', + 'Path or http(s) URL of the app (.apk, .ipa, .app, .zip or an EAS iOS .tar.gz)', + ) .option( '--ignore-checksum-check', 'Skip checksum verification and always upload the app.', @@ -1148,7 +1182,8 @@ withFlags( if (args.debug) enableDebugLogging(); const maestro = new Maestro( credentials, - new MaestroOptions(appFile, [], undefined, { + new MaestroOptions(isAppUrl(appFile) ? '' : appFile, [], undefined, { + appUrl: isAppUrl(appFile) ? appFile : undefined, quiet: args.quiet || jsonOptions.json || jsonOptions.jsonFile, ignoreChecksumCheck: args.ignoreChecksumCheck, debug: args.debug, diff --git a/src/models/maestro_options.ts b/src/models/maestro_options.ts index d8adca8..35a864c 100644 --- a/src/models/maestro_options.ts +++ b/src/models/maestro_options.ts @@ -81,6 +81,7 @@ export default class MaestroOptions { private _app: string; private _appBinaryId?: number; + private _appUrl?: string; private _flows: string[]; private _otherApps: string[]; private _device?: string; @@ -158,10 +159,12 @@ export default class MaestroOptions { metadata?: RunMetadata; otherApps?: string[]; appBinaryId?: number; + appUrl?: string; }, ) { this._app = app; this._appBinaryId = options?.appBinaryId; + this._appUrl = options?.appUrl; this._flows = flows ? (Array.isArray(flows) ? flows : [flows]) : []; this._otherApps = options?.otherApps ?? []; if (this._otherApps.length > MAX_OTHER_APPS) { @@ -231,6 +234,11 @@ export default class MaestroOptions { return this._appBinaryId; } + /** Download URL for the app under test (--app-url), used instead of `app`. */ + public get appUrl(): string | undefined { + return this._appUrl; + } + public get flows(): string[] { return this._flows; } diff --git a/src/providers/maestro.ts b/src/providers/maestro.ts index 9801d23..6065a45 100644 --- a/src/providers/maestro.ts +++ b/src/providers/maestro.ts @@ -16,6 +16,14 @@ import pc from 'picocolors'; import BaseProvider, { ProviderResult } from './base_provider'; import type { JsonFlowResult, JsonRunResult } from '../utils/json_output'; import { junitToAllureResults, writeAllureResults } from '../utils/allure'; +import { + APP_EXTENSIONS, + appExtension, + downloadApp, + extractAppBundle, + isAppUrl, + isSupportedAppExtension, +} from '../utils/app_source'; import { setTitle } from '../ui/terminal-title'; import { HTTP, SOCKET } from '../config/constants'; @@ -171,22 +179,39 @@ export default class Maestro extends BaseProvider { super(credentials, options); } - private static readonly SUPPORTED_APP_EXTENSIONS = [ - '.apk', - '.apks', - '.ipa', - '.app', - '.zip', - ]; + private static readonly SUPPORTED_APP_EXTENSIONS = APP_EXTENSIONS; - /** Rejects an app path that is missing, has an unsupported extension, or is unreadable. */ + // Local path of the app once a URL was downloaded or a .tar.gz extracted; + // the upload pipeline reads this instead of options.app. + private resolvedAppPath: string | undefined = undefined; + private appTempDirs: string[] = []; + + /** The app path to upload: the materialized download/extraction, else the option. */ + private get appPath(): string { + return this.resolvedAppPath ?? this.options.app; + } + + /** + * Rejects an app path that is missing, has an unsupported extension, or is + * unreadable. With --app-url only the URL syntax is checked here; the file + * is validated after download. + */ private async validateAppFile(): Promise { + if (this.options.appUrl) { + if (!isAppUrl(this.options.appUrl)) { + throw new TestingBotError( + `Invalid --app-url: "${this.options.appUrl}" is not an http(s) URL.`, + ); + } + return; + } + if (!this.options.app) { throw new TestingBotError(`app option is required`); } - const appExt = path.extname(this.options.app).toLowerCase(); - if (!Maestro.SUPPORTED_APP_EXTENSIONS.includes(appExt)) { + if (!isSupportedAppExtension(this.options.app)) { + const appExt = appExtension(this.options.app); throw new TestingBotError( `Unsupported app file format: ${appExt || '(no extension)'}. ` + `Supported formats: ${Maestro.SUPPORTED_APP_EXTENSIONS.join(', ')}`, @@ -200,11 +225,68 @@ export default class Maestro extends BaseProvider { }); } + /** + * Turns --app-url and .tar.gz inputs into a local app the upload pipeline + * understands: downloads the URL, then extracts a .tar.gz to its .app + * bundle. No-op for plain local files. Temp directories are removed by + * cleanupAppTemp() once the upload is done. + */ + private async materializeApp(): Promise { + if (this.options.appBinaryId != null) return; + + let current = this.options.app; + if (this.options.appUrl) { + const downloaded = await downloadApp(this.options.appUrl, { + quiet: this.options.quiet, + log: (message) => logger.info(message), + }); + this.appTempDirs.push(downloaded.tmpDir); + current = downloaded.filePath; + if (!isSupportedAppExtension(current)) { + throw new TestingBotError( + `Downloaded file ${path.basename(current)} is not a supported app format (${APP_EXTENSIONS.join(', ')}).`, + ); + } + } + + if (appExtension(current) === '.tar.gz') { + if (!this.options.quiet) { + logger.info(`Extracting ${path.basename(current)}`); + } + const extracted = await extractAppBundle(current); + this.appTempDirs.push(extracted.tmpDir); + current = extracted.appPath; + if (!this.options.quiet) { + logger.info(`Found app bundle ${path.basename(current)}`); + } + } + + this.resolvedAppPath = current === this.options.app ? undefined : current; + } + + private async cleanupAppTemp(): Promise { + const dirs = this.appTempDirs.splice(0); + await Promise.all( + dirs.map((dir) => + fs.promises.rm(dir, { recursive: true, force: true }).catch((err) => { + logger.warn( + `Failed to clean up temporary app dir ${dir}: ${err instanceof Error ? err.message : err}`, + ); + }), + ), + ); + } + private async validate(): Promise { const reusingApp = this.options.appBinaryId != null; - if (reusingApp && this.options.app) { + const sources = [ + this.options.app ? 'an app file' : null, + this.options.appUrl ? '--app-url' : null, + reusingApp ? '--app-binary-id' : null, + ].filter(Boolean); + if (sources.length > 1) { throw new TestingBotError( - 'Pass either an app file or --app-binary-id, not both.', + `Pass only one app source, not ${sources.join(' and ')}.`, ); } if (!reusingApp) { @@ -325,7 +407,7 @@ export default class Maestro extends BaseProvider { * Detect platform from app file content using magic bytes */ private async detectPlatform(): Promise<'Android' | 'iOS' | undefined> { - const appPath = this.options.app; + const appPath = this.appPath; if (!appPath) return undefined; return detectPlatformFromFile(appPath); @@ -377,7 +459,9 @@ export default class Maestro extends BaseProvider { } : { label: 'App', - filePath: this.options.app, + filePath: this.options.appUrl + ? `${this.options.appUrl} (downloaded at run time)` + : this.options.app, endpoint: `${this.URL}/app`, }, ...otherAppPaths.map((p, i) => ({ @@ -443,13 +527,22 @@ export default class Maestro extends BaseProvider { // Quick connectivity check before starting uploads await this.ensureConnectivity(); + // Download --app-url / extract .tar.gz so detection and upload see a + // plain local app. + setTitle('maestro · preparing app'); + await this.materializeApp(); + // Detect platform from file content if not explicitly provided if (!this.options.platformName) { this.detectedPlatform = await this.detectPlatform(); } setTitle('maestro · uploading app'); - await this.uploadApp(); + try { + await this.uploadApp(); + } finally { + await this.cleanupAppTemp(); + } if (!this.options.quiet) { logger.info( `App ready. Project ID: ${this.appId} (reuse this app later with --app-binary-id ${this.appId})`, @@ -520,6 +613,7 @@ export default class Maestro extends BaseProvider { this.disconnectFromUpdateServer(); this.removeSignalHandlers(); await this.stopTunnel(); + await this.cleanupAppTemp(); setTitle('maestro · ✘ error'); logger.error(error instanceof Error ? error.message : error); @@ -549,6 +643,7 @@ export default class Maestro extends BaseProvider { try { await this.validateAppFile(); await this.ensureConnectivity(); + await this.materializeApp(); await this.uploadApp(); if (this.appId == null) { throw new TestingBotError('Upload did not return a project id'); @@ -558,6 +653,8 @@ export default class Maestro extends BaseProvider { this.spinner.stop(); const result = this.errorResult(error); return { success: false, error: result.error ?? 'Upload failed' }; + } finally { + await this.cleanupAppTemp(); } } @@ -617,8 +714,8 @@ export default class Maestro extends BaseProvider { return true; } - let appPath = this.options.app; - const ext = path.extname(appPath).toLowerCase(); + let appPath = this.appPath; + const ext = appExtension(appPath); let tempZipDir: string | null = null; // If .app bundle (directory), zip it first diff --git a/src/utils/app_source.ts b/src/utils/app_source.ts new file mode 100644 index 0000000..4a2d9d3 --- /dev/null +++ b/src/utils/app_source.ts @@ -0,0 +1,265 @@ +import axios from 'axios'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { execFile } from 'node:child_process'; +import { promisify } from 'node:util'; +import { pipeline } from 'node:stream/promises'; +import TestingBotError from '../models/testingbot_error'; +import utils from '../utils'; + +const execFileAsync = promisify(execFile); + +/** + * Where an app under test can come from besides a local .apk/.ipa/.app/.zip: + * a download URL (EAS Build, an artifact store, a release page) and .tar.gz + * archives, which is what EAS produces for iOS simulator builds. Both are + * turned into a local path the rest of the upload pipeline already handles. + */ + +export const APP_EXTENSIONS = [ + '.apk', + '.apks', + '.ipa', + '.app', + '.zip', + '.tar.gz', +]; + +const URL_PATTERN = /^https?:\/\//i; +const DOWNLOAD_TIMEOUT_MS = 30 * 60 * 1000; + +const CONTENT_TYPE_EXTENSIONS: Record = { + 'application/vnd.android.package-archive': '.apk', + 'application/gzip': '.tar.gz', + 'application/x-gzip': '.tar.gz', + 'application/x-tar': '.tar.gz', + 'application/zip': '.zip', + 'application/x-zip-compressed': '.zip', + 'application/octet-stream+ipa': '.ipa', +}; + +export function isAppUrl(value: string | undefined): boolean { + return !!value && URL_PATTERN.test(value.trim()); +} + +/** + * The app extension of a path, treating `.tar.gz` as one extension (Node's + * path.extname would report `.gz`). Lowercase, including the dot. + */ +export function appExtension(filePath: string): string { + const lower = filePath.toLowerCase(); + if (lower.endsWith('.tar.gz')) return '.tar.gz'; + return path.extname(lower); +} + +export function isSupportedAppExtension(filePath: string): boolean { + return APP_EXTENSIONS.includes(appExtension(filePath)); +} + +/** Filename hint from the URL path, ignoring the query string. */ +function filenameFromUrl(url: URL): string { + const last = url.pathname.split('/').filter(Boolean).pop() ?? ''; + try { + return decodeURIComponent(last); + } catch { + return last; + } +} + +function filenameFromContentDisposition(header: unknown): string | undefined { + if (typeof header !== 'string') return undefined; + const utf8 = /filename\*=(?:UTF-8'')?([^;]+)/i.exec(header); + if (utf8) { + try { + return decodeURIComponent(utf8[1].trim().replace(/^"|"$/g, '')); + } catch { + /* fall through */ + } + } + const plain = /filename="?([^";]+)"?/i.exec(header); + return plain ? plain[1].trim() : undefined; +} + +/** + * Downloads an app from `url` into a fresh temp directory and returns the + * local path. The file keeps a recognisable extension so platform detection + * and the upload content type work exactly as for a local file. + * + * Signed URLs (EAS Build, S3, GCS) expire; a 401/403 is reported as such + * instead of a generic HTTP failure because that is the common cause. + */ +export async function downloadApp( + rawUrl: string, + options: { quiet?: boolean; log?: (message: string) => void } = {}, +): Promise<{ filePath: string; tmpDir: string }> { + let url: URL; + try { + url = new URL(rawUrl.trim()); + } catch { + throw new TestingBotError(`Invalid --app-url: "${rawUrl}" is not a URL.`); + } + if (url.protocol !== 'https:' && url.protocol !== 'http:') { + throw new TestingBotError( + `Invalid --app-url: only http(s) URLs are supported, got ${url.protocol}`, + ); + } + + const log = options.log ?? (() => {}); + if (!options.quiet) log(`Downloading app from ${url.host}...`); + + let response; + try { + response = await axios.get(url.toString(), { + responseType: 'stream', + timeout: DOWNLOAD_TIMEOUT_MS, + maxRedirects: 10, + headers: { 'User-Agent': utils.getUserAgent() }, + }); + } catch (error) { + if (axios.isAxiosError(error) && error.response) { + const status = error.response.status; + if (status === 401 || status === 403) { + throw new TestingBotError( + `The app URL was rejected (HTTP ${status}). Signed download links such as EAS Build URLs expire after about an hour; request a fresh URL and try again.`, + ); + } + if (status === 404) { + throw new TestingBotError( + `The app URL was not found (HTTP 404): ${url.origin}${url.pathname}`, + ); + } + throw new TestingBotError( + `Failed to download the app (HTTP ${status}) from ${url.host}`, + ); + } + throw new TestingBotError( + `Failed to download the app from ${url.host}: ${error instanceof Error ? error.message : String(error)}`, + ); + } + + // Work out a filename with a usable extension: URL path, then + // Content-Disposition, then Content-Type. + let filename = filenameFromUrl(url); + if (!isSupportedAppExtension(filename)) { + const fromHeader = filenameFromContentDisposition( + response.headers?.['content-disposition'], + ); + if (fromHeader && isSupportedAppExtension(fromHeader)) + filename = fromHeader; + } + if (!isSupportedAppExtension(filename)) { + const contentType = String(response.headers?.['content-type'] ?? '') + .split(';')[0] + .trim() + .toLowerCase(); + const ext = CONTENT_TYPE_EXTENSIONS[contentType]; + if (ext) filename = `app${ext}`; + } + if (!isSupportedAppExtension(filename)) { + response.data.destroy?.(); + throw new TestingBotError( + `Cannot tell the app type from the URL (${filename || url.pathname}) or its headers. ` + + `Expected one of ${APP_EXTENSIONS.join(', ')} in the file name.`, + ); + } + + const tmpDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'testingbot-app-'), + ); + const filePath = path.join(tmpDir, path.basename(filename)); + try { + await pipeline(response.data, fs.createWriteStream(filePath)); + } catch (error) { + await fs.promises + .rm(tmpDir, { recursive: true, force: true }) + .catch(() => {}); + throw new TestingBotError( + `Failed to save the downloaded app: ${error instanceof Error ? error.message : String(error)}`, + ); + } + + const size = (await fs.promises.stat(filePath)).size; + if (size === 0) { + await fs.promises + .rm(tmpDir, { recursive: true, force: true }) + .catch(() => {}); + throw new TestingBotError(`The app URL returned an empty file: ${rawUrl}`); + } + if (!options.quiet) { + log( + `Downloaded ${path.basename(filePath)} (${(size / 1024 / 1024).toFixed(1)} MB)`, + ); + } + return { filePath, tmpDir }; +} + +/** + * Finds the shallowest `.app` bundle below `dir` without descending into + * bundles themselves (they contain no nested apps we want). + */ +export async function findAppBundle(dir: string): Promise { + const queue: string[] = [dir]; + while (queue.length > 0) { + const current = queue.shift() as string; + let entries: fs.Dirent[]; + try { + entries = await fs.promises.readdir(current, { withFileTypes: true }); + } catch { + continue; + } + // Check this level fully before going deeper so the shallowest bundle wins. + for (const entry of entries) { + if (entry.isDirectory() && entry.name.toLowerCase().endsWith('.app')) { + return path.join(current, entry.name); + } + } + for (const entry of entries) { + if (entry.isDirectory()) queue.push(path.join(current, entry.name)); + } + } + return undefined; +} + +/** + * Extracts a `.tar.gz` (an EAS Build iOS simulator archive, typically) and + * returns the `.app` bundle inside it. Uses the system `tar`, present on + * macOS, Linux and Windows 10+ alike, so no extra dependency is needed. + */ +export async function extractAppBundle( + archivePath: string, +): Promise<{ appPath: string; tmpDir: string }> { + const tmpDir = await fs.promises.mkdtemp( + path.join(os.tmpdir(), 'testingbot-untar-'), + ); + try { + await execFileAsync('tar', ['-xzf', archivePath, '-C', tmpDir], { + maxBuffer: 10 * 1024 * 1024, + }); + } catch (error) { + await fs.promises + .rm(tmpDir, { recursive: true, force: true }) + .catch(() => {}); + const detail = + error instanceof Error && 'code' in error && error.code === 'ENOENT' + ? 'the "tar" command is not available on this machine' + : error instanceof Error + ? error.message + : String(error); + throw new TestingBotError( + `Failed to extract ${path.basename(archivePath)}: ${detail}`, + ); + } + + const appPath = await findAppBundle(tmpDir); + if (!appPath) { + await fs.promises + .rm(tmpDir, { recursive: true, force: true }) + .catch(() => {}); + throw new TestingBotError( + `No .app bundle found inside ${path.basename(archivePath)}. ` + + 'For Expo, use an iOS simulator build (an EAS profile with "ios.simulator": true); device builds produce an .ipa instead.', + ); + } + return { appPath, tmpDir }; +} diff --git a/tests/cli.test.ts b/tests/cli.test.ts index 03d2605..2c3f5c6 100644 --- a/tests/cli.test.ts +++ b/tests/cli.test.ts @@ -1781,6 +1781,132 @@ describe('TestingBotCTL CLI', () => { }); }); + describe('--app-url and EAS metadata', () => { + beforeEach(() => { + mockGetCredentials.mockResolvedValue({ apiKey: 'test-api-key' }); + mockMaestroRun.mockResolvedValue({ + success: true, + outcome: 'passed', + runs: [], + }); + delete process.env.EAS_BUILD; + delete process.env.EAS_BUILD_ID; + delete process.env.EAS_BUILD_PROFILE; + delete process.env.EAS_BUILD_PLATFORM; + delete process.env.EAS_BUILD_GIT_COMMIT_HASH; + }); + + type Opts = { + app: string; + appUrl?: string; + flows: string[]; + metadata?: Record; + }; + + test('maestro --app-url treats every positional as a flow', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + '--app-url', + 'https://expo.dev/artifacts/build.tar.gz', + './flows', + './more', + ]); + const opts = lastConstructorOptions(Maestro); + expect(opts.appUrl).toBe('https://expo.dev/artifacts/build.tar.gz'); + expect(opts.app).toBe(''); + expect(opts.flows).toEqual(['./flows', './more']); + expect(mockMaestroRun).toHaveBeenCalledTimes(1); + }); + + test('maestro --app-url still requires flows', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + '--app-url', + 'https://x.test/app.apk', + ]); + expect(mockMaestroRun).not.toHaveBeenCalled(); + expect(logger.error).toHaveBeenCalledWith( + expect.stringContaining(''), + ); + expect(process.exitCode).toBe(1); + }); + + test('upload accepts a URL argument', async () => { + const mockUploadOnly = jest + .fn() + .mockResolvedValue({ success: true, appId: 9 }); + Maestro.prototype.uploadOnly = mockUploadOnly; + await program.parseAsync([ + 'node', + 'cli', + 'upload', + 'https://testingbot.com/appium/sample.apk', + ]); + const opts = lastConstructorOptions(Maestro); + expect(opts.app).toBe(''); + expect(opts.appUrl).toBe('https://testingbot.com/appium/sample.apk'); + expect(mockUploadOnly).toHaveBeenCalledTimes(1); + }); + + test('EAS Build env fills in the commit and build metadata', async () => { + process.env.EAS_BUILD = 'true'; + process.env.EAS_BUILD_ID = 'b-123'; + process.env.EAS_BUILD_PROFILE = 'preview'; + process.env.EAS_BUILD_PLATFORM = 'ios'; + process.env.EAS_BUILD_GIT_COMMIT_HASH = 'a'.repeat(40); + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + '-m', + 'team=mobile', + ]); + expect(lastConstructorOptions(Maestro).metadata).toEqual({ + commitSha: 'a'.repeat(40), + custom: { + easBuildId: 'b-123', + easBuildProfile: 'preview', + easBuildPlatform: 'ios', + team: 'mobile', + }, + }); + }); + + test('explicit --commit-sha wins over the EAS commit', async () => { + process.env.EAS_BUILD = 'true'; + process.env.EAS_BUILD_GIT_COMMIT_HASH = 'a'.repeat(40); + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + '--commit-sha', + 'b'.repeat(40), + ]); + expect(lastConstructorOptions(Maestro).metadata?.commitSha).toBe( + 'b'.repeat(40), + ); + }); + + test('no EAS metadata outside EAS Build', async () => { + await program.parseAsync([ + 'node', + 'cli', + 'maestro', + 'app.apk', + './flows', + ]); + expect(lastConstructorOptions(Maestro).metadata).toBeUndefined(); + }); + }); + test('unknown command should show help', async () => { const exitSpy = jest .spyOn(process, 'exit') diff --git a/tests/providers/maestro.test.ts b/tests/providers/maestro.test.ts index 191c128..6ed4f1a 100644 --- a/tests/providers/maestro.test.ts +++ b/tests/providers/maestro.test.ts @@ -29,6 +29,11 @@ jest.mock('socket.io-client', () => ({ io: jest.fn(() => mockSocket), })); jest.mock('../../src/utils/file-type-detector'); +jest.mock('../../src/utils/app_source', () => ({ + ...jest.requireActual('../../src/utils/app_source'), + downloadApp: jest.fn(), + extractAppBundle: jest.fn(), +})); jest.mock('../../src/utils', () => ({ __esModule: true, default: { @@ -7112,7 +7117,7 @@ onFlowStart: }), ); await expect(m['validate']()).rejects.toThrow( - 'Pass either an app file or --app-binary-id, not both.', + 'Pass only one app source, not an app file and --app-binary-id.', ); }); @@ -7449,4 +7454,169 @@ onFlowStart: ); }); }); + + describe('--app-url and .tar.gz apps', () => { + const appSource = jest.requireMock('../../src/utils/app_source') as { + downloadApp: jest.Mock; + extractAppBundle: jest.Mock; + }; + + beforeEach(() => { + appSource.downloadApp.mockReset(); + appSource.extractAppBundle.mockReset(); + fs.promises.rm = jest.fn().mockResolvedValue(undefined); + }); + + it('validate() accepts --app-url without a local file and rejects a bad URL', async () => { + const ok = new Maestro( + mockCredentials, + new MaestroOptions('', 'flows', undefined, { + appUrl: 'https://x.test/app.apk', + }), + ); + fs.promises.access = jest.fn().mockResolvedValue(undefined); + fs.promises.stat = jest + .fn() + .mockResolvedValue({ isFile: () => false, isDirectory: () => true }); + await expect(ok['validate']()).resolves.toBe(true); + + const bad = new Maestro( + mockCredentials, + new MaestroOptions('', 'flows', undefined, { + appUrl: 'ftp://x.test/app.apk', + }), + ); + await expect(bad['validate']()).rejects.toThrow('not an http(s) URL'); + }); + + it('validate() rejects --app-url combined with a file or --app-binary-id', async () => { + const both = new Maestro( + mockCredentials, + new MaestroOptions('app.apk', 'flows', undefined, { + appUrl: 'https://x.test/app.apk', + }), + ); + await expect(both['validate']()).rejects.toThrow( + 'Pass only one app source, not an app file and --app-url.', + ); + const three = new Maestro( + mockCredentials, + new MaestroOptions('', 'flows', undefined, { + appUrl: 'https://x.test/app.apk', + appBinaryId: 1, + }), + ); + await expect(three['validate']()).rejects.toThrow( + '--app-url and --app-binary-id', + ); + }); + + it('materializeApp() downloads a URL and uses the file for detection and upload', async () => { + appSource.downloadApp.mockResolvedValue({ + filePath: '/tmp/dl/app.apk', + tmpDir: '/tmp/dl', + }); + const m = new Maestro( + mockCredentials, + new MaestroOptions('', 'flows', undefined, { + appUrl: 'https://x.test/app.apk', + quiet: true, + }), + ); + await m['materializeApp'](); + expect(appSource.downloadApp).toHaveBeenCalledWith( + 'https://x.test/app.apk', + expect.objectContaining({ quiet: true }), + ); + expect(m['appPath']).toBe('/tmp/dl/app.apk'); + expect(appSource.extractAppBundle).not.toHaveBeenCalled(); + await m['cleanupAppTemp'](); + expect(fs.promises.rm).toHaveBeenCalledWith('/tmp/dl', { + recursive: true, + force: true, + }); + }); + + it('materializeApp() extracts a downloaded .tar.gz to its .app bundle', async () => { + appSource.downloadApp.mockResolvedValue({ + filePath: '/tmp/dl/build.tar.gz', + tmpDir: '/tmp/dl', + }); + appSource.extractAppBundle.mockResolvedValue({ + appPath: '/tmp/x/MyApp.app', + tmpDir: '/tmp/x', + }); + const m = new Maestro( + mockCredentials, + new MaestroOptions('', 'flows', undefined, { + appUrl: 'https://expo.dev/artifacts/build.tar.gz', + quiet: true, + }), + ); + await m['materializeApp'](); + expect(appSource.extractAppBundle).toHaveBeenCalledWith( + '/tmp/dl/build.tar.gz', + ); + expect(m['appPath']).toBe('/tmp/x/MyApp.app'); + await m['cleanupAppTemp'](); + expect(fs.promises.rm).toHaveBeenCalledTimes(2); + }); + + it('materializeApp() extracts a local .tar.gz and leaves plain files alone', async () => { + appSource.extractAppBundle.mockResolvedValue({ + appPath: '/tmp/x/MyApp.app', + tmpDir: '/tmp/x', + }); + const tar = new Maestro( + mockCredentials, + new MaestroOptions('build.tar.gz', 'flows', undefined, { quiet: true }), + ); + await tar['materializeApp'](); + expect(tar['appPath']).toBe('/tmp/x/MyApp.app'); + + const plain = new Maestro( + mockCredentials, + new MaestroOptions('app.apk', 'flows', undefined, { quiet: true }), + ); + await plain['materializeApp'](); + expect(plain['appPath']).toBe('app.apk'); + expect(appSource.downloadApp).not.toHaveBeenCalled(); + }); + + it('materializeApp() is a no-op with --app-binary-id', async () => { + const m = new Maestro( + mockCredentials, + new MaestroOptions('', 'flows', undefined, { appBinaryId: 5 }), + ); + await m['materializeApp'](); + expect(appSource.downloadApp).not.toHaveBeenCalled(); + expect(appSource.extractAppBundle).not.toHaveBeenCalled(); + }); + + it('run() cleans temp dirs up when the upload fails', async () => { + appSource.downloadApp.mockResolvedValue({ + filePath: '/tmp/dl/app.apk', + tmpDir: '/tmp/dl', + }); + const m = new Maestro( + mockCredentials, + new MaestroOptions('', 'flows', undefined, { + appUrl: 'https://x.test/app.apk', + quiet: true, + }), + ); + m['validate'] = jest.fn().mockResolvedValue(true); + m['ensureConnectivity'] = jest.fn().mockResolvedValue(undefined); + m['detectPlatform'] = jest.fn().mockResolvedValue('Android'); + m['uploadApp'] = jest + .fn() + .mockRejectedValue(new TestingBotError('Upload failed')); + const result = await m.run(); + expect(result.outcome).toBe('error'); + expect(fs.promises.rm).toHaveBeenCalledWith('/tmp/dl', { + recursive: true, + force: true, + }); + }); + }); }); diff --git a/tests/utils/app_source.test.ts b/tests/utils/app_source.test.ts new file mode 100644 index 0000000..b841dcd --- /dev/null +++ b/tests/utils/app_source.test.ts @@ -0,0 +1,213 @@ +import axios from 'axios'; +import fs from 'node:fs'; +import os from 'node:os'; +import path from 'node:path'; +import { execFileSync } from 'node:child_process'; +import { Readable } from 'node:stream'; +import { + appExtension, + downloadApp, + extractAppBundle, + findAppBundle, + isAppUrl, + isSupportedAppExtension, +} from '../../src/utils/app_source'; + +jest.mock('axios'); +jest.mock('../../src/utils', () => ({ + __esModule: true, + default: { getUserAgent: jest.fn().mockReturnValue('TestingBot-CTL-test') }, +})); + +const mockedAxios = axios as jest.Mocked; + +function streamOf(text: string): Readable { + return Readable.from([Buffer.from(text)]); +} + +describe('app_source helpers', () => { + it('recognises URLs and app extensions, including .tar.gz', () => { + expect(isAppUrl('https://expo.dev/x.tar.gz')).toBe(true); + expect(isAppUrl('HTTP://x.test/app.apk')).toBe(true); + expect(isAppUrl('./app.apk')).toBe(false); + expect(isAppUrl(undefined)).toBe(false); + expect(appExtension('Build.TAR.GZ')).toBe('.tar.gz'); + expect(appExtension('app.apk')).toBe('.apk'); + expect(appExtension('archive.gz')).toBe('.gz'); + expect(isSupportedAppExtension('a.tar.gz')).toBe(true); + expect(isSupportedAppExtension('a.txt')).toBe(false); + }); +}); + +describe('downloadApp', () => { + const tmpDirs: string[] = []; + afterEach(async () => { + jest.resetAllMocks(); + for (const d of tmpDirs.splice(0)) { + await fs.promises.rm(d, { recursive: true, force: true }); + } + }); + + it('saves the file with the extension from the URL path, ignoring the query', async () => { + mockedAxios.get.mockResolvedValue({ + data: streamOf('apk-bytes'), + headers: {}, + }); + const log = jest.fn(); + const result = await downloadApp( + 'https://cdn.test/builds/app-release.apk?X-Signature=abc', + { log }, + ); + tmpDirs.push(result.tmpDir); + expect(path.basename(result.filePath)).toBe('app-release.apk'); + expect(await fs.promises.readFile(result.filePath, 'utf8')).toBe( + 'apk-bytes', + ); + expect(mockedAxios.get).toHaveBeenCalledWith( + 'https://cdn.test/builds/app-release.apk?X-Signature=abc', + expect.objectContaining({ responseType: 'stream' }), + ); + expect(log).toHaveBeenCalledWith('Downloading app from cdn.test...'); + expect(log).toHaveBeenCalledWith( + expect.stringMatching(/^Downloaded app-release\.apk/), + ); + }); + + it('falls back to Content-Disposition, then Content-Type, for the extension', async () => { + mockedAxios.get.mockResolvedValueOnce({ + data: streamOf('x'), + headers: { 'content-disposition': 'attachment; filename="build.tar.gz"' }, + }); + const a = await downloadApp('https://expo.dev/artifacts/eas/abc123', { + quiet: true, + }); + tmpDirs.push(a.tmpDir); + expect(path.basename(a.filePath)).toBe('build.tar.gz'); + + mockedAxios.get.mockResolvedValueOnce({ + data: streamOf('x'), + headers: { 'content-type': 'application/vnd.android.package-archive' }, + }); + const b = await downloadApp('https://cdn.test/download/12345', { + quiet: true, + }); + tmpDirs.push(b.tmpDir); + expect(path.basename(b.filePath)).toBe('app.apk'); + }); + + it('rejects when no extension can be determined', async () => { + mockedAxios.get.mockResolvedValue({ + data: streamOf('x'), + headers: { 'content-type': 'text/html' }, + }); + await expect( + downloadApp('https://cdn.test/download/12345', { quiet: true }), + ).rejects.toThrow('Cannot tell the app type'); + }); + + it('rejects an empty download', async () => { + mockedAxios.get.mockResolvedValue({ data: streamOf(''), headers: {} }); + await expect( + downloadApp('https://cdn.test/app.apk', { quiet: true }), + ).rejects.toThrow('empty file'); + }); + + it('explains expired signed URLs on 403 and reports 404s', async () => { + mockedAxios.isAxiosError.mockReturnValue(true); + mockedAxios.get.mockRejectedValueOnce({ response: { status: 403 } }); + await expect( + downloadApp('https://expo.dev/artifacts/build.tar.gz', { quiet: true }), + ).rejects.toThrow('expire after about an hour'); + mockedAxios.get.mockRejectedValueOnce({ response: { status: 404 } }); + await expect( + downloadApp('https://cdn.test/app.apk', { quiet: true }), + ).rejects.toThrow('HTTP 404'); + }); + + it('rejects non-http URLs and garbage', async () => { + await expect(downloadApp('ftp://x.test/app.apk')).rejects.toThrow( + 'only http(s) URLs', + ); + await expect(downloadApp('not a url')).rejects.toThrow('is not a URL'); + expect(mockedAxios.get).not.toHaveBeenCalled(); + }); +}); + +describe('extractAppBundle', () => { + const tmpDirs: string[] = []; + afterEach(async () => { + for (const d of tmpDirs.splice(0)) { + await fs.promises.rm(d, { recursive: true, force: true }); + } + }); + + async function makeArchive( + layout: (root: string) => Promise, + ): Promise { + const work = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'tb-tar-')); + tmpDirs.push(work); + const src = path.join(work, 'src'); + await fs.promises.mkdir(src); + await layout(src); + const archive = path.join(work, 'build.tar.gz'); + execFileSync('tar', ['-czf', archive, '-C', src, '.']); + return archive; + } + + it('finds a root-level .app bundle', async () => { + const archive = await makeArchive(async (root) => { + await fs.promises.mkdir(path.join(root, 'MyApp.app')); + await fs.promises.writeFile( + path.join(root, 'MyApp.app', 'Info.plist'), + 'x', + ); + }); + const { appPath, tmpDir } = await extractAppBundle(archive); + tmpDirs.push(tmpDir); + expect(path.basename(appPath)).toBe('MyApp.app'); + expect(fs.existsSync(path.join(appPath, 'Info.plist'))).toBe(true); + }); + + it('finds a nested Payload/*.app bundle and prefers the shallowest', async () => { + const archive = await makeArchive(async (root) => { + await fs.promises.mkdir(path.join(root, 'Payload', 'Deep.app'), { + recursive: true, + }); + await fs.promises.mkdir( + path.join(root, 'Payload', 'Deep.app', 'Frameworks', 'Inner.app'), + { recursive: true }, + ); + }); + const { appPath, tmpDir } = await extractAppBundle(archive); + tmpDirs.push(tmpDir); + expect(path.basename(appPath)).toBe('Deep.app'); + }); + + it('fails clearly when the archive holds no .app', async () => { + const archive = await makeArchive(async (root) => { + await fs.promises.writeFile( + path.join(root, 'readme.txt'), + 'nothing here', + ); + }); + await expect(extractAppBundle(archive)).rejects.toThrow( + 'No .app bundle found', + ); + }); + + it('fails clearly on a corrupt archive', async () => { + const work = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'tb-tar-')); + tmpDirs.push(work); + const bad = path.join(work, 'bad.tar.gz'); + await fs.promises.writeFile(bad, 'this is not gzip'); + await expect(extractAppBundle(bad)).rejects.toThrow( + 'Failed to extract bad.tar.gz', + ); + }); + + it('findAppBundle returns undefined for an empty tree', async () => { + const work = await fs.promises.mkdtemp(path.join(os.tmpdir(), 'tb-tar-')); + tmpDirs.push(work); + expect(await findAppBundle(work)).toBeUndefined(); + }); +});