diff --git a/.github/justfile b/.github/justfile index 94be4d2eb6b..a6dfa14af16 100644 --- a/.github/justfile +++ b/.github/justfile @@ -31,6 +31,7 @@ check: # Tests of repository tooling; every `just check` and `just ci` runs them. test: + python3 test/smoke/runner.test.py python3 sh/coverage.test.py python3 sh/size/report.test.py python3 sh/size/release.test.py diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 18f64bff500..df4d98ec673 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -399,7 +399,8 @@ jobs: cancel-in-progress: false queue: max permissions: - # Delete a Pages artifact left by an earlier run attempt. + # Delete a Pages artifact left by an earlier run attempt or by this run's + # upstream runner report. actions: write contents: write pages: write @@ -521,3 +522,109 @@ jobs: # failing the job; remove this once that protocol mismatch is fixed. continue-on-error: true run: just moxygen + + interop-runner: + name: Upstream Docker interoperability + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + outputs: + report: ${{ steps.upload.outcome }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0 + + - name: Build this checkout's relay image + run: just runner-build + + # Upstream follows its default branch. This job has no secrets and neither + # checkout persists credentials; its report is the only publishing input. + - name: Checkout upstream runner HEAD + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: englishm/moq-interop-runner + path: runner-upstream + persist-credentials: false + + # Any failed pair or setup step fails the job. A complete run still writes + # its report first, since a failing run is the one worth looking at. + - name: Run upstream Docker matrices + run: just runner "$GITHUB_WORKSPACE/runner-upstream" "$GITHUB_WORKSPACE/interop-runner-report" + + - name: Upload upstream report + id: upload + if: ${{ !cancelled() && hashFiles('interop-runner-report/index.html') != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: interop-runner-report + path: interop-runner-report/ + if-no-files-found: error + + - name: Upload runner diagnostics + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: interop-runner-diagnostics + path: runner-upstream/results + if-no-files-found: ignore + + interop-runner-report: + name: Publish upstream runner report + needs: interop-runner + if: ${{ !cancelled() && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' && needs.interop-runner.outputs.report == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + concurrency: + group: benchmark-pages + cancel-in-progress: false + queue: max + permissions: + # Delete a Pages artifact left by an earlier run attempt or by this run's + # relay report. + actions: write + contents: write + pages: write + id-token: write + defaults: + run: + working-directory: . + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: true + + - uses: ./.github/actions/just + + - name: Download upstream report + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: interop-runner-report + path: ${{ runner.temp }}/interop-runner-report + + - name: Commit upstream report + run: just gh pages-publish interop-runner "$RUNNER_TEMP/interop-runner-report" "$RUNNER_TEMP/pages" + + - name: Drop a stale Pages artifact + run: just gh pages-clean + env: + GH_TOKEN: ${{ github.token }} + + - name: Upload site + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + path: ${{ runner.temp }}/pages + + - name: Deploy to GitHub Pages + id: deploy + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/quest/m1/README.md b/quest/m1/README.md index 2e0126409dd..6e3befb0913 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -150,7 +150,6 @@ tracks, so per-group stats stop costing cluster announces. - [Generated JS API reference](/quest/m1/js-api-reference.md) - the doc site renders an API reference for every `@moq/*` package from API Extractor's doc model - [IETF lazy discovery](/quest/m1/ietf-lazy-discovery.md) - moq-transport sessions ask for announcements only under watched prefixes, starting with UNSUBSCRIBE_NAMESPACE on d14/15 - [Interop coverage](/quest/m1/interop-coverage/README.md) - the riskiest moq-transport features moq-net exposes get cross-implementation tests over the even drafts 14 to 22 -- [Runner nightly](/quest/m1/interop-runner-nightly.md) - a nightly job runs the upstream interop runner against our relay and clients, so newly registered implementations with a published image are tested without a change here - [One Annex-B exporter](/quest/m1/annexb-export.md) - H.264 and H.265 Annex-B export share one private lifecycle, which also follows a rebound rendition - [One CLI duration type](/quest/m1/cli-duration.md) - moq-cli, moq-bench and moq-relay reuse moq-tokio's public `cli::Duration` instead of their own copies - [Shared watch and publish UI](/quest/m1/js-ui.md) - a new `@moq/ui` package holds the graph, buttons, and support banner that watch and publish duplicate diff --git a/quest/m1/audit-interop.md b/quest/m1/audit-interop.md index 2adf01abeee..ea987aa9df7 100644 --- a/quest/m1/audit-interop.md +++ b/quest/m1/audit-interop.md @@ -30,10 +30,10 @@ Decided with the maintainer on 2026-10-21: through every registered public relay), plus every cell of the runner's report with a moq.dev side: our clients, our docker relay, or `cdn.moq.dev`, including third-party clients failing against our relay. -- Sync new registry entries. Once the [runner nightly](/quest/m1/interop-runner-nightly.md) - lands, it runs the runner's HEAD, so check that newly registered clients - and docker-only relays show up in its report, and ask upstream for a - published image for any it lists as not run. The relays page reads every +- Sync new registry entries. The smoke workflow's upstream runner job runs + the runner's HEAD, so check that newly registered clients and docker-only + relays show up in its [report](https://git.moq.dev/interop-runner/), + and ask upstream for a published image for any it lists as not run. The relays page reads every remote relay from the registry; give new relays the per-relay config they need (`publish_path`, `WEBSOCKET_KEYS` in `test/smoke/relays.sh`). - Standing approval to open PRs on englishm/moq-interop-runner that touch only @@ -120,7 +120,6 @@ waits on [JavaScript FETCH](/quest/m1/js-fetch.md). ## Related -- [Runner nightly](/quest/m1/interop-runner-nightly.md) - runs every registered client and docker relay with a published image against us - [Runner teardown](/quest/m3/interop-runner-approval.md) - the runner's moq.dev clients move to the withdrawing `close()` once it ships - [Moxygen minimal interop](/quest/m1/smoke-moxygen.md) - Moxygen's filter type, waiting upstream - [JavaScript FETCH](/quest/m1/js-fetch.md) - the JS publisher serves FETCH, which most red `js-web` publisher cells need diff --git a/quest/m1/interop-runner-nightly.md b/quest/m1/interop-runner-nightly.md deleted file mode 100644 index 6cbcebcc8e9..00000000000 --- a/quest/m1/interop-runner-nightly.md +++ /dev/null @@ -1,51 +0,0 @@ -# [M] Nightly runs the upstream interop runner - -## Goal - -A nightly job runs [englishm/moq-interop-runner](https://github.com/englishm/moq-interop-runner) -in its docker mode at its default branch's HEAD, with a moq-relay image -built from `main` standing in for the `moq-dev-rs` relay. Every client and docker-only -relay registered upstream with a published image is exercised against our -relay and our clients, so such an implementation added to the registry is -tested here without a change on our side. Entries with only a local-build -recipe are reported as not run. The runner's own HTML report publishes to -. Third-party verdicts never fail the -nightly, but a failure in the runner's own setup (checkout, startup, or -report generation) does. - -## Plan - -Decided with the maintainer on 2026-10-21. - -- Reuse the runner's tooling rather than teaching `test/smoke/relays.sh` to - start docker relays and clients. That keeps one implementation of the - registry's client contract. -- Track the runner's HEAD, not a pinned commit. The registry lives in the - runner's checkout, so a pin would freeze it and hide every implementation - registered after it until the next [interop audit](/quest/m1/audit-interop.md). - The Cloudflare and Moxygen smoke lanes already build and run unpinned - third-party HEAD the same way. -- Split the work the way `interop.yml` does, so upstream code never runs - with credentials. The runner job has read-only permissions and no - secrets, and uploads the generated report as an artifact. A separate - publish job only copies that artifact under its own Pages directory with - the per-directory publisher (`sh/gh/pages-publish.sh`); it is another - `gh-pages` writer, so it joins the `benchmark-pages` concurrency group - with `queue: max`. The remaining exposure is upstream-written HTML on a - static site that holds no credentials. The Pages root lists it. -- Report only for third-party verdicts. Pairs fail for reasons outside this - repo, and the interop audit owns the follow-up, so no cell gates the run. - A failure in the runner's own setup (checkout, image startup, or report - generation) fails the job, so a total loss of coverage or a stale report - shows up. -- Recommendation: run it from `smoke.yml` next to the public relays lane, - since both exercise third-party implementations and share its schedule. -- Images the registry only describes as a local build (no published image) - are skipped and listed as not run, not built here. -- Add a lane to `test/smoke/README.md`'s table. - -Public API: none. Wire: none. - -## Related - -- [Interop audit](/quest/m1/audit-interop.md) - audits this job's moq.dev cells each run diff --git a/sh/gh/pages-clean.sh b/sh/gh/pages-clean.sh index 9cc86ecfe36..e62ba40d995 100755 --- a/sh/gh/pages-clean.sh +++ b/sh/gh/pages-clean.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -# deploy-pages refuses to choose between artifacts from different run attempts. +# deploy-pages refuses to choose between a run's Pages artifacts, left by earlier +# attempts or by another publishing job in the same workflow. # Scope deletion to this run so other Pages publishers keep their artifacts. : "${GITHUB_REPOSITORY:?}" "${GITHUB_RUN_ID:?}" artifacts=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?name=github-pages" --jq '.artifacts[].id') diff --git a/test/smoke/README.md b/test/smoke/README.md index c06491401e5..1e8461460eb 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -12,6 +12,7 @@ This suite installs the published moq packages and checks that they exchange med | --- | --- | --- | | Packages | `just smoke`, `just full` | Published clients through a local `moq-relay`. `smoke` is Rust only. `full` is the cross-language matrix. | | Public relays | `just relays` | Rust publisher and subscriber through every relay in the [moq-interop-runner](https://github.com/englishm/moq-interop-runner) registry. CI publishes with Rust and the browser (`js-web`) and subscribes with Rust, `js-web`, and `js-bun`. On those relays `js-web` and `js-bun` run only on WebTransport (`@moq/web-transport` has no raw QUIC mode); a client that does not belong is omitted. WebSocket is its own column for the relays in `WEBSOCKET_KEYS` (`moq-dev-rs`): each of their `http(s)` endpoints also runs as `ws(s)`, and every other relay keeps that fallback off. A relay that refuses anonymous root broadcasts is dialed on the path `publish_path` names (`stitcher-moq` publishes under `/moq-test`). That run is what the [results page](https://moq-dev.github.io/moq/relays/) shows. Only `--required` relays fail the run; the default is moq-dev's `cdn.moq.dev`, including its WebSocket endpoint. | +| Upstream Docker runner | `just runner-build`, `just runner UPSTREAM OUTPUT` | Every published registry client against this checkout's relay, and the moq-dev clients against published Docker relays. Uses unpinned upstream HEAD and its HTML report. Local-build-only adapters appear as not run. Any failed pair or setup step fails the run, after the report is written. CI publishes it to [interop-runner/](https://git.moq.dev/interop-runner/). | | Cloudflare | `just cloudflare` | Relays built from the default branches of [cloudflare/moq-rs](https://github.com/cloudflare/moq-rs) and moq-dev/moq, driven by a Cloudflare client over WebTransport and raw QUIC. | | moxygen | `just moxygen` | Meta's [`moxygen`](https://github.com/facebookexperimental/moxygen) interop client through the latest moq-dev relay. Needs Linux Docker. | | Tokens | `just token`, `just token-full` | `moq auth` and npm [`@moq/auth`](https://www.npmjs.com/package/@moq/auth) mint and verify each other's JWTs, and each verifier rejects a tampered token. | diff --git a/test/smoke/justfile b/test/smoke/justfile index ddfad6484db..c5b19ffa10b 100644 --- a/test/smoke/justfile +++ b/test/smoke/justfile @@ -133,3 +133,16 @@ check-toolchain: nix develop --command just freshness cd clients/js && nix develop --command bun install cd clients/js && nix develop --command bun -e 'import {chromium} from "playwright"; const browser = await chromium.launch({channel: "chromium", headless: true}); await browser.close();' + +# Build this checkout's relay image and its upstream interop adapter. +runner-build: + docker build --build-arg package=moq-relay -t moq-interop-relay-base:local ../.. + docker build -f runner.Dockerfile -t moq-interop-relay:local . + +# Run the upstream Docker matrices touching moq-dev and render upstream HTML. +runner upstream output: + python3 runner.py {{ quote(upstream) }} {{ quote(output) }} + +# Check dynamic matrix selection and fail-closed coverage validation. +runner-check: + python3 runner.test.py diff --git a/test/smoke/runner.Dockerfile b/test/smoke/runner.Dockerfile new file mode 100644 index 00000000000..3e437da2bb1 --- /dev/null +++ b/test/smoke/runner.Dockerfile @@ -0,0 +1,10 @@ +# The source-built relay follows the runner's /certs convention. Its scratch +# image needs a shell and grep for Docker Compose's UDP healthcheck; the musl +# busybox is static, since the image has no system loader. +FROM moq-interop-relay-base:local +COPY --from=docker.io/library/busybox:stable-musl /bin /bin +ENV MOQ_LISTEN=[::]:4443 \ + MOQ_LISTEN_TLS_CERT=/certs/cert.pem \ + MOQ_LISTEN_TLS_KEY=/certs/priv.key \ + MOQ_AUTH_PUBLIC=** +EXPOSE 4443/udp diff --git a/test/smoke/runner.py b/test/smoke/runner.py new file mode 100644 index 00000000000..0225cc0809b --- /dev/null +++ b/test/smoke/runner.py @@ -0,0 +1,151 @@ +#!/usr/bin/env python3 +"""Run the upstream Docker interop matrices that touch moq-dev.""" + +import argparse +import copy +import html +import json +from pathlib import Path +import shutil +import subprocess +import time + + +OWN = ('moq-dev-rs', 'moq-dev-js') +RELAY_IMAGE = 'moq-interop-relay:local' + + +def matrices(registry): + """Keep published images and partition the pairs around our relay and clients.""" + registry = copy.deepcopy(registry) + implementations = registry['implementations'] + relay = implementations['moq-dev-rs']['roles']['relay']['docker'] + relay['image'] = RELAY_IMAGE + relay.pop('build', None) + inventory = [] + for name, implementation in implementations.items(): + for role in ('relay', 'client'): + docker = implementation.get('roles', {}).get(role, {}).get('docker') + if not docker: + continue + image = docker['image'] + # Local adapter tags need a build even when their base is published. + local = bool(docker.get('build')) and '/' not in image + inventory.append({'implementation': name, 'role': role, 'image': image, + 'status': 'not run: local build only' if local else 'selected'}) + if local: + del implementation['roles'][role]['docker'] + if any('docker' not in implementations[name]['roles']['client'] for name in OWN): + raise ValueError('the registry must provide both moq-dev client images') + + to_relay = copy.deepcopy(registry) + from_clients = copy.deepcopy(registry) + for name in implementations: + if name != 'moq-dev-rs': + to_relay['implementations'][name]['roles'].pop('relay', None) + if name not in OWN: + from_clients['implementations'][name]['roles'].pop('client', None) + # The first matrix already covers both of our clients against our relay. + from_clients['implementations']['moq-dev-rs']['roles'].pop('relay', None) + return (to_relay, from_clients), inventory + + +def expected_pairs(registry): + implementations = registry['implementations'] + return {(client, relay) for client, client_info in implementations.items() + if client_info.get('roles', {}).get('client', {}).get('docker') + for relay, relay_info in implementations.items() + if relay_info.get('roles', {}).get('relay', {}).get('docker') + and set(client_info['draft_versions']) & set(relay_info['draft_versions'])} + + +def validate_summary(summary, expected): + runs = summary['runs'] + actual = {(run['client'], run['relay']) for run in runs} + if actual != expected or len(runs) != len(expected): + raise ValueError(f'incomplete runner coverage: expected {sorted(expected)}, got {sorted(actual)}') + if any(run['mode'] != 'docker' or run['status'] not in ('pass', 'fail', 'timeout') for run in runs): + raise ValueError('unexpected mode, skipped image, or unknown verdict in runner summary') + + +def run(upstream, output): + if output.exists(): + raise ValueError(f'report destination already exists: {output}') + registry_file = upstream / 'implementations.json' + original = registry_file.read_text() + plans, inventory = matrices(json.loads(original)) + for command in (['docker', 'info'], ['docker', 'compose', 'version']): + subprocess.run(command, check=True) + images = {entry['image'] for entry in inventory if entry['status'] == 'selected'} + for image in sorted(images - {RELAY_IMAGE}): + # A registry blip would otherwise abort both matrices before any report. + for attempt in range(1, 4): + if not subprocess.run(['docker', 'pull', image]).returncode: + break + if attempt == 3: + raise SystemExit(f'failed to pull {image}') + time.sleep(attempt * 2) + subprocess.run(['docker', 'image', 'inspect', RELAY_IMAGE], check=True) + subprocess.run(['make', 'certs'], cwd=upstream, check=True) + results = upstream / 'results' + if results.exists(): + raise ValueError(f'runner checkout already contains results: {results}') + combined = None + target = None + failed = [] + try: + for index, plan in enumerate(plans): + registry_file.write_text(json.dumps(plan)) + expected = expected_pairs(plan) + if not expected: + raise ValueError(f'runner matrix {index} has no compatible pairs') + if subprocess.run(['bash', 'run-interop-tests.sh', '--docker-only'], cwd=upstream).returncode: + failed.append(index) + summaries = list(results.glob('*/summary.json')) + if len(summaries) != 1: + raise ValueError('runner must produce exactly one summary per matrix') + summary_file = summaries[0] + summary = json.loads(summary_file.read_text()) + validate_summary(summary, expected) + if combined is None: + combined = summary + target = upstream / 'combined' + summary_file.parent.rename(target) + else: + combined['runs'].extend(summary['runs']) + for log in summary_file.parent.glob('*.log'): + shutil.move(log, target / log.name) + shutil.rmtree(summary_file.parent) + target.rename(results / 'nightly') + (results / 'nightly' / 'summary.json').write_text(json.dumps(combined, indent=2) + '\n') + subprocess.run(['bash', 'generate-report.sh', 'results/nightly'], cwd=upstream, check=True) + if not (results / 'index.html').is_file() or not (results / 'nightly' / 'report.html').is_file(): + raise ValueError('upstream HTML report is missing') + (results / 'inventory.json').write_text(json.dumps(inventory, indent=2) + '\n') + skipped = [entry for entry in inventory if entry['status'] != 'selected'] + if skipped: + rows = ''.join(f'
  • {html.escape(entry["implementation"])} {entry["role"]}: ' + f'{html.escape(entry["image"])} (local build only)
    • ' for entry in skipped) + index_file = results / 'index.html' + index_file.write_text(index_file.read_text().replace('', + f'

    Not run

    ')) + shutil.copytree(results, output) + finally: + registry_file.write_text(original) + subprocess.run(['docker', 'compose', '-f', 'docker-compose.test.yml', 'down', '--timeout', '1'], + cwd=upstream, check=True) + # Any failure fails the run, but only after the report shows which pairs failed. + if failed: + raise SystemExit(f'runner matrices {failed} exited nonzero; see the report') + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('upstream', type=Path) + parser.add_argument('output', type=Path) + args = parser.parse_args() + run(args.upstream.resolve(), args.output.resolve()) + + +if __name__ == '__main__': + main() diff --git a/test/smoke/runner.test.py b/test/smoke/runner.test.py new file mode 100644 index 00000000000..3f509af9916 --- /dev/null +++ b/test/smoke/runner.test.py @@ -0,0 +1,97 @@ +#!/usr/bin/env python3 +"""The upstream registry grows without changing our matrix selection.""" + +import importlib.util +import json +import subprocess +import tempfile +from unittest.mock import patch +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location('runner', Path(__file__).with_name('runner.py')) +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +def implementation(**roles): + return {'draft_versions': ['draft-18'], 'roles': { + role: {'docker': {'image': image}} for role, image in roles.items()}} + + +class Runner(unittest.TestCase): + def setUp(self): + self.registry = {'implementations': { + 'moq-dev-rs': implementation(client='ghcr.io/ours/rs', relay='moq-local'), + 'moq-dev-js': implementation(client='ghcr.io/ours/js'), + 'new': implementation(client='docker.io/new/client', relay='docker.io/new/relay'), + 'adapter': implementation(relay='local-adapter'), + }} + self.registry['implementations']['adapter']['roles']['relay']['docker']['build'] = {'context': '.'} + + def test_new_published_implementations_join_both_directions_once(self): + plans, inventory = runner.matrices(self.registry) + pairs = [runner.expected_pairs(plan) for plan in plans] + self.assertEqual(pairs[0], {('moq-dev-rs', 'moq-dev-rs'), ('moq-dev-js', 'moq-dev-rs'), + ('new', 'moq-dev-rs')}) + self.assertEqual(pairs[1], {('moq-dev-rs', 'new'), ('moq-dev-js', 'new')}) + self.assertFalse(pairs[0] & pairs[1]) + self.assertEqual(next(entry['status'] for entry in inventory + if entry['implementation'] == 'adapter'), 'not run: local build only') + self.assertEqual(self.registry['implementations']['moq-dev-rs']['roles']['relay']['docker']['image'], + 'moq-local') + + def test_published_image_with_source_recipe_is_still_selected(self): + self.registry['implementations']['new']['roles']['relay']['docker']['build'] = {'context': '.'} + plans, _ = runner.matrices(self.registry) + self.assertIn(('moq-dev-rs', 'new'), runner.expected_pairs(plans[1])) + + def test_missing_skipped_duplicate_or_unexpected_cells_fail(self): + cell = {'client': 'a', 'relay': 'b', 'mode': 'docker', 'status': 'fail'} + runner.validate_summary({'runs': [cell]}, {('a', 'b')}) + for cells in ([], [cell, cell], [dict(cell, status='skip')], [dict(cell, relay='c')]): + with self.subTest(cells=cells), self.assertRaises(ValueError): + runner.validate_summary({'runs': cells}, {('a', 'b')}) + + def test_failed_matrix_still_reports_every_matrix_then_fails(self): + with tempfile.TemporaryDirectory() as temporary: + upstream = Path(temporary) + registry = upstream / 'implementations.json' + original = json.dumps(self.registry) + registry.write_text(original) + matrices = [] + + def command(args, **kwargs): + results = upstream / 'results' + if args[:2] == ['bash', 'run-interop-tests.sh']: + pairs = runner.expected_pairs(json.loads(registry.read_text())) + result = results / str(len(matrices)) + result.mkdir(parents=True) + (result / 'summary.json').write_text(json.dumps({'runs': [ + {'client': client, 'relay': relay, 'mode': 'docker', 'status': 'fail'} + for client, relay in pairs]})) + matrices.append(pairs) + return subprocess.CompletedProcess(args, 1 if len(matrices) == 1 else 0) + if args[:2] == ['bash', 'generate-report.sh']: + (results / 'index.html').write_text('') + (results / 'nightly' / 'report.html').write_text('') + return subprocess.CompletedProcess(args, 0) + + with patch.object(runner.subprocess, 'run', side_effect=command): + with self.assertRaises(SystemExit): + runner.run(upstream, upstream / 'report') + self.assertEqual(len(matrices), 2) + self.assertEqual(registry.read_text(), original) + runs = json.loads((upstream / 'report' / 'nightly' / 'summary.json').read_text())['runs'] + self.assertEqual(len(runs), sum(len(pairs) for pairs in matrices)) + self.assertIn('local-adapter', (upstream / 'report' / 'index.html').read_text()) + + def test_drafts_without_overlap_have_no_pair(self): + self.registry['implementations']['new']['draft_versions'] = ['draft-22'] + plans, _ = runner.matrices(self.registry) + self.assertNotIn(('new', 'moq-dev-rs'), runner.expected_pairs(plans[0])) + self.assertFalse(runner.expected_pairs(plans[1])) + + +if __name__ == '__main__': + unittest.main()