From 629ddc456cf8d07deebf594cf6645ff3d3a77984 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 22:05:14 -0700 Subject: [PATCH 1/5] chore(quest): claim quest/m1/interop-runner-nightly.md Quest-Claim: 698d6690ca0378b0 From efff8619a1f4beb8ec3ed3aa99be3c967c62eceb Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 21 Oct 2026 22:13:03 -0700 Subject: [PATCH 2/5] ci(smoke): checkpoint upstream runner integration Preserve the nightly runner integration as a draft while upstream distinguishes setup errors from third-party verdicts. Co-Authored-By: GPT-6 --- .github/justfile | 1 + .github/workflows/smoke.yml | 96 +++++++++++++++++++ quest/m1/interop-runner-nightly.md | 22 +++++ test/smoke/README.md | 1 + test/smoke/justfile | 13 +++ test/smoke/runner.Dockerfile | 9 ++ test/smoke/runner.py | 143 +++++++++++++++++++++++++++++ test/smoke/runner.test.py | 88 ++++++++++++++++++ 8 files changed, 373 insertions(+) create mode 100644 test/smoke/runner.Dockerfile create mode 100644 test/smoke/runner.py create mode 100644 test/smoke/runner.test.py diff --git a/.github/justfile b/.github/justfile index b9fb18921b5..d5ceaf45a82 100644 --- a/.github/justfile +++ b/.github/justfile @@ -14,6 +14,7 @@ check: # Tests of repository tooling; nightly, not on every pull request. test: + python3 test/smoke/runner.test.py python3 sh/coverage.test.py python3 sh/size/report.test.py python3 sh/size/release.test.py diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 55020cebb67..365e374d429 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -513,3 +513,99 @@ jobs: # failing the job; remove this once that protocol mismatch is fixed. continue-on-error: true run: just moxygen + + interop-runner: + name: Upstream Docker interoperability + runs-on: ubuntu-latest + timeout-minutes: 90 + permissions: + contents: read + outputs: + report: ${{ steps.runner.outcome }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0 + + - name: Build this checkout's relay image + run: just runner-build + + # Upstream follows its default branch. This job has no secrets and neither + # checkout persists credentials; its report is the only publishing input. + - name: Checkout upstream runner HEAD + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + repository: englishm/moq-interop-runner + path: runner-upstream + persist-credentials: false + + - name: Run upstream Docker matrices + id: runner + run: just runner "$GITHUB_WORKSPACE/runner-upstream" "$RUNNER_TEMP/interop-runner-report" + + - name: Upload upstream report + if: ${{ !cancelled() && steps.runner.outcome == 'success' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: interop-runner-report + path: ${{ runner.temp }}/interop-runner-report + if-no-files-found: error + + - name: Upload runner diagnostics + if: failure() + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + with: + name: interop-runner-diagnostics + path: runner-upstream/results + if-no-files-found: ignore + + interop-runner-report: + name: Publish upstream runner report + needs: interop-runner + if: ${{ !cancelled() && github.ref == 'refs/heads/main' && github.event_name == 'schedule' && needs.interop-runner.outputs.report == 'success' }} + runs-on: ubuntu-latest + timeout-minutes: 15 + concurrency: + group: benchmark-pages + cancel-in-progress: false + queue: max + permissions: + contents: write + pages: write + id-token: write + defaults: + run: + working-directory: . + environment: + name: github-pages + url: ${{ steps.deploy.outputs.page_url }} + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: true + + - uses: ./.github/actions/just + + - name: Download upstream report + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8 + with: + name: interop-runner-report + path: ${{ runner.temp }}/interop-runner-report + + - name: Commit upstream report + run: just gh pages-publish interop-runner "$RUNNER_TEMP/interop-runner-report" "$RUNNER_TEMP/pages" + + - name: Upload site + uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 + with: + path: ${{ runner.temp }}/pages + + - name: Deploy to GitHub Pages + id: deploy + uses: actions/deploy-pages@368f82528645a54fb793d4d04e342629a3f51346 # v5.0.1 diff --git a/quest/m1/interop-runner-nightly.md b/quest/m1/interop-runner-nightly.md index ae70629664e..d72244f6eca 100644 --- a/quest/m1/interop-runner-nightly.md +++ b/quest/m1/interop-runner-nightly.md @@ -15,6 +15,28 @@ report generation) does. ## Plan + +Implementation checkpoint: [moq-dev/moq#6257](https://github.com/moq-dev/moq/pull/6257) +prepares the dynamic published-image matrices and the separate artifact publisher. +It is unfinished and must stay a draft until these conditions are met: + +- Upstream must distinguish setup failures from client verdicts. Today + `run-interop-tests.sh` returns 1 and records `status: fail` for both a + failed Docker Compose startup and a failed third-party test. A complete + summary cannot distinguish these. The checkpoint fails closed on all + nonzero runner exits; do not make it report-only by accepting exit 1. + Recommendation: an upstream summary field or distinct exit code for setup + failures, then accept only explicitly classified test verdicts locally. +- Run both matrices with real Docker Compose and the source-built relay, + including an injected Compose startup failure and a third-party test + failure. This host has rootless Podman but no Compose provider. +- Reuse the stale Pages artifact cleanup from + [moq-dev/moq#6207](https://github.com/moq-dev/moq/pull/6207) once available; + do not duplicate its helper here. Verify publishing and reruns. +- Confirm the new directory appears in the shared Pages root after the + benchmark writer regenerates its directory-based index. + + Decided with the maintainer on 2026-10-21. - Reuse the runner's tooling rather than teaching `test/smoke/relays.sh` to diff --git a/test/smoke/README.md b/test/smoke/README.md index c06491401e5..06871b104da 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -12,6 +12,7 @@ This suite installs the published moq packages and checks that they exchange med | --- | --- | --- | | Packages | `just smoke`, `just full` | Published clients through a local `moq-relay`. `smoke` is Rust only. `full` is the cross-language matrix. | | Public relays | `just relays` | Rust publisher and subscriber through every relay in the [moq-interop-runner](https://github.com/englishm/moq-interop-runner) registry. CI publishes with Rust and the browser (`js-web`) and subscribes with Rust, `js-web`, and `js-bun`. On those relays `js-web` and `js-bun` run only on WebTransport (`@moq/web-transport` has no raw QUIC mode); a client that does not belong is omitted. WebSocket is its own column for the relays in `WEBSOCKET_KEYS` (`moq-dev-rs`): each of their `http(s)` endpoints also runs as `ws(s)`, and every other relay keeps that fallback off. A relay that refuses anonymous root broadcasts is dialed on the path `publish_path` names (`stitcher-moq` publishes under `/moq-test`). That run is what the [results page](https://moq-dev.github.io/moq/relays/) shows. Only `--required` relays fail the run; the default is moq-dev's `cdn.moq.dev`, including its WebSocket endpoint. | +| Upstream Docker runner | `just runner-build`, `just runner UPSTREAM OUTPUT` | Every published registry client against this checkout's relay, and the moq-dev clients against published Docker relays. Uses unpinned upstream HEAD and its HTML report. Local-build-only adapters appear as not run. The draft integration still fails closed on upstream's ambiguous nonzero exit pending a setup/verdict contract. | | Cloudflare | `just cloudflare` | Relays built from the default branches of [cloudflare/moq-rs](https://github.com/cloudflare/moq-rs) and moq-dev/moq, driven by a Cloudflare client over WebTransport and raw QUIC. | | moxygen | `just moxygen` | Meta's [`moxygen`](https://github.com/facebookexperimental/moxygen) interop client through the latest moq-dev relay. Needs Linux Docker. | | Tokens | `just token`, `just token-full` | `moq auth` and npm [`@moq/auth`](https://www.npmjs.com/package/@moq/auth) mint and verify each other's JWTs, and each verifier rejects a tampered token. | diff --git a/test/smoke/justfile b/test/smoke/justfile index ddfad6484db..c5b19ffa10b 100644 --- a/test/smoke/justfile +++ b/test/smoke/justfile @@ -133,3 +133,16 @@ check-toolchain: nix develop --command just freshness cd clients/js && nix develop --command bun install cd clients/js && nix develop --command bun -e 'import {chromium} from "playwright"; const browser = await chromium.launch({channel: "chromium", headless: true}); await browser.close();' + +# Build this checkout's relay image and its upstream interop adapter. +runner-build: + docker build --build-arg package=moq-relay -t moq-interop-relay-base:local ../.. + docker build -f runner.Dockerfile -t moq-interop-relay:local . + +# Run the upstream Docker matrices touching moq-dev and render upstream HTML. +runner upstream output: + python3 runner.py {{ quote(upstream) }} {{ quote(output) }} + +# Check dynamic matrix selection and fail-closed coverage validation. +runner-check: + python3 runner.test.py diff --git a/test/smoke/runner.Dockerfile b/test/smoke/runner.Dockerfile new file mode 100644 index 00000000000..ec541fc512d --- /dev/null +++ b/test/smoke/runner.Dockerfile @@ -0,0 +1,9 @@ +# The source-built relay follows the runner's /certs convention. Its scratch +# image needs a shell and grep for Docker Compose's UDP healthcheck. +FROM moq-interop-relay-base:local +COPY --from=busybox:stable /bin /bin +ENV MOQ_LISTEN=[::]:4443 \ + MOQ_LISTEN_TLS_CERT=/certs/cert.pem \ + MOQ_LISTEN_TLS_KEY=/certs/priv.key \ + MOQ_AUTH_PUBLIC=** +EXPOSE 4443/udp diff --git a/test/smoke/runner.py b/test/smoke/runner.py new file mode 100644 index 00000000000..c516971473d --- /dev/null +++ b/test/smoke/runner.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python3 +"""Run the upstream Docker interop matrices that touch moq-dev.""" + +import argparse +import copy +import html +import json +from pathlib import Path +import shutil +import subprocess + + +OWN = ('moq-dev-rs', 'moq-dev-js') +RELAY_IMAGE = 'moq-interop-relay:local' + + +def matrices(registry): + """Keep published images and partition the pairs around our relay and clients.""" + registry = copy.deepcopy(registry) + implementations = registry['implementations'] + relay = implementations['moq-dev-rs']['roles']['relay']['docker'] + relay['image'] = RELAY_IMAGE + relay.pop('build', None) + inventory = [] + for name, implementation in implementations.items(): + for role in ('relay', 'client'): + docker = implementation.get('roles', {}).get(role, {}).get('docker') + if not docker: + continue + image = docker['image'] + # Local adapter tags need a build even when their base is published. + local = bool(docker.get('build')) and '/' not in image + inventory.append({'implementation': name, 'role': role, 'image': image, + 'status': 'not run: local build only' if local else 'selected'}) + if local: + del implementation['roles'][role]['docker'] + if any('docker' not in implementations[name]['roles']['client'] for name in OWN): + raise ValueError('the registry must provide both moq-dev client images') + + to_relay = copy.deepcopy(registry) + from_clients = copy.deepcopy(registry) + for name in implementations: + if name != 'moq-dev-rs': + to_relay['implementations'][name]['roles'].pop('relay', None) + if name not in OWN: + from_clients['implementations'][name]['roles'].pop('client', None) + # The first matrix already covers both of our clients against our relay. + from_clients['implementations']['moq-dev-rs']['roles'].pop('relay', None) + return (to_relay, from_clients), inventory + + +def expected_pairs(registry): + implementations = registry['implementations'] + return {(client, relay) for client, client_info in implementations.items() + if client_info.get('roles', {}).get('client', {}).get('docker') + for relay, relay_info in implementations.items() + if relay_info.get('roles', {}).get('relay', {}).get('docker') + and set(client_info['draft_versions']) & set(relay_info['draft_versions'])} + + +def validate_summary(summary, expected): + runs = summary['runs'] + actual = {(run['client'], run['relay']) for run in runs} + if actual != expected or len(runs) != len(expected): + raise ValueError(f'incomplete runner coverage: expected {sorted(expected)}, got {sorted(actual)}') + if any(run['mode'] != 'docker' or run['status'] not in ('pass', 'fail', 'timeout') for run in runs): + raise ValueError('unexpected mode, skipped image, or unknown verdict in runner summary') + + +def run(upstream, output): + if output.exists(): + raise ValueError(f'report destination already exists: {output}') + registry_file = upstream / 'implementations.json' + original = registry_file.read_text() + plans, inventory = matrices(json.loads(original)) + for command in (['docker', 'info'], ['docker', 'compose', 'version']): + subprocess.run(command, check=True) + images = {entry['image'] for entry in inventory if entry['status'] == 'selected'} + for image in sorted(images - {RELAY_IMAGE}): + subprocess.run(['docker', 'pull', image], check=True) + subprocess.run(['docker', 'image', 'inspect', RELAY_IMAGE], check=True) + subprocess.run(['make', 'certs'], cwd=upstream, check=True) + results = upstream / 'results' + if results.exists(): + raise ValueError(f'runner checkout already contains results: {results}') + combined = None + target = None + try: + for index, plan in enumerate(plans): + registry_file.write_text(json.dumps(plan)) + expected = expected_pairs(plan) + if not expected: + raise ValueError(f'runner matrix {index} has no compatible pairs') + verdict = subprocess.run(['bash', 'run-interop-tests.sh', '--docker-only'], cwd=upstream) + summaries = list(results.glob('*/summary.json')) + if len(summaries) != 1: + raise ValueError('runner must produce exactly one summary per matrix') + summary_file = summaries[0] + summary = json.loads(summary_file.read_text()) + validate_summary(summary, expected) + # Upstream exit 1 conflates setup and test failures. Until its contract + # distinguishes them, a complete summary alone cannot make it safe to + # treat a nonzero exit as a report-only third-party verdict. + verdict.check_returncode() + if combined is None: + combined = summary + target = upstream / 'combined' + summary_file.parent.rename(target) + else: + combined['runs'].extend(summary['runs']) + for log in summary_file.parent.glob('*.log'): + shutil.move(log, target / log.name) + shutil.rmtree(summary_file.parent) + target.rename(results / 'nightly') + (results / 'nightly' / 'summary.json').write_text(json.dumps(combined, indent=2) + '\n') + subprocess.run(['bash', 'generate-report.sh', 'results/nightly'], cwd=upstream, check=True) + if not (results / 'index.html').is_file() or not (results / 'nightly' / 'report.html').is_file(): + raise ValueError('upstream HTML report is missing') + (results / 'inventory.json').write_text(json.dumps(inventory, indent=2) + '\n') + skipped = [entry for entry in inventory if entry['status'] != 'selected'] + if skipped: + rows = ''.join(f'
  • {html.escape(entry["implementation"])} {entry["role"]}: ' + f'{html.escape(entry["image"])} (local build only)
    • ' for entry in skipped) + index_file = results / 'index.html' + index_file.write_text(index_file.read_text().replace('', + f'

    Not run

      {rows}
    ')) + shutil.copytree(results, output) + finally: + registry_file.write_text(original) + subprocess.run(['docker', 'compose', '-f', 'docker-compose.test.yml', 'down', '--timeout', '1'], + cwd=upstream, check=True) + + +def main(): + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument('upstream', type=Path) + parser.add_argument('output', type=Path) + args = parser.parse_args() + run(args.upstream.resolve(), args.output.resolve()) + + +if __name__ == '__main__': + main() diff --git a/test/smoke/runner.test.py b/test/smoke/runner.test.py new file mode 100644 index 00000000000..89b0fbf0a3b --- /dev/null +++ b/test/smoke/runner.test.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 +"""The upstream registry grows without changing our matrix selection.""" + +import importlib.util +import json +import subprocess +import tempfile +from unittest.mock import patch +from pathlib import Path +import unittest + +spec = importlib.util.spec_from_file_location('runner', Path(__file__).with_name('runner.py')) +runner = importlib.util.module_from_spec(spec) +spec.loader.exec_module(runner) + + +def implementation(**roles): + return {'draft_versions': ['draft-18'], 'roles': { + role: {'docker': {'image': image}} for role, image in roles.items()}} + + +class Runner(unittest.TestCase): + def setUp(self): + self.registry = {'implementations': { + 'moq-dev-rs': implementation(client='ghcr.io/ours/rs', relay='moq-local'), + 'moq-dev-js': implementation(client='ghcr.io/ours/js'), + 'new': implementation(client='docker.io/new/client', relay='docker.io/new/relay'), + 'adapter': implementation(relay='local-adapter'), + }} + self.registry['implementations']['adapter']['roles']['relay']['docker']['build'] = {'context': '.'} + + def test_new_published_implementations_join_both_directions_once(self): + plans, inventory = runner.matrices(self.registry) + pairs = [runner.expected_pairs(plan) for plan in plans] + self.assertEqual(pairs[0], {('moq-dev-rs', 'moq-dev-rs'), ('moq-dev-js', 'moq-dev-rs'), + ('new', 'moq-dev-rs')}) + self.assertEqual(pairs[1], {('moq-dev-rs', 'new'), ('moq-dev-js', 'new')}) + self.assertFalse(pairs[0] & pairs[1]) + self.assertEqual(next(entry['status'] for entry in inventory + if entry['implementation'] == 'adapter'), 'not run: local build only') + self.assertEqual(self.registry['implementations']['moq-dev-rs']['roles']['relay']['docker']['image'], + 'moq-local') + + def test_published_image_with_source_recipe_is_still_selected(self): + self.registry['implementations']['new']['roles']['relay']['docker']['build'] = {'context': '.'} + plans, _ = runner.matrices(self.registry) + self.assertIn(('moq-dev-rs', 'new'), runner.expected_pairs(plans[1])) + + def test_missing_skipped_duplicate_or_unexpected_cells_fail(self): + cell = {'client': 'a', 'relay': 'b', 'mode': 'docker', 'status': 'fail'} + runner.validate_summary({'runs': [cell]}, {('a', 'b')}) + for cells in ([], [cell, cell], [dict(cell, status='skip')], [dict(cell, relay='c')]): + with self.subTest(cells=cells), self.assertRaises(ValueError): + runner.validate_summary({'runs': cells}, {('a', 'b')}) + + def test_complete_summary_does_not_mask_ambiguous_nonzero_exit(self): + with tempfile.TemporaryDirectory() as temporary: + upstream = Path(temporary) + registry = upstream / 'implementations.json' + original = json.dumps(self.registry) + registry.write_text(original) + + def command(args, **kwargs): + if args[:2] == ['bash', 'run-interop-tests.sh']: + pairs = runner.expected_pairs(json.loads(registry.read_text())) + result = upstream / 'results' / 'one' + result.mkdir(parents=True) + (result / 'summary.json').write_text(json.dumps({'runs': [ + {'client': client, 'relay': relay, 'mode': 'docker', 'status': 'fail'} + for client, relay in pairs]})) + return subprocess.CompletedProcess(args, 1) + return subprocess.CompletedProcess(args, 0) + + with patch.object(runner.subprocess, 'run', side_effect=command): + with self.assertRaises(subprocess.CalledProcessError): + runner.run(upstream, upstream / 'report') + self.assertEqual(registry.read_text(), original) + self.assertFalse((upstream / 'report').exists()) + + def test_drafts_without_overlap_have_no_pair(self): + self.registry['implementations']['new']['draft_versions'] = ['draft-22'] + plans, _ = runner.matrices(self.registry) + self.assertNotIn(('new', 'moq-dev-rs'), runner.expected_pairs(plans[0])) + self.assertFalse(runner.expected_pairs(plans[1])) + + +if __name__ == '__main__': + unittest.main() From 35262bc7fd46ff642999253f79155acdf929b019 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 13:44:39 -0700 Subject: [PATCH 3/5] ci(smoke): fail the upstream runner job on any failure Any failed pair or setup step fails the job, after the report from both matrices is written and uploaded. The publisher drops this run's earlier Pages artifact with the shared pages-clean helper, since the relay report publishes from the same run. This finishes the runner nightly quest. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/smoke.yml | 29 ++++++++---- quest/m1/README.md | 1 - quest/m1/audit-interop.md | 9 ++-- quest/m1/interop-runner-nightly.md | 73 ------------------------------ sh/gh/pages-clean.sh | 3 +- test/smoke/README.md | 2 +- test/smoke/runner.py | 11 +++-- test/smoke/runner.test.py | 19 ++++++-- 8 files changed, 47 insertions(+), 100 deletions(-) delete mode 100644 quest/m1/interop-runner-nightly.md diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 65f37b49974..df4d98ec673 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -399,7 +399,8 @@ jobs: cancel-in-progress: false queue: max permissions: - # Delete a Pages artifact left by an earlier run attempt. + # Delete a Pages artifact left by an earlier run attempt or by this run's + # upstream runner report. actions: write contents: write pages: write @@ -529,7 +530,7 @@ jobs: permissions: contents: read outputs: - report: ${{ steps.runner.outcome }} + report: ${{ steps.upload.outcome }} steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -552,21 +553,23 @@ jobs: path: runner-upstream persist-credentials: false + # Any failed pair or setup step fails the job. A complete run still writes + # its report first, since a failing run is the one worth looking at. - name: Run upstream Docker matrices - id: runner - run: just runner "$GITHUB_WORKSPACE/runner-upstream" "$RUNNER_TEMP/interop-runner-report" + run: just runner "$GITHUB_WORKSPACE/runner-upstream" "$GITHUB_WORKSPACE/interop-runner-report" - name: Upload upstream report - if: ${{ !cancelled() && steps.runner.outcome == 'success' }} - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + id: upload + if: ${{ !cancelled() && hashFiles('interop-runner-report/index.html') != '' }} + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: interop-runner-report - path: ${{ runner.temp }}/interop-runner-report + path: interop-runner-report/ if-no-files-found: error - name: Upload runner diagnostics if: failure() - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7 + uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 with: name: interop-runner-diagnostics path: runner-upstream/results @@ -575,7 +578,7 @@ jobs: interop-runner-report: name: Publish upstream runner report needs: interop-runner - if: ${{ !cancelled() && github.ref == 'refs/heads/main' && github.event_name == 'schedule' && needs.interop-runner.outputs.report == 'success' }} + if: ${{ !cancelled() && github.ref == 'refs/heads/main' && github.event_name != 'pull_request' && needs.interop-runner.outputs.report == 'success' }} runs-on: ubuntu-latest timeout-minutes: 15 concurrency: @@ -583,6 +586,9 @@ jobs: cancel-in-progress: false queue: max permissions: + # Delete a Pages artifact left by an earlier run attempt or by this run's + # relay report. + actions: write contents: write pages: write id-token: write @@ -609,6 +615,11 @@ jobs: - name: Commit upstream report run: just gh pages-publish interop-runner "$RUNNER_TEMP/interop-runner-report" "$RUNNER_TEMP/pages" + - name: Drop a stale Pages artifact + run: just gh pages-clean + env: + GH_TOKEN: ${{ github.token }} + - name: Upload site uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0 with: diff --git a/quest/m1/README.md b/quest/m1/README.md index 851363759a4..f0ba7737b88 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -138,7 +138,6 @@ tracks, so per-group stats stop costing cluster announces. - [Generated JS API reference](/quest/m1/js-api-reference.md) - the doc site renders an API reference for every `@moq/*` package from API Extractor's doc model - [IETF lazy discovery](/quest/m1/ietf-lazy-discovery.md) - moq-transport sessions ask for announcements only under watched prefixes, starting with UNSUBSCRIBE_NAMESPACE on d14/15 - [Interop coverage](/quest/m1/interop-coverage/README.md) - the riskiest moq-transport features moq-net exposes get cross-implementation tests over the even drafts 14 to 22 -- [Runner nightly](/quest/m1/interop-runner-nightly.md) - a nightly job runs the upstream interop runner against our relay and clients, so newly registered implementations with a published image are tested without a change here - [One Annex-B exporter](/quest/m1/annexb-export.md) - H.264 and H.265 Annex-B export share one private lifecycle, which also follows a rebound rendition - [One CLI duration type](/quest/m1/cli-duration.md) - moq-cli, moq-bench and moq-relay reuse moq-tokio's public `cli::Duration` instead of their own copies - [Shared watch and publish UI](/quest/m1/js-ui.md) - a new `@moq/ui` package holds the graph, buttons, and support banner that watch and publish duplicate diff --git a/quest/m1/audit-interop.md b/quest/m1/audit-interop.md index 2adf01abeee..ea987aa9df7 100644 --- a/quest/m1/audit-interop.md +++ b/quest/m1/audit-interop.md @@ -30,10 +30,10 @@ Decided with the maintainer on 2026-10-21: through every registered public relay), plus every cell of the runner's report with a moq.dev side: our clients, our docker relay, or `cdn.moq.dev`, including third-party clients failing against our relay. -- Sync new registry entries. Once the [runner nightly](/quest/m1/interop-runner-nightly.md) - lands, it runs the runner's HEAD, so check that newly registered clients - and docker-only relays show up in its report, and ask upstream for a - published image for any it lists as not run. The relays page reads every +- Sync new registry entries. The smoke workflow's upstream runner job runs + the runner's HEAD, so check that newly registered clients and docker-only + relays show up in its [report](https://git.moq.dev/interop-runner/), + and ask upstream for a published image for any it lists as not run. The relays page reads every remote relay from the registry; give new relays the per-relay config they need (`publish_path`, `WEBSOCKET_KEYS` in `test/smoke/relays.sh`). - Standing approval to open PRs on englishm/moq-interop-runner that touch only @@ -120,7 +120,6 @@ waits on [JavaScript FETCH](/quest/m1/js-fetch.md). ## Related -- [Runner nightly](/quest/m1/interop-runner-nightly.md) - runs every registered client and docker relay with a published image against us - [Runner teardown](/quest/m3/interop-runner-approval.md) - the runner's moq.dev clients move to the withdrawing `close()` once it ships - [Moxygen minimal interop](/quest/m1/smoke-moxygen.md) - Moxygen's filter type, waiting upstream - [JavaScript FETCH](/quest/m1/js-fetch.md) - the JS publisher serves FETCH, which most red `js-web` publisher cells need diff --git a/quest/m1/interop-runner-nightly.md b/quest/m1/interop-runner-nightly.md deleted file mode 100644 index 44e308a5448..00000000000 --- a/quest/m1/interop-runner-nightly.md +++ /dev/null @@ -1,73 +0,0 @@ -# [M] Nightly runs the upstream interop runner - -## Goal - -A nightly job runs [englishm/moq-interop-runner](https://github.com/englishm/moq-interop-runner) -in its docker mode at its default branch's HEAD, with a moq-relay image -built from `main` standing in for the `moq-dev-rs` relay. Every client and docker-only -relay registered upstream with a published image is exercised against our -relay and our clients, so such an implementation added to the registry is -tested here without a change on our side. Entries with only a local-build -recipe are reported as not run. The runner's own HTML report publishes to -. Third-party verdicts never fail the -nightly, but a failure in the runner's own setup (checkout, startup, or -report generation) does. - -## Plan - - -Implementation checkpoint: [moq-dev/moq#6257](https://github.com/moq-dev/moq/pull/6257) -prepares the dynamic published-image matrices and the separate artifact publisher. -It is unfinished and must stay a draft until these conditions are met: - -- Upstream must distinguish setup failures from client verdicts. Today - `run-interop-tests.sh` returns 1 and records `status: fail` for both a - failed Docker Compose startup and a failed third-party test. A complete - summary cannot distinguish these. The checkpoint fails closed on all - nonzero runner exits; do not make it report-only by accepting exit 1. - Recommendation: an upstream summary field or distinct exit code for setup - failures, then accept only explicitly classified test verdicts locally. -- Run both matrices with real Docker Compose and the source-built relay, - including an injected Compose startup failure and a third-party test - failure. This host has rootless Podman but no Compose provider. -- Reuse the stale Pages artifact cleanup from - [moq-dev/moq#6207](https://github.com/moq-dev/moq/pull/6207) once available; - do not duplicate its helper here. Verify publishing and reruns. -- Confirm the new directory appears in the shared Pages root after the - benchmark writer regenerates its directory-based index. - - -Decided with the maintainer on 2026-10-21. - -- Reuse the runner's tooling rather than teaching `test/smoke/relays.sh` to - start docker relays and clients. That keeps one implementation of the - registry's client contract. -- Track the runner's HEAD, not a pinned commit. The registry lives in the - runner's checkout, so a pin would freeze it and hide every implementation - registered after it until the next [interop audit](/quest/m1/audit-interop.md). - The Cloudflare and Moxygen smoke lanes already build and run unpinned - third-party HEAD the same way. -- Split the work the way `interop.yml` does, so upstream code never runs - with credentials. The runner job has read-only permissions and no - secrets, and uploads the generated report as an artifact. A separate - publish job only copies that artifact under its own Pages directory with - the per-directory publisher (`sh/gh/pages-publish.sh`); it is another - `gh-pages` writer, so it joins the `benchmark-pages` concurrency group - with `queue: max`. The remaining exposure is upstream-written HTML on a - static site that holds no credentials. The Pages root lists it. -- Report only for third-party verdicts. Pairs fail for reasons outside this - repo, and the interop audit owns the follow-up, so no cell gates the run. - A failure in the runner's own setup (checkout, image startup, or report - generation) fails the job, so a total loss of coverage or a stale report - shows up. -- Recommendation: run it from `smoke.yml` next to the public relays lane, - since both exercise third-party implementations and share its schedule. -- Images the registry only describes as a local build (no published image) - are skipped and listed as not run, not built here. -- Add a lane to `test/smoke/README.md`'s table. - -Public API: none. Wire: none. - -## Related - -- [Interop audit](/quest/m1/audit-interop.md) - audits this job's moq.dev cells each run diff --git a/sh/gh/pages-clean.sh b/sh/gh/pages-clean.sh index 9cc86ecfe36..e62ba40d995 100755 --- a/sh/gh/pages-clean.sh +++ b/sh/gh/pages-clean.sh @@ -1,7 +1,8 @@ #!/usr/bin/env bash set -euo pipefail -# deploy-pages refuses to choose between artifacts from different run attempts. +# deploy-pages refuses to choose between a run's Pages artifacts, left by earlier +# attempts or by another publishing job in the same workflow. # Scope deletion to this run so other Pages publishers keep their artifacts. : "${GITHUB_REPOSITORY:?}" "${GITHUB_RUN_ID:?}" artifacts=$(gh api --paginate "repos/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID/artifacts?name=github-pages" --jq '.artifacts[].id') diff --git a/test/smoke/README.md b/test/smoke/README.md index 06871b104da..1e8461460eb 100644 --- a/test/smoke/README.md +++ b/test/smoke/README.md @@ -12,7 +12,7 @@ This suite installs the published moq packages and checks that they exchange med | --- | --- | --- | | Packages | `just smoke`, `just full` | Published clients through a local `moq-relay`. `smoke` is Rust only. `full` is the cross-language matrix. | | Public relays | `just relays` | Rust publisher and subscriber through every relay in the [moq-interop-runner](https://github.com/englishm/moq-interop-runner) registry. CI publishes with Rust and the browser (`js-web`) and subscribes with Rust, `js-web`, and `js-bun`. On those relays `js-web` and `js-bun` run only on WebTransport (`@moq/web-transport` has no raw QUIC mode); a client that does not belong is omitted. WebSocket is its own column for the relays in `WEBSOCKET_KEYS` (`moq-dev-rs`): each of their `http(s)` endpoints also runs as `ws(s)`, and every other relay keeps that fallback off. A relay that refuses anonymous root broadcasts is dialed on the path `publish_path` names (`stitcher-moq` publishes under `/moq-test`). That run is what the [results page](https://moq-dev.github.io/moq/relays/) shows. Only `--required` relays fail the run; the default is moq-dev's `cdn.moq.dev`, including its WebSocket endpoint. | -| Upstream Docker runner | `just runner-build`, `just runner UPSTREAM OUTPUT` | Every published registry client against this checkout's relay, and the moq-dev clients against published Docker relays. Uses unpinned upstream HEAD and its HTML report. Local-build-only adapters appear as not run. The draft integration still fails closed on upstream's ambiguous nonzero exit pending a setup/verdict contract. | +| Upstream Docker runner | `just runner-build`, `just runner UPSTREAM OUTPUT` | Every published registry client against this checkout's relay, and the moq-dev clients against published Docker relays. Uses unpinned upstream HEAD and its HTML report. Local-build-only adapters appear as not run. Any failed pair or setup step fails the run, after the report is written. CI publishes it to [interop-runner/](https://git.moq.dev/interop-runner/). | | Cloudflare | `just cloudflare` | Relays built from the default branches of [cloudflare/moq-rs](https://github.com/cloudflare/moq-rs) and moq-dev/moq, driven by a Cloudflare client over WebTransport and raw QUIC. | | moxygen | `just moxygen` | Meta's [`moxygen`](https://github.com/facebookexperimental/moxygen) interop client through the latest moq-dev relay. Needs Linux Docker. | | Tokens | `just token`, `just token-full` | `moq auth` and npm [`@moq/auth`](https://www.npmjs.com/package/@moq/auth) mint and verify each other's JWTs, and each verifier rejects a tampered token. | diff --git a/test/smoke/runner.py b/test/smoke/runner.py index c516971473d..45f1f49cd98 100644 --- a/test/smoke/runner.py +++ b/test/smoke/runner.py @@ -85,23 +85,21 @@ def run(upstream, output): raise ValueError(f'runner checkout already contains results: {results}') combined = None target = None + failed = [] try: for index, plan in enumerate(plans): registry_file.write_text(json.dumps(plan)) expected = expected_pairs(plan) if not expected: raise ValueError(f'runner matrix {index} has no compatible pairs') - verdict = subprocess.run(['bash', 'run-interop-tests.sh', '--docker-only'], cwd=upstream) + if subprocess.run(['bash', 'run-interop-tests.sh', '--docker-only'], cwd=upstream).returncode: + failed.append(index) summaries = list(results.glob('*/summary.json')) if len(summaries) != 1: raise ValueError('runner must produce exactly one summary per matrix') summary_file = summaries[0] summary = json.loads(summary_file.read_text()) validate_summary(summary, expected) - # Upstream exit 1 conflates setup and test failures. Until its contract - # distinguishes them, a complete summary alone cannot make it safe to - # treat a nonzero exit as a report-only third-party verdict. - verdict.check_returncode() if combined is None: combined = summary target = upstream / 'combined' @@ -129,6 +127,9 @@ def run(upstream, output): registry_file.write_text(original) subprocess.run(['docker', 'compose', '-f', 'docker-compose.test.yml', 'down', '--timeout', '1'], cwd=upstream, check=True) + # Any failure fails the run, but only after the report shows which pairs failed. + if failed: + raise SystemExit(f'runner matrices {failed} exited nonzero; see the report') def main(): diff --git a/test/smoke/runner.test.py b/test/smoke/runner.test.py index 89b0fbf0a3b..3f509af9916 100644 --- a/test/smoke/runner.test.py +++ b/test/smoke/runner.test.py @@ -53,29 +53,38 @@ def test_missing_skipped_duplicate_or_unexpected_cells_fail(self): with self.subTest(cells=cells), self.assertRaises(ValueError): runner.validate_summary({'runs': cells}, {('a', 'b')}) - def test_complete_summary_does_not_mask_ambiguous_nonzero_exit(self): + def test_failed_matrix_still_reports_every_matrix_then_fails(self): with tempfile.TemporaryDirectory() as temporary: upstream = Path(temporary) registry = upstream / 'implementations.json' original = json.dumps(self.registry) registry.write_text(original) + matrices = [] def command(args, **kwargs): + results = upstream / 'results' if args[:2] == ['bash', 'run-interop-tests.sh']: pairs = runner.expected_pairs(json.loads(registry.read_text())) - result = upstream / 'results' / 'one' + result = results / str(len(matrices)) result.mkdir(parents=True) (result / 'summary.json').write_text(json.dumps({'runs': [ {'client': client, 'relay': relay, 'mode': 'docker', 'status': 'fail'} for client, relay in pairs]})) - return subprocess.CompletedProcess(args, 1) + matrices.append(pairs) + return subprocess.CompletedProcess(args, 1 if len(matrices) == 1 else 0) + if args[:2] == ['bash', 'generate-report.sh']: + (results / 'index.html').write_text('') + (results / 'nightly' / 'report.html').write_text('') return subprocess.CompletedProcess(args, 0) with patch.object(runner.subprocess, 'run', side_effect=command): - with self.assertRaises(subprocess.CalledProcessError): + with self.assertRaises(SystemExit): runner.run(upstream, upstream / 'report') + self.assertEqual(len(matrices), 2) self.assertEqual(registry.read_text(), original) - self.assertFalse((upstream / 'report').exists()) + runs = json.loads((upstream / 'report' / 'nightly' / 'summary.json').read_text())['runs'] + self.assertEqual(len(runs), sum(len(pairs) for pairs in matrices)) + self.assertIn('local-adapter', (upstream / 'report' / 'index.html').read_text()) def test_drafts_without_overlap_have_no_pair(self): self.registry['implementations']['new']['draft_versions'] = ['draft-22'] From 8c7ec1342386dfba9cae56fcb1532a52b3eb8642 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 14:25:21 -0700 Subject: [PATCH 4/5] fix(smoke): give the runner relay a static busybox busybox:stable is glibc-linked, and the scratch relay image has no system loader, so Compose's healthcheck could not exec `sh` and every pair failed to start. The musl variant is static. Co-Authored-By: Claude Opus 5.5 --- test/smoke/runner.Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/test/smoke/runner.Dockerfile b/test/smoke/runner.Dockerfile index ec541fc512d..3e437da2bb1 100644 --- a/test/smoke/runner.Dockerfile +++ b/test/smoke/runner.Dockerfile @@ -1,7 +1,8 @@ # The source-built relay follows the runner's /certs convention. Its scratch -# image needs a shell and grep for Docker Compose's UDP healthcheck. +# image needs a shell and grep for Docker Compose's UDP healthcheck; the musl +# busybox is static, since the image has no system loader. FROM moq-interop-relay-base:local -COPY --from=busybox:stable /bin /bin +COPY --from=docker.io/library/busybox:stable-musl /bin /bin ENV MOQ_LISTEN=[::]:4443 \ MOQ_LISTEN_TLS_CERT=/certs/cert.pem \ MOQ_LISTEN_TLS_KEY=/certs/priv.key \ From aabe50c85af704ab3f42059fb11ea67a61f1e936 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 23 Oct 2026 16:03:02 -0700 Subject: [PATCH 5/5] fix(smoke): retry transient runner image pulls Co-Authored-By: Claude Opus 5.5 --- test/smoke/runner.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/test/smoke/runner.py b/test/smoke/runner.py index 45f1f49cd98..0225cc0809b 100644 --- a/test/smoke/runner.py +++ b/test/smoke/runner.py @@ -8,6 +8,7 @@ from pathlib import Path import shutil import subprocess +import time OWN = ('moq-dev-rs', 'moq-dev-js') @@ -77,7 +78,13 @@ def run(upstream, output): subprocess.run(command, check=True) images = {entry['image'] for entry in inventory if entry['status'] == 'selected'} for image in sorted(images - {RELAY_IMAGE}): - subprocess.run(['docker', 'pull', image], check=True) + # A registry blip would otherwise abort both matrices before any report. + for attempt in range(1, 4): + if not subprocess.run(['docker', 'pull', image]).returncode: + break + if attempt == 3: + raise SystemExit(f'failed to pull {image}') + time.sleep(attempt * 2) subprocess.run(['docker', 'image', 'inspect', RELAY_IMAGE], check=True) subprocess.run(['make', 'certs'], cwd=upstream, check=True) results = upstream / 'results'