From e40a0c957452da478024c8007f0daba6a083c97d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 24 Sep 2026 10:36:08 -0700 Subject: [PATCH 01/18] quest: open the tooling line Co-Authored-By: Claude Opus 5.5 From 5dda9362257a3a99bd8880fca063dcf7bc70369d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 24 Sep 2026 17:09:14 -0700 Subject: [PATCH 02/18] chore(just): thin justfiles with one impact map (#4058) Co-authored-by: Claude Opus 5.5 --- .github/justfile | 22 +- .github/scripts/package-binary.test.sh | 44 - .github/workflows/alert.yml | 2 +- .github/workflows/docker.yml | 2 +- .github/workflows/interop.yml | 2 +- .github/workflows/libmoq.yml | 14 +- .github/workflows/moq-cli.yml | 26 +- .github/workflows/moq-gst.yml | 16 +- .github/workflows/moq-relay.yml | 24 +- .github/workflows/nightly.yml | 3 +- .github/workflows/obs.yml | 1 + .github/workflows/release-brew.yml | 4 +- .github/workflows/release-dart-ffi.yml | 10 +- .github/workflows/release-dart.yml | 6 +- .github/workflows/release-go-ffi.yml | 14 +- .github/workflows/release-go.yml | 6 +- .github/workflows/release-kt-ffi.yml | 4 +- .github/workflows/release-kt-lib.yml | 12 +- .github/workflows/release-py-ffi.yml | 2 +- .github/workflows/release-py.yml | 10 +- .github/workflows/release-swift-ffi.yml | 12 +- .github/workflows/release-swift-lib.yml | 32 +- .github/workflows/swift.yml | 4 +- .github/workflows/wasm.yml | 1 + .gitignore | 5 +- Cargo.toml | 2 +- bench/justfile | 19 - cpp/obs/justfile | 386 +------- dart/justfile | 15 +- dart/moq_ffi/hook/build.dart | 2 +- dart/moq_ffi/pubspec.yaml | 2 +- demo/justfile | 6 - doc/setup/dev.md | 5 + drafts/justfile | 79 +- flake.nix | 13 +- go/.gitignore | 2 +- go/ffi/README.md | 8 +- go/ffi/moq/cgo.go | 6 +- go/justfile | 28 +- go/wrapper/README.md | 2 +- go/wrapper/go.mod | 2 +- js/common/workers.test.ts | 45 - js/justfile | 98 +- js/wasm/.gitignore | 2 +- js/wasm/README.md | 2 +- justfile | 822 +--------------- kt/.gitignore | 2 +- kt/README.md | 1 - kt/justfile | 32 +- kt/moq-ffi/build.gradle.kts | 2 +- py/justfile | 61 +- quest/m1/browser-benchmarks.md | 2 +- quest/m1/dart-leak.md | 2 +- quest/m1/performance-comparisons.md | 8 +- quest/m1/performance-profiles.md | 2 +- quest/m1/tooling/README.md | 1 - quest/m1/tooling/justfiles.md | 93 -- quest/m1/tooling/workflows-call-just.md | 4 - rs/justfile | 902 ++---------------- rs/moq-gst/package.sh | 2 +- rs/moq-gst/scrub.sh | 2 +- rs/moq-wasm/README.md | 4 +- rs/scripts/doc-names.jq | 31 - rs/scripts/package-nfpm.test.sh | 68 -- sh/clean.sh | 72 ++ {dart/scripts => sh/dart}/check.sh | 11 +- {dart/scripts => sh/dart}/generate.sh | 2 +- {dart/scripts => sh/dart}/package.sh | 2 +- {dart/scripts => sh/dart}/publish.sh | 4 +- sh/dispatch.sh | 194 ++++ sh/drafts/build.sh | 20 + sh/drafts/check.sh | 31 + sh/drafts/publish.sh | 44 + {.github/scripts => sh/gh}/alert.sh | 2 +- {.github/scripts => sh/gh}/install-formula.py | 0 .../scripts => sh/gh}/install-formula.test.py | 0 {.github/scripts => sh/gh}/release.sh | 0 {.github/scripts => sh/gh}/release.test.sh | 0 {.github/scripts => sh/gh}/render-formula.sh | 0 .../scripts => sh/gh}/trigger-repo-publish.sh | 0 {go/scripts => sh/go}/check-errors.sh | 0 {go/scripts => sh/go}/check.sh | 8 +- {go/scripts => sh/go}/package-ffi.sh | 4 +- {go/scripts => sh/go}/package-wrapper.sh | 4 +- {go/scripts => sh/go}/publish-ffi.sh | 0 {go/scripts => sh/go}/publish-wrapper.sh | 0 {go/scripts => sh/go}/publish-wrapper.test.sh | 6 +- {go/scripts => sh/go}/stage.sh | 4 +- sh/js/workers.sh | 16 + {kt/scripts => sh/kt}/check.sh | 4 +- {kt/scripts => sh/kt}/generate.sh | 4 +- {kt/scripts => sh/kt}/package.sh | 4 +- sh/markdown.sh | 69 ++ sh/obs/check.sh | 68 ++ sh/obs/compile.sh | 67 ++ sh/obs/fix.sh | 12 + sh/obs/includes.sh | 76 ++ sh/obs/preset.sh | 22 + sh/obs/test.sh | 43 + sh/obs/unit.sh | 54 ++ sh/rs/c-tests.sh | 60 ++ sh/rs/fuzz.sh | 40 + {rs/scripts => sh/rs}/media-features.sh | 0 {rs/scripts => sh/rs}/package-binary.sh | 2 +- {rs/scripts => sh/rs}/package-nfpm.sh | 0 {rs/scripts => sh/rs}/package-rename.test.py | 2 +- {rs/scripts => sh/rs}/package-windows.sh | 2 +- sh/rs/package.sh | 51 + {rs/scripts => sh/rs}/scrub-macho.sh | 0 sh/rs/select.sh | 136 +++ {rs/scripts => sh/rs}/stats-docs.py | 0 sh/rs/tokio-features.sh | 28 + sh/rs/vulkan-cuda.sh | 17 + sh/shell.sh | 27 + {swift/scripts => sh/swift}/check.sh | 2 +- {swift/scripts => sh/swift}/package-ffi.sh | 4 +- {swift/scripts => sh/swift}/package.sh | 4 +- {swift/scripts => sh/swift}/publish-ffi.sh | 0 {swift/scripts => sh/swift}/publish.sh | 0 {swift/scripts => sh/swift}/verify-ffi.sh | 4 +- {swift/scripts => sh/swift}/verify.sh | 4 +- swift/.gitignore | 2 +- swift/.spi.yml | 2 +- swift/Package.swift | 4 +- swift/Package.swift.template | 2 +- swift/README.md | 7 +- swift/ffi/Package.swift.template | 2 +- swift/justfile | 24 +- test/README.md | 22 - test/interop/README.md | 4 +- test/interop/clients/go/go.mod | 2 +- test/interop/clients/js-native/subscribe.ts | 13 +- test/interop/interop.sh | 14 +- test/justfile | 68 +- test/wasm/README.md | 2 +- test/wasm/run.sh | 2 +- 136 files changed, 1537 insertions(+), 2906 deletions(-) delete mode 100755 .github/scripts/package-binary.test.sh delete mode 100755 bench/justfile delete mode 100644 js/common/workers.test.ts delete mode 100644 quest/m1/tooling/justfiles.md delete mode 100644 rs/scripts/doc-names.jq delete mode 100755 rs/scripts/package-nfpm.test.sh create mode 100755 sh/clean.sh rename {dart/scripts => sh/dart}/check.sh (88%) rename {dart/scripts => sh/dart}/generate.sh (96%) rename {dart/scripts => sh/dart}/package.sh (97%) rename {dart/scripts => sh/dart}/publish.sh (84%) create mode 100755 sh/dispatch.sh create mode 100755 sh/drafts/build.sh create mode 100755 sh/drafts/check.sh create mode 100755 sh/drafts/publish.sh rename {.github/scripts => sh/gh}/alert.sh (99%) rename {.github/scripts => sh/gh}/install-formula.py (100%) rename {.github/scripts => sh/gh}/install-formula.test.py (100%) rename {.github/scripts => sh/gh}/release.sh (100%) rename {.github/scripts => sh/gh}/release.test.sh (100%) rename {.github/scripts => sh/gh}/render-formula.sh (100%) rename {.github/scripts => sh/gh}/trigger-repo-publish.sh (100%) rename {go/scripts => sh/go}/check-errors.sh (100%) rename {go/scripts => sh/go}/check.sh (84%) rename {go/scripts => sh/go}/package-ffi.sh (98%) rename {go/scripts => sh/go}/package-wrapper.sh (97%) rename {go/scripts => sh/go}/publish-ffi.sh (100%) rename {go/scripts => sh/go}/publish-wrapper.sh (100%) rename {go/scripts => sh/go}/publish-wrapper.test.sh (97%) rename {go/scripts => sh/go}/stage.sh (98%) create mode 100755 sh/js/workers.sh rename {kt/scripts => sh/kt}/check.sh (89%) rename {kt/scripts => sh/kt}/generate.sh (96%) rename {kt/scripts => sh/kt}/package.sh (97%) create mode 100755 sh/markdown.sh create mode 100755 sh/obs/check.sh create mode 100755 sh/obs/compile.sh create mode 100755 sh/obs/fix.sh create mode 100755 sh/obs/includes.sh create mode 100755 sh/obs/preset.sh create mode 100755 sh/obs/test.sh create mode 100755 sh/obs/unit.sh create mode 100755 sh/rs/c-tests.sh create mode 100755 sh/rs/fuzz.sh rename {rs/scripts => sh/rs}/media-features.sh (100%) rename {rs/scripts => sh/rs}/package-binary.sh (99%) rename {rs/scripts => sh/rs}/package-nfpm.sh (100%) rename {rs/scripts => sh/rs}/package-rename.test.py (98%) rename {rs/scripts => sh/rs}/package-windows.sh (98%) create mode 100755 sh/rs/package.sh rename {rs/scripts => sh/rs}/scrub-macho.sh (100%) create mode 100755 sh/rs/select.sh rename {rs/scripts => sh/rs}/stats-docs.py (100%) create mode 100755 sh/rs/tokio-features.sh create mode 100755 sh/rs/vulkan-cuda.sh create mode 100755 sh/shell.sh rename {swift/scripts => sh/swift}/check.sh (98%) rename {swift/scripts => sh/swift}/package-ffi.sh (98%) rename {swift/scripts => sh/swift}/package.sh (98%) rename {swift/scripts => sh/swift}/publish-ffi.sh (100%) rename {swift/scripts => sh/swift}/publish.sh (100%) rename {swift/scripts => sh/swift}/verify-ffi.sh (96%) rename {swift/scripts => sh/swift}/verify.sh (96%) diff --git a/.github/justfile b/.github/justfile index f341da77296..8a3fb9ff678 100644 --- a/.github/justfile +++ b/.github/justfile @@ -1,13 +1,13 @@ -set fallback +set working-directory := '..' -# actionlint is skipped when absent locally; the root `_tools` preflight makes -# it a hard requirement under MOQ_STRICT, so CI can't skip it. - -# Lint GitHub Actions workflows. +# Lint GitHub Actions workflows and their alert coverage. check: - python3 {{ source_directory() }}/scripts/install-formula.test.py - {{ source_directory() }}/scripts/release.test.sh - @if command -v nfpm >/dev/null 2>&1 && command -v envsubst >/dev/null 2>&1 && command -v dpkg-deb >/dev/null 2>&1 && command -v rpm >/dev/null 2>&1; then python3 {{ source_directory() }}/../rs/scripts/package-rename.test.py; fi - @if command -v actionlint >/dev/null 2>&1; then actionlint; fi - {{ source_directory() }}/scripts/alert.sh check-coverage - @if command -v rustc >/dev/null 2>&1; then {{ source_directory() }}/scripts/package-binary.test.sh; fi + actionlint + sh/gh/alert.sh check-coverage + +# Tests of the release tooling itself; nightly, not on every pull request. +test: + python3 sh/gh/install-formula.test.py + sh/gh/release.test.sh + python3 sh/rs/package-rename.test.py + sh/go/publish-wrapper.test.sh diff --git a/.github/scripts/package-binary.test.sh b/.github/scripts/package-binary.test.sh deleted file mode 100755 index 2b2554b6c1a..00000000000 --- a/.github/scripts/package-binary.test.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKSPACE_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" - -tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT - -binary="$tmp/moq-relay" -printf '#!/usr/bin/env sh\necho test\n' >"$binary" -chmod 0755 "$binary" -target="$(rustc -vV | awk '/^host:/ {print $2}')" - -# This test covers archive staging, not the macOS Mach-O rewrite. -mkdir "$tmp/bin" -printf '#!/usr/bin/env sh\nprintf "Linux\\n"\n' >"$tmp/bin/uname" -chmod 0755 "$tmp/bin/uname" - -PATH="$tmp/bin:$PATH" "$WORKSPACE_DIR/rs/scripts/package-binary.sh" \ - --crate moq-relay \ - --bin moq-relay \ - --binary "$binary" \ - --bare \ - --version 0.14.13 \ - --target "$target" \ - --output "$tmp/dist" - -name="moq-relay-v0.14.13-$target" -bare="$tmp/dist/$name" -archive="$tmp/dist/$name.tar.gz" - -[[ -x "$bare" ]] -[[ -f "$archive" ]] - -mkdir "$tmp/extracted" -tar -xzf "$archive" -C "$tmp/extracted" -[[ -x "$tmp/extracted/$name/bin/moq-relay" ]] -[[ -f "$tmp/extracted/$name/LICENSE-MIT" ]] -[[ -f "$tmp/extracted/$name/LICENSE-APACHE" ]] -cmp "$binary" "$bare" -cmp "$binary" "$tmp/extracted/$name/bin/moq-relay" - -echo "release assets package together without path collisions" diff --git a/.github/workflows/alert.yml b/.github/workflows/alert.yml index 4180d6bd844..78d9c481259 100644 --- a/.github/workflows/alert.yml +++ b/.github/workflows/alert.yml @@ -88,4 +88,4 @@ jobs: RUN_REF: ${{ github.event.workflow_run.head_branch }} RUN_TITLE: ${{ github.event.workflow_run.display_title }} RUN_ACTOR: ${{ github.event.workflow_run.triggering_actor.login }} - run: .github/scripts/alert.sh discord + run: sh/gh/alert.sh discord diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 2a3c3df447d..a512d5696c5 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -53,7 +53,7 @@ jobs: exit 1 fi echo "target=${target}" >> "$GITHUB_OUTPUT" - .github/scripts/release.sh parse-version "$target" + sh/gh/release.sh parse-version "$target" build: name: Build ${{ matrix.platform }} diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index 7116bd39c19..e7eab95dee6 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -24,7 +24,7 @@ on: - "test/justfile" - "package.json" # The Go client builds against the modules this script stages. - - "go/scripts/**" + - "sh/go/**" - ".github/workflows/interop.yml" concurrency: diff --git a/.github/workflows/libmoq.yml b/.github/workflows/libmoq.yml index 7eb12759de1..3ad2daf514e 100644 --- a/.github/workflows/libmoq.yml +++ b/.github/workflows/libmoq.yml @@ -55,7 +55,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version libmoq + run: sh/gh/release.sh parse-version libmoq - name: Build and package shell: bash @@ -87,11 +87,11 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version libmoq + run: sh/gh/release.sh parse-version libmoq - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag libmoq + run: sh/gh/release.sh prev-tag libmoq - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -105,7 +105,7 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "libmoq v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts # The OBS plugin (cpp/obs) statically links libmoq, so it ships with every # libmoq release: rebuild it against the release we just published (no @@ -140,7 +140,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version libmoq + run: sh/gh/release.sh parse-version libmoq # Native Xcode / Visual Studio; libobs/Qt6/ffmpeg come from the obs-deps # bundle and libmoq from the release we just published (--libmoq-release). @@ -173,7 +173,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version libmoq + run: sh/gh/release.sh parse-version libmoq # Only the obs-moq archives: the libmoq-* artifacts from the build job # belong to the libmoq release, not this one. @@ -208,4 +208,4 @@ jobs: RELEASE_TAG: "obs-moq-v${{ steps.parse.outputs.version }}" RELEASE_TITLE: "obs-moq v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts diff --git a/.github/workflows/moq-cli.yml b/.github/workflows/moq-cli.yml index 141e47ecb8d..9e829309910 100644 --- a/.github/workflows/moq-cli.yml +++ b/.github/workflows/moq-cli.yml @@ -47,7 +47,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-cli + run: sh/gh/release.sh parse-version moq-cli - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -64,7 +64,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} TARGET: ${{ matrix.target }} run: | - ./rs/scripts/package-binary.sh \ + ./sh/rs/package-binary.sh \ --crate moq-cli \ --bin moq \ --binary "target/${TARGET}/release/moq" \ @@ -80,8 +80,8 @@ jobs: ARCH: ${{ matrix.deb-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq run: | - rs/scripts/package-nfpm.sh packaging/moq-cli/nfpm.yaml deb dist/ - rs/scripts/package-nfpm.sh packaging/moq-cli/transition.yaml deb dist/ + sh/rs/package-nfpm.sh packaging/moq-cli/nfpm.yaml deb dist/ + sh/rs/package-nfpm.sh packaging/moq-cli/transition.yaml deb dist/ - name: Package .rpm shell: bash @@ -89,7 +89,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.rpm-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq - run: rs/scripts/package-nfpm.sh packaging/moq-cli/nfpm.yaml rpm dist/ + run: sh/rs/package-nfpm.sh packaging/moq-cli/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -121,12 +121,12 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-cli + run: sh/gh/release.sh parse-version moq-cli - name: Build and package tarball shell: bash run: | - ./rs/scripts/package-binary.sh \ + ./sh/rs/package-binary.sh \ --crate moq-cli \ --bin moq \ --target ${{ matrix.target }} \ @@ -169,12 +169,12 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-cli + run: sh/gh/release.sh parse-version moq-cli - name: Build and package zip shell: bash run: | - ./rs/scripts/package-windows.sh \ + ./sh/rs/package-windows.sh \ --crate moq-cli \ --bin moq \ --target ${{ matrix.target }} \ @@ -204,11 +204,11 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-cli + run: sh/gh/release.sh parse-version moq-cli - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-cli + run: sh/gh/release.sh prev-tag moq-cli - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -222,10 +222,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-cli v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: .github/scripts/trigger-repo-publish.sh + run: sh/gh/trigger-repo-publish.sh diff --git a/.github/workflows/moq-gst.yml b/.github/workflows/moq-gst.yml index 2c93d1ac9a7..139d066c0fc 100644 --- a/.github/workflows/moq-gst.yml +++ b/.github/workflows/moq-gst.yml @@ -41,7 +41,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-gst + run: sh/gh/release.sh parse-version moq-gst - name: Build and package (tarball) shell: bash @@ -123,7 +123,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-gst + run: sh/gh/release.sh parse-version moq-gst - name: Install GStreamer dev libraries shell: bash @@ -160,7 +160,7 @@ jobs: PLUGIN_DIR: /usr/lib/${{ matrix.deb-multiarch }}/gstreamer-1.0 run: | mkdir -p dist - rs/scripts/package-nfpm.sh packaging/moq-gst/nfpm.yaml deb dist/ + sh/rs/package-nfpm.sh packaging/moq-gst/nfpm.yaml deb dist/ - name: Package .rpm shell: bash @@ -172,7 +172,7 @@ jobs: PLUGIN_DIR: /usr/lib64/gstreamer-1.0 run: | mkdir -p dist - rs/scripts/package-nfpm.sh packaging/moq-gst/nfpm.yaml rpm dist/ + sh/rs/package-nfpm.sh packaging/moq-gst/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -197,11 +197,11 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-gst + run: sh/gh/release.sh parse-version moq-gst - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-gst + run: sh/gh/release.sh prev-tag moq-gst - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -215,10 +215,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-gst v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: .github/scripts/trigger-repo-publish.sh + run: sh/gh/trigger-repo-publish.sh diff --git a/.github/workflows/moq-relay.yml b/.github/workflows/moq-relay.yml index 761ee0aba35..96496d97319 100644 --- a/.github/workflows/moq-relay.yml +++ b/.github/workflows/moq-relay.yml @@ -50,7 +50,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-relay + run: sh/gh/release.sh parse-version moq-relay - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -67,7 +67,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} TARGET: ${{ matrix.target }} run: | - ./rs/scripts/package-binary.sh \ + ./sh/rs/package-binary.sh \ --crate moq-relay \ --bin moq-relay \ --binary "target/${TARGET}/release/moq-relay" \ @@ -82,7 +82,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.deb-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: rs/scripts/package-nfpm.sh packaging/moq-relay/nfpm.yaml deb dist/ + run: sh/rs/package-nfpm.sh packaging/moq-relay/nfpm.yaml deb dist/ - name: Package .rpm shell: bash @@ -90,7 +90,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.rpm-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: rs/scripts/package-nfpm.sh packaging/moq-relay/nfpm.yaml rpm dist/ + run: sh/rs/package-nfpm.sh packaging/moq-relay/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -122,12 +122,12 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-relay + run: sh/gh/release.sh parse-version moq-relay - name: Build and package tarball shell: bash run: | - ./rs/scripts/package-binary.sh \ + ./sh/rs/package-binary.sh \ --crate moq-relay \ --target ${{ matrix.target }} \ --version ${{ steps.parse.outputs.version }} \ @@ -169,12 +169,12 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-relay + run: sh/gh/release.sh parse-version moq-relay - name: Build and package zip shell: bash run: | - ./rs/scripts/package-windows.sh \ + ./sh/rs/package-windows.sh \ --crate moq-relay \ --target ${{ matrix.target }} \ --version ${{ steps.parse.outputs.version }} \ @@ -203,11 +203,11 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-relay + run: sh/gh/release.sh parse-version moq-relay - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-relay + run: sh/gh/release.sh prev-tag moq-relay - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -221,10 +221,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-relay v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: .github/scripts/trigger-repo-publish.sh + run: sh/gh/trigger-repo-publish.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 9f73f461090..431d89002e8 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -121,7 +121,8 @@ jobs: matrix: # `rs uring` is a separate feature compile (off the default set) and # kernel-gated below 6.12. The embedding tests for io_uring live there. - recipe: ["rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring"] + # `gh test` exercises the release tooling, which no pull request runs. + recipe: ["rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "gh test"] steps: - name: Free disk space uses: jlumbroso/free-disk-space@54081f138730dfa15788a46383842cd2f914a1be # main diff --git a/.github/workflows/obs.yml b/.github/workflows/obs.yml index 139a60cfebd..0f583c95b02 100644 --- a/.github/workflows/obs.yml +++ b/.github/workflows/obs.yml @@ -48,6 +48,7 @@ on: types: [opened, synchronize, reopened, closed] paths: - "cpp/obs/**" + - "sh/obs/**" - "rs/libmoq/**" - "rs/**/Cargo.toml" - "rs/**/build.rs" diff --git a/.github/workflows/release-brew.yml b/.github/workflows/release-brew.yml index 20fe7211562..fb06fe444f1 100644 --- a/.github/workflows/release-brew.yml +++ b/.github/workflows/release-brew.yml @@ -98,7 +98,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} run: | mkdir -p rendered - ./.github/scripts/render-formula.sh \ + ./sh/gh/render-formula.sh \ --template ".github/homebrew/Formula/${CRATE}.rb.tmpl" \ --version "$VERSION" \ --release-dir release-assets \ @@ -156,7 +156,7 @@ jobs: EOF fi - python3 ../.github/scripts/install-formula.py "$CRATE" "../rendered/${CRATE}.rb" . + python3 ../sh/gh/install-formula.py "$CRATE" "../rendered/${CRATE}.rb" . if git diff --quiet && [[ -z "$(git status --porcelain)" ]]; then echo "No changes to commit (formula already up to date)." diff --git a/.github/workflows/release-dart-ffi.yml b/.github/workflows/release-dart-ffi.yml index 85f2fff849d..59f6c7a61c0 100644 --- a/.github/workflows/release-dart-ffi.yml +++ b/.github/workflows/release-dart-ffi.yml @@ -16,7 +16,7 @@ on: paths: - ".github/workflows/release-dart-ffi.yml" - "dart/moq_ffi/**" - - "dart/scripts/**" + - "sh/dart/**" - "rs/moq-ffi/**" - "!rs/moq-ffi/Cargo.toml" @@ -48,7 +48,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-ffi + run: sh/gh/release.sh parse-version moq-ffi build: name: Build Native Asset (${{ matrix.target }}) @@ -157,7 +157,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./dart/scripts/package.sh \ + ./sh/dart/package.sh \ --package moq_ffi \ --version "$BUILD_VERSION" \ --output release-out @@ -188,7 +188,7 @@ jobs: - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-ffi + run: sh/gh/release.sh prev-tag moq-ffi - name: Download Native Assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -204,7 +204,7 @@ jobs: RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} RELEASE_CHECKSUMS: SHA256SUMS-dart - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts publish: name: Publish moq_ffi to pub.dev diff --git a/.github/workflows/release-dart.yml b/.github/workflows/release-dart.yml index 2c099bc3188..4df8f205f43 100644 --- a/.github/workflows/release-dart.yml +++ b/.github/workflows/release-dart.yml @@ -12,7 +12,7 @@ on: paths: - ".github/workflows/release-dart.yml" - "dart/moq/**" - - "dart/scripts/**" + - "sh/dart/**" permissions: contents: read @@ -47,7 +47,7 @@ jobs: if [[ "$EVENT_NAME" == "pull_request" ]]; then VERSION=$(sed -n 's/^version: //p' dart/moq/pubspec.yaml | head -1) else - .github/scripts/release.sh parse-version moq-dart + sh/gh/release.sh parse-version moq-dart exit 0 fi echo "version=$VERSION" >> "$GITHUB_OUTPUT" @@ -69,7 +69,7 @@ jobs: env: BUILD_VERSION: ${{ steps.version.outputs.version }} run: | - ./dart/scripts/package.sh \ + ./sh/dart/package.sh \ --package moq \ --version "$BUILD_VERSION" \ --output release-out diff --git a/.github/workflows/release-go-ffi.yml b/.github/workflows/release-go-ffi.yml index 33e21c0c02d..d1d8477d387 100644 --- a/.github/workflows/release-go-ffi.yml +++ b/.github/workflows/release-go-ffi.yml @@ -16,10 +16,10 @@ on: pull_request: paths: - ".github/workflows/release-go-ffi.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" - "go/ffi/**" - - "go/scripts/package-ffi.sh" - - "go/scripts/publish-ffi.sh" + - "sh/go/package-ffi.sh" + - "sh/go/publish-ffi.sh" - "rs/moq-ffi/build.sh" permissions: @@ -53,7 +53,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-ffi + run: sh/gh/release.sh parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -208,7 +208,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./go/scripts/package-ffi.sh \ + ./sh/go/package-ffi.sh \ --version "$BUILD_VERSION" \ --source-dir go/ffi \ --lib-dir libs \ @@ -256,7 +256,7 @@ jobs: GO_FFI_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./go/scripts/publish-ffi.sh + run: ./sh/go/publish-ffi.sh publish-dry-run: name: Publish dry-run @@ -278,4 +278,4 @@ jobs: - name: Dry-run publish to mirror env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: ./go/scripts/publish-ffi.sh --dry-run + run: ./sh/go/publish-ffi.sh --dry-run diff --git a/.github/workflows/release-go.yml b/.github/workflows/release-go.yml index 0f87ed9b743..92300e5f131 100644 --- a/.github/workflows/release-go.yml +++ b/.github/workflows/release-go.yml @@ -105,7 +105,7 @@ jobs: if: github.event_name == 'push' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: .github/scripts/release.sh ffi-unreleased "$FFI_VERSION" + run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" # Skip on unreleased ffi changes, and when the pinned ffi's mirror tag is # missing (release-plz merged but Release Go FFI hasn't published yet): @@ -160,7 +160,7 @@ jobs: LINE: ${{ steps.line.outputs.line }} FFI_VERSION: ${{ steps.ffi.outputs.version }} run: | - ./go/scripts/package-wrapper.sh --line "$LINE" --ffi-version "$FFI_VERSION" --output release-out + ./sh/go/package-wrapper.sh --line "$LINE" --ffi-version "$FFI_VERSION" --output release-out - name: Upload wrapper module if: steps.gate.outputs.skip != 'true' @@ -203,4 +203,4 @@ jobs: GO_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./go/scripts/publish-wrapper.sh + run: ./sh/go/publish-wrapper.sh diff --git a/.github/workflows/release-kt-ffi.yml b/.github/workflows/release-kt-ffi.yml index 2d72acfbf08..31a6979587a 100644 --- a/.github/workflows/release-kt-ffi.yml +++ b/.github/workflows/release-kt-ffi.yml @@ -39,7 +39,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-ffi + run: sh/gh/release.sh parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -199,7 +199,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./kt/scripts/package.sh \ + ./sh/kt/package.sh \ --version "$BUILD_VERSION" \ --lib-dir libs \ --bindings-dir bindings \ diff --git a/.github/workflows/release-kt-lib.yml b/.github/workflows/release-kt-lib.yml index d1357543296..1bec21db752 100644 --- a/.github/workflows/release-kt-lib.yml +++ b/.github/workflows/release-kt-lib.yml @@ -30,7 +30,7 @@ on: - "kt/build.gradle.kts" - "kt/settings.gradle.kts" - ".github/workflows/release-kt-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" pull_request: paths: - "kt/moq/**" @@ -38,7 +38,7 @@ on: - "kt/build.gradle.kts" - "kt/settings.gradle.kts" - ".github/workflows/release-kt-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" - "rs/moq-ffi/build.sh" # Manual trigger so a wrapper bump can be published once the bindings it # depends on land on Maven Central (the guard below skips otherwise). @@ -103,18 +103,18 @@ jobs: - name: Check for unreleased FFI changes id: ffi-diff if: github.event_name != 'workflow_run' - run: .github/scripts/release.sh ffi-unreleased "${{ steps.ffi.outputs.version }}" + run: sh/gh/release.sh ffi-unreleased "${{ steps.ffi.outputs.version }}" - name: Check wrapper publication id: wrapper - run: .github/scripts/release.sh maven-exists dev.moq moq "${{ steps.version.outputs.version }}" + run: sh/gh/release.sh maven-exists dev.moq moq "${{ steps.version.outputs.version }}" # The bindings this checkout generates must be on Maven Central, or the # wrapper's POM range resolves to an older moq-ffi without them. - name: Check bindings publication id: bindings if: github.event_name != 'workflow_run' && steps.wrapper.outputs.exists != 'true' && steps.ffi-diff.outputs.unreleased != 'true' - run: .github/scripts/release.sh maven-exists dev.moq moq-ffi "${{ steps.ffi.outputs.version }}" + run: sh/gh/release.sh maven-exists dev.moq moq-ffi "${{ steps.ffi.outputs.version }}" - name: Decide publish id: decide @@ -202,7 +202,7 @@ jobs: - name: Recheck wrapper publication id: recheck if: github.event_name != 'pull_request' - run: .github/scripts/release.sh maven-exists dev.moq moq "${{ needs.check-version.outputs.version }}" + run: sh/gh/release.sh maven-exists dev.moq moq "${{ needs.check-version.outputs.version }}" - name: Publish to Maven Central if: github.event_name != 'pull_request' && steps.recheck.outputs.exists != 'true' diff --git a/.github/workflows/release-py-ffi.yml b/.github/workflows/release-py-ffi.yml index 2a79e7096ee..2e1fa1bb49e 100644 --- a/.github/workflows/release-py-ffi.yml +++ b/.github/workflows/release-py-ffi.yml @@ -36,7 +36,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-ffi + run: sh/gh/release.sh parse-version moq-ffi # Maturin reads the wheel version from rs/moq-ffi/Cargo.toml (via # `dynamic = ["version"]` in pyproject.toml). If a tag is pushed by diff --git a/.github/workflows/release-py.yml b/.github/workflows/release-py.yml index b2ba9f0cd2f..67a2741e8bf 100644 --- a/.github/workflows/release-py.yml +++ b/.github/workflows/release-py.yml @@ -19,7 +19,7 @@ on: paths: - "py/moq-rs/**" - ".github/workflows/release-py.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" workflow_run: workflows: ["Release Py FFI"] types: @@ -58,7 +58,7 @@ jobs: - name: Read version id: version - run: .github/scripts/release.sh read-version py/moq-rs/pyproject.toml + run: sh/gh/release.sh read-version py/moq-rs/pyproject.toml - name: Resolve FFI version id: ffi @@ -78,14 +78,14 @@ jobs: if: github.event_name == 'push' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: .github/scripts/release.sh ffi-unreleased "$FFI_VERSION" + run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" - name: Check FFI on PyPI id: ffi-pypi if: github.event_name == 'push' && steps.ffi-gate.outputs.unreleased != 'true' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: .github/scripts/release.sh pypi-exists moq-ffi "$FFI_VERSION" + run: sh/gh/release.sh pypi-exists moq-ffi "$FFI_VERSION" - name: Unpublished FFI notice if: github.event_name == 'push' && steps.ffi-gate.outputs.unreleased != 'true' && steps.ffi-pypi.outputs.exists != 'true' @@ -100,7 +100,7 @@ jobs: id: pypi env: VERSION: ${{ steps.version.outputs.version }} - run: .github/scripts/release.sh pypi-exists moq-rs "$VERSION" + run: sh/gh/release.sh pypi-exists moq-rs "$VERSION" - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: diff --git a/.github/workflows/release-swift-ffi.yml b/.github/workflows/release-swift-ffi.yml index 658c06b6541..966a1b86c35 100644 --- a/.github/workflows/release-swift-ffi.yml +++ b/.github/workflows/release-swift-ffi.yml @@ -36,7 +36,7 @@ jobs: - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-ffi + run: sh/gh/release.sh parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -149,7 +149,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./swift/scripts/package-ffi.sh \ + ./sh/swift/package-ffi.sh \ --version "$BUILD_VERSION" \ --lib-dir libs \ --bindings-dir bindings \ @@ -183,7 +183,7 @@ jobs: - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-ffi + run: sh/gh/release.sh prev-tag moq-ffi - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -197,7 +197,7 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: sh/gh/release.sh create artifacts verify: # Gate the mirror push on actually being able to resolve the staged package @@ -223,7 +223,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./swift/scripts/verify-ffi.sh --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" + ./sh/swift/verify-ffi.sh --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" publish: name: Publish to Swift Package mirror @@ -259,4 +259,4 @@ jobs: SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./swift/scripts/publish-ffi.sh + run: ./sh/swift/publish-ffi.sh diff --git a/.github/workflows/release-swift-lib.yml b/.github/workflows/release-swift-lib.yml index 6c5b07fc5a5..6361566a574 100644 --- a/.github/workflows/release-swift-lib.yml +++ b/.github/workflows/release-swift-lib.yml @@ -22,22 +22,22 @@ on: - "swift/Package.swift.template" - "swift/Sources/Moq/**" - "swift/Tests/**" - - "swift/scripts/package.sh" - - "swift/scripts/publish.sh" - - "swift/scripts/verify.sh" + - "sh/swift/package.sh" + - "sh/swift/publish.sh" + - "sh/swift/verify.sh" - ".github/workflows/release-swift-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" pull_request: paths: - "swift/VERSION" - "swift/Package.swift.template" - "swift/Sources/Moq/**" - "swift/Tests/**" - - "swift/scripts/package.sh" - - "swift/scripts/publish.sh" - - "swift/scripts/verify.sh" + - "sh/swift/package.sh" + - "sh/swift/publish.sh" + - "sh/swift/verify.sh" - ".github/workflows/release-swift-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" workflow_run: workflows: ["Release Swift FFI"] types: @@ -101,13 +101,13 @@ jobs: if: github.event_name != 'workflow_run' env: FFI_VERSION: ${{ steps.version.outputs.ffi_version }} - run: .github/scripts/release.sh ffi-unreleased "$FFI_VERSION" + run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" - name: Check FFI mirror tag id: ffi-gate env: FFI_VERSION: ${{ steps.version.outputs.ffi_version }} - run: .github/scripts/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" + run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" - name: Unpublished FFI notice if: steps.ffi-diff.outputs.unreleased != 'true' && steps.ffi-gate.outputs.exists != 'true' && github.event_name != 'pull_request' @@ -121,12 +121,12 @@ jobs: id: gate env: VERSION: ${{ steps.version.outputs.version }} - run: .github/scripts/release.sh git-tag-exists moq-dev/moq-swift "$VERSION" + run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift "$VERSION" - name: Package env: BUILD_VERSION: ${{ steps.version.outputs.version }} - run: ./swift/scripts/package.sh --version "$BUILD_VERSION" --output release-out + run: ./sh/swift/package.sh --version "$BUILD_VERSION" --output release-out - name: Upload Swift package uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -158,7 +158,7 @@ jobs: # rather than a hard failure, so the first bootstrap (before # moq-swift-ffi exists) skips the cross-package resolve instead of # failing the job. publish.sh stays idempotent regardless. - run: .github/scripts/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" || echo "exists=false" >> "$GITHUB_OUTPUT" + run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" || echo "exists=false" >> "$GITHUB_OUTPUT" - name: Download staged package if: steps.ffi.outputs.exists == 'true' @@ -171,7 +171,7 @@ jobs: if: steps.ffi.outputs.exists == 'true' env: BUILD_VERSION: ${{ needs.build.outputs.version }} - run: ./swift/scripts/verify.sh --tarball "artifacts/moq-${BUILD_VERSION}-swift.tar.gz" + run: ./sh/swift/verify.sh --tarball "artifacts/moq-${BUILD_VERSION}-swift.tar.gz" - name: Skip notice if: needs.build.outputs.ffi_unreleased == 'true' @@ -218,7 +218,7 @@ jobs: SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./swift/scripts/publish.sh + run: ./sh/swift/publish.sh publish-dry-run: name: Publish dry-run @@ -240,4 +240,4 @@ jobs: - name: Dry-run publish to mirror env: BUILD_VERSION: ${{ needs.build.outputs.version }} - run: ./swift/scripts/publish.sh --dry-run + run: ./sh/swift/publish.sh --dry-run diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 7092f0d955f..3c20d759076 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -14,6 +14,7 @@ on: branches: [main] paths: - "swift/**" + - "sh/swift/**" - "rs/moq-ffi/**" - "rs/moq-video/**" - "Cargo.lock" @@ -24,6 +25,7 @@ on: types: [opened, synchronize, reopened, closed] paths: - "swift/**" + - "sh/swift/**" - "rs/moq-ffi/**" - "rs/moq-video/**" - "Cargo.lock" @@ -77,4 +79,4 @@ jobs: done - name: Swift check - run: ./swift/scripts/check.sh + run: ./sh/swift/check.sh diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index 6cfb329d565..a1291df9dd2 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -43,6 +43,7 @@ on: - "test/wasm/**" - "test/lib/**" - "justfile" + - "js/justfile" - "test/justfile" - ".cargo/config.toml" - "package.json" diff --git a/.gitignore b/.gitignore index 414686073cf..357feca35ad 100644 --- a/.gitignore +++ b/.gitignore @@ -7,9 +7,8 @@ *.local.json # Claude Code scratch space. These are untracked working directories, and -# `just _changed` counts untracked files, so leaving them visible pushes the -# changed-file list past its cap and silently turns every scoped `just check` -# into `check --all`. The worktrees also carry their own biome.jsonc, which the +# `just check` counts untracked files, so leaving them visible would put their +# whole tree in scope. The worktrees also carry their own biome.jsonc, which the # unscoped run then rejects as a nested root config. /.claude/cache/ /.claude/tmp/ diff --git a/Cargo.toml b/Cargo.toml index 5a529c9cd67..009381361b3 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -318,7 +318,7 @@ debug = true # Size-optimized build for the browser/WASM bindings (rs/moq-wasm). Shipping over # the wire, so optimize for size, not speed (~579KB -> ~471KB gzipped vs the -# default release profile). `just wasm` builds with `--profile wasm-release`. +# default release profile). `just js wasm` builds with `--profile wasm-release`. [profile.wasm-release] inherits = "release" opt-level = "z" diff --git a/bench/justfile b/bench/justfile deleted file mode 100755 index b3fa64c3929..00000000000 --- a/bench/justfile +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env just --justfile - -set working-directory := '..' - -# Benchmark the current tree, or compare it with a commit. -compare $BASE="": - #!/usr/bin/env bash - exec bench/run.sh "$BASE" - -# Compare the shared Tokio runtime with independent Tokio and io_uring workers. -runtime $ROUNDS="3" $WORKERS="": - #!/usr/bin/env bash - MOQ_BENCH_RUNTIME_ROUNDS="$ROUNDS" MOQ_BENCH_RUNTIME_WORKERS="$WORKERS" \ - exec bench/run.sh --runtime - -# Compile every binary and feature used by the repository-level harness. -check: - cargo check --locked --package moq-relay --package moq-bench \ - --features moq-relay/io-uring diff --git a/cpp/obs/justfile b/cpp/obs/justfile index 99b658406f0..eed0d373444 100644 --- a/cpp/obs/justfile +++ b/cpp/obs/justfile @@ -21,393 +21,39 @@ default: # Configure via CMake presets. MOQ_LOCAL defaults to the repo root inside # CMakeLists.txt (so libmoq builds from rs/libmoq); pass `path=/some/moq` to -# override. We deliberately don't compute the path here: `justfile_directory()` -# resolves to the root justfile when this runs as `just obs setup`, so the - -# relative math would point outside the repo. +# override it. setup preset="" path="": - #!/usr/bin/env bash - set -euo pipefail - PRESET=$(just preset "{{ preset }}") - if [ -n "{{ path }}" ]; then - echo "Configuring with preset: $PRESET and MOQ_LOCAL={{ path }}" - cmake --preset "$PRESET" -DMOQ_LOCAL="{{ path }}" - else - echo "Configuring with preset: $PRESET (MOQ_LOCAL defaults to the repo root)" - cmake --preset "$PRESET" - fi + cmake --preset "$(../../sh/obs/preset.sh '{{ preset }}')" {{ if path == "" { "" } else { "-DMOQ_LOCAL=" + quote(path) } }} # Build via CMake presets. Run `just obs setup` first (it configures + downloads - # deps); not chained here because on macOS setup reconfigures OBS, which is slow. build preset="": - #!/usr/bin/env bash - set -euo pipefail - PRESET=$(just preset "{{ preset }}") - cmake --build --preset "$PRESET" - -# Copy the freshly built plugin into the OBS user plugin dir and launch OBS. - -# Uses the installed OBS (no local OBS build required). macOS only for now. -run: - #!/usr/bin/env bash - set -euo pipefail - if [[ "$OSTYPE" != "darwin"* ]]; then - echo "just run is macOS-only for now; copy build_*/obs-moq.* into your OBS plugin dir manually" >&2 - exit 1 - fi - dest="$HOME/Library/Application Support/obs-studio/plugins" - mkdir -p "$dest" - cp -a build_macos/RelWithDebInfo/obs-moq.plugin "$dest/" - RUST_LOG=debug RUST_BACKTRACE=1 OBS_LOG_LEVEL=debug /Applications/OBS.app/Contents/MacOS/OBS + cmake --build --preset "$(../../sh/obs/preset.sh '{{ preset }}')" # Type-check every plugin source and unit test, without linking or an obs-deps -# download. -# -# This is the gate to run in a worktree. `just obs build` needs the -# multi-hundred-MB obs-deps bundle (macOS/Windows) and a full cargo build, per -# worktree; this needs neither, because the dev shell carries all three header -# sets: libobs pinned to the same OBS release buildspec.json downloads (see -# `obs-headers` in flake.nix), Qt6, and ffmpeg. CI links the real thing on Linux -# (.github/workflows/obs.yml). -# -# It compiles the Qt sources too, which the CMake build only does when -# ENABLE_QT and ENABLE_FRONTEND_API are on, so the definitions they gate on are -# set here to match. -# -# test/ is in scope because each test file defines the libmoq entry points the -# plugin calls, so a signature that drifts from the generated moq.h is a -# conflicting C declaration. Catching that needs only headers, which is why it -# belongs here: `just obs ci` finds the same drift, but only where obs.yml's -# path filter reaches, and `just obs test` is manual. +# download. The gate to run in a worktree. compile: - #!/usr/bin/env bash - set -euo pipefail - # Never skips, for the reason `test` doesn't: a compile gate that reports - # success without compiling is worse than no gate. Everything it needs is in - # the dev shell, so the fix is always to enter it. - cxx="${CXX:-c++}" - if ! command -v "$cxx" >/dev/null; then - echo "no C++ compiler at '$cxx'" >&2 - exit 1 - fi - for pkgs in "Qt6Widgets Qt6Gui Qt6Core" "libavcodec libavutil libswscale libswresample"; do - # shellcheck disable=SC2086 - if ! pkg-config --exists $pkgs; then - echo "missing headers for: $pkgs" >&2 - echo "run inside 'nix develop', which supplies Qt6 and ffmpeg on every platform" >&2 - exit 1 - fi - done - - # Assign first so a failure in `_includes` still aborts under `set -e`, - # then split on newlines only: an include path may contain spaces. - includes_raw=$(just _includes) - includes=() - while IFS= read -r flag; do includes+=("$flag"); done <<< "$includes_raw" - - qt=$(pkg-config --cflags Qt6Widgets Qt6Gui Qt6Core) - ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale libswresample) - - # MOQ_VERSION_STRING only reaches a label in the dock, so any value - # type-checks the same; CMake stamps the real libmoq version. - status=0 - for source in src/*.cpp test/*.cpp; do - # shellcheck disable=SC2086 - # -Wno-unused-command-line-argument: the nix compiler wrapper injects - # link flags, which a syntax-only run reports as unused, once per file. - # $qt and $ffmpeg stay unquoted on purpose: pkg-config hands back one - # space-separated string, and splitting it is the intended reading. - if ! "$cxx" -std=c++17 -fsyntax-only -Wno-unused-command-line-argument \ - "${includes[@]}" $qt $ffmpeg \ - -DMOQ_FRONTEND_ENABLED -DMOQ_VERSION_STRING='"0.0.0"' "$source"; then - status=1 - fi - done - exit $status + ../../sh/obs/compile.sh # Compile and link the plugin the way CI does, against the dev shell's own # libobs/Qt6/ffmpeg, then run the unit tests. Linux-only in practice: it's the -# one platform where nixpkgs has obs-studio, so nothing has to be downloaded. -# Manual elsewhere, like `just rs macos`. -# -# The tests run here without ThreadSanitizer because this is the only recipe CI -# invokes (.github/workflows/obs.yml). `just obs test` is manual, so on its own -# it can't make a build red; the sanitizer finds more, but only what runs in CI -# gates a merge. -# -# CMAKE_BUILD_TYPE overrides the preset's RelWithDebInfo because the Rust cache -# CI restores holds dev-profile artifacts. A release build of libmoq would -# rebuild its whole dependency tree on every run, and the gate is about whether -# the plugin compiles, which the profile doesn't change. +# one platform where nixpkgs has obs-studio. The Debug build type reuses the +# dev-profile Rust artifacts CI restores, where a release libmoq would rebuild +# its whole dependency tree. ci preset="": - #!/usr/bin/env bash - set -euo pipefail - PRESET=$(just preset "{{ preset }}" ci) - cmake --preset "$PRESET" -DCMAKE_BUILD_TYPE=Debug - cmake --build --preset "$PRESET" - just _unit - -# Print the -I flags for libobs and libmoq, shared by `compile` and `_unit`. -# -# moq.h is generated by cbindgen from rs/libmoq/src, so regenerate rather than -# just locating it: a stale header is how a plugin call to a since-changed -# libmoq function type-checks anyway. -[private] -_includes: - #!/usr/bin/env bash - set -euo pipefail - # The dev shell's pinned headers first, so every platform checks against the - # same libobs. Then the obs-deps framework (macOS/Windows), the OBS sources - # CMake unpacks beside it, and a system install (Linux without nix). - obs_include="" - for candidate in \ - "${OBS_INCLUDE_DIR:-}" \ - .deps/Frameworks/libobs.framework/Versions/A/Headers \ - .deps/obs-studio-*/libobs; do - if [ -n "$candidate" ] && [ -f "$candidate/obs.h" ]; then - obs_include="$candidate" - break - fi - done - if [ -z "$obs_include" ] && command -v pkg-config >/dev/null && pkg-config --exists libobs; then - obs_include=$(pkg-config --variable=includedir libobs)/obs - fi - if [ -z "$obs_include" ]; then - echo "libobs headers not found; run inside 'nix develop' (or set OBS_INCLUDE_DIR)" >&2 - exit 1 - fi - - # obs.h reaches as of OBS 32, which used to be vendored - # under libobs/util/ and so came along with the headers above. The dev shell - # carries it (see flake.nix); the obs-deps bundle ships its own copy beside - # libobs, which is how the macOS and Windows builds get it. - simde_include="" - if command -v pkg-config >/dev/null && pkg-config --exists simde; then - simde_include=$(pkg-config --variable=includedir simde) - else - for candidate in .deps/obs-deps-*/include; do - if [ -f "$candidate/simde/x86/sse2.h" ]; then - simde_include="$candidate" - break - fi - done - fi - if [ -z "$simde_include" ]; then - echo "simde headers not found; run inside 'nix develop', which supplies them" >&2 - exit 1 - fi - - (cd ../.. && cargo check --locked --quiet -p libmoq) - - # Ask cargo where its build script ran rather than reconstructing the path. - # CARGO_TARGET_DIR, a `--target` triple and a custom profile each move the - # generated header, and guessing wrong doesn't fail: it type-checks against - # whatever stale copy the default directory still holds. Replays the check - # above from cache, so it costs a process rather than a compile. - out_dir=$(cd ../.. && cargo check --locked -p libmoq --message-format=json | - jq -r 'select(.reason == "build-script-executed") | select(.package_id | tostring | test("libmoq")) | .out_dir' | - tail -1) - if [ -z "$out_dir" ]; then - echo "cargo reported no build script output for libmoq" >&2 - exit 1 - fi - # //build/libmoq-/out, and the header sits beside the - # profile directory. rs/libmoq/build.rs derives it from OUT_DIR the same way. - moq_include="$(dirname "$(dirname "$(dirname "$(dirname "$out_dir")")")")/include" - if [ ! -f "$moq_include/moq.h" ]; then - echo "$moq_include/moq.h missing after 'cargo check --locked -p libmoq'" >&2 - exit 1 - fi - - # One flag per line, path glued to the -I, so a caller can rebuild the - # arguments without splitting on the spaces in a path. - printf -- '-I%s\n' src "$obs_include" "$simde_include" "$moq_include" + cmake --preset "$(../../sh/obs/preset.sh '{{ preset }}' ci)" -DCMAKE_BUILD_TYPE=Debug + cmake --build --preset "$(../../sh/obs/preset.sh '{{ preset }}' ci)" + ../../sh/obs/unit.sh # Unit-test the plugin sources against stubbed libobs/libmoq/ffmpeg, under -# ThreadSanitizer. -# -# Manual, like `just rs macos` and `just rs windows`, because ThreadSanitizer -# needs its own build. `just obs ci` runs the same tests without it, so a -# regression these assertions catch still turns PR CI red; the sanitizer is what -# adds the races on top. Run this whenever you touch src/, especially the session -# callback plumbing, whose orderings the build can't check. +# ThreadSanitizer. Manual; `just obs ci` runs them without the sanitizer. test: - #!/usr/bin/env bash - set -euo pipefail - # Unlike the lint recipes, this one never skips: a test gate that reports - # success without running anything is worse than no gate. - cxx="${CXX:-c++}" - unsupported="'$cxx' cannot build and run -fsanitize=thread binaries. These tests need a - Clang or GCC whose ThreadSanitizer runtime works on this host: set CXX, or on Windows run - them from WSL, since neither MSVC nor Clang on Windows implements ThreadSanitizer. - They also run without the sanitizer as part of 'just obs ci'." - if ! command -v "$cxx" >/dev/null; then - echo "no C++ compiler at '$cxx'." >&2 - echo "$unsupported" >&2 - exit 1 - fi - - probe=$(mktemp -d) - trap 'rm -rf "$probe"' EXIT - - # Run the probe, don't just link it: a mismatch between the compiler's TSan - # runtime and the host kernel (macOS is where this shows up) links fine and - # then segfaults before main, which is an unreadable way for the real test - # to fail. - # The subshell keeps bash's own "Segmentation fault" job message out of the - # output, so the explanation below is all the reader gets. - if ! printf 'int main(){}\n' | "$cxx" -x c++ -fsanitize=thread -o "$probe/probe" - >/dev/null 2>&1 \ - || ! ("$probe/probe" >/dev/null 2>&1) 2>/dev/null; then - echo "$unsupported" >&2 - exit 1 - fi - - just _unit tsan + ../../sh/obs/test.sh -# Compile and run the unit tests, plain or under ThreadSanitizer. Shared by -# `test` and `ci` so both run the same assertions. -[private] -_unit mode="plain": - #!/usr/bin/env bash - set -euo pipefail - cxx="${CXX:-c++}" - if ! command -v "$cxx" >/dev/null; then - echo "no C++ compiler at '$cxx'" >&2 - exit 1 - fi - # moq-source.cpp decodes with ffmpeg, so its test needs the headers even - # though it stubs every function out of them. - if ! pkg-config --exists libavcodec libavutil libswscale; then - echo "missing ffmpeg headers" >&2 - echo "run inside 'nix develop', which supplies ffmpeg on every platform" >&2 - exit 1 - fi - - includes_raw=$(just _includes) - includes=() - while IFS= read -r flag; do includes+=("$flag"); done <<< "$includes_raw" - - ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale) - - flags=(-std=c++17 -g -O0 -pthread) - if [ "{{ mode }}" = "tsan" ]; then - flags+=(-fsanitize=thread) - fi - - out=$(mktemp -d) - trap 'rm -rf "$out"' EXIT - - # One binary per source under test: each test file defines its own libobs and - # libmoq stubs, so two of them can't share a link. Header-only helpers - # (quality defaults) build as a third binary with no plugin source. - for name in moq-output moq-source; do - # shellcheck disable=SC2086 - # $ffmpeg stays unquoted on purpose: pkg-config hands back one - # space-separated string, and splitting it is the intended reading. - "$cxx" "${flags[@]}" "${includes[@]}" $ffmpeg \ - -o "$out/$name-test" "test/$name-test.cpp" "src/$name.cpp" - TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" - done - for name in moq-quality-defaults moq-dock-stop moq-error moq-encoder-latency moq-dial moq-spark; do - "$cxx" "${flags[@]}" "${includes[@]}" \ - -o "$out/$name-test" "test/$name-test.cpp" - TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" - done - -# Lint formatting. Skips a tool silently if it isn't on $PATH (matches repo convention). +# Lint formatting and the OBS version pins. check: - #!/usr/bin/env bash - set -euo pipefail - if grep -Fq '"MOQ_VERSION"' CMakePresets.json; then - echo "CMakePresets.json must not pin MOQ_VERSION" >&2 - exit 1 - fi - # The headers `just obs compile` type-checks against are a second copy of - # the OBS version this spec downloads, so a bump here has to reach flake.nix - # or the two gates check different libobs. - spec_obs=$(jq -r '.dependencies["obs-studio"].version // ""' buildspec.json) - flake_obs=$(sed -n '/pname = "libobs-headers"/,/}/p' ../../flake.nix | sed -n 's/.*version = "\([^"]*\)".*/\1/p') - # Both are pattern matches against files this recipe doesn't own the shape - # of. An empty parse compares equal to an empty parse, so the guard would - # pass by reporting nothing rather than by finding the versions in sync. - if [ -z "$spec_obs" ] || [ -z "$flake_obs" ]; then - echo "couldn't read the obs-studio version: '$spec_obs' from buildspec.json, '$flake_obs' from flake.nix" >&2 - exit 1 - fi - if [ "$spec_obs" != "$flake_obs" ]; then - echo "obs-studio is $spec_obs in buildspec.json but $flake_obs in flake.nix (libobs-headers)" >&2 - exit 1 - fi - # And a third: `just obs ci` links against nixpkgs' obs-studio, the one OBS - # this repo doesn't pick. Without this the released macOS/Windows binaries - # could be a whole libobs API apart from what CI ever compiles. Patch - # releases carry no API change, so only the major.minor has to agree; that - # also keeps the guard quiet until flake.lock actually moves OBS, which is - # the change that opens the gap and so the one that should fail. - # OBS_LINKED_VERSION comes from the dev shell (see flake.nix), so this leg - # is skipped outside it, where there is no nixpkgs obs-studio to compare. - # MOQ_STRICT turns that skip into an error for the same reason it turns a - # missing tool into one: in CI a leg that checks nothing still reports green. - if [ -z "${OBS_LINKED_VERSION:-}" ]; then - if [ -n "${MOQ_STRICT:-}" ]; then - echo "MOQ_STRICT is set but OBS_LINKED_VERSION is unset; run inside 'nix develop', which exports it" >&2 - exit 1 - fi - else - spec_api=$(echo "$spec_obs" | cut -d. -f1,2) - linked_api=$(echo "$OBS_LINKED_VERSION" | cut -d. -f1,2) - if [ "$spec_api" != "$linked_api" ]; then - echo "obs-studio is $spec_obs in buildspec.json but $OBS_LINKED_VERSION in nixpkgs, which is what 'just obs ci' links" >&2 - echo "bump buildspec.json and flake.nix's libobs-headers to $OBS_LINKED_VERSION, or pin nixpkgs back" >&2 - exit 1 - fi - fi - if command -v cmake >/dev/null; then - build_dir=$(mktemp -d) - trap 'rm -rf "$build_dir"' EXIT - if cmake -S . -B "$build_dir" -DMOQ_LOCAL= >"$build_dir/configure.log" 2>&1; then - echo "CMake accepted a release build without MOQ_VERSION" >&2 - exit 1 - fi - if ! grep -Fq "MOQ_VERSION is required when MOQ_LOCAL does not contain rs/libmoq" "$build_dir/configure.log"; then - cat "$build_dir/configure.log" >&2 - exit 1 - fi - fi - if command -v clang-format >/dev/null; then - git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format --dry-run --Werror - fi - if command -v gersemi >/dev/null; then - gersemi --check CMakeLists.txt cmake - fi + ../../sh/obs/check.sh # Auto-fix formatting. fix: - #!/usr/bin/env bash - set -euo pipefail - if command -v clang-format >/dev/null; then - git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format -i - fi - if command -v gersemi >/dev/null; then - gersemi --in-place CMakeLists.txt cmake - fi - -# Detect the CMake preset for the current platform (or use the override). -# Pass `ci` as the second argument for the warnings-as-errors variant, whose -# name upstream spells differently on each platform. -preset override="" variant="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "{{ override }}" ]]; then - echo "{{ override }}" - elif [[ "$OSTYPE" == "darwin"* ]]; then - [[ "{{ variant }}" == "ci" ]] && echo "macos-ci" || echo "macos" - elif [[ "$OSTYPE" == "linux-gnu"* ]]; then - [[ "{{ variant }}" == "ci" ]] && echo "ubuntu-ci-x86_64" || echo "ubuntu-x86_64" - elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then - [[ "{{ variant }}" == "ci" ]] && echo "windows-ci-x64" || echo "windows-x64" - else - echo "Unknown platform: $OSTYPE" >&2 - exit 1 - fi + ../../sh/obs/fix.sh diff --git a/dart/justfile b/dart/justfile index 9823e8cf324..4387f113c81 100644 --- a/dart/justfile +++ b/dart/justfile @@ -1,18 +1,13 @@ #!/usr/bin/env just --justfile # Check the generated bindings and both Dart packages. -check $FILES="": - ./scripts/check.sh "{{ FILES }}" +check: + ../sh/dart/check.sh # Format Dart sources. -fix $FILES="": - ./scripts/check.sh --fix "{{ FILES }}" +fix: + ../sh/dart/check.sh --fix # Regenerate the raw UniFFI Dart bindings. generate: - ./scripts/generate.sh - -# Remove Dart tool state and build output. -clean: - rm -rf moq/.dart_tool moq_ffi/.dart_tool - rm -rf moq/build moq_ffi/build + ../sh/dart/generate.sh diff --git a/dart/moq_ffi/hook/build.dart b/dart/moq_ffi/hook/build.dart index 6916ea46a70..bfb465d6036 100644 --- a/dart/moq_ffi/hook/build.dart +++ b/dart/moq_ffi/hook/build.dart @@ -7,7 +7,7 @@ import 'package:http/http.dart' as http; /// The moq-ffi release whose native assets this package downloads. /// -/// dart/scripts/package.sh rewrites this when staging a release. The sentinel +/// sh/dart/package.sh rewrites this when staging a release. The sentinel /// is never used to download: a checkout builds from source instead, and a /// package built without the injection fails loudly rather than fetching a /// library that does not match these bindings. diff --git a/dart/moq_ffi/pubspec.yaml b/dart/moq_ffi/pubspec.yaml index cf7a8ecadc2..0f53a4f24cb 100644 --- a/dart/moq_ffi/pubspec.yaml +++ b/dart/moq_ffi/pubspec.yaml @@ -1,6 +1,6 @@ name: moq_ffi description: Raw Dart and Flutter bindings for Media over QUIC. -# Overridden by dart/scripts/package.sh from the moq-ffi-v* tag. Committing a +# Overridden by sh/dart/package.sh from the moq-ffi-v* tag. Committing a # real version would go stale the moment release-plz bumps the crate. version: 0.0.0-dev repository: https://github.com/moq-dev/moq/tree/main/dart/moq_ffi diff --git a/demo/justfile b/demo/justfile index bfd725f82d1..b6128734659 100644 --- a/demo/justfile +++ b/demo/justfile @@ -1,11 +1,5 @@ set fallback -mod boy -mod pub -mod relay -mod sub -mod web - # Run the web demo (default). # # Picks the first free port at/after 4443 (both UDP for QUIC and TCP for HTTP) diff --git a/doc/setup/dev.md b/doc/setup/dev.md index 9aaf57bad60..0eb3852d7f7 100644 --- a/doc/setup/dev.md +++ b/doc/setup/dev.md @@ -63,6 +63,11 @@ just check just test ``` +These diff the branch against its upstream (or `origin/main`) and run only the +modules the diff reaches. The map from paths to modules lives in +`sh/dispatch.sh`; every recipe is one line, and any logic lives in a script +under `sh/`. + See [CONTRIBUTING.md](https://github.com/moq-dev/moq/blob/main/CONTRIBUTING.md) for branch targeting, commit messages, and reviews, and [Agent setup](/setup/agent) if an AI coding agent is doing the work. diff --git a/drafts/justfile b/drafts/justfile index 8ca61beadc6..bdb39c6e4df 100644 --- a/drafts/justfile +++ b/drafts/justfile @@ -5,87 +5,28 @@ # # The toolchain (kramdown-rfc, xml2rfc, mmark) comes from the nix dev shell # (see `draftsDeps` in flake.nix), so `nix develop --command just drafts build` -# renders with the same tool versions everywhere. There is no submodule or -# venv to bootstrap: `kramdown-rfc` turns the markdown into RFC XML, `xml2rfc` -# renders the txt/html. +# renders with the same tool versions everywhere. `kramdown-rfc` turns the +# markdown into RFC XML, `xml2rfc` renders the txt/html. set working-directory := '.' -# Common xml2rfc flags. --allow-local-file-access lets it read the CSS/refs. -xml2rfc := "xml2rfc -q --allow-local-file-access" - # List the draft names (the argument to `build` / `publish`). default: - @for f in draft-*.md; do echo "${f%.md}"; done + @ls draft-*.md | sed 's/\.md$//' -# Render one draft to .txt and .html (local editor's copy, -# gitignored). Renders the -latest docname as-is; use `publish` for a version. +# Render one draft to .txt and .html (local editor's copy, gitignored). build name: - kramdown-rfc --v3 < {{ name }}.md > {{ name }}.xml - {{ xml2rfc }} --text {{ name }}.xml -o {{ name }}.txt - {{ xml2rfc }} --html {{ name }}.xml -o {{ name }}.html - @echo "built {{ name }}.txt and {{ name }}.html" + ../sh/drafts/build.sh {{ name }} # Render every draft. all: - @for f in draft-*.md; do just build "${f%.md}"; done + ../sh/drafts/build.sh # Validate that every draft parses and that its generated XML passes validation. -# -# Both halves are needed. kramdown-rfc catches markdown and frontmatter errors, -# but happily emits XML for a cross-reference that points at no such section; -# xml2rfc is what rejects the dangling IDREF. Checking only the first half lets -# a broken `[text](#anchor)` pass here and then fail at `publish` time, which is -# the worst place to find it. -# -# --preptool stops after validation instead of rendering txt/html, and -N keeps -# it off the network (kramdown-rfc already inlined the references), which is -# what makes this a couple of seconds per draft rather than a minute. check: - bun install --frozen-lockfile --cwd .. - @for f in draft-*.md; do \ - echo "checking $f"; \ - kramdown-rfc --v3 < "$f" > "${f%.md}.xml" || exit 1; \ - {{ xml2rfc }} --preptool -N "${f%.md}.xml" -o "${f%.md}.prepped.xml" \ - || { rm -f "${f%.md}.prepped.xml"; exit 1; }; \ - rm -f "${f%.md}.prepped.xml"; \ - done - @for f in moq-e2ee-*.ts; do \ - bun tsc --noEmit --skipLibCheck --target esnext --module esnext --moduleResolution bundler --types bun "$f" || exit 1; \ - bun "$f" || exit 1; \ - done - -# Remove generated files and the reference cache. -clean: - rm -f draft-*.xml draft-*.txt draft-*.html - rm -rf .refcache + ../sh/drafts/check.sh -# Submit a new draft version to the IETF datatracker. The datatracker emails -# the submitter a confirmation link; the submission is not final until that -# link is clicked. For a brand-new draft (-00) set "Replaces" on the -# confirmation page. -# Example: just drafts publish draft-lcurley-moq-lite 05 you@example.com +# Submit a new draft version to the IETF datatracker, e.g. +# `just drafts publish draft-lcurley-moq-lite 05 you@example.com`. publish name version email: - #!/usr/bin/env bash - set -euo pipefail - case "{{ version }}" in - [0-9][0-9]) ;; - *) echo "version must be two digits, e.g. 05" >&2; exit 1 ;; - esac - doc="{{ name }}-{{ version }}" - if [ ! -f "{{ name }}.md" ]; then - echo "no such draft: {{ name }}.md" >&2; exit 1 - fi - echo "Building $doc.xml" - sed "s/{{ name }}-latest/$doc/g" "{{ name }}.md" | kramdown-rfc --v3 > "$doc.xml" - echo "Submitting $doc.xml to the datatracker as {{ email }}" - resp="$(mktemp)" - code="$(curl -sS -o "$resp" -w '%{http_code}' \ - -F "user={{ email }}" -F "xml=@$doc.xml" \ - https://datatracker.ietf.org/api/submission)" - echo "HTTP $code" - cat "$resp"; echo; rm -f "$resp" - case "$code" in - 200|201) echo "Submitted. Check {{ email }} for the confirmation link." ;; - *) echo "Submission failed." >&2; exit 1 ;; - esac + ../sh/drafts/publish.sh {{ name }} {{ version }} {{ email }} diff --git a/flake.nix b/flake.nix index 48971bef39a..7e557312254 100644 --- a/flake.nix +++ b/flake.nix @@ -121,7 +121,7 @@ # unification would otherwise hide a broken single-crate build. cargo-hack cargo-nextest - # Browser/WASM bindings (rs/moq-wasm -> @moq/wasm via `just wasm`). + # Browser/WASM bindings (rs/moq-wasm -> @moq/wasm via `just js wasm`). # wasm-bindgen-cli must match the `wasm-bindgen` crate version (the # crate is pinned to nixpkgs' CLI version); bump both together. wasm-bindgen-cli @@ -231,10 +231,9 @@ ]; # Linters / formatters used by `just check` and `just fix`, which - # guard each tool with `command -v` so they skip silently when the - # binary isn't on $PATH. CI sets MOQ_STRICT=1, which turns that skip - # into an error (see `_tools` in the root justfile), so this list and - # that one have to stay in step. + # skip a module whose tools aren't on $PATH. CI sets MOQ_STRICT=1, + # which turns that skip into an error (see the tools map in + # sh/dispatch.sh), so this list and that one have to stay in step. lintDeps = with pkgs; [ shellcheck shfmt @@ -263,8 +262,8 @@ # dependency in rs/moq-ffi/Cargo.toml. Five other places name the same # generator version and must be bumped together: the repo and revision # in release-go-ffi.yml, and the `cargo install` line in - # rs/moq-ffi/build.sh, go/ffi/README.md, go/scripts/check.sh, and - # go/scripts/stage.sh. + # rs/moq-ffi/build.sh, go/ffi/README.md, sh/go/check.sh, and + # sh/go/stage.sh. # # This points at a fork rather than NordSecurity because upstream has no # uniffi 0.32 generator: the metadata encoding changed in 0.32 even diff --git a/go/.gitignore b/go/.gitignore index 9d1c2961a27..4e2b41f06ea 100644 --- a/go/.gitignore +++ b/go/.gitignore @@ -1,4 +1,4 @@ -# Generated by scripts/{check,package-ffi}.sh; the main repo stays binary-free. +# Generated by sh/go/{check,package-ffi}.sh; the main repo stays binary-free. ffi/moq/moq.go ffi/moq/moq.h ffi/moq/lib/ diff --git a/go/ffi/README.md b/go/ffi/README.md index 23aae8b0253..c45b1e70264 100644 --- a/go/ffi/README.md +++ b/go/ffi/README.md @@ -20,9 +20,9 @@ The published module ships prebuilt `libmoq_ffi.a` for `linux/amd64`, `linux/arm ## Local development -`go/scripts/stage.sh` builds `moq-ffi` for the host, runs `uniffi-bindgen-go` to regenerate `moq.go`, and stages this module plus the wrapper into `dist/` with the wrapper wired to the local ffi by a `replace`. It prints the two staged module directories, so anything that needs to build Go against this checkout (`go/scripts/check.sh`, `test/interop`) shares one staging path. +`sh/go/stage.sh` builds `moq-ffi` for the host, runs `uniffi-bindgen-go` to regenerate `moq.go`, and stages this module plus the wrapper into `dist/` with the wrapper wired to the local ffi by a `replace`. It prints the two staged module directories, so anything that needs to build Go against this checkout (`sh/go/check.sh`, `test/interop`) shares one staging path. -`go/scripts/check.sh` stages that way and then runs `go build`/`go vet`/`go test`. It also runs `publish-wrapper.test.sh`, which drives the wrapper publisher against a scratch bare repo (no cargo, no network). Run via `just go check`. Skips cleanly without `cargo`, `go`, or `uniffi-bindgen-go`. +`sh/go/check.sh` stages that way and then runs `go build`/`go vet`/`go test`. Run via `just go check`. Skips cleanly without `cargo`, `go`, or `uniffi-bindgen-go`. The dev shell provides both `go` and `uniffi-bindgen-go`, so `nix develop --command just go check` needs no setup. Without Nix, install `uniffi-bindgen-go` once: @@ -49,8 +49,8 @@ go/ffi/ lib/_/libmoq_ffi.a (gitignored, staged at release time) ``` -Compiled binaries never live in the source tree. `scripts/check.sh` stages into the gitignored `dist/` working dir under the repo root; CI does the equivalent assembly into the [moq-dev/moq-go-ffi](https://github.com/moq-dev/moq-go-ffi) mirror. +Compiled binaries never live in the source tree. `sh/go/check.sh` stages into the gitignored `dist/` working dir under the repo root; CI does the equivalent assembly into the [moq-dev/moq-go-ffi](https://github.com/moq-dev/moq-go-ffi) mirror. ## Release -The `release-go-ffi.yml` workflow fires on every `moq-ffi-v*` tag, builds per-target static libraries, runs `uniffi-bindgen-go`, calls `go/scripts/package-ffi.sh` to assemble the module, and `go/scripts/publish-ffi.sh` to push the result to `moq-dev/moq-go-ffi` with a bare-semver tag (e.g. `v0.2.18`) lockstep with the crate. Go's module proxy picks up the new tag automatically. +The `release-go-ffi.yml` workflow fires on every `moq-ffi-v*` tag, builds per-target static libraries, runs `uniffi-bindgen-go`, calls `sh/go/package-ffi.sh` to assemble the module, and `sh/go/publish-ffi.sh` to push the result to `moq-dev/moq-go-ffi` with a bare-semver tag (e.g. `v0.2.18`) lockstep with the crate. Go's module proxy picks up the new tag automatically. diff --git a/go/ffi/moq/cgo.go b/go/ffi/moq/cgo.go index 8be6f790a68..50dc423b50e 100644 --- a/go/ffi/moq/cgo.go +++ b/go/ffi/moq/cgo.go @@ -3,12 +3,12 @@ // moq.dev/moq wrapper, which builds on this package. // // The exported API is generated from rs/moq-ffi via uniffi-bindgen-go and -// dropped in alongside this file (moq.go) by scripts/package-ffi.sh; the in-tree -// source therefore does not build on its own. Run scripts/check.sh to stage a +// dropped in alongside this file (moq.go) by sh/go/package-ffi.sh; the in-tree +// source therefore does not build on its own. Run sh/go/check.sh to stage a // complete copy into dist/ and exercise it. // // The per-platform static archive is loaded from moq/lib/_/ -// inside the staged module, populated by scripts/package-ffi.sh from the release +// inside the staged module, populated by sh/go/package-ffi.sh from the release // build matrix. package moq diff --git a/go/justfile b/go/justfile index d842e27d103..ad26a9875d9 100644 --- a/go/justfile +++ b/go/justfile @@ -10,36 +10,18 @@ default: # Stages both the ffi and wrapper modules into dist/ (wiring the wrapper to the # local ffi via replace). Skips cleanly if cargo, go, or uniffi-bindgen-go is -# missing. -# -# This runs the tests too, so there is no separate `just go test`. Takes an -# optional newline-separated list of changed files and skips when none are -# Go-scoped; `just go check` (no FILES) always runs. +# missing. This runs the tests too, so there is no separate `just go test`. # Build moq-ffi, run uniffi-bindgen-go, then `go build`/`go vet`/`go test`. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '^(go/|rs/moq-ffi/)' <<< "$FILES"; then - echo "go: no Go changes; skipping." - exit 0 - fi - bash scripts/check.sh +check: + bash ../sh/go/check.sh # Stage the in-tree go/ffi source + per-target moq-ffi libs + generated # bindings into the moq.dev/moq-ffi module ready for publish. package-ffi *args: - bash scripts/package-ffi.sh {{ args }} + bash ../sh/go/package-ffi.sh {{ args }} # Stage the in-tree go/wrapper source into the moq.dev/moq module, rewriting its # moq.dev/moq-ffi require to the target ffi version, ready for publish. package-wrapper *args: - bash scripts/package-wrapper.sh {{ args }} - -# The paths are the gitignored ones in go/.gitignore. `just go check` stages -# into the workspace dist/ instead, which `just js clean` sweeps, so these only -# show up when uniffi-bindgen-go or `go mod` is pointed at the tree directly. - -# Remove the generated bindings, native libs, and vendoring from go/. -clean: - rm -rf ffi/moq/moq.go ffi/moq/moq.h ffi/moq/lib ffi/go.sum ffi/vendor wrapper/go.sum wrapper/vendor + bash ../sh/go/package-wrapper.sh {{ args }} diff --git a/go/wrapper/README.md b/go/wrapper/README.md index 52ed3ce69df..9a69365d694 100644 --- a/go/wrapper/README.md +++ b/go/wrapper/README.md @@ -117,4 +117,4 @@ The committed `go.mod` carries a `require moq.dev/moq-ffi v0.0.0` **placeholder* ## Local development -Run `just go check`: it builds `moq-ffi` for the host, regenerates the bindings, stages both modules into `dist/` with a `replace` wiring the wrapper to the local ffi, and runs `go build`/`go vet`/`go test`. It also runs `scripts/publish-wrapper.test.sh`, which exercises the publisher's release/no-op/recovery paths against a scratch bare repo standing in for the mirror. See [../ffi/README.md](../ffi/README.md) for the `uniffi-bindgen-go` install. +Run `just go check`: it builds `moq-ffi` for the host, regenerates the bindings, stages both modules into `dist/` with a `replace` wiring the wrapper to the local ffi, and runs `go build`/`go vet`/`go test`. See [../ffi/README.md](../ffi/README.md) for the `uniffi-bindgen-go` install. diff --git a/go/wrapper/go.mod b/go/wrapper/go.mod index e306de284d7..eba975c62c7 100644 --- a/go/wrapper/go.mod +++ b/go/wrapper/go.mod @@ -3,7 +3,7 @@ module moq.dev/moq go 1.23 // The require below is a placeholder. The real version is rewritten at release -// time by go/scripts/package-wrapper.sh to the latest published moq-ffi, and +// time by sh/go/package-wrapper.sh to the latest published moq-ffi, and // `just go check` injects a local `replace` to the freshly-generated bindings. // Do not "fix" this by hand or add a replace directive to the committed file. require moq.dev/moq-ffi v0.0.0 diff --git a/js/common/workers.test.ts b/js/common/workers.test.ts deleted file mode 100644 index 5b11c5e1fba..00000000000 --- a/js/common/workers.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { expect, test } from "bun:test"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; - -test("worker checks select changed projects and propagate build failures", () => { - const temp = mkdtempSync(join(tmpdir(), "moq-workers-")); - const log = join(temp, "calls"); - writeFileSync( - join(temp, "bun"), - '#!/bin/sh\nprintf "%s %s\\n" "$PWD" "$*" >> "$WORKER_TEST_LOG"\nif [ "$*" = "run deploy --dry-run" ]; then exit "$WORKER_TEST_EXIT"; fi\n', - { mode: 0o755 }, - ); - try { - for (const project of ["infra/apt", "infra/rpm", "demo/pub"]) { - writeFileSync(log, ""); - const result = Bun.spawnSync(["just", "js", "check", `${project}/bun.lock`], { - cwd: resolve(import.meta.dir, "../.."), - env: { - ...process.env, - PATH: `${temp}:${process.env.PATH}`, - WORKER_TEST_LOG: log, - WORKER_TEST_EXIT: "0", - }, - }); - expect(result.exitCode).toBe(0); - expect(readFileSync(log, "utf8").trim().split("\n")).toEqual([ - `${resolve(import.meta.dir, "../..", project)} install --frozen-lockfile`, - `${resolve(import.meta.dir, "../..", project)} run deploy --dry-run`, - ]); - } - const failed = Bun.spawnSync(["just", "js", "check", "infra/apt/bun.lock"], { - cwd: resolve(import.meta.dir, "../.."), - env: { - ...process.env, - PATH: `${temp}:${process.env.PATH}`, - WORKER_TEST_LOG: log, - WORKER_TEST_EXIT: "1", - }, - }); - expect(failed.exitCode).not.toBe(0); - } finally { - rmSync(temp, { recursive: true, force: true }); - } -}); diff --git a/js/justfile b/js/justfile index bcad24af599..3cc52a9e22a 100644 --- a/js/justfile +++ b/js/justfile @@ -6,108 +6,48 @@ set working-directory := '.' -# Paths a JS check depends on: every bun workspace (they are not all under js/, -# see the root package.json) plus the config shared by all of them. `check`, -# `fix`, and `ci` all skip themselves when a diff misses this. -scope := '^(js/|doc/|drafts/|demo/(boy|web)/|test/interop/clients/js|test/wasm/|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' - # Run all checks by default. default: just check -install: - bun install - -# Takes an optional newline-separated list of changed files and skips if none -# match the JS scope; `just js check` (no FILES) always runs. -# -# FILES is an exported parameter, not `{{ FILES }}`: just interpolates the -# latter into the recipe source, where a filename like `$(...)` would run as a -# command. Git-derived paths are untrusted input, so they stay data. - # `build` runs here rather than as its own gate: the per-package `check` script # is `tsc --noEmit`, which never runs declaration emit, so the errors that only # surface when emitting a `.d.ts` (TS4023 and friends) pass `check` and then # break a JSR/npm publish. It costs about 20s for every package. - +# # `common/deps.ts` runs first and cheaply: bundlers resolve an undeclared # workspace import through hoisting, so nothing else in `check` notices one. +# doc/ is in scope, so the binding pages' stats fields are checked here too. # Lint, type-check, biome-format, and build all packages. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - # These Workers have independent lockfiles outside the Bun workspace. - # Validate the deploy bundle without publishing or requiring credentials. - for project in infra/apt infra/rpm demo/pub; do - if [[ -z "$FILES" ]] || grep -qE "^($project/|js/justfile$)" <<< "$FILES"; then - ( - cd "../$project" - bun install --frozen-lockfile - bun run deploy --dry-run - ) - fi - done - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "js: no JS changes; skipping." - exit 0 - fi - # doc/ is in this scope, so the binding pages are checked here. - python3 ../rs/scripts/stats-docs.py +check: + python3 ../sh/rs/stats-docs.py bun install --frozen-lockfile bun common/deps.ts - if tty -s; then - bun run --filter='*' --elide-lines=0 check - else - bun run --filter='*' check - fi + bun run --filter='*' --elide-lines=0 check bun biome check just build -# Auto-fix lint and formatting issues. Skips on the same terms as `check`. -fix $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "js: no JS changes; skipping." - exit 0 - fi +# Validate the Worker deploy bundles outside the Bun workspace. +workers: + ../sh/js/workers.sh + +# Auto-fix lint and formatting issues. +fix: bun install bun biome check --write -# Run all unit tests. Skips on the same terms as `check`. -test $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "js: no JS changes; skipping tests." - exit 0 - fi +# Run all unit tests. +test: bun install --frozen-lockfile - bun test common/deps.test.ts common/workers.test.ts - if tty -s; then - bun run --filter='*' --elide-lines=0 test - else - bun run --filter='*' test - fi + bun test common/deps.test.ts + bun run --filter='*' --elide-lines=0 test # Build all packages. build: bun run --filter='*' build -# Remove node_modules and build output. The bun workspace spans the repo - -# (deps hoist to the root, not into js/), so sweep from the workspace root. -clean: - #!/usr/bin/env bash - set -euo pipefail - cd .. - find . -name .claude -prune -o \ - -type d \( -name node_modules -o -name dist -o -name out -o -name pkg \) \ - -prune -exec rm -rf {} + - find . -name .claude -prune -o -type f -name '*.tsbuildinfo' -exec rm -f {} + - -# Pull in new versions and report what's outdated. -update: - bun update - bun outdated +# Build browser/WASM bindings into @moq/wasm using the pinned wasm-bindgen toolchain. +wasm: + cargo build --locked -p moq-wasm --target wasm32-unknown-unknown --profile wasm-release + wasm-bindgen --target web --out-name moq --out-dir wasm/dist "${CARGO_TARGET_DIR:-../target}/wasm32-unknown-unknown/wasm-release/moq_wasm.wasm" diff --git a/js/wasm/.gitignore b/js/wasm/.gitignore index d3d540b01fb..b7322370aeb 100644 --- a/js/wasm/.gitignore +++ b/js/wasm/.gitignore @@ -1,3 +1,3 @@ -# Generated by `just wasm` (wasm-bindgen). The .wasm binary and JS glue are +# Generated by `just js wasm` (wasm-bindgen). The .wasm binary and JS glue are # build artifacts, not source: build them, never commit them. /dist/ diff --git a/js/wasm/README.md b/js/wasm/README.md index b836032e6c0..bd554effd73 100644 --- a/js/wasm/README.md +++ b/js/wasm/README.md @@ -29,7 +29,7 @@ The classes (`Moq.Session`, `Moq.Broadcast`, `Moq.Track`, `Moq.Group`) drop the `dist/` is generated, not committed. Build it from the repo root: ```bash -just wasm +just js wasm ``` That compiles `rs/moq-wasm` for `wasm32-unknown-unknown`, runs `wasm-bindgen` diff --git a/justfile b/justfile index a48833cafee..4da8f933acf 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,8 @@ #!/usr/bin/env just --justfile # Using Just: https://github.com/casey/just?tab=readme-ov-file#installation +# +# Every recipe is a menu entry: one line that runs a tool, or a script under +# sh/. Logic belongs in the script. set unstable @@ -29,10 +32,6 @@ mod relay 'demo/relay' mod sub 'demo/sub' mod web 'demo/web' -# Byte budget for a changed-file list, sized so it survives being passed as a -# single argv/env string on every hop of the dispatch. See `_changed`. -changed_max := '65536' - # Run the demo by default. default: just demo @@ -43,826 +42,39 @@ dev: # Benchmark the current tree, or compare it with a commit: `just bench origin/main`. bench $BASE="": - #!/usr/bin/env bash - exec just --justfile bench/justfile compare "$BASE" + bench/run.sh "$BASE" # Compare one multi-threaded Tokio runtime with the same number of independent -# Tokio/epoll and io_uring workers. Defaults to every logical CPU. -bench-runtime $ROUNDS="3" $WORKERS="": - #!/usr/bin/env bash - exec just --justfile bench/justfile runtime "$ROUNDS" "$WORKERS" - -# A linked worktree's Git metadata does not live under its own root: the -# per-worktree directory is `--git-dir` and everything shared (objects, remote -# refs, the branch namespace) is under `--git-common-dir`, which for an agent -# checkout is inside the main repository. Write access to the source tree -# therefore says nothing about whether this checkout can fetch, branch, or -# rebase; the answer is a property of those two directories, and finding out by -# running `git fetch` and reading the error is the slow way. -# -# `check` reports; `setup` fetches, points the branch at its base, and records -# the SHA it fetched under the per-worktree Git directory, where `check` reads it -# back to say how stale the recorded base has become. Neither ever resets, -# rebases, cleans, or checks anything out: a dirty tree is someone's work in -# progress, and adopting a checkout must not be able to destroy it. -# -# BASE follows the same rule as the rest of the repo (see `_base`): `main` -# unless the branch's upstream says otherwise. - -# Report a worktree's base, Git metadata access, and state; `setup` also fetches. -worktree ACTION="check" $BASE="": - #!/usr/bin/env bash - set -euo pipefail - - case "{{ ACTION }}" in - check | setup) ;; - *) - echo "usage: just worktree [check|setup] [BASE]" >&2 - exit 2 - ;; - esac - - root=$(git rev-parse --show-toplevel) - git_dir=$(cd "$(git rev-parse --git-dir)" && pwd) - common_dir=$(cd "$(git rev-parse --git-common-dir)" && pwd) - branch=$(git branch --show-current || true) - - # A written probe rather than `[[ -w ]]`: the directory can be readable and - # nominally writable while the sandbox, a read-only mount, or an ACL refuses - # the create, and it is the create that fetch and branch creation need. - access() { - local dir="$1" probe - [[ -d "$dir" ]] || { echo missing; return; } - [[ -r "$dir" ]] || { echo denied; return; } - probe="$dir/.moq-access-probe.$$" - if (umask 077 && : > "$probe") 2>/dev/null; then - rm -f "$probe" - echo write - else - echo read-only - fi - } - - # A directory git will create on demand is only as writable as its nearest - # existing parent, so probe upward instead of reporting it missing. - access_or_parent() { - local dir="$1" parent - while [[ ! -d "$dir" ]]; do - parent=$(dirname "$dir") - [[ "$parent" != "$dir" ]] || { echo missing; return; } - dir="$parent" - done - access "$dir" - } - - base=$(just _base "$BASE") - stamp="$git_dir/moq-base" - # Whichever remote provides the base, not always origin. A base whose first - # segment names no remote (a local branch, a tag) leaves origin. - remote="${base%%/*}" - git remote | grep -qx "$remote" || remote=origin - - objects=$(access "$common_dir/objects") - heads=$(access "$common_dir/refs/heads") - worktree_meta=$(access "$git_dir") - worktree=$(access "$root") - # Tracking is `branch..remote`/`.merge` in the repository config, which - # lives in the common directory alongside the `config.lock` the write needs, - # not in the branch's ref. Probing refs/heads for it would refuse on a - # writable config and, worse, proceed on a read-only one. - config=$(access "$common_dir") - # A fetch writes three places, not one: objects, the remote-tracking refs it - # updates, and FETCH_HEAD in the per-worktree directory. The refs are the - # narrow one: `refs/remotes/` is where `.lock` is created, so - # probing `refs/remotes` stops a directory short and passes a split that git - # then fails on. - tracking=$(access_or_parent "$common_dir/refs/remotes/$remote") - - echo "worktree: $root" - echo "branch: ${branch:-(detached)} $(git rev-parse --short HEAD)" - echo "git-dir: $git_dir ($worktree_meta)" - echo "common-dir: $common_dir ($config)" - echo " fetch needs $common_dir/objects: $objects" - echo " $common_dir/refs/remotes/$remote: $tracking" - echo " $git_dir (FETCH_HEAD): $worktree_meta" - echo " branch needs $common_dir/refs/heads: $heads" - echo " upstream needs $common_dir/config: $config" - echo " rebase needs $git_dir: $worktree_meta" - echo " $root: $worktree" - echo " $common_dir/refs/heads: $heads" +# Tokio/epoll and io_uring workers; WORKERS defaults to every logical CPU. - dirty=$(git status --porcelain | wc -l | tr -d ' ') - echo "dirty: $dirty tracked/untracked path(s)" - - if [[ "{{ ACTION }}" == setup ]]; then - blocked="" - [[ "$objects" == write ]] || blocked="$blocked $common_dir/objects ($objects)" - [[ "$tracking" == write ]] || blocked="$blocked $common_dir/refs/remotes/$remote ($tracking)" - [[ "$worktree_meta" == write ]] || blocked="$blocked $git_dir ($worktree_meta)" - if [[ -n "$blocked" ]]; then - echo "error: cannot fetch;$blocked" >&2 - echo " grant write access to the main repository's Git directory, not just this worktree" >&2 - exit 1 - fi - # The remote resolved above, not always origin: recording a stamp against a - # ref nobody refreshed is worse than recording none, and `just check` would - # scope the branch against a base that has since moved. - git fetch --quiet "$remote" - # Repointing an upstream someone chose would silently change what `just - # check` scopes against, so only three cases write it: no upstream, an - # upstream `_base` discards anyway (the branch's own remote copy, which - # `git push -u` leaves behind and which says nothing about what the branch - # merges into), and a base the caller named on the command line. - upstream=$(git rev-parse --abbrev-ref '@{upstream}' 2> /dev/null || true) - # Two things stop the write, and they fail identically: a detached HEAD has - # no branch to hang an upstream on, and the config it lands in may be - # read-only. Skipping either silently would report a setup that recorded - # the caller's base while `just check` still scoped against origin/main. - blocker="" - if [[ -z "$branch" ]]; then - blocker="HEAD is detached" - elif [[ "$config" != write ]]; then - blocker="$common_dir/config is $config" - fi - if [[ -z "$upstream" ]] || [[ "$upstream" == */"$branch" ]] || [[ -n "$BASE" ]]; then - if [[ -z "$blocker" ]]; then - git branch --set-upstream-to "$base" "$branch" - elif [[ "$base" == origin/main ]]; then - # Nothing is lost: with no upstream `_base` falls back to - # origin/main, which is what this would have written. - echo "warning: cannot record the upstream; $blocker" >&2 - else - # The upstream is the only place this choice survives, so a setup - # that could not write it did not do what it was asked. - echo "error: cannot set the upstream to $base; $blocker" >&2 - echo " the branch would keep scoping against origin/main" >&2 - exit 1 - fi - fi - # Resolved, written, then renamed into place. A redirect straight into the - # stamp truncates it before git runs, so a failure there would leave an - # empty file that reads back as a recorded base that never existed. Each - # setup needs its own temporary file so concurrent runs cannot rename or - # overwrite each other's in-progress stamp. - if [[ "$worktree_meta" == write ]]; then - stamp_tmp=$(mktemp "$git_dir/.moq-base.XXXXXXXX") - if ! git rev-parse "$base" > "$stamp_tmp"; then - rm -f "$stamp_tmp" - exit 1 - fi - mv "$stamp_tmp" "$stamp" - fi - fi - - if ! git rev-parse --verify --quiet "$base^{commit}" > /dev/null; then - echo "base: $base (NOT FETCHED; run 'just worktree setup')" - exit 0 - fi - - head=$(git rev-parse "$base") - echo "base: $base $(git rev-parse --short "$base")" - echo "upstream: $(git rev-parse --abbrev-ref '@{upstream}' 2> /dev/null || echo '(unset)')" - echo "behind: $(git rev-list --count "HEAD..$base") commit(s)" - - recorded="" - [[ -f "$stamp" ]] && recorded=$(cat "$stamp") - - # A stamp that no longer names a commit is worse than none: reporting it would - # abort here on the `rev-parse --short` rather than say what to do about it. - # An interrupted setup, or a base garbage-collected out of the repository. - if [[ -z "$recorded" ]]; then - echo "recorded: (none; run 'just worktree setup')" - elif ! git rev-parse --verify --quiet "$recorded^{commit}" > /dev/null; then - echo "recorded: $recorded (UNKNOWN COMMIT; run 'just worktree setup')" - elif [[ "$recorded" == "$head" ]]; then - echo "recorded: $(git rev-parse --short "$recorded") (current)" - else - echo "recorded: $(git rev-parse --short "$recorded") (STALE; $base has moved since setup)" - fi +# Compare a shared Tokio runtime with independent workers: `just bench-runtime 5 16`. +bench-runtime $ROUNDS="3" $WORKERS="": + MOQ_BENCH_RUNTIME_ROUNDS="$ROUNDS" MOQ_BENCH_RUNTIME_WORKERS="$WORKERS" bench/run.sh --runtime # Install repo-wide tooling. Per-language deps install on first check. install: bun install - cargo install --locked cargo-shear cargo-sort cargo-upgrades cargo-edit cargo-semver-checks release-plz - -# Resolve BASE: arg > $GITHUB_BASE_REF > upstream > origin/main. A branch's -# upstream is the branch it merges into, which is the base a `dev`-targeted -# branch needs. `git push -u` repoints upstream at the branch's own remote copy, -# which would diff HEAD against itself, so ignore that case (see CLAUDE.md). -# GITHUB_BASE_REF outranks the upstream because a PR checkout has no upstream -# configured, and the branch being merged into is exactly the base GitHub is -# asking about. -# -# Shared by `_changed` and `worktree`, so a checkout's scope and its reported -# base can never disagree. - -# Print the ref this branch is based on. -[private] -_base $BASE="": - #!/usr/bin/env bash - set -euo pipefail - - base="$BASE" - if [[ -z "$base" && -n "${GITHUB_BASE_REF:-}" ]]; then - base="origin/${GITHUB_BASE_REF}" - fi - if [[ -z "$base" ]]; then - base=$(git rev-parse --abbrev-ref '@{upstream}' 2>/dev/null || true) - if [[ -z "$base" || "$base" == */"$(git branch --show-current)" ]]; then - base="origin/main" - fi - fi - printf '%s\n' "$base" - -# Reports the base it picked on stderr, so a surprising scope is traceable. -# -# LIMIT is the byte budget for the printed list, and exists as a parameter so -# `_changed-test` can force the oversized path without a synthetic 30k-file diff. - -# Print the files this branch changed relative to BASE, one per line, or `ALL`. -[private] -_changed $BASE $LIMIT=changed_max: - #!/usr/bin/env bash - set -euo pipefail - - base=$(just _base "$BASE") - - merge_base=$(git merge-base "$base" HEAD) || { - echo "error: cannot resolve merge-base against $base (is full history fetched?)" >&2 - exit 1 - } - echo "base: $base" >&2 - - # Untracked files count too: a brand new crate or module is the whole change. - files=$({ - git diff --name-only "$merge_base" - git ls-files --others --exclude-standard - } | sort -u) - - # Assigned rather than tested inline, so a failing `_changed-cap` aborts here - # under `set -e`. Inside `[[ ]]` its exit status is discarded, and the caller - # would then scope to a list that never got budgeted -- straight back into the - # E2BIG this exists to prevent. - cap=$(printf '%s' "$files" | just _changed-cap "$LIMIT") - if [[ "$cap" == ALL ]]; then - echo ALL - exit 0 - fi - - # Guarded because a bare printf of an empty list prints a newline, and the - # callers test the result for emptiness to decide whether anything changed. - if [[ -n "$files" ]]; then - printf '%s\n' "$files" - fi - -# Every hop of the dispatch takes the changed-file list as one argument, and -# just exports recipe parameters into the child's environment, so the whole list -# has to fit in a single execve string. Linux caps one string at MAX_ARG_STRLEN -# (32 pages, 131072 bytes) however large ARG_MAX is, so past that each hop dies -# with E2BIG, which just reports as exit code 126 and no mention of the diff. A -# diff that large selects most of the workspace anyway, so the callers widen to -# the unscoped suite, which passes no list at all. -# -# Split out from `_changed` so `_changed-test` can drive the decision with a -# synthetic list, rather than needing the working tree to hold a diff of a -# particular size. -# -# The list arrives on stdin, which is both the only channel that can carry an -# oversized one and the only way to measure it honestly: `${#var}` counts -# CHARACTERS under a UTF-8 locale while execve counts BYTES, so a path set of -# 3-byte characters would read as a third of its real size and sail past a -# budget it actually blows. - -# Print `ALL` when the changed-file list on stdin is too long for one argument. -[private] -_changed-cap $LIMIT=changed_max: - #!/usr/bin/env bash - set -euo pipefail - - [[ "$LIMIT" =~ ^[0-9]+$ ]] || { - echo "changed: not a byte count: $LIMIT" >&2 - exit 2 - } - - bytes=$(wc -c | tr -d '[:space:]') - if ((bytes > LIMIT)); then - echo "changed: $bytes bytes of paths exceeds the $LIMIT budget; selecting everything." >&2 - echo ALL - fi - -# Guards the thing that fails LOUDLY but unhelpfully: past the budget every -# `just` hop dies with "Argument list too long" and exit 126, naming neither the -# diff nor the recipe that could not receive it. Both halves matter -- a budget -# that never trips scopes nothing, and one above what execve accepts still dies. - -# Check that an oversized diff widens to `ALL`, and that the budget fits in argv. -[private] -_changed-test $LIMIT=changed_max: - #!/usr/bin/env bash - set -euo pipefail - - fail() { echo "changed: _changed-test: $1" >&2; exit 1; } - - # Synthetic sizes rather than whatever the working tree happens to hold: a - # clean checkout has no diff at all, and `check --all` runs there (cache.yml - # warms the cache from `main`), so a test keyed on the real list would take - # down the one job allowed to write the shared Rust cache. - [[ "$(printf 'aaaa' | just _changed-cap 3)" == ALL ]] || fail "over budget must print ALL" - [[ -z "$(printf 'aaa' | just _changed-cap 3)" ]] || fail "at budget must print nothing" - [[ -z "$(printf '' | just _changed-cap 3)" ]] || fail "an empty diff must print nothing" - - # The smallest nonempty list there is, against the only budget below it. - [[ "$(printf 'x' | just _changed-cap 0)" == ALL ]] || fail "a 1-byte list must exceed a 0 budget" - - # execve counts bytes, so the budget has to as well. Spelled as raw bytes - # rather than as characters: this is one CJK character, 3 bytes wide, which - # `${#var}` would count as 1 under a UTF-8 locale and let past a 2-byte - # budget it actually blows. - [[ "$(printf '\xe6\x97\xa5' | just _changed-cap 2)" == ALL ]] \ - || fail "the budget must count bytes, not characters" - - # A byte count is the whole input, so anything else is a caller bug, not a - # reason to silently scope to nothing. - ! printf '' | just _changed-cap not-a-number 2> /dev/null || fail "a bad budget must be rejected" - - # ...and that rejection has to reach the caller. A swallowed one would hand - # back an unbudgeted list, which is the failure this whole recipe prevents. - ! just _changed "" not-a-number 2> /dev/null || fail "a rejected budget must fail _changed" - - # Linux caps a single argv/env string at MAX_ARG_STRLEN (32 pages), whatever - # ARG_MAX says, and the list travels as one string. Asserted rather than - # probed because this repo's CI is the Linux host and a dev box may be laxer. - ((LIMIT <= 131072)) || fail "budget $LIMIT exceeds Linux MAX_ARG_STRLEN" - - # The budget still has to survive the hop it was sized for, which the checks - # above cannot show: they never pass a list that big to anything. - payload=$(head -c "$LIMIT" /dev/zero | tr '\0' x) - [[ "$(just _echo "$payload" | wc -c)" -eq $((LIMIT + 1)) ]] \ - || fail "a $LIMIT-byte list does not survive an argv hop" - - # End to end, but only when there is a diff to be oversized: see above. The - # budget is zero rather than one because the real list is whatever the - # checkout holds, and a single one-character root path is a one-byte list - # that a one-byte budget does not exceed. Zero is below every nonempty list. - if [[ -n "$(just _changed "" 100000000)" ]]; then - [[ "$(just _changed "" 0)" == ALL ]] || fail "a diff over budget must print ALL" - fi - - echo "changed: budget ok" - -# Print an argument back, to measure what survives a `just` invocation. -[private] -_echo $VALUE: - @printf '%s\n' "$VALUE" - -# Tools every scope guards with `command -v`, so an incomplete local toolchain -# checks less instead of failing. That trade is wrong in CI, where a skip is -# indistinguishable from a pass, so CI exports MOQ_STRICT=1 and this turns the -# required set into a precondition. Checked up front, and as one list, so a -# missing tool is reported before a long compile rather than after it. -# -# Required per scope, mirroring what `check` actually dispatches for a given -# diff: demanding gradle on a docs-only PR would fail a run that was never going -# to invoke it. Takes the same file list as the dispatch, or `ALL` to require -# everything (`check --all`). -# -# One deliberate absence: swift exists only on macOS, and `swift check` skips -# off-macOS by design; swift.yml is its real gate. - -# Fail when a tool the diff's scopes need is missing. No-op unless MOQ_STRICT. -[private] -_tools $FILES="": - #!/usr/bin/env bash - set -euo pipefail - [[ -n "${MOQ_STRICT:-}" ]] || exit 0 - - scoped() { [[ "$FILES" == ALL ]] || grep -qE "$1" <<< "$FILES"; } - - # `_check-common` runs on every invocation, so its tools are unconditional. - tools=(actionlint bun jq nix nixfmt shellcheck shfmt taplo python3 nfpm dpkg-deb envsubst rpm) - scoped '^(drafts/|doc/\.vitepress/drafts\.ts$)' && tools+=(kramdown-rfc xml2rfc) - scoped '^(bench/|quest/|rs/|Cargo\.(toml|lock)$|rust-toolchain\.toml$)' && tools+=(cargo envsubst) - scoped '^(py/|pyproject\.toml$|uv\.lock$|rs/moq-ffi/)' && tools+=(uv) - scoped '^(kt/|rs/moq-ffi/)' && tools+=(gradle java) - # cargo because `go check` builds moq-ffi for the host, and skips on a - # missing cargo the same way it skips on a missing go. rsync because the - # publish scripts stage the mirror tree with it, so the publisher test skips - # without it, and a skip that keeps `just check` green is what MOQ_STRICT is - # here to prevent. - scoped '^(go/|rs/moq-ffi/)' && tools+=(go uniffi-bindgen-go cargo rsync) - scoped '^(dart/|rs/moq-ffi/)' && tools+=(cargo dart uniffi_bindgen_dart) - # Two obs recipes with two dispatch scopes, so two lines: over-requiring - # would fail a diff that never runs the recipe. `just obs compile` needs - # cargo to regenerate moq.h and pkg-config to locate Qt6 and ffmpeg. Every - # platform: the plugin type-checks against headers, and the dev shell ships - # those even on Darwin, where obs-studio can't build. - scoped '^(cpp/obs/|rs/libmoq/|flake\.nix$)' && tools+=(pkg-config cargo) - # `just obs check` lints with clang-format and gersemi, validates the CMake - # release configuration, and compares the three OBS pins, one of which moves - # on a flake.lock bump alone. - scoped '^(cpp/obs/|flake\.(nix|lock)$)' && tools+=(clang-format gersemi cmake) - - # Scopes overlap (rs/moq-ffi/ is in five of them), so the same tool can land - # in the list twice and be reported missing twice. Splitting on whitespace is - # safe: every entry is a bare command name. - tools=($(printf '%s\n' "${tools[@]}" | sort -u)) - - missing=() - for tool in "${tools[@]}"; do - command -v "$tool" >/dev/null 2>&1 || missing+=("$tool") - done - if ((${#missing[@]})); then - echo "error: MOQ_STRICT is set but these tools are missing: ${missing[*]}" >&2 - echo " run inside 'nix develop', or unset MOQ_STRICT to skip what isn't installed" >&2 - exit 1 - fi + cargo install --locked cargo-shear cargo-sort cargo-semver-checks release-plz # Lints and compiles only the packages the branch changed plus everything -# depending on them, so several worktrees can build at once. This is also what -# CI runs (with MOQ_STRICT=1), so there is no second, drifting definition of -# "checked". Tests are the sibling `just test`; `check --all` is the unscoped suite. +# depending on them; `check --all` is the unscoped suite. CI runs the same +# thing with MOQ_STRICT=1. BASE defaults to the branch's upstream. # Lint and compile what the branch changed since BASE, plus its dependents. -check $BASE="" *args: - #!/usr/bin/env bash - set -euo pipefail - - if [[ "$BASE" == --all ]]; then - files=ALL - else - files=$(just _changed "$BASE") - fi - - # `_changed` says ALL when the list outgrew what argv can carry. The unscoped - # suite is the path that passes no list at all, so it is the one that works. - # The dispatch below lives in these two files, and neither matches any - # language scope, so a PR that rewrites how CI dispatches would otherwise - # validate none of it. Widen to the unscoped suite instead. - if [[ "$files" != ALL ]] && grep -qE '^(justfile|test/justfile)$' <<< "$files"; then - echo "check: root orchestration changed; checking everything." >&2 - files=ALL - fi - - just _tools "$files" - - if [[ "$files" == ALL ]]; then - just js check - just drafts check - just rs check --workspace --exclude moq-net-fuzz {{ args }} - just rs tokio-features - just --justfile bench/justfile check - cargo run --quiet --locked --package quest -- check - # Not covered by the line above: moq-wasm only exists on the wasm32 target. - just rs wasm - just py check - just kt check - just swift check - just go check - just dart check - just obs check - just obs compile - just _flake - elif [[ -n "$files" ]]; then - just js check "$files" - just rs check-changed "$files" - if echo "$files" | grep -q '^bench/'; then - just --justfile bench/justfile check - fi - # Draft sources render into the doc site and carry their own kramdown-rfc - # plus vector checks; a drafts-only diff would otherwise skip both. - if echo "$files" | grep -qE '^(drafts/|doc/\.vitepress/drafts\.ts$)'; then - just drafts check - fi - # Quest documents form one graph, so validate the whole living tree when - # either a quest or its validator changes. - if echo "$files" | grep -qE '^(quest/|rs/quest/)'; then - cargo run --quiet --locked --package quest -- check - fi - just py check "$files" - just kt check "$files" - just swift check "$files" - just go check "$files" - just dart check "$files" - # Type-checking the plugin and its unit tests needs only headers, so it - # runs here rather than waiting for obs.yml to link them on Linux. libmoq - # is in scope because the plugin calls through its generated C header, and - # the tests restate those entry points as stubs, so an ABI change breaks - # both. flake.nix because it owns the libobs headers this compiles - # against -- obs.yml links against nixpkgs' obs-studio instead, so nothing - # else would notice that package going bad. - if echo "$files" | grep -qE '^(cpp/obs/|rs/libmoq/|flake\.nix$)'; then - just obs compile - fi - # flake.nix is in scope because `just obs check` is what compares the OBS - # version pinned there against buildspec.json, and either side can move. - # flake.lock too, because the third OBS the guard compares is nixpkgs' - # obs-studio, the one `just obs ci` links: it moves on a lock bump alone, - # and that bump is the change that opens the gap. - if echo "$files" | grep -qE '^(cpp/obs/|flake\.(nix|lock)$)'; then - just obs check - fi - # Validates flake eval + dev shell build; it no longer compiles the - # workspace, so it's cheap. Gated anyway: a pure doc/JS PR can't - # affect flake eval. - if echo "$files" | grep -qE '(^rs/|^Cargo\.(toml|lock)$|^flake\.lock$|\.nix$)'; then - just _flake - fi - else - echo "check: nothing changed." - fi - - just _check-common - -# Skips when nix is absent: the flake is not a precondition for working on the -# repo, and `_tools` already makes it required under MOQ_STRICT. - -# Validate flake evaluation and the dev shell build. -[private] -_flake: - @if command -v nix >/dev/null 2>&1; then nix flake check; fi - -# Repository-wide non-compiling checks shared by scoped and unscoped `check`. -# Optional shell, workflow, TOML, Nix, and justfile lints skip if missing. -# -# `bun install` because remark-cli lives in node_modules and `just js check` is -# where it would otherwise be installed, which a Rust-only diff skips. - -# Run shell checks or formatting over tracked files that exist in the worktree. -[private] -_shell $ACTION: - #!/usr/bin/env bash - set -euo pipefail - - if ! command -v shfmt >/dev/null 2>&1; then - exit 0 - fi - if [[ "$ACTION" == "check" ]] && ! command -v shellcheck >/dev/null 2>&1; then - exit 0 - fi - - scripts_file=$(mktemp) - trap 'rm -f "$scripts_file"' EXIT - shfmt -f=0 . > "$scripts_file" - - scripts=() - while IFS= read -r -d '' file; do - if git --literal-pathspecs ls-files --error-unmatch -- "$file" >/dev/null 2>&1; then - scripts+=("$file") - fi - done < "$scripts_file" - ((${#scripts[@]})) || exit 0 - - case "$ACTION" in - check) - shfmt --diff "${scripts[@]}" - shellcheck "${scripts[@]}" - ;; - fix) - shfmt --write "${scripts[@]}" - ;; - *) - echo "invalid shell action: $ACTION" >&2 - exit 2 - ;; - esac - -# remark-cli has no `--check`. `--frail` raises the exit code on lint messages, -# and only `--output` formats, so a file that is merely misformatted passes both -# ways. Format a scratch mirror and diff that, so the check stays read-only -# while `fix` still writes in place. - -# Run Markdown lints over the worktree, either checking formatting or applying it. -[private] -_markdown $ACTION: - #!/usr/bin/env bash - set -euo pipefail - - case "$ACTION" in - check) ;; - fix) - bun remark . --quiet --output - exit 0 - ;; - *) - echo "invalid markdown action: $ACTION" >&2 - exit 2 - ;; - esac - - mirror=$(mktemp -d) - trap 'rm -rf "$mirror"' EXIT - - extension_list=$(bun -e \ - 'import extensions from "markdown-extensions"; console.log(extensions.join("\n"))') - patterns=() - while IFS= read -r extension; do - [[ -n "$extension" ]] && patterns+=("*.$extension") - done <<< "$extension_list" - ((${#patterns[@]})) || { - echo "error: remark reported no Markdown extensions" >&2 - exit 1 - } - - # Untracked files are in scope so a new doc is linted before it is staged; - # --exclude-standard keeps build output out. - files=() - while IFS= read -r -d '' file; do - [[ -f "$file" ]] || continue - files+=("$file") - mkdir -p "$mirror/$(dirname "$file")" - cp "$file" "$mirror/$file" - done < <(git ls-files -z --cached --others --exclude-standard -- "${patterns[@]}") - ((${#files[@]})) || exit 0 - - # The config rides along so every .remarkignore pattern resolves against the - # mirror root exactly as it does here, and node_modules is where the plugins - # named by .remarkrc.mjs come from. - cp .remarkrc.mjs .remarkignore "$mirror/" - ln -s "$PWD/node_modules" "$mirror/node_modules" - - status=0 - (cd "$mirror" && bun remark . --quiet --frail --output) || status=$? - - stale=() - for file in "${files[@]}"; do - cmp -s "$file" "$mirror/$file" || stale+=("$file") - done - - if ((${#stale[@]})); then - echo "error: these files are not formatted, run 'just fix':" >&2 - printf ' %s\n' "${stale[@]}" >&2 - status=1 - fi - - exit "$status" - -# Check Markdown formatting detection without touching the caller's worktree. -[private] -_markdown-test: - #!/usr/bin/env bash - set -euo pipefail - - fail() { echo "markdown: _markdown-test: $1" >&2; exit 1; } - - repo=$PWD - fixture=$(mktemp -d) - trap 'rm -rf "$fixture"' EXIT - - git -C "$fixture" init --quiet - cp .remarkrc.mjs .remarkignore "$fixture/" - ln -s "$repo/node_modules" "$fixture/node_modules" - for file in dirty.md dirty.markdown; do - printf '# dirty_heading\n' > "$fixture/$file" - git -C "$fixture" add "$file" - done - - original='# dirty_heading' - if just --justfile "$repo/justfile" --working-directory "$fixture" \ - _markdown check > "$fixture/check.log" 2>&1; then - fail "formatter-only drift must fail the check" - fi - for file in dirty.md dirty.markdown; do - [[ "$(<"$fixture/$file")" == "$original" ]] \ - || fail "the check modified $file" - grep -q "$file" "$fixture/check.log" \ - || fail "the check did not name $file" - done - - just --justfile "$repo/justfile" --working-directory "$fixture" _markdown fix - for file in dirty.md dirty.markdown; do - [[ "$(<"$fixture/$file")" == '# dirty\_heading' ]] \ - || fail "the fix did not format $file" - done - just --justfile "$repo/justfile" --working-directory "$fixture" _markdown check - - echo "markdown: check/fix regression ok" - -# Repository-wide lints, shared by scoped and unscoped `check`. -[private] -_check-common: - just _changed-test - bun install --frozen-lockfile - just _markdown-test - just _markdown check - just _shell check - @if command -v taplo >/dev/null 2>&1; then RUST_LOG=error taplo format --check; fi - @if command -v nixfmt >/dev/null 2>&1; then nixfmt --check $(find . -name '*.nix' -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); fi - @for f in $(find . -name justfile -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); do just --fmt --check --justfile "$f"; done - just gh check - -# Scoped exactly like `check`, because `clippy --fix` compiles what it fixes. -# Pass `--all` for the unscoped version. +check $BASE="": + sh/dispatch.sh check "$BASE" # Auto-fix lint and formatting for what the branch changed since BASE. fix $BASE="": - #!/usr/bin/env bash - set -euo pipefail - - if [[ "$BASE" == --all ]]; then - files=ALL - else - files=$(just _changed "$BASE") - fi - - # Mirrors `check`: too long for argv means fix everything instead. - if [[ "$files" == ALL ]]; then - just js fix - just rs fix --workspace --exclude moq-net-fuzz - just py fix - just dart fix - just obs fix - elif [[ -n "$files" ]]; then - just js fix "$files" - just rs fix-changed "$files" - just py fix "$files" - just dart fix "$files" - if echo "$files" | grep -q '^cpp/obs/'; then - just obs fix - fi - else - echo "fix: nothing changed." - fi - - just _fix-common - -# Optional tools skip if missing locally. `bun install` for the same reason as -# `_check-common`. - -# Repository-wide fixes, shared by scoped and unscoped `fix`. -[private] -_fix-common: - bun install - just _markdown fix - just _shell fix - @if command -v taplo >/dev/null 2>&1; then RUST_LOG=error taplo format; fi - @if command -v nixfmt >/dev/null 2>&1; then nixfmt $(find . -name '*.nix' -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); fi - @for f in $(find . -name justfile -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); do just --fmt --justfile "$f"; done + sh/dispatch.sh fix "$BASE" # Build the packages. build: just js build just rs build - if command -v uv &> /dev/null; then just py build; fi - if command -v wasm-bindgen &> /dev/null; then just wasm; fi - -# Build browser/WASM bindings into @moq/wasm using the pinned wasm-bindgen toolchain. -wasm: - cargo build --locked -p moq-wasm --target wasm32-unknown-unknown --profile wasm-release - wasm-bindgen --target web --out-name moq \ - --out-dir js/wasm/dist "${CARGO_TARGET_DIR:-target}/wasm32-unknown-unknown/wasm-release/moq_wasm.wasm" - -# Only this checkout by default. Agent worktrees each carry their own artifacts -# now that the shared target dir is gone, and another agent is usually building -# in one right now: `cargo clean` under a running build fails it, and there is no -# way to tell a finished worktree from a busy one from out here. `just clean all` -# is the explicit opt-in for a machine the caller knows is idle. -# -# Source is never touched either way, dirty or untracked: this deletes build -# output, not work. Nothing here reaches a machine-wide store -- no Nix garbage -# collection, no cargo/bun/uv home cache -- because those are shared with every -# other checkout and rebuilding them costs far more than the space they hold. + just py build + just js wasm # Delete this checkout's build artifacts and caches; `all` includes agent worktrees. clean SCOPE="here": - #!/usr/bin/env bash - set -euo pipefail - - case "{{ SCOPE }}" in - here | all) ;; - *) - echo "usage: just clean [here|all]" >&2 - exit 2 - ;; - esac - - just rs clean - just js clean - just py clean - just kt clean - just swift clean - just go clean - just dart clean - - # Caches not owned by any one language: nix build result, direnv, wrangler. - rm -rf result .direnv - find . -name .claude -prune -o -type d -name .wrangler -prune -exec rm -rf {} + - - # Worktrees don't nest, so this recurses exactly one level. Tolerate stale - # worktrees on branches that predate this recipe. - if [[ "{{ SCOPE }}" == all ]]; then - for wt in .claude/worktrees/*/; do - [ -f "${wt}justfile" ] || continue - echo "==> cleaning ${wt}" - (cd "$wt" && just clean) || echo " (skipped: just clean failed in ${wt})" - done - fi - -# Upgrade any tooling -update: - just js update - just rs update - nix flake update - -# Serve the documentation locally. -doc: - cd doc && bun run dev + sh/clean.sh {{ SCOPE }} diff --git a/kt/.gitignore b/kt/.gitignore index f9ad4114a98..2a7da565b63 100644 --- a/kt/.gitignore +++ b/kt/.gitignore @@ -4,7 +4,7 @@ build/ local.properties .kotlin/ -# Generated by kt/scripts/{check,package}.sh. Never check these in. +# Generated by sh/kt/{check,package}.sh. Never check these in. moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi/ moq-ffi/src/jvmMain/resources/ moq-ffi/src/androidMain/jniLibs/ diff --git a/kt/README.md b/kt/README.md index b25426f977c..f8a55837c61 100644 --- a/kt/README.md +++ b/kt/README.md @@ -89,7 +89,6 @@ kt/ src/ jvmAndAndroidMain/kotlin/dev/moq/ Wrapper sources (Moq, Server, Aliases, Flows, Durations, Errors) jvmAndAndroidTest/ Facade smoke test - scripts/ check.sh, package.sh ``` The native/UniFFI layer stays in a single `dev.moq:moq-ffi` artifact because uniffi-linked libraries can't be split across separately packaged artifacts (the Python `moq-rs` wheel is one umbrella for the same reason). That constraint is about the *native* layer; the pure-Kotlin `dev.moq:moq` wrapper sits cleanly on top of it as its own artifact because it ships no native code. diff --git a/kt/justfile b/kt/justfile index d88d307c07e..af419fb33e0 100644 --- a/kt/justfile +++ b/kt/justfile @@ -9,42 +9,20 @@ set working-directory := '.' default: just check -# Missing tools are errors so wrapper drift cannot slip past a green check. -# # This runs the tests too, so there is no separate `just kt test`: the script -# builds, generates, and tests as one gradle invocation. Takes an optional -# newline-separated list of changed files and skips when none are Kotlin-scoped; -# `just kt check` (no FILES) always runs. +# builds, generates, and tests as one gradle invocation. # Build moq-ffi, regenerate the bindings, and run :moq-ffi:jvmTest + :moq:jvmTest. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '^(kt/|rs/moq-ffi/)' <<< "$FILES"; then - echo "kt: no Kotlin changes; skipping." - exit 0 - fi - bash scripts/check.sh +check: + bash ../sh/kt/check.sh # Regenerate the checked-in uniffi bindings and host native library without # compiling or testing. For environments that intentionally lack Gradle. generate: - bash scripts/generate.sh + bash ../sh/kt/generate.sh # Assemble the KMP module from per-target moq-ffi binaries + bindings. # Used by .github/workflows/release-kt-ffi.yml; see kt/README.md for the - # expected --lib-dir layout. package *args: - bash scripts/package.sh {{ args }} - -# Remove gradle output plus the moq-ffi bindings and native libs that -# scripts/check.sh generates into the source tree (see kt/.gitignore). -clean: - #!/usr/bin/env bash - set -euo pipefail - find . -name .claude -prune -o -type d \( -name build -o -name .gradle -o -name .kotlin \) -prune -exec rm -rf {} + - rm -rf local.properties \ - moq/src/jvmAndAndroidMain/kotlin/uniffi \ - moq/src/jvmMain/resources \ - moq/src/androidMain/jniLibs + bash ../sh/kt/package.sh {{ args }} diff --git a/kt/moq-ffi/build.gradle.kts b/kt/moq-ffi/build.gradle.kts index c5e76a9f0bd..4aa387e97fe 100644 --- a/kt/moq-ffi/build.gradle.kts +++ b/kt/moq-ffi/build.gradle.kts @@ -11,7 +11,7 @@ // ├─ jvmMain JVM-specific: native libs as JAR resources // └─ androidMain Android-specific: native libs in jniLibs // -// Native libraries + bindings are populated by `kt/scripts/package.sh`: +// Native libraries + bindings are populated by `sh/kt/package.sh`: // src/jvmMain/resources/-/ (JNA classpath layout) // src/androidMain/jniLibs//libmoq_ffi.so (Android packaging layout) // src/jvmAndAndroidMain/kotlin/uniffi/moq/moq.kt (uniffi-bindgen output) diff --git a/py/justfile b/py/justfile index 2af24340e6d..3f28e9029d3 100644 --- a/py/justfile +++ b/py/justfile @@ -10,15 +10,6 @@ set working-directory := '.' -# Paths a Python check depends on. `rs/moq-ffi` is in scope because maturin -# bundles it into the moq-ffi wheel. `check`, `fix`, and `test` all skip -# themselves when a diff misses this; no FILES at all means force-run. -# -# They also skip when uv is absent, matching the kt/swift/go check scripts: an -# incomplete local toolchain should check less rather than fail. The root -# `_tools` preflight makes uv a hard requirement under MOQ_STRICT. -scope := '^(py/|pyproject\.toml$|uv\.lock$|rs/moq-ffi/)' - default: just check @@ -39,17 +30,7 @@ _develop: # the Docs after the fact. It mocks the native module, so it costs no Rust build. # Lint, type-check, editable-build, and render the docs. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "py: no Python changes; skipping." - exit 0 - fi - if ! command -v uv >/dev/null 2>&1; then - echo "py: uv not on PATH, skipping" >&2 - exit 0 - fi +check: uv sync --locked --no-install-workspace uv run --no-sync ruff check . uv run --no-sync ruff format --check . @@ -57,34 +38,14 @@ check $FILES="": uv run --no-sync pyright just docs -# Auto-fix lint and formatting issues. Skips on the same terms as `check`. -fix $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "py: no Python changes; skipping." - exit 0 - fi - if ! command -v uv >/dev/null 2>&1; then - echo "py: uv not on PATH, skipping" >&2 - exit 0 - fi +# Auto-fix lint and formatting issues. +fix: uv sync --no-install-workspace uv run --no-sync ruff check --fix . uv run --no-sync ruff format . -# Run the unit tests. Skips on the same terms as `check`. -test $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "py: no Python changes; skipping tests." - exit 0 - fi - if ! command -v uv >/dev/null 2>&1; then - echo "py: uv not on PATH, skipping" >&2 - exit 0 - fi +# Run the unit tests. +test: uv sync --locked --no-install-workspace just _develop uv run --no-sync pytest moq-rs/tests/ moq-ffi/tests/ @@ -96,18 +57,6 @@ build: uv sync --locked --no-install-workspace just _develop -# Remove the virtualenv, release dist, bytecode caches, and the uniffi -# bindings maturin drops in during editable installs. The uv workspace venv - -# lives at the repo root, so reach up for it. -clean: - #!/usr/bin/env bash - set -euo pipefail - rm -rf dist moq-ffi/moq_ffi/_uniffi - rm -rf ../.venv .venv - find . -name .claude -prune -o -type d -name __pycache__ -prune -exec rm -rf {} + - find . -name .claude -prune -o -type f -name '*.pyc' -exec rm -f {} + - # Build the pure-python moq-rs wrapper sdist + wheel into py/dist (for release). # moq-ffi is built separately by maturin (see release-py.yml); the wrapper is diff --git a/quest/m1/browser-benchmarks.md b/quest/m1/browser-benchmarks.md index 60c5dcb212c..06043fac8d0 100644 --- a/quest/m1/browser-benchmarks.md +++ b/quest/m1/browser-benchmarks.md @@ -7,7 +7,7 @@ Criterion targets and native relay load generator do not exercise. ## Plan -`rs/scripts/bench.sh::criterion_targets` discovers Cargo targets only. Existing JS unit +`bench/run.sh::criterion_targets` discovers Cargo targets only. Existing JS unit tests validate behavior, and `test/wasm` validates browser interop, but neither provides a repeatable JS performance comparison. Reuse the existing relay/browser harness pieces and add a focused recipe with artifacts under the benchmark diff --git a/quest/m1/dart-leak.md b/quest/m1/dart-leak.md index 9feb833b86b..8a70080748c 100644 --- a/quest/m1/dart-leak.md +++ b/quest/m1/dart-leak.md @@ -18,7 +18,7 @@ bound. path. The fix belongs upstream, not here: `dart/moq_ffi/lib/src/uniffi_runtime.dart` -is generated, and `dart/scripts/check.sh` diffs it against a fresh +is generated, and `sh/dart/check.sh` diffs it against a fresh `generate.sh` run, so an in-tree patch fails the staleness check by design. That upstream fix is merged as diff --git a/quest/m1/performance-comparisons.md b/quest/m1/performance-comparisons.md index 0169acbf11e..3e53d674fe7 100644 --- a/quest/m1/performance-comparisons.md +++ b/quest/m1/performance-comparisons.md @@ -7,7 +7,7 @@ estimate and preserved evidence, so a small reported speedup can be evaluated. ## Plan -`rs/scripts/bench.sh` runs each relay workload once as base then current, +`bench/run.sh` runs each relay workload once as base then current, without repeated rounds or alternating execution order. `cleanup` deletes the run directory, including Criterion estimates, load/host JSONL, relay logs, and summaries. Preserve the existing default command @@ -24,7 +24,7 @@ while extending this harness rather than creating another benchmark runner. hardware/kernel, allocator, affinity, workload, and execution order. Preserve partial evidence on failure while still cleaning up owned processes/worktrees. - Distinguish throughput-window counters from cumulative latency/loss. Today - `rs/scripts/bench.sh::summarize_load` differences bytes over the last five seconds + `bench/relay.sh::summarize_load` differences bytes over the last five seconds but reads final lifetime latency and group-loss counters. Label that explicitly; consume windowed data when the existing latency quest supplies it. Never subtract percentiles or call cumulative loss a steady-state sample. @@ -36,10 +36,6 @@ while extending this harness rather than creating another benchmark runner. samples. Validate an unchanged-revision A/A run and a deliberately degraded fixture; keep normal machine timing informational rather than a flaky CI gate. -## Required - -- [Thin justfiles](/quest/m1/tooling/justfiles.md) - finish benchmark script relocation before changing its lifecycle - ## Related - [Windowed latency](/quest/m1/3126-moq-bench-every-readme-example-fails-to-parse-and.md) - owns histogram/window semantics diff --git a/quest/m1/performance-profiles.md b/quest/m1/performance-profiles.md index 8fe119b4eec..eb52ef23b11 100644 --- a/quest/m1/performance-profiles.md +++ b/quest/m1/performance-profiles.md @@ -8,7 +8,7 @@ cost. Profiling is opt-in and has no production overhead when disabled. ## Plan -`rs/scripts/bench.sh` already owns the builds, relay PID, workload, and host +`bench/run.sh` already owns the builds, relay PID, workload, and host samples, but has no profiler integration. Reuse that lifecycle instead of adding a second launcher. `Cargo.toml` already has a `profiling` profile and `rs/moq-native/src/jemalloc.rs` already supports on-demand heap dumps. diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index 9f2ce96ac43..a0c524e0198 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -19,7 +19,6 @@ requires the one before it, so they land as one line of pull requests. ## Quests -- [Thin justfiles](/quest/m1/tooling/justfiles.md) - recipe bodies move to `sh/`, one impact map scopes check/fix/test, self-tests and guards are deleted - [Workflows call just](/quest/m1/tooling/workflows-call-just.md) - no workflow `run:` step names a `.sh`; every script a workflow needs has a recipe - [Binary release workflow](/quest/m1/tooling/release-binary.md) - moq-cli and moq-relay share one reusable workflow behind two thin callers - [FFI release workflow](/quest/m1/tooling/release-ffi.md) - the five `release-*-ffi.yml` share the moq-ffi target matrix and artifact staging diff --git a/quest/m1/tooling/justfiles.md b/quest/m1/tooling/justfiles.md deleted file mode 100644 index de77164f404..00000000000 --- a/quest/m1/tooling/justfiles.md +++ /dev/null @@ -1,93 +0,0 @@ -# [L] Thin justfiles with one impact map - -## Goal - -Every justfile is a menu: each recipe is a single line that runs a tool or a -script under `sh/`, `just --list` shows each recipe once, and no recipe body -is bash. The scoped `check`, `fix`, and `test` resolve the branch diff once, -in one impact map, instead of a scope regex per language justfile plus a -mirror of them in the strict-tools preflight. Nothing that tests the tooling -itself runs on a pull request. A recipe that no workflow, doc, skill, or other -recipe invokes is gone, except the hand-run demo and infra ops recipes, which -stay as they are. - -## Plan - -Layout: - -- New root `sh/`. Root helpers sit at the top (`sh/dispatch.sh`, - `sh/markdown.sh`, `sh/shell.sh`, `sh/clean.sh`); per-area tooling moves - under `sh//`: `rs/scripts` to `sh/rs`, `.github/scripts` to `sh/gh`, - `go/scripts` to `sh/go`, `kt/scripts` to `sh/kt`, `swift/scripts` to - `sh/swift`, `dart/scripts` to `sh/dart`, and the OBS recipe bodies to - `sh/obs`. Workflow references to moved paths are updated mechanically here; - the next quest replaces them with recipes. -- Package build scripts that nix, CMake, gradle, and the dart build hook - point at stay put: `rs/*/build.sh`, `rs/moq-gst/{package,scrub,smoke}.sh`, - `cpp/obs/build.sh`, `infra/*/publish.sh`. The harness under `test/` is - harness code, not tooling; it stays too. -- `shfmt -f .` already enumerates `sh/`, so the shell lint covers everything - the justfiles stop containing. - -Dispatch: - -- `just check [BASE]`, `just fix [BASE]`, and `just test [BASE]` are one line - each: `sh/dispatch.sh check|fix|test "$BASE"`. The script resolves BASE - (arg, `GITHUB_BASE_REF`, upstream, `origin/main`), lists changed files into - a temp file, and applies one impact map: which language modules run, which - tools `MOQ_STRICT` demands, and whether the root orchestration changed and - everything runs. The map lives in one place; the `scope` variables in the - js, py, kt, swift, go, and dart justfiles and the regex table in `_tools` - are deleted with it. -- Per-language `check`, `fix`, and `test` lose their `$FILES` parameter and - skip logic. Only rs still receives the list: `just rs check-changed - ` (and `fix-changed`, `test-changed`) hand the path to - `sh/rs/select.sh`, which does the crate selection today spread over - `_select`, `_names`, and `_wants-wasm`. -- Passing a path instead of the list kills the argv budget: - `changed_max`, `_changed-cap`, `_changed-test`, `_echo`, and the E2BIG - commentary go. -- `check --all`, `fix --all`, and `test all` keep their names; cache.yml, - nightly.yml, and the docs call them. - -Delete: - -- Self-tests: `_changed-test`, `_markdown-test`, `_select-test`, - `_doc-names-test`, `_publish-test`, `sh/rs/package-nfpm.test.sh`, - `sh/gh/package-binary.test.sh`. -- Guards: `_doc-names` and `doc-names.jq`, and the publish lower-bound - check. This is a deliberate trade: the guards were incidents' worth of - bash plus their self-tests on every pull request. Where each surfaces - instead: a doc-directory collision shows up as an intermittent `failed to - remove directory` in the `cargo doc` step of whichever pull request selects - both crates, not deterministically (resolve with `doc = false` as before); - a stale internal lower bound fails release-plz's publish. `_fuzz-lock` is - already gone: #3543 folded the fuzz harness into the workspace so the root - lockfile and `--locked` cover it. `alert.sh check-coverage` stays: it is a - lint of alert.yml, not of a recipe. -- The `worktree` recipe and its 170 lines, plus the Worktrees section of - `test/README.md`; keep `_base` folded into `sh/dispatch.sh`. -- `rs bump` and `rs semver`. `rs release` stays for release-rs.yml. -- The `mod` lines in `demo/justfile`: `just pub`, `just relay`, `just boy`, - `just sub`, `just web` are the only spelling, and `--list` stops showing - each twice. `just demo` remains the default recipe and `just dev` its - documented alias. - -Rename: - -- Root `wasm` becomes `js wasm`: it emits `js/wasm/dist`, so it is a JS - package build. `rs wasm` stays the compile gate and `test wasm` the browser - run. Update wasm.yml, root `build`, `test/wasm/run.sh` (which runs `just - wasm` to build the package under test), and the four doc references. - -Keep, moved into scripts unchanged in behavior: the remark mirror-and-diff in -`sh/markdown.sh` (remark-cli has no check mode and the lint presets are -wanted), `_shell`, `_flake` as its one-liner, `clean`, the rs `package` and -`fuzz` bodies, and the OBS `compile`, `_includes`, `_unit`, `test`, `check`, -and `preset` bodies. - -Docs: `doc/setup/dev.md`, `CONTRIBUTING.md`, `test/README.md`, and the -`CLAUDE.md` mentions of `just wasm` follow the survivors. Verify with `just -check`, `just test`, and `just check --all`, and confirm every recipe name -check.yml, cache.yml, nightly.yml, interop.yml, wasm.yml, obs.yml, swift.yml, -and release-*.yml invoke still resolves. diff --git a/quest/m1/tooling/workflows-call-just.md b/quest/m1/tooling/workflows-call-just.md index 4d6c79c4e8e..a38fcc5217d 100644 --- a/quest/m1/tooling/workflows-call-just.md +++ b/quest/m1/tooling/workflows-call-just.md @@ -39,7 +39,3 @@ the swift `package`, `package-ffi`, `publish`, `publish-ffi`, `verify`, `workflow_dispatch` of apt-repo.yml, rpm-repo.yml, and release-winget.yml where the inputs allow a dry run. The tag-triggered release workflows are verified by the next release; say so in the PR. - -## Required - -- [Thin justfiles](/quest/m1/tooling/justfiles.md) - establishes `sh/` and the module recipes the workflows will call diff --git a/rs/justfile b/rs/justfile index dba4cbbf3d4..4a2a3f3ad63 100644 --- a/rs/justfile +++ b/rs/justfile @@ -11,120 +11,30 @@ set working-directory := '..' # `moq-net-fuzz` is a workspace member only so it shares the root lockfile; libFuzzer # needs a nightly toolchain and sanitizer flags, so `just rs fuzz` is the one recipe -# that may compile it. Every `--workspace` below therefore carries this, and a new -# one that forgets it pays a vendored C++ libFuzzer build for nothing. The root -# justfile's `check --all` and `fix --all` spell it out, since a module cannot read this. +# that may compile it. Every `--workspace` below therefore carries this, as does +# sh/rs/select.sh, since it cannot read this. no_fuzz := "--exclude moq-net-fuzz" default: just check -# Default features for the selected workspace packages, here and in CI. The full -# workspace permutations (`--all-features`, `--no-default-features`) each cost a -# separate compile that shares almost nothing with this one, so `features` below -# covers them nightly. `check-changed` makes one scoped exception for moq-tokio, -# whose feature-gated transport surface must compile in isolation to avoid -# workspace feature unification hiding a broken configuration. -# -# No `cargo check` pass: clippy is a superset of it, and the two use different -# rustc wrappers, so running both means compiling the workspace twice. +# Default features for the selected packages. The full workspace permutations +# each cost a separate compile, so `features` covers them nightly. No `cargo +# check` pass: clippy is a superset of it, and the two use different rustc +# wrappers, so running both compiles the workspace twice. # # A bare `just rs check` is default-members, which skips libmoq, moq-ffi, -# moq-gst, and moq-wasm (they need cbindgen, a uniffi build, GStreamer, and a -# wasm32 target respectively, so they stay out of a casual local check). Every -# recipe that means "all of it" therefore passes `--workspace` explicitly, since -# `check` is the only gate now and nothing else would compile them. +# moq-gst, and moq-wasm; pass `--workspace` for all of it. # Compile, lint, format-check, doc-check, and verify dependency hygiene. check *args: - just rs _select-test - just rs _publish-test - just rs _doc-names-test - # A new MoqConnectionStats field has to reach every binding's doc page. - python3 rs/scripts/stats-docs.py - rs/scripts/package-nfpm.test.sh + python3 sh/rs/stats-docs.py cargo clippy --locked --all-targets {{ args }} -- -D warnings cargo fmt --all --check - just rs _doc-names RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps {{ args }} cargo shear cargo sort --workspace --check --no-format -# Two targets whose names differ only by `-` vs `_` render to the same -# `target/doc/`, and a `cargo doc` that selects both races to clean it: the -# collision warning escalates to `error: failed to remove directory` depending on -# which rustdoc process gets there first. That reads as a flake, and because the doc -# pass above is diff-scoped it surfaces on whichever unrelated PR happens to select -# both crates. Resolve a pair by setting `doc = false` on whichever target has no -# published API to render (usually the binary). -# -# Metadata only, so this costs no compilation and can sit in the PR gate. Takes an -# optional metadata file so `_doc-names-test` can feed it synthetic workspaces. -_doc-names metadata="": - #!/usr/bin/env bash - set -euo pipefail - - if [[ -n "{{ metadata }}" ]]; then - collisions=$(jq -r -f rs/scripts/doc-names.jq < "{{ metadata }}") - else - collisions=$(cargo metadata --locked --no-deps --format-version 1 | jq -r -f rs/scripts/doc-names.jq) - fi - - if [[ -n "$collisions" ]]; then - echo "rs: these targets render to the same doc directory:" >&2 - echo "$collisions" >&2 - echo "set doc = false on whichever has no published API." >&2 - exit 1 - fi - -# Both directions of `_doc-names`, since the interesting half is what it must NOT -# report: cargo documents only the library when a binary shares its own package's -# name, so flagging that would reject the conventional src/lib.rs + src/main.rs layout. -_doc-names-test: - #!/usr/bin/env bash - set -euo pipefail - - scratch=$(mktemp -d) - trap 'rm -rf "$scratch"' EXIT - - # Same package, same name: cargo suppresses the bin, so this must pass. - cat > "$scratch/ok.json" <<'JSON' - {"packages":[{"name":"solo","targets":[ - {"name":"solo","kind":["lib"],"doc":true}, - {"name":"solo","kind":["bin"],"doc":true}]}]} - JSON - just rs _doc-names "$scratch/ok.json" - - # Same package, differing only by `-` vs `_`: cargo folds before suppressing, so - # this must pass too. Comparing the names as spelled would report it. - cat > "$scratch/folded.json" <<'JSON' - {"packages":[{"name":"folded","targets":[ - {"name":"foo_bar","kind":["lib"],"doc":true}, - {"name":"foo-bar","kind":["bin"],"doc":true}]}]} - JSON - just rs _doc-names "$scratch/folded.json" - - # Different packages: a real collision, must be reported. - cat > "$scratch/bad.json" <<'JSON' - {"packages":[ - {"name":"a-lib","targets":[{"name":"shared_name","kind":["lib"],"doc":true}]}, - {"name":"b-cli","targets":[{"name":"shared-name","kind":["bin"],"doc":true}]}]} - JSON - if just rs _doc-names "$scratch/bad.json" 2>/dev/null; then - echo "rs: _doc-names missed a cross-package collision" >&2 - exit 1 - fi - - # `doc = false` is how a pair gets resolved, so it must stop being reported. - cat > "$scratch/fixed.json" <<'JSON' - {"packages":[ - {"name":"a-lib","targets":[{"name":"shared_name","kind":["lib"],"doc":true}]}, - {"name":"b-cli","targets":[{"name":"shared-name","kind":["bin"],"doc":false}]}]} - JSON - just rs _doc-names "$scratch/fixed.json" - - echo "rs: _doc-names ok" - # Auto-fix clippy/format/shear/sort. fix *args: cargo clippy --locked --fix --allow-staged --allow-dirty --all-targets {{ args }} @@ -132,391 +42,27 @@ fix *args: cargo shear --fix cargo sort --workspace --no-format -# Prints `ALL` when the diff hits something every crate depends on, and nothing -# when no crate is affected. -# -# FILES is an exported parameter, not `{{ FILES }}`: just interpolates the -# latter into the recipe source, where a filename like `$(...)` would run as a -# command. Git-derived paths are untrusted input, so they stay data. - -# Print `--package` flags for the crates a diff touches, plus their dependents. -[private] -_select $FILES: - #!/usr/bin/env bash - set -euo pipefail - - # Workspace-wide inputs affect every crate, so there is nothing to scope to. - # - # `rs/justfile` and the nextest config are in here because they define how - # every crate is checked and tested. They also match `^rs/` below while - # seeding no crate (the seed pattern needs `rs//...`), so without this - # a PR editing only the recipes would compile and test nothing at all, and a - # broken cargo flag would pass CI on a format check alone. - if [[ -z "$FILES" ]] || grep -qE '^(Cargo\.(toml|lock)|rust-toolchain\.toml|rs/justfile|\.config/nextest\.toml)$' <<< "$FILES"; then - echo ALL - exit 0 - fi - - grep -q '^rs/' <<< "$FILES" || exit 0 - - # `--no-deps` keeps this to workspace members and off the network. - metadata=$(cargo metadata --format-version 1 --no-deps) - - # A seed is the first path segment under `rs/`, so only a crate whose manifest - # sits directly at `rs//Cargo.toml` can ever be one. Drop the rest here: - # `moq-net-fuzz` lives at `rs/moq-net/fuzz`, so its seed is already `moq-net`, - # and leaving it in the graph would select it as a dependent of the crate it - # fuzzes and compile libFuzzer on every moq-net diff. - # - # `manifest_path` is a native path, so normalize the separator before splitting - # on it. Windows would otherwise match no segment, drop every package, and - # select nothing at all, where the assertion below fails safe to ALL. - metadata=$(jq '.packages |= map((.manifest_path |= gsub("\\\\"; "/")) | select(.manifest_path | split("/")[-3] == "rs"))' <<< "$metadata") - - # Every remaining crate directory is named after the crate it holds, which is - # what makes a changed path a seed below. Verify it rather than trust it: under- - # selecting silently checks nothing, so fall back to checking everything. - if jq -e '.packages[] | select((.manifest_path | split("/")[-2]) != .name)' <<< "$metadata" > /dev/null; then - echo "rs: a crate directory no longer matches its crate name; selecting everything." >&2 - echo ALL - exit 0 - fi - - seeds=$(sed -n 's|^rs/\([^/]*\)/.*|\1|p' <<< "$FILES" | sort -u) - - # Every workspace member, not just the default ones. `--no-deps` already - # limits this to the workspace. The non-default members (libmoq, moq-ffi, - # moq-gst, moq-wasm) have to participate as both seeds and dependents: they - # used to drop out here because `check` matched a bare `cargo check`, but - # `check` is the only gate now, and the `cargo clippy --workspace` in the - # deleted `just rs ci` is what used to cover them. Without them, a PR - # touching only rs/libmoq selects nothing and compiles nothing. - edges=$(jq -r ' - .packages[] - | .name as $name - | .dependencies[] - | "\($name) \(.name)" - ' <<< "$metadata") - - # A crate has to be rebuilt when anything it depends on changed, so walk the - # edges backwards until the selection stops growing. - selected=$(awk -v seeds="$seeds" ' - BEGIN { split(seeds, s, "\n"); for (i in s) want[s[i]] = 1 } - { pkg[NR] = $1; dep[NR] = $2 } - END { - do { - grew = 0 - for (i = 1; i <= NR; i++) - if (want[dep[i]] && !want[pkg[i]]) { want[pkg[i]] = 1; grew = 1 } - } while (grew) - # Every wanted crate, not just those that appear as an edge source: a - # crate with no dependencies of its own emits no edge, so keying the - # output on `pkg[i]` would drop it and select nothing for a direct - # change to it. Test the VALUE, not the key -- awk creates a key on - # every `want[dep[i]]` read above, so iterating keys alone would emit - # every crate the workspace mentions. The lookup below drops names that - # are not workspace crates, which is what keeps a seed like the - # non-crate `rs/scripts` from reaching cargo. - for (name in want) if (want[name]) print name - } - ' <<< "$edges" | sort -u) - - [[ -n "$selected" ]] || exit 0 - - # Emit cargo's own package ids, not bare names. A workspace crate we also - # publish collides with its crates.io copy as soon as any dependency pulls - # that copy in (the web-transport-* crates depend on a published `kio`), and - # `--package kio` is then ambiguous: cargo refuses during package selection, - # before compiling anything, so the whole gate fails. Ids from `cargo - # metadata` carry the source, so they are unambiguous by construction. - jq -r --arg selected "$selected" ' - ($selected | split("\n")) as $want - | .packages[] - | select(.name as $name | $want | index($name) != null) - | "--package \(.id)" - ' <<< "$metadata" | sort -u | tr '\n' ' ' - -# Bare crate names for the `--package ` flags `_select` emits, for log lines -# and for the wasm gate's alternation below. Matching names rather than the flag -# spelling keeps those call sites working whatever spec form `_select` uses; the -# previous alternation matched `--package ` literally and silently stopped -# firing the moment the spec form changed. Ids are `path+file://#` -# and `_select` only emits crates whose directory it has verified matches -# their crate name, so the last path segment is the name. - -# True when a selection includes a crate the wasm32 pass covers. One predicate -# rather than the same alternation in each gate, so the test below exercises what -# the gates actually run instead of a copy of it. -[private] -_wants-wasm $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qwE '(moq-wasm|moq-mux|moq-ffi)' <<< "$(just rs _names "$PACKAGES")" - -# True when the selection includes moq-tokio, whose zero-feature build is hidden -# by workspace feature unification whenever any dependent enables a transport. -[private] -_wants-tokio-features $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qw 'moq-tokio' <<< "$(just rs _names "$PACKAGES")" - -# True when the selection includes a crate with a `capture` feature. Selection -# pulls in dependents, so a diff to moq-mux or moq-net lands here too, which is -# how a rename in either reaches the device code that calls it. -[private] -_wants-capture $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qwE '(moq-video|moq-audio)' <<< "$(just rs _names "$PACKAGES")" - -# True when the selection includes moq-relay, whose io_uring listener is off the -# default feature set. Selection pulls in dependents, so a diff to any crate the -# relay is built from (moq-uring, moq-sock, moq-tokio) lands here too. -[private] -_wants-uring $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qw 'moq-relay' <<< "$(just rs _names "$PACKAGES")" - -# Print the crate names behind a list of `--package` flags. -[private] -_names $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - tr ' ' '\n' <<< "$PACKAGES" | sed -n 's|.*/\([^/#]*\)#.*|\1|p' | sort -u | tr '\n' ' ' - -# Guards the two things about selection that fail SILENTLY rather than loudly: -# a `--package` spec cargo cannot resolve takes the gate down before it compiles -# anything, and a wasm gate that stops matching skips the only pass that ever -# looks at moq-wasm's code (its crate root is `#![cfg(target_arch = "wasm32")]`, -# so the host pass sees an empty crate and reports success). Both have happened. - -# Check that selection emits resolvable specs and still drives the wasm gate. -[private] -_select-test: - #!/usr/bin/env bash - set -euo pipefail - - fail() { echo "rs: _select-test: $1" >&2; exit 1; } - - # `kio` is published and pulled from crates.io by the web-transport-* crates, - # so the workspace copy and the registry copy share a name: a bare name is - # ambiguous and cargo refuses. Every emitted spec must name its source, and - # cargo must accept it. - packages=$(just rs _select "rs/kio/src/waiter.rs") - for spec in $(sed 's/--package //g' <<< "$packages"); do - [[ "$spec" == *"file://"* ]] || fail "expected a source-qualified id, got: $spec" - cargo pkgid --offline "$spec" > /dev/null 2>&1 || fail "cargo cannot resolve: $spec" - done - - # The wasm gate keys on names, so it survives whatever spec form the ids use. - just rs _wants-wasm "$packages" \ - || fail "a kio diff must still drive the wasm gate: $(just rs _names "$packages")" - - # ...and it must still be able to say no, or it is not a gate. - relay=$(just rs _select "rs/moq-relay/src/web.rs") - if just rs _wants-wasm "$relay"; then - fail "a moq-relay diff must not drive it: $(just rs _names "$relay")" - fi - - # The break the capture gate was added for: a moq-mux API change that missed - # its callers behind `capture`. A moq-mux diff has to reach it. - mux=$(just rs _select "rs/moq-mux/src/clock.rs") - just rs _wants-capture "$mux" \ - || fail "a moq-mux diff must drive the capture gate: $(just rs _names "$mux")" - if just rs _wants-capture "$relay"; then - fail "a moq-relay diff must not drive the capture gate: $(just rs _names "$relay")" - fi - - # The break the uring gate was added for: a change that breaks moq-relay's - # io_uring listener, which the default pass above never compiles. A moq-relay - # diff has to reach it, and a diff that cannot affect the relay must not. - just rs _wants-uring "$relay" \ - || fail "a moq-relay diff must drive the uring gate: $(just rs _names "$relay")" - hls=$(just rs _select "rs/moq-hls/src/lib.rs") - if just rs _wants-uring "$hls"; then - fail "a moq-hls diff must not drive the uring gate: $(just rs _names "$hls")" - fi - - tokio=$(just rs _select "rs/moq-tokio/src/lib.rs") - just rs _wants-tokio-features "$tokio" \ - || fail "a moq-tokio diff must drive its feature matrix: $(just rs _names "$tokio")" - if just rs _wants-tokio-features "$relay"; then - fail "a moq-relay-only selection must not drive the moq-tokio feature matrix" - fi - - # A non-crate directory under rs/ seeds a name that is not a package. It must - # select nothing rather than reach cargo as a bogus spec. - [[ -z "$(just rs _select "rs/scripts/package-binary.sh")" ]] \ - || fail "a non-crate seed must select nothing" - - # A crate nested inside another crate's directory is covered by its parent's - # seed. Selecting it directly would compile libFuzzer on every moq-net diff, - # and its directory name is not its crate name, so it would also mis-report. - fuzz=$(just rs _names "$(just rs _select "rs/moq-net/fuzz/fuzz_targets/lite.rs")") - grep -qw moq-net <<< "$fuzz" || fail "a fuzz diff must select moq-net: $fuzz" - ! grep -qw fuzz <<< "$fuzz" || fail "a fuzz diff must not select the harness: $fuzz" - - echo "rs: selection ok" - -# Published crates need the current workspace version as the lower bound for -# internal dependencies. The workspace only tests that version set, and a looser -# bound lets package verification combine incompatible older releases. -# -# Internal dev-dependencies must be path-only. `cargo publish` keeps a versioned -# dev-dependency and requires it on crates.io, but release-plz orders publishes -# by normal dependencies only, so a bumped dev-dependency is often not there yet. -[private] -_publish-test: - #!/usr/bin/env bash - set -euo pipefail - - metadata=$(cargo metadata --locked --format-version 1 --no-deps) - stale=$(jq -r ' - .packages as $packages - | $packages[] as $package - | select($package.publish != []) - | $package.dependencies[] - | select(.source == null and .path != null and .kind != "dev") - | . as $dependency - | ($packages[] | select(.name == $dependency.name)) as $current - | select($dependency.req != ("^" + $current.version)) - | "\($package.name): \($dependency.name) requires \($dependency.req), current is ^\($current.version)" - ' <<< "$metadata") - - if [[ -n "$stale" ]]; then - echo "rs: published workspace dependency lower bounds are stale:" >&2 - echo "$stale" >&2 - exit 1 - fi - - versioned=$(jq -r ' - .packages[] - | select(.publish != []) - | .name as $package - | .dependencies[] - | select(.source == null and .path != null and .kind == "dev" and .req != "*") - | "\($package): \(.name) \(.req)" - ' <<< "$metadata") - - if [[ -n "$versioned" ]]; then - echo "rs: published crates have versioned workspace dev-dependencies:" >&2 - echo "$versioned" >&2 - echo "use { path = \"../\" } instead of { workspace = true }." >&2 - exit 1 - fi - - echo "rs: published workspace dependencies current" - -# Takes a newline-separated list of changed files; skips when no crate is -# affected. The `just rs ...` calls below go through the root justfile's -# `mod rs` because these recipes run from the repo root, where a bare `check` -# would resolve to the root recipe of that name. - -# Like `check`, but only the crates a diff touches plus their dependents. -check-changed $FILES: - #!/usr/bin/env bash - set -euo pipefail - packages=$(just rs _select "$FILES") - case "$packages" in - "") echo "rs: no crates affected; skipping." ;; - ALL) just rs check --workspace {{ no_fuzz }} ;; - *) echo "rs: checking $(just rs _names "$packages")"; just rs check $packages ;; - esac - - if [[ "$packages" == "ALL" ]] || just rs _wants-tokio-features "$packages"; then - just rs tokio-features - fi - - # moq-wasm's crate root is entirely `#![cfg(target_arch = "wasm32")]`, so - # however it got selected the host-target pass above compiled it to nothing - # and saw no errors in it. This wasm32 pass is what actually checks it. - # - # The alternation must list every package `wasm` compiles, not just moq-wasm: - # nothing in the workspace depends on moq-ffi, so an moq-ffi-only diff selects - # only itself and would never reach the gate that exists to cover it. moq-mux - # is selected by its dependents, none of which are moq-wasm. - if [[ "$packages" == "ALL" ]] || just rs _wants-wasm "$packages"; then - just rs wasm - fi - - # moq-video's and moq-audio's device code is behind `capture`, which is off - # by default, so the pass above never compiles it. Only nightly's - # `--all-features` did, and a break there lands on main. - if [[ "$packages" == "ALL" ]] || just rs _wants-capture "$packages"; then - just rs capture - fi - - # moq-relay's io_uring listener is behind `io-uring`, off by default, so the - # pass above never compiles it. `check --all` reaches it through `bench - # check`, but only for a diff under `bench/`; a relay change would otherwise - # land on the nightly. - if [[ "$packages" == "ALL" ]] || just rs _wants-uring "$packages"; then - just rs uring-check - fi +# Like `check`, but only the crates in a changed-file list plus their dependents. +check-changed $LIST: + sh/rs/select.sh check "$LIST" + +# Like `fix`, but only the crates in a changed-file list plus their dependents. +fix-changed $LIST: + sh/rs/select.sh fix "$LIST" + +# Like `test`, but only the crates in a changed-file list plus their dependents. +test-changed $LIST: + sh/rs/select.sh test "$LIST" # Compile moq-tokio by itself at the feature extremes and with each crypto provider. -# Its default-feature build is already part of the ordinary clippy pass above. tokio-features: - #!/usr/bin/env bash - set -euo pipefail - - cargo clippy --locked -p moq-tokio --no-default-features -- -D warnings - cargo clippy --locked -p moq-tokio --all-features -- -D warnings - cargo clippy --locked -p moq-tokio --no-default-features --features noq,aws-lc-rs -- -D warnings - cargo clippy --locked -p moq-tokio --no-default-features --features noq,ring -- -D warnings - - for feature in aws-lc-rs ring tcp uds websocket; do - cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings - done - - want='a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' - if output=$(cargo check --locked -p moq-tokio --no-default-features --features noq 2>&1); then - echo "rs: moq-tokio with only noq should reject its missing crypto provider" >&2 - exit 1 - fi - if ! grep -qF "$want" <<< "$output"; then - echo "rs: moq-tokio with only noq should fail on: $want" >&2 - echo "$output" | grep -E '^error' >&2 || true - exit 1 - fi - -# Same as `fix`, but scoped the same way as `check-changed`. -fix-changed $FILES: - #!/usr/bin/env bash - set -euo pipefail - packages=$(just rs _select "$FILES") - case "$packages" in - "") echo "rs: no crates affected; skipping." ;; - ALL) just rs fix --workspace {{ no_fuzz }} ;; - *) echo "rs: fixing $(just rs _names "$packages")"; just rs fix $packages ;; - esac - - # Mirrors `check-changed`'s gate, alternation included: without this a wasm32 - # lint is never auto-fixed, and `check`'s wasm32 clippy pass fails on it later. - if [[ "$packages" == "ALL" ]] || just rs _wants-wasm "$packages"; then - just rs wasm-fix - fi - -# Runs the `#[cfg(target_os = "windows")]` code past the compiler: moq-video's -# Media Foundation capture/encode/decode and its D3D11 frames, which the Linux -# gate skips entirely. Cross-compiling can't stand in, because openh264-sys2 -# builds vendored C++ that needs an MSVC toolchain and openh264 is a -# non-optional dependency. -# -# Windows runners are throttled too hard for a per-PR gate, so nightly.yml runs -# this once a day instead. A break there lands on main rather than being caught -# in review, which is the same trade the other nightly gates make. -# -# Default features rather than `--all-features`: jemalloc doesn't build on -# MSVC, while moq-video's Linux-only `nvidia` default -# are already no-ops off Linux. moq-gst is excluded because it links -# GStreamer via pkg-config, which the Windows runner doesn't have. -# -# `moq-cli/play` and `moq-cli/capture` are named explicitly because they are -# off by default, and they are the only thing that compiles the cli's own -# device and render code. + sh/rs/tokio-features.sh + +# Runs the `#[cfg(target_os = "windows")]` code past the compiler, which the +# Linux gate skips; nightly.yml runs it on a Windows runner. Default features, +# since jemalloc doesn't build on MSVC, and moq-gst is excluded because the +# runner has no GStreamer. `moq-cli/play` and `moq-cli/capture` are the only +# thing that compiles the cli's device and render code. # Compile the whole workspace on a Windows host. Must run ON Windows. windows *args: @@ -524,111 +70,52 @@ windows *args: # Compile and test the io_uring relay path, which is off the default feature set. # Linux-only, and the tests below the 6.12 kernel floor skip themselves. -# -# `qlog` is a second axis, and the traces are what a debugging run depends on, -# so it gets a separate pass. uring *args: just rs uring-check {{ args }} cargo nextest run --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets {{ args }} cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring,moq-relay/qlog --all-targets {{ args }} -- -D warnings cargo nextest run --locked -p moq-uring --features qlog --all-targets {{ args }} -# Compile the io_uring relay path, off the default feature set. `check-changed` -# runs this whenever moq-relay is selected, so a change that breaks the ring -# listener fails its own PR rather than landing on the nightly. +# Compile the io_uring relay path, off the default feature set. uring-check *args: cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets {{ args }} -- -D warnings -# Runs the `#[cfg(target_os = "macos")]` code past the compiler: moq-video's -# VideoToolbox encode/decode and its ScreenCaptureKit / AVFoundation capture, -# plus moq-audio's ScreenCaptureKit system audio and TCC permission pre-check. -# -# Same trade as `windows`: nightly.yml runs this once a day rather than gating a -# merge on a Mac runner. The moq-video half also has a release-time backstop, -# since libmoq's `libmoq-v*` tag build compiles it on Apple Silicon. The -# moq-audio half has none: libmoq and moq-ffi take it codecs-only, so no release -# build compiles its capture backend and this recipe is the only thing that does. -# -# Scoped to those two crates instead of the workspace, because they hold all -# the Apple-gated code the Linux gate misses. moq-ffi (and through it the -# Swift/Kotlin/Go wrappers) already compiles on macOS in swift.yml. -# `--all-features` also compiles the off-by-default cpal hosts and PipeWire -# capture, which are Linux-only and no-ops here, so it costs nothing extra. +# moq-video's VideoToolbox/ScreenCaptureKit/AVFoundation code and moq-audio's +# ScreenCaptureKit capture, which nightly.yml compiles on a Mac runner. # Compile the Apple-only code paths. Must run ON macOS. macos *args: cargo check --locked -p moq-video -p moq-audio --all-targets --all-features {{ args }} -# The Linux device code: moq-video's V4L2 and X11 capture and moq-audio's cpal -# capture, all behind the off-by-default `capture` feature. `check-changed` runs -# this whenever either crate is selected. The per-crate `--features` spelling -# keeps it to those two instead of widening what their dependencies compile. -# `--all-features` would also build the bindgen codecs, which nightly covers. - # Compile and lint the capture feature, which the default check skips. capture *args: cargo clippy --locked -p moq-video -p moq-audio --all-targets --features moq-video/capture,moq-audio/capture {{ args }} -- -D warnings -# Same idea as `windows`/`macos`, for Android: moq-video's MediaCodec encoder and -# decoder, its `Surface::HardwareBuffer` variant, and the `frame::android` module -# are all `#[cfg(target_os = "android")]`, so a host-target check compiles none of -# them and every error in them stays invisible. -# -# Unlike `windows` and `macos` this needs no special host: the Android targets -# cross-compile from Linux in about half a minute. It needs an NDK, which -# `cargo-ndk` finds through `ANDROID_NDK_HOME` or the standard SDK layout. The -# dev shell provides neither, so android.yml runs this outside Nix on a GitHub -# runner, whose image ships an NDK, for every PR that touches moq-video or -# anything it is built from. -# -# arm64-v8a alone: it is the only ABI anything here ships, and the code is not -# ABI-specific, so the other two would compile the same source twice more. -# -# `check` rather than a `-D warnings` clippy, matching `windows` and `macos`: -# the capture module's plumbing is dead on a target with no capture backend, so -# a denying lint would fail on code this recipe is not here to police. +# moq-video's MediaCodec code is `#[cfg(target_os = "android")]`. Needs an NDK, +# which the dev shell lacks, so android.yml runs it outside Nix. `check` rather +# than a denying clippy: the capture plumbing is dead on a target without a +# capture backend. # Compile the Android backends, which the host-target check skips. android *args: cargo ndk -t arm64-v8a --platform 26 check --locked -p moq-video --all-targets {{ args }} -# Same idea as `windows`/`macos`, for the browser: moq-wasm's whole crate root is -# `#![cfg(target_arch = "wasm32")]`, so a host-target `cargo check --workspace` -# compiles an empty crate and every error in it stays invisible. Only a wasm32 -# build sees the code. -# -# Unlike those two this needs no special host, so `ci` runs it on every Rust PR. -# The wasm32 target and the `getrandom`/`web-sys` cfg flags come from the Nix dev -# shell and `.cargo/config.toml`. Not to be confused with the root `just wasm`, -# which builds the shippable `@moq/wasm` package (wasm-bindgen, release profile); -# this is the compile gate. - -# `moq-mux` and `moq-ffi` ride along in the same invocation on purpose: `moq-mux` -# needs `getrandom`'s `wasm_js` backend that only its siblings declare, and feature -# unification only happens within one cargo invocation. Splitting them fails on -# getrandom. `--lib` rather than `--all-targets` because moq-mux's tests use tokio's -# native timers and moq-ffi's drive a native relay; moq-wasm has no test targets, so -# it loses no coverage. +# moq-wasm's crate root is `#![cfg(target_arch = "wasm32")]`, so only a wasm32 +# build sees the code. Not to be confused with `just js wasm`, which builds the +# shippable `@moq/wasm` package. `moq-mux` and `moq-ffi` share the invocation +# because moq-mux needs the getrandom backend only its siblings declare, and +# features unify only within one cargo invocation. # Compile and lint the browser/WASM bindings, which the host-target check skips. wasm *args: cargo clippy --locked -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib {{ args }} -- -D warnings -# The `fix` counterpart to `wasm`. `cargo fmt` is scoped rather than `--all` -# because this runs alongside `fix`, which already formats everything else. - # Auto-fix the browser/WASM bindings. wasm-fix *args: cargo clippy --locked --fix --allow-staged --allow-dirty -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib {{ args }} cargo fmt -p moq-wasm -p moq-mux -p moq-ffi -# Dependency advisories plus the license/ban policy. Time-based rather than -# diff-based: an advisory lands without this repo changing, so a per-PR run both -# fires on PRs that cannot affect it and stays silent on the days nobody pushes. -# Runs nightly instead (.github/workflows/nightly.yml), which also keeps a fresh -# advisory from blocking an unrelated PR mid-review. - -# Audit dependencies against deny.toml. +# Audit dependencies against deny.toml. Nightly: advisories land without a diff. audit: cargo deny check --show-stats @@ -636,96 +123,16 @@ audit: relay-minimal: cargo nextest run --locked -p moq-relay --no-default-features --lib -E 'test(workers_require_a_quic_backend)' -# libmoq's C fixtures (`rs/libmoq/c-tests/*.c`) link `libmoq.a` the way an -# embedder does: an external `cc` against the generated `moq.h`, plus the native -# libraries from `rs/libmoq/native-libs` that cargo can't inject into a link it -# doesn't drive. Each fixture pins part of the C ABI at compile time and -# exercises it at runtime, so it has to run, not just compile. -# -# Outside nextest on purpose. The staticlib link alone takes ~2 minutes, past the -# hang budget in `.config/nextest.toml` once the rest of the workspace builds -# beside it, and `rs/CLAUDE.md` says to make a slow test fast, not to raise the -# ceiling. obs.yml runs this on every PR that touches rs/libmoq, right after it -# has built the same debug libmoq.a for the plugin, and nightly.yml covers the -# paths that filter misses. -# -# Unix only: linking a Rust staticlib from MSVC spells the flags and the -# archive (`moq.lib`) differently, and this recipe doesn't model that. +# Outside nextest because the staticlib link alone outlasts its hang budget. # Compile and run libmoq's C fixtures against libmoq.a. Must run on Unix. c-tests: - #!/usr/bin/env bash - set -euo pipefail - cc="${CC:-cc}" - if ! command -v "$cc" >/dev/null; then - echo "no C compiler at '$cc'" >&2 - exit 1 - fi - - cargo build --locked -p libmoq - - # Ask cargo where it put the staticlib rather than assuming target/debug: - # CARGO_TARGET_DIR and a configured build target (`target//debug`) - # both move it, and guessing wrong would link a stale copy or nothing. - # Replays the build above from cache, so it costs a process, not a compile. - lib=$(cargo build --locked -p libmoq --message-format=json | - jq -r 'select(.reason == "compiler-artifact") | .filenames[] | select(endswith("/libmoq.a"))' | - tail -1) - if [ -z "$lib" ]; then - echo "cargo reported no libmoq.a for libmoq" >&2 - exit 1 - fi - # The header sits beside the profile directory: rs/libmoq/build.rs writes it - # to `/include`, one level above `//libmoq.a`. - profile=$(dirname "$lib") - include="$(dirname "$profile")/include" - if [ ! -f "$include/moq.h" ]; then - echo "$include/moq.h missing after 'cargo build --locked -p libmoq'" >&2 - exit 1 - fi - - # Same list build.rs (moq.pc), CMakeLists.txt, and test/interop/interop.sh read: - # `framework:Foo` is a linker framework flag, anything else a plain library. - case "$(uname -s)" in - Darwin) native_libs=rs/libmoq/native-libs/apple.txt ;; - *) native_libs=rs/libmoq/native-libs/linux.txt ;; - esac - libs=() - while read -r entry; do - case "$entry" in - '' | '#'*) continue ;; - framework:*) libs+=(-framework "${entry#framework:}") ;; - *) libs+=("-l$entry") ;; - esac - done <"$native_libs" - - out=$(mktemp -d) - trap 'rm -rf "$out"' EXIT - for source in rs/libmoq/c-tests/*.c; do - bin="$out/$(basename "$source" .c)" - "$cc" "$source" -I"$include" -L"$profile" -lmoq "${libs[@]}" -o "$bin" - echo "running $source" - "$bin" - done - -# The feature permutations `check` leaves out. Each is a full workspace compile -# at a feature set that shares almost no artifacts with the default one (~6min -# for the all-features clippy alone), which is why this is nightly rather than -# part of the per-PR gate. -# -# `--all-features` is the only thing that compiles moq-cli's play/capture, the -# moq-video and moq-audio device features, the Linux bindgen codecs (vaapi, -# v4l2), PipeWire, and jemalloc. `--no-default-features` is -# the only thing that compiles the `#[cfg(not(feature = ...))]` arms. -# -# Cargo unifies features across a workspace selection. Check each package alone -# to catch missing dependencies in minimal builds. The nightly sweep adds each -# feature to aws-lc-rs; tokio-features checks provider-free transports, standalone -# providers, and the intentional refusal of rustls QUIC without a provider. -# -# Both carry `--all-targets`. A `#[cfg(test)]` module leans on whatever the -# default features happened to import, so a test that only compiles by accident -# is invisible to a lib-only check. + sh/rs/c-tests.sh + +# The feature permutations `check` leaves out, nightly. `--all-features` is the +# only thing that compiles moq-cli's play/capture, the device features, the +# Linux bindgen codecs, PipeWire, and jemalloc; `--no-default-features` the only +# thing that compiles the `#[cfg(not(feature = ...))]` arms. # Compile the workspace at the feature extremes. features: @@ -743,162 +150,47 @@ features: # Compile the media crates at supported feature boundaries and prove expensive # native/render dependencies are absent when their features are excluded. media-features: - rs/scripts/media-features.sh + sh/rs/media-features.sh -# nextest, not `cargo test`, so a wedged test is killed instead of pinning a core -# until someone notices (`.config/nextest.toml` sets the timeout). The trade is -# doctests, which nextest does not execute; `just rs doctest` covers those. -# -# Takes cargo arguments, so `just rs test -p moq-net` works. `test-changed` is -# the diff-scoped variant that `just test` dispatches to. -# -# clippy-driver as the compiler wrapper is rustc plus the lint passes, so this -# one compilation both builds the test binaries and reports clippy findings. -# `cargo clippy` can't be reused here: it asks cargo for metadata only, and a -# metadata build and a codegen build have different fingerprints, so running -# both compiles the dependency tree twice. Linting here is free, which makes -# `just test` a single-compile substitute for `check` + `test` on a laptop. -# -# RUSTC_WORKSPACE_WRAPPER, not RUSTC_WRAPPER, for two reasons. Cargo folds it -# into the fingerprint, so a crate already built by a plain `cargo test` is -# rebuilt through clippy rather than silently reported as fresh with no -# findings. And it applies to workspace crates only, so dependencies keep the -# artifacts they share with plain cargo instead of duplicating the whole tree. -# It is also what `cargo clippy` sets internally. -# -# Findings stay warnings rather than errors. `check` is the gate (it runs -# `clippy -D warnings`), and a hard failure here would block running the tests -# over an unused import while you are mid-edit. -# -# No RUSTFLAGS: it is part of cargo's fingerprint too, so setting it here would -# split the artifact cache against every other cargo invocation, including CI's -# warm cache. Skips the wrapper entirely when clippy-driver is absent. +# nextest kills a wedged test (`.config/nextest.toml` sets the timeout), but +# skips doctests; `just rs doctest` covers those. clippy-driver as the workspace +# wrapper reports lint findings from the same compile, as warnings: `check` is +# the gate. It is part of cargo's fingerprint, and RUSTFLAGS would split the +# artifact cache, which is why neither is set any other way. # Run the test suite, reporting clippy findings from the same compile. test *args: RUSTC_WORKSPACE_WRAPPER="$(command -v clippy-driver || true)" cargo nextest run --locked --all-targets {{ args }} -# Exercise the Vulkan producer -> CUDA image contract, the GPU color conversion -# and resize, and NVENC encoding of the result on Linux/NVIDIA hardware. The -# driver directory is added explicitly because the Nix shell's dynamic loader -# path does not include Ubuntu's host driver directory. +# Exercise Vulkan -> CUDA -> NVENC on Linux/NVIDIA hardware. vulkan-cuda: - #!/usr/bin/env bash - set -euo pipefail - driver=$(/usr/sbin/ldconfig -p | awk '/libcuda\.so\.1/{print $NF; exit}') - if [[ -z "$driver" ]]; then - echo "libcuda.so.1 is not installed" >&2 - exit 1 - fi - driver_dir=$(dirname "$driver") - export LD_LIBRARY_PATH="$driver_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - cargo nextest run --locked -p moq-video --run-ignored only -E 'test(/^frame::(vulkan|cuda)::tests::vulkan_cuda_/)' - -# Same selection as `check-changed`: compiling a crate's test binaries is the -# expensive half of a test run, so scoping matters more here than anywhere else. -# -# `--no-tests=pass` only on the scoped branch. A selection can legitimately hold -# nothing testable (moq-wasm has no host-target tests, since its crate root is -# `#![cfg(target_arch = "wasm32")]`), and nextest exits 4 on that by default. -# The unscoped branch keeps the default, where finding no tests really is wrong. - -# Like `test`, but only the crates a diff touches plus their dependents. -test-changed $FILES: - #!/usr/bin/env bash - set -euo pipefail - packages=$(just rs _select "$FILES") - case "$packages" in - "") echo "rs: no crates affected; skipping tests." ;; - ALL) just rs test --workspace {{ no_fuzz }} ;; - *) echo "rs: testing $(just rs _names "$packages")"; just rs test --no-tests=pass $packages ;; - esac - - if [[ "$packages" == "ALL" ]] || just rs _names "$packages" | grep -qw moq-relay; then - just rs relay-minimal - fi + sh/rs/vulkan-cuda.sh # Compile and run the `/// ```` examples, which nextest skips. doctest *args: cargo test --locked --doc {{ args }} +# `--cfg loom` swaps kio's primitives for loom's, so it rebuilds the world and +# stays out of `check`. `--release` because a model check runs the body once +# per interleaving (411s -> 51s); the assertions are plain `assert!`. kio's own +# unit tests are `cfg(not(loom))`. + # Permutation-test the concurrent handoffs with loom. -# -# `--cfg loom` swaps kio's Mutex/atomics for loom's instrumented ones, so it -# rebuilds the world and can't share artifacts with a normal `cargo test`, so -# it stays separate from `check`/`ci`. -# -# `--release` because a model check runs the body once per interleaving, so the -# optimizer pays for itself many times over: 411s -> 51s across the two suites. -# Our assertions are plain `assert!`, not `debug_assert!`, so nothing is -# compiled out (verified by re-running the mutation check in release). -# -# The search is deliberately unbounded (no `preemption_bound`), so it covers -# every interleaving rather than just the short ones. -# -# kio's own unit tests are `cfg(not(loom))`: they'd construct loom primitives -# outside `loom::model`, which panics. loom *args: RUSTFLAGS="--cfg loom" cargo test --locked --release -p kio --lib loom:: {{ args }} RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test loom {{ args }} -# Coverage-guided fuzzing of the moq-net wire codecs (libFuzzer, via cargo-fuzz). -# -# The one recipe that steps off the pinned toolchain: libFuzzer needs `-Zsanitizer`, -# which is nightly-only, so this stays out of `check` and `test` the same way `loom` -# does. What CI gets instead is the replay: the target bodies live in moq-net's hidden -# `fuzz` module, and its own tests run the generated corpus plus everything committed -# under `fuzz/regressions/` on stable. A crash found here becomes a regression test -# nobody has to install cargo-fuzz to run. -# -# Seeds are regenerated first so the corpus follows the dispatch rather than a stale -# run, and libFuzzer writes what it discovers to the FIRST corpus directory, which is -# why the generated seeds are passed second and stay read-only. -# -# TARGET is lite, ietf, varint, or path. Arguments pass through, so -# `just rs fuzz lite -- -max_total_time=300` bounds a run. See rs/moq-net/fuzz/README.md. -fuzz target *args: - #!/usr/bin/env bash - set -euo pipefail - - if ! command -v cargo-fuzz > /dev/null 2>&1; then - echo "rs: cargo-fuzz is missing: cargo install --locked cargo-fuzz" >&2 - exit 1 - fi - - if ! rustup run nightly rustc --version > /dev/null 2>&1; then - echo "rs: the nightly toolchain is missing: rustup toolchain install nightly" >&2 - exit 1 - fi - - cargo run --locked -q -p moq-net --features fuzz --example fuzz-seeds -- rs/moq-net/fuzz/seeds - mkdir -p rs/moq-net/fuzz/corpus/{{ target }} +# Needs nightly and cargo-fuzz; CI replays the corpus on stable through +# moq-net's own tests instead. See rs/moq-net/fuzz/README.md. - # Nightly by PATH rather than by `cargo +nightly`: the dev shell puts a - # Nix-provided cargo ahead of the rustup shim, so the `+` form is not - # understood, and cargo-fuzz shells out to a bare `cargo` and `rustc` anyway. - # Both have to be the nightly ones or the sanitizer flags are rejected. - export PATH="$(rustup run nightly rustc --print sysroot)/bin:$PATH" - - cargo fuzz run --fuzz-dir rs/moq-net/fuzz {{ target }} \ - rs/moq-net/fuzz/corpus/{{ target }} rs/moq-net/fuzz/seeds/{{ target }} {{ args }} +# Fuzz the moq-net wire codecs: `just rs fuzz lite -- -max_total_time=300`. +fuzz target *args: + sh/rs/fuzz.sh {{ target }} {{ args }} +# Build the default workspace members. build: cargo build --locked -# Run a moq-bench preset against a relay, e.g. `just rs bench chat https://relay.example.com`. -# Extra args pass through, so `--connections 500 --output stats.jsonl` work as-is. -bench preset url *args: - cargo run --release -p moq-bench -- --file 'rs/moq-bench/config/{{ preset }}.toml' --connect '{{ url }}' {{ args }} - -# Measure the Linux Tokio/epoll UDP syscall path. Extra args pass to Criterion, -# e.g. `just rs bench-udp-tokio --quick`. -bench-udp-tokio *args: - cargo bench -p moq-uring --bench udp_tokio -- {{ args }} - -# Measure the Linux io_uring UDP completion path with multishot recvmsg. -bench-udp-uring *args: - cargo bench -p moq-uring --bench udp_uring -- {{ args }} - # Run the comparable epoll and io_uring UDP matrices. bench-udp *args: cargo bench -p moq-uring --bench udp_tokio -- {{ args }} @@ -915,71 +207,11 @@ bench-echo *args: bench-session *args: cargo bench -p moq-uring --bench session_lite -- {{ args }} -# Sample a process's CPU/memory/context switches on this host (run it where the -# relay runs; it only reads /proc). E.g. `just rs bench-host --name moq-relay`. -bench-host *args: - cargo run --release -p moq-bench --bin moq-bench-host -- {{ args }} - -# Remove the Rust target directory. -clean: - cargo clean - -# Check semver compatibility against crates.io (default-members only). -semver: - cargo semver-checks check-release - -# Update versions and changelogs via release-plz. -bump: - release-plz update - # Create release PRs and publish crates via release-plz. release: release-plz release-pr --git-token "$GITHUB_TOKEN" release-plz release --git-token "$GITHUB_TOKEN" -update: - cargo update - cargo upgrade --incompatible - -# Build a .deb or .rpm for one of the Rust binaries locally. nfpm comes -# from the flake's dev shell (`nix develop`). For .rpm produced this way, -# the linkage matches the host's glibc; CI uses an AlmaLinux 9 container -# to produce broadly compatible artifacts instead. -# -# Examples: -# just rs package moq-relay deb - -# just rs package moq-cli rpm +# Build a .deb or .rpm locally: `just rs package moq-relay deb`. package crate packager: - #!/usr/bin/env bash - set -euo pipefail - case "{{ crate }}" in - moq-relay) bin=moq-relay ;; - moq-cli) bin=moq ;; - *) echo "Unknown crate: {{ crate }} (use moq-relay or moq-cli)" >&2; exit 1 ;; - esac - case "{{ packager }}" in - deb) - if command -v dpkg >/dev/null 2>&1; then - arch=$(dpkg --print-architecture) - else - case "$(uname -m)" in - x86_64) arch=amd64 ;; - aarch64|arm64) arch=arm64 ;; - *) echo "Cannot infer deb arch from host $(uname -m)" >&2; exit 1 ;; - esac - fi - ;; - rpm) arch=$(uname -m) ;; - *) echo "Unknown packager: {{ packager }} (use deb or rpm)" >&2; exit 1 ;; - esac - version=$(grep -m1 '^version' rs/{{ crate }}/Cargo.toml | sed 's/.*"\(.*\)".*/\1/') - cargo build --locked --release -p {{ crate }} - mkdir -p dist - VERSION="$version" ARCH="$arch" BINARY_PATH="target/release/$bin" \ - rs/scripts/package-nfpm.sh packaging/{{ crate }}/nfpm.yaml {{ packager }} dist/ - if [[ "{{ packager }}" == deb && -f "packaging/{{ crate }}/transition.yaml" ]]; then - VERSION="$version" ARCH="$arch" \ - rs/scripts/package-nfpm.sh packaging/{{ crate }}/transition.yaml deb dist/ - fi - ls -1 dist/ + sh/rs/package.sh {{ crate }} {{ packager }} diff --git a/rs/moq-gst/package.sh b/rs/moq-gst/package.sh index b48f392eb20..85ff0731b67 100755 --- a/rs/moq-gst/package.sh +++ b/rs/moq-gst/package.sh @@ -145,7 +145,7 @@ mkdir -p "$OUTPUT_DIR" echo ">> Running nfpm ($PACKAGER)..." export VERSION ARCH="$PKG_ARCH" PKG_NAME PLUGIN_PATH="$BUILT_SO" PLUGIN_DIR -"$WORKSPACE_DIR/rs/scripts/package-nfpm.sh" \ +"$WORKSPACE_DIR/sh/rs/package-nfpm.sh" \ "$WORKSPACE_DIR/packaging/moq-gst/nfpm.yaml" "$PACKAGER" "$OUTPUT_DIR/" echo ">> Done. Artifacts in: $OUTPUT_DIR" diff --git a/rs/moq-gst/scrub.sh b/rs/moq-gst/scrub.sh index 5d4699deee1..09ec56ad34a 100755 --- a/rs/moq-gst/scrub.sh +++ b/rs/moq-gst/scrub.sh @@ -40,7 +40,7 @@ scrub_macos() { # Framework path covers the official .pkg installer, and /usr/lib lets # dyld resolve system libs (libiconv, libc++) via the dyld_shared_cache at # @rpath substitution time. - "$SCRIPT_DIR/../scripts/scrub-macho.sh" "$dylib" \ + "$SCRIPT_DIR/../../sh/rs/scrub-macho.sh" "$dylib" \ /opt/homebrew/lib \ /Library/Frameworks/GStreamer.framework/Libraries \ /usr/lib diff --git a/rs/moq-wasm/README.md b/rs/moq-wasm/README.md index 7a0609c710b..34a90b2ccc5 100644 --- a/rs/moq-wasm/README.md +++ b/rs/moq-wasm/README.md @@ -7,7 +7,7 @@ wire implementation in `@moq/net` (~10k LOC) with the canonical Rust one, so the protocol lives in exactly one place. This crate is the Rust half; the generated JS package is -[`@moq/wasm`](../../js/wasm) (`just wasm` builds it). It is **not** the same as +[`@moq/wasm`](../../js/wasm) (`just js wasm` builds it). It is **not** the same as `moq-ffi`: that crate uses UniFFI, which targets the C ABI (Kotlin/Swift/Python/ Go). Browsers need `wasm-bindgen`, so this is a separate sibling crate. (For *React Native* JS, `uniffi-bindgen-react-native` can reuse `moq-ffi` directly; @@ -66,7 +66,7 @@ dependency is commented out in `Cargo.toml` until then. ## Building -`just wasm` (from the repo root) does everything: builds for wasm and runs +`just js wasm` (from the repo root) does everything: builds for wasm and runs `wasm-bindgen` (web target) into `js/wasm/dist`. The wasm target, the cfg flags (`getrandom` wasm backend + web-sys unstable WebTransport APIs), and the `wasm-bindgen-cli` tool come from `.cargo/config.toml` and the Nix dev shell. diff --git a/rs/scripts/doc-names.jq b/rs/scripts/doc-names.jq deleted file mode 100644 index 5dfad17c57c..00000000000 --- a/rs/scripts/doc-names.jq +++ /dev/null @@ -1,31 +0,0 @@ -# Find workspace targets that would render to the same `target/doc/`. -# -# `cargo doc` writes each documented target to a directory named after the target with -# dashes folded to underscores, so two targets that differ only there collide: both -# rustdoc processes race to clean it, and the warning escalates to a hard error -# depending on which wins. -# -# Reads `cargo metadata --no-deps` and prints one line per colliding group. -[ .packages[] - | . as $pkg - # Cargo skips a binary that would land on the same directory as a library in its own - # package, documenting only the library, so such a pair never actually collides. - # Compared after folding, matching cargo: a `foo_bar` lib beside a `foo-bar` bin is - # suppressed too (verified against cargo 1.95). Only cross-package pairs collide. - | [ $pkg.targets[] - | select([.kind[]] | any(. == "lib" or . == "rlib" or . == "dylib" - or . == "cdylib" or . == "staticlib" or . == "proc-macro")) - | (.name | gsub("-"; "_")) - ] as $libs - | $pkg.targets[] - # `doc = false` in the manifest, which is how a reported pair gets resolved. - | select(.doc) - | select([.kind[]] | any(. == "lib" or . == "rlib" or . == "dylib" - or . == "cdylib" or . == "staticlib" or . == "proc-macro" or . == "bin")) - | select((([.kind[]] | any(. == "bin")) and ((.name | gsub("-"; "_")) | IN($libs[]))) | not) - | { dir: (.name | gsub("-"; "_")), package: $pkg.name, target: .name, kind: .kind[0] } -] -| group_by(.dir) -| map(select(length > 1)) -| .[] -| " target/doc/\(.[0].dir) <- " + (map("\(.package) [\(.kind)] \(.target)") | join(" and ")) diff --git a/rs/scripts/package-nfpm.test.sh b/rs/scripts/package-nfpm.test.sh deleted file mode 100755 index 5582669426e..00000000000 --- a/rs/scripts/package-nfpm.test.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if ! command -v envsubst >/dev/null 2>&1; then - exit 0 -fi - -scratch=$(mktemp -d) -trap 'rm -rf "$scratch"' EXIT -mkdir -p "$scratch/bin" "$scratch/capture" - -cat >"$scratch/bin/nfpm" <<'SH' -#!/usr/bin/env bash -set -euo pipefail - -while (($#)); do - case "$1" in - --config) - config=$2 - shift 2 - ;; - --packager) - printf '%s\n' "$2" >"$NFPM_CAPTURE/packager" - shift 2 - ;; - --target) - printf '%s\n' "$2" >"$NFPM_CAPTURE/target" - shift 2 - ;; - *) shift ;; - esac -done - -cp "$config" "$NFPM_CAPTURE/config.yaml" -SH -chmod +x "$scratch/bin/nfpm" - -cat >"$scratch/nfpm.yaml" <<'YAML' -name: test -version: ${VERSION} -arch: ${ARCH} -contents: - - src: ${BINARY_PATH} - dst: /usr/bin/test -YAML - -PATH="$scratch/bin:$PATH" \ - NFPM_CAPTURE="$scratch/capture" \ - VERSION=1.2.3 \ - ARCH=amd64 \ - BINARY_PATH=target/release/test \ - rs/scripts/package-nfpm.sh "$scratch/nfpm.yaml" deb "$scratch/dist" - -grep -qx 'version: 1.2.3' "$scratch/capture/config.yaml" -grep -qx 'arch: amd64' "$scratch/capture/config.yaml" -grep -qx ' - src: target/release/test' "$scratch/capture/config.yaml" -grep -qx 'deb' "$scratch/capture/packager" -grep -qx "$scratch/dist" "$scratch/capture/target" - -if PATH="$scratch/bin:$PATH" \ - NFPM_CAPTURE="$scratch/capture" \ - VERSION=1.2.3 \ - ARCH=amd64 \ - rs/scripts/package-nfpm.sh "$scratch/nfpm.yaml" deb "$scratch/dist" 2>"$scratch/missing.log"; then - echo "package-nfpm accepted a missing BINARY_PATH" >&2 - exit 1 -fi -grep -qx "Missing environment variable in $scratch/nfpm.yaml: BINARY_PATH" "$scratch/missing.log" diff --git a/sh/clean.sh b/sh/clean.sh new file mode 100755 index 00000000000..24f495cafe7 --- /dev/null +++ b/sh/clean.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Delete build artifacts and caches: sh/clean.sh [here|all] +# +# Only this checkout by default. Agent worktrees each carry their own +# artifacts, and another agent is usually building in one: `cargo clean` under +# a running build fails it, and there is no telling a finished worktree from a +# busy one from out here. `all` is the explicit opt-in for an idle machine. +# +# Source is never touched, dirty or untracked: this deletes build output, not +# work. Nothing here reaches a machine-wide store (Nix, cargo/bun/uv home +# caches), because those are shared with every other checkout. +set -euo pipefail + +scope=${1:-here} +case "$scope" in + here | all) ;; + *) + echo "usage: sh/clean.sh [here|all]" >&2 + exit 2 + ;; +esac + +cd "$(git rev-parse --show-toplevel)" + +# Rust. +cargo clean + +# JS: the bun workspace spans the repo, so sweep from the root. +find . -name .claude -prune -o \ + -type d \( -name node_modules -o -name dist -o -name out -o -name pkg \) \ + -prune -exec rm -rf {} + +find . -name .claude -prune -o -type f -name '*.tsbuildinfo' -exec rm -f {} + + +# Python: the venv, release dist, bytecode, and the bindings maturin drops in. +rm -rf .venv py/.venv py/dist py/moq-ffi/moq_ffi/_uniffi +find py -name .claude -prune -o -type d -name __pycache__ -prune -exec rm -rf {} + +find py -name .claude -prune -o -type f -name '*.pyc' -exec rm -f {} + + +# Kotlin: gradle output plus the bindings and native libs sh/kt generates. +find kt -type d \( -name build -o -name .gradle -o -name .kotlin \) -prune -exec rm -rf {} + +rm -rf kt/local.properties \ + kt/moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi \ + kt/moq-ffi/src/jvmMain/resources \ + kt/moq-ffi/src/androidMain/jniLibs + +# Swift: SwiftPM output plus the XCFramework and bindings sh/swift lays down. +rm -rf swift/.build swift/.swiftpm swift/Package.resolved \ + swift/Sources/MoqFFI/Generated.swift swift/MoqFFI.xcframework + +# Go: the gitignored bindings, native libs, and vendoring in go/. +rm -rf go/ffi/moq/moq.go go/ffi/moq/moq.h go/ffi/moq/lib go/ffi/go.sum \ + go/ffi/vendor go/wrapper/go.sum go/wrapper/vendor + +# Dart: tool state and build output. +rm -rf dart/moq/.dart_tool dart/moq_ffi/.dart_tool dart/moq/build dart/moq_ffi/build + +# Rendered drafts and their reference cache. +rm -rf drafts/draft-*.xml drafts/draft-*.txt drafts/draft-*.html drafts/.refcache + +# Caches no one language owns: nix build result, direnv, wrangler. +rm -rf result .direnv +find . -name .claude -prune -o -type d -name .wrangler -prune -exec rm -rf {} + + +# Worktrees don't nest, so this recurses exactly one level. Tolerate stale +# worktrees on branches that predate this script. +if [[ "$scope" == all ]]; then + for wt in .claude/worktrees/*/; do + [[ -f "${wt}justfile" ]] || continue + echo "==> cleaning ${wt}" + (cd "$wt" && just clean) || echo " (skipped: just clean failed in ${wt})" + done +fi diff --git a/dart/scripts/check.sh b/sh/dart/check.sh similarity index 88% rename from dart/scripts/check.sh rename to sh/dart/check.sh index 5e658e9b187..eaa4f579bef 100755 --- a/dart/scripts/check.sh +++ b/sh/dart/check.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) ACTION=check if [[ "${1:-}" == --fix ]]; then @@ -9,13 +9,6 @@ if [[ "${1:-}" == --fix ]]; then shift fi -# `just check` passes the whole changed-file list, so match the scope anywhere in -# it rather than only at the start. -files="${1:-}" -if [[ -n "$files" ]] && ! grep -qE '(^|[[:space:]])(dart/|rs/moq-ffi/)' <<<"$files"; then - exit 0 -fi - for tool in cargo dart uniffi_bindgen_dart; do if ! command -v "$tool" >/dev/null 2>&1; then echo "dart check: skipping, $tool is not installed" >&2 @@ -58,7 +51,7 @@ done generated=$(mktemp -d) trap 'rm -rf "$generated"' EXIT -"$DART_DIR/scripts/generate.sh" "$generated" +"$DART_DIR/../sh/dart/generate.sh" "$generated" for file in moq.dart uniffi_runtime.dart; do if ! diff -u -B "$DART_DIR/moq_ffi/lib/src/$file" "$generated/$file"; then diff --git a/dart/scripts/generate.sh b/sh/dart/generate.sh similarity index 96% rename from dart/scripts/generate.sh rename to sh/dart/generate.sh index bbbe65ad976..7a2c9c24fc3 100755 --- a/dart/scripts/generate.sh +++ b/sh/dart/generate.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) WORKSPACE_DIR=$(cd "$DART_DIR/.." && pwd) OUTPUT_DIR="${1:-$DART_DIR/moq_ffi/lib/src}" diff --git a/dart/scripts/package.sh b/sh/dart/package.sh similarity index 97% rename from dart/scripts/package.sh rename to sh/dart/package.sh index 21c4d7b135c..cb984776957 100755 --- a/dart/scripts/package.sh +++ b/sh/dart/package.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) PACKAGE="" VERSION="" OUTPUT_DIR="dist" diff --git a/dart/scripts/publish.sh b/sh/dart/publish.sh similarity index 84% rename from dart/scripts/publish.sh rename to sh/dart/publish.sh index 6a6caaa50e8..1348e0ec64b 100755 --- a/dart/scripts/publish.sh +++ b/sh/dart/publish.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) PACKAGE="${1:?usage: publish.sh }" VERSION="${2:?usage: publish.sh }" OUTPUT=$(mktemp -d) trap 'rm -rf "$OUTPUT"' EXIT package_dir=$( - "$DART_DIR/scripts/package.sh" \ + "$DART_DIR/../sh/dart/package.sh" \ --package "$PACKAGE" \ --version "$VERSION" \ --output "$OUTPUT" diff --git a/sh/dispatch.sh b/sh/dispatch.sh new file mode 100755 index 00000000000..7cfcc9d6a0f --- /dev/null +++ b/sh/dispatch.sh @@ -0,0 +1,194 @@ +#!/usr/bin/env bash +# Run `just check`, `just fix`, or `just test` over what the branch changed. +# +# Usage: sh/dispatch.sh check|fix|test [BASE|--all] +# +# The branch diff is resolved once and matched against the impact map below, +# the one place that says which paths put which module in scope. CI runs the +# same thing with MOQ_STRICT=1, so there is no second definition of "checked". +set -euo pipefail + +usage="usage: sh/dispatch.sh check|fix|test [BASE|--all]" +action=${1:?$usage} +base=${2:-} +case "$action" in + check | fix | test) ;; + *) + echo "$usage" >&2 + exit 2 + ;; +esac + +cd "$(git rev-parse --show-toplevel)" + +changed=$(mktemp) +trap 'rm -f "$changed"' EXIT + +all= +if [[ "$base" == --all ]]; then + all=1 +else + # BASE: the argument, then $GITHUB_BASE_REF (a PR checkout has no + # upstream), then the upstream, then origin/main. `git push -u` points the + # upstream at the branch's own remote copy, which would diff HEAD against + # itself, so that case falls through to origin/main. + if [[ -z "$base" && -n "${GITHUB_BASE_REF:-}" ]]; then + base="origin/$GITHUB_BASE_REF" + fi + if [[ -z "$base" ]]; then + base=$(git rev-parse --abbrev-ref '@{upstream}' 2>/dev/null || true) + if [[ -z "$base" || "$base" == */"$(git branch --show-current)" ]]; then + base=origin/main + fi + fi + merge_base=$(git merge-base "$base" HEAD) || { + echo "error: cannot resolve merge-base against $base (is full history fetched?)" >&2 + exit 1 + } + echo "$action: base $base" >&2 + + # Untracked files count too: a brand new crate or module is the whole change. + { + git diff --name-only "$merge_base" + git ls-files --others --exclude-standard + } | sort -u >"$changed" + + # These files hold the map and the recipes that call it, and match no + # module, so a change to them would otherwise validate none of it. + if grep -qE '^(justfile|test/justfile|sh/dispatch\.sh)$' "$changed"; then + echo "$action: root orchestration changed; running everything." >&2 + all=1 + fi +fi + +# The impact map: a module is in scope when a changed path matches its +# pattern. An empty pattern is a repository-wide lint that runs on every diff. +declare -A scope=( + [js]='^(js/|doc/|drafts/|demo/(boy|web)/|test/interop/clients/js|test/wasm/|sh/js/|sh/rs/stats-docs\.py$|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' + # Workers with lockfiles outside the Bun workspace. + [workers]='^(infra/apt/|infra/rpm/|demo/pub/|sh/js/workers\.sh$)' + # sh/rs/select.sh widens to the whole workspace for inputs every crate shares. + [rs]='^(rs/|sh/rs/|Cargo\.(toml|lock)$|rust-toolchain\.toml$|\.config/nextest\.toml$)' + [bench]='^bench/' + [drafts]='^(drafts/|sh/drafts/|doc/\.vitepress/drafts\.ts$)' + # Quest documents form one graph, so any change validates the whole tree. + [quest]='^(quest/|rs/quest/)' + # maturin bundles rs/moq-ffi into the moq-ffi wheel, and the other + # bindings generate from it. + [py]='^(py/|pyproject\.toml$|uv\.lock$|rs/moq-ffi/)' + [kt]='^(kt/|sh/kt/|rs/moq-ffi/)' + [swift]='^(swift/|sh/swift/|rs/moq-ffi/)' + [go]='^(go/|sh/go/|rs/moq-ffi/)' + [dart]='^(dart/|sh/dart/|rs/moq-ffi/)' + # The plugin calls libmoq through its generated header, and flake.nix owns + # the libobs headers it compiles against. + [obs_compile]='^(cpp/obs/|sh/obs/|rs/libmoq/|flake\.nix$)' + # `obs check` compares the OBS pinned in buildspec.json, flake.nix, and + # nixpkgs, and the last moves on a flake.lock bump alone. + [obs]='^(cpp/obs/|sh/obs/|flake\.(nix|lock)$)' + [flake]='(^rs/|^Cargo\.(toml|lock)$|^flake\.lock$|\.nix$)' + [markdown]='' + [shell]='' + [toml]='' + [nix]='' + [justfile]='' + [gh]='' +) + +# The tools each module needs. A module missing one is skipped locally, so an +# incomplete toolchain checks less; under MOQ_STRICT (CI) it is an error, up +# front, because there a skip is indistinguishable from a pass. swift needs +# none: it skips off macOS by design, and swift.yml is its real gate. +declare -A tools=( + [js]='bun python3' + [workers]='bun' + [rs]='cargo jq' + [bench]='cargo' + [drafts]='bun kramdown-rfc xml2rfc' + [quest]='cargo' + [py]='uv' + [kt]='cargo gradle java' + [swift]='' + [go]='cargo go uniffi-bindgen-go' + [dart]='cargo dart uniffi_bindgen_dart' + [obs_compile]='cargo jq pkg-config' + [obs]='clang-format cmake gersemi jq' + [flake]='nix' + [markdown]='bun' + [shell]='shellcheck shfmt' + [toml]='taplo' + [nix]='nixfmt' + [justfile]='' + [gh]='actionlint bun' +) + +case "$action" in + check) modules=(js workers drafts rs bench quest py kt swift go dart obs_compile obs flake markdown shell toml nix justfile gh) ;; + fix) modules=(js rs py dart obs markdown shell toml nix justfile) ;; + test) modules=(js rs py) ;; +esac + +selected=() +missing=() +for module in "${modules[@]}"; do + pattern=${scope[$module]} + if [[ -z "$all" && -n "$pattern" ]] && ! grep -qE "$pattern" "$changed"; then + continue + fi + absent=() + for tool in ${tools[$module]}; do + command -v "$tool" >/dev/null 2>&1 || absent+=("$tool") + done + if ((${#absent[@]} == 0)); then + selected+=("$module") + elif [[ -n "${MOQ_STRICT:-}" ]]; then + missing+=("${absent[@]}") + else + echo "$action: skipping $module; missing ${absent[*]}" >&2 + fi +done + +if ((${#missing[@]})); then + echo "error: MOQ_STRICT is set but these tools are missing: $(printf '%s\n' "${missing[@]}" | sort -u | tr '\n' ' ')" >&2 + echo " run inside 'nix develop', or unset MOQ_STRICT to skip what isn't installed" >&2 + exit 1 +fi + +# rs selects crates from the same list, or the whole workspace. +rs_list=$changed +[[ -z "$all" ]] || rs_list=--all + +# Tracked and new files, skipping whatever .gitignore does (node_modules, target). +files() { + git ls-files -z --cached --others --exclude-standard -- "$@" +} + +for module in "${selected[@]}"; do + case "$action:$module" in + check:js) just js check ;; + check:workers) just js workers ;; + check:rs) just rs check-changed "$rs_list" ;; + check:bench) cargo check --locked --package moq-relay --package moq-bench --features moq-relay/io-uring ;; + check:quest) cargo run --quiet --locked --package quest -- check ;; + check:obs_compile) just obs compile ;; + check:flake) nix flake check ;; + check:markdown) + bun install --frozen-lockfile + sh/markdown.sh check + ;; + check:toml) RUST_LOG=error taplo format --check ;; + check:nix) files '*.nix' | xargs -0 nixfmt --check ;; + check:justfile) files justfile '*/justfile' | xargs -0 -n1 just --fmt --check --justfile ;; + fix:rs) just rs fix-changed "$rs_list" ;; + fix:markdown) + bun install + sh/markdown.sh fix + ;; + fix:toml) RUST_LOG=error taplo format ;; + fix:nix) files '*.nix' | xargs -0 nixfmt ;; + fix:justfile) files justfile '*/justfile' | xargs -0 -n1 just --fmt --justfile ;; + test:rs) just rs test-changed "$rs_list" ;; + *:shell) sh/shell.sh "$action" ;; + *) just "$module" "$action" ;; + esac +done diff --git a/sh/drafts/build.sh b/sh/drafts/build.sh new file mode 100755 index 00000000000..e050b63a3c1 --- /dev/null +++ b/sh/drafts/build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Render drafts to .txt and .html: sh/drafts/build.sh [NAME...] +# +# The local editor's copy (gitignored), with the -latest docname as-is; use +# `just drafts publish` for a version. No NAME renders every draft. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)/drafts" + +names=("$@") +if ((${#names[@]} == 0)); then + for f in draft-*.md; do names+=("${f%.md}"); done +fi + +for name in "${names[@]}"; do + kramdown-rfc --v3 <"$name.md" >"$name.xml" + xml2rfc -q --allow-local-file-access --text "$name.xml" -o "$name.txt" + xml2rfc -q --allow-local-file-access --html "$name.xml" -o "$name.html" + echo "built $name.txt and $name.html" +done diff --git a/sh/drafts/check.sh b/sh/drafts/check.sh new file mode 100755 index 00000000000..15ad5bd6c0c --- /dev/null +++ b/sh/drafts/check.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Validate that every draft parses and that its generated XML passes validation. +# +# Both halves are needed. kramdown-rfc catches markdown and frontmatter errors, +# but happily emits XML for a cross-reference that points at no such section; +# xml2rfc is what rejects the dangling IDREF. Checking only the first half lets +# a broken `[text](#anchor)` pass here and then fail at `publish` time. +# +# --preptool stops after validation instead of rendering txt/html, and -N keeps +# it off the network (kramdown-rfc already inlined the references), which is +# what makes this a couple of seconds per draft rather than a minute. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" +bun install --frozen-lockfile +cd drafts + +for f in draft-*.md; do + echo "checking $f" + kramdown-rfc --v3 <"$f" >"${f%.md}.xml" + xml2rfc -q --allow-local-file-access --preptool -N "${f%.md}.xml" -o "${f%.md}.prepped.xml" || { + rm -f "${f%.md}.prepped.xml" + exit 1 + } + rm -f "${f%.md}.prepped.xml" +done + +for f in moq-e2ee-*.ts; do + bun tsc --noEmit --skipLibCheck --target esnext --module esnext --moduleResolution bundler --types bun "$f" + bun "$f" +done diff --git a/sh/drafts/publish.sh b/sh/drafts/publish.sh new file mode 100755 index 00000000000..9ef5e8ad5c0 --- /dev/null +++ b/sh/drafts/publish.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Submit a new draft version to the IETF datatracker: sh/drafts/publish.sh NAME VERSION EMAIL +# +# The datatracker emails the submitter a confirmation link; the submission is +# not final until that link is clicked. For a brand-new draft (-00) set +# "Replaces" on the confirmation page. +set -euo pipefail + +usage="usage: sh/drafts/publish.sh NAME VERSION EMAIL" +name=${1:?$usage} +version=${2:?$usage} +email=${3:?$usage} + +cd "$(git rev-parse --show-toplevel)/drafts" +case "$version" in + [0-9][0-9]) ;; + *) + echo "version must be two digits, e.g. 05" >&2 + exit 1 + ;; +esac +doc="$name-$version" +if [ ! -f "$name.md" ]; then + echo "no such draft: $name.md" >&2 + exit 1 +fi +echo "Building $doc.xml" +sed "s/$name-latest/$doc/g" "$name.md" | kramdown-rfc --v3 >"$doc.xml" +echo "Submitting $doc.xml to the datatracker as $email" +resp="$(mktemp)" +code="$(curl -sS -o "$resp" -w '%{http_code}' \ + -F "user=$email" -F "xml=@$doc.xml" \ + https://datatracker.ietf.org/api/submission)" +echo "HTTP $code" +cat "$resp" +echo +rm -f "$resp" +case "$code" in + 200 | 201) echo "Submitted. Check $email for the confirmation link." ;; + *) + echo "Submission failed." >&2 + exit 1 + ;; +esac diff --git a/.github/scripts/alert.sh b/sh/gh/alert.sh similarity index 99% rename from .github/scripts/alert.sh rename to sh/gh/alert.sh index 6a795dc47ad..cfdec039c48 100755 --- a/.github/scripts/alert.sh +++ b/sh/gh/alert.sh @@ -18,7 +18,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKFLOWS_DIR="$(cd "$SCRIPT_DIR/../workflows" && pwd)" +WORKFLOWS_DIR="$(cd "$SCRIPT_DIR/../../.github/workflows" && pwd)" # Post a red embed naming the workflow, what it was reacting to, and a link. # jq builds the body so a commit subject with quotes or newlines can't produce diff --git a/.github/scripts/install-formula.py b/sh/gh/install-formula.py similarity index 100% rename from .github/scripts/install-formula.py rename to sh/gh/install-formula.py diff --git a/.github/scripts/install-formula.test.py b/sh/gh/install-formula.test.py similarity index 100% rename from .github/scripts/install-formula.test.py rename to sh/gh/install-formula.test.py diff --git a/.github/scripts/release.sh b/sh/gh/release.sh similarity index 100% rename from .github/scripts/release.sh rename to sh/gh/release.sh diff --git a/.github/scripts/release.test.sh b/sh/gh/release.test.sh similarity index 100% rename from .github/scripts/release.test.sh rename to sh/gh/release.test.sh diff --git a/.github/scripts/render-formula.sh b/sh/gh/render-formula.sh similarity index 100% rename from .github/scripts/render-formula.sh rename to sh/gh/render-formula.sh diff --git a/.github/scripts/trigger-repo-publish.sh b/sh/gh/trigger-repo-publish.sh similarity index 100% rename from .github/scripts/trigger-repo-publish.sh rename to sh/gh/trigger-repo-publish.sh diff --git a/go/scripts/check-errors.sh b/sh/go/check-errors.sh similarity index 100% rename from go/scripts/check-errors.sh rename to sh/go/check-errors.sh diff --git a/go/scripts/check.sh b/sh/go/check.sh similarity index 84% rename from go/scripts/check.sh rename to sh/go/check.sh index 5f5901c0a78..86f45396d9e 100755 --- a/go/scripts/check.sh +++ b/sh/go/check.sh @@ -3,7 +3,7 @@ set -euo pipefail # Local smoke check for the Go module. # -# Stages the ffi + wrapper modules from this checkout (scripts/stage.sh) and +# Stages the ffi + wrapper modules from this checkout (sh/go/stage.sh) and # runs `go build`/`go vet`/`go test` against them. Intended for `just go check`. # # The main repo stays binary-free: no `.a` or generated `.go` files land @@ -13,17 +13,13 @@ set -euo pipefail # Skipped cleanly on hosts without `go`, `cargo`, or `uniffi-bindgen-go`. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_DIR="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_DIR/.." && pwd)" if ! command -v go >/dev/null 2>&1; then echo "go check: no go on PATH, skipping" >&2 exit 0 fi -# The publish state machine needs neither cargo nor generated bindings, so it -# runs ahead of the gates that skip a partial toolchain. -bash "$SCRIPT_DIR/publish-wrapper.test.sh" - if ! command -v cargo >/dev/null 2>&1; then echo "go check: no cargo on PATH, skipping" >&2 exit 0 diff --git a/go/scripts/package-ffi.sh b/sh/go/package-ffi.sh similarity index 98% rename from go/scripts/package-ffi.sh rename to sh/go/package-ffi.sh index 04450fd28f1..b2d89cd51d2 100755 --- a/go/scripts/package-ffi.sh +++ b/sh/go/package-ffi.sh @@ -10,7 +10,7 @@ set -euo pipefail # output at $BINDINGS_DIR/moq/moq.go. # # Usage: -# go/scripts/package-ffi.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist +# sh/go/package-ffi.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist # # Optional: # --source-dir DIR in-tree ffi module skeleton (default: go/ffi) @@ -24,7 +24,7 @@ set -euo pipefail # $LIB_DIR/x86_64-pc-windows-msvc/moq_ffi.lib SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_ROOT="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_ROOT/.." && pwd)" VERSION="" diff --git a/go/scripts/package-wrapper.sh b/sh/go/package-wrapper.sh similarity index 97% rename from go/scripts/package-wrapper.sh rename to sh/go/package-wrapper.sh index 210d0db854c..52159e7932e 100755 --- a/go/scripts/package-wrapper.sh +++ b/sh/go/package-wrapper.sh @@ -13,7 +13,7 @@ set -euo pipefail # produces an identical tree and no release. # # Usage: -# go/scripts/package-wrapper.sh --line 0.3 --ffi-version 0.2.18 --output dist +# sh/go/package-wrapper.sh --line 0.3 --ffi-version 0.2.18 --output dist # # Optional: # --source-dir DIR in-tree wrapper skeleton (default: go/wrapper) @@ -22,7 +22,7 @@ set -euo pipefail # --no-archive SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_ROOT="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_ROOT/.." && pwd)" LINE="" diff --git a/go/scripts/publish-ffi.sh b/sh/go/publish-ffi.sh similarity index 100% rename from go/scripts/publish-ffi.sh rename to sh/go/publish-ffi.sh diff --git a/go/scripts/publish-wrapper.sh b/sh/go/publish-wrapper.sh similarity index 100% rename from go/scripts/publish-wrapper.sh rename to sh/go/publish-wrapper.sh diff --git a/go/scripts/publish-wrapper.test.sh b/sh/go/publish-wrapper.test.sh similarity index 97% rename from go/scripts/publish-wrapper.test.sh rename to sh/go/publish-wrapper.test.sh index 423778218be..39e57fe7943 100755 --- a/go/scripts/publish-wrapper.test.sh +++ b/sh/go/publish-wrapper.test.sh @@ -12,7 +12,7 @@ set -euo pipefail # missing. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_DIR="$(cd "$SCRIPT_DIR/../../go" && pwd)" if ! command -v go >/dev/null 2>&1; then echo "publish-wrapper test: no go on PATH, skipping" >&2 @@ -75,7 +75,7 @@ LINE=$(tr -d '[:space:]' <"$GO_DIR/wrapper/VERSION") # Stage a real wrapper tarball. --skip-tidy keeps it offline (no go.sum), which # is fine: publish-wrapper.sh only reads VERSION out of the staged tree. -bash "$GO_DIR/scripts/package-wrapper.sh" \ +bash "$SCRIPT_DIR/package-wrapper.sh" \ --line "$LINE" \ --ffi-version "0.0.0-test" \ --source-dir "$GO_DIR/wrapper" \ @@ -93,7 +93,7 @@ publish() { ( cd "$WORK/run" GO_MIRROR_URL="$MIRROR" GO_MIRROR_TOKEN=unused \ - bash "$GO_DIR/scripts/publish-wrapper.sh" + bash "$SCRIPT_DIR/publish-wrapper.sh" ) } diff --git a/go/scripts/stage.sh b/sh/go/stage.sh similarity index 98% rename from go/scripts/stage.sh rename to sh/go/stage.sh index ca72d92c676..5992049805a 100755 --- a/go/scripts/stage.sh +++ b/sh/go/stage.sh @@ -10,7 +10,7 @@ set -euo pipefail # its own module against them and build Go code against these bindings. # # Usage: -# go/scripts/stage.sh [--output DIR] +# sh/go/stage.sh [--output DIR] # # Optional: # --output DIR stage parent (default: /dist, gitignored) @@ -19,7 +19,7 @@ set -euo pipefail # for an optimized cdylib. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_DIR="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_DIR/.." && pwd)" STAGE_PARENT="$WORKSPACE_DIR/dist" diff --git a/sh/js/workers.sh b/sh/js/workers.sh new file mode 100755 index 00000000000..4fb2178c2ac --- /dev/null +++ b/sh/js/workers.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Validate each Worker's deploy bundle without publishing or credentials. +# +# These Workers have lockfiles outside the Bun workspace, so `just js check` +# never installs or builds them. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +for project in infra/apt infra/rpm demo/pub; do + ( + cd "$project" + bun install --frozen-lockfile + bun run deploy --dry-run + ) +done diff --git a/kt/scripts/check.sh b/sh/kt/check.sh similarity index 89% rename from kt/scripts/check.sh rename to sh/kt/check.sh index ee43ebe5cdb..78e960dd3b5 100644 --- a/kt/scripts/check.sh +++ b/sh/kt/check.sh @@ -2,7 +2,7 @@ set -euo pipefail # Full check for the Kotlin packages: regenerate the bindings + native lib -# (scripts/generate.sh), then run the raw-bindings and wrapper JVM tests. +# (sh/kt/generate.sh), then run the raw-bindings and wrapper JVM tests. # # Unlike generation, this needs a JDK and Gradle. Both ship in the `nix # develop` dev shell (see flake.nix ktDeps), so a missing one is an error @@ -11,7 +11,7 @@ set -euo pipefail # should run `just kt generate` instead. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +KT_DIR="$(cd "$SCRIPT_DIR/../../kt" && pwd)" bash "$SCRIPT_DIR/generate.sh" diff --git a/kt/scripts/generate.sh b/sh/kt/generate.sh similarity index 96% rename from kt/scripts/generate.sh rename to sh/kt/generate.sh index c98dddd32dc..fcae7f80808 100755 --- a/kt/scripts/generate.sh +++ b/sh/kt/generate.sh @@ -6,11 +6,11 @@ set -euo pipefail # and regenerate the uniffi bindings. No Gradle, no JDK required. # # Intended for environments that intentionally lack Gradle (e.g. regenerating -# checked-in bindings after a moq-ffi change). `scripts/check.sh` calls this +# checked-in bindings after a moq-ffi change). `sh/kt/check.sh` calls this # first and then compiles + tests the result. Requires cargo. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +KT_DIR="$(cd "$SCRIPT_DIR/../../kt" && pwd)" WORKSPACE_DIR="$(cd "$KT_DIR/.." && pwd)" if ! command -v cargo >/dev/null 2>&1; then diff --git a/kt/scripts/package.sh b/sh/kt/package.sh similarity index 97% rename from kt/scripts/package.sh rename to sh/kt/package.sh index 73e9011d708..03c9cc13382 100755 --- a/kt/scripts/package.sh +++ b/sh/kt/package.sh @@ -10,7 +10,7 @@ set -euo pipefail # uniffi-bindgen kotlin output at $BINDINGS_DIR/uniffi/moq/moq.kt. # # Usage: -# kt/scripts/package.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist +# sh/kt/package.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist # # Expected $LIB_DIR layout (per target, populated by the build matrix): # $LIB_DIR/aarch64-linux-android/libmoq_ffi.so @@ -20,7 +20,7 @@ set -euo pipefail # ... etc. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +KT_DIR="$(cd "$SCRIPT_DIR/../../kt" && pwd)" VERSION="" LIB_DIR="" diff --git a/sh/markdown.sh b/sh/markdown.sh new file mode 100755 index 00000000000..cd811fa9119 --- /dev/null +++ b/sh/markdown.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Lint or format every Markdown file: sh/markdown.sh check|fix +# +# remark-cli has no check mode: `--frail` raises the exit code on lint +# messages, and only `--output` formats, so a file that is merely misformatted +# passes both ways. `check` formats a scratch mirror and diffs it, so it stays +# read-only while `fix` writes in place. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +case "${1:-}" in + check) ;; + fix) + bun remark . --quiet --output + exit 0 + ;; + *) + echo "usage: sh/markdown.sh check|fix" >&2 + exit 2 + ;; +esac + +mirror=$(mktemp -d) +trap 'rm -rf "$mirror"' EXIT + +extension_list=$(bun -e \ + 'import extensions from "markdown-extensions"; console.log(extensions.join("\n"))') +patterns=() +while IFS= read -r extension; do + [[ -n "$extension" ]] && patterns+=("*.$extension") +done <<<"$extension_list" +((${#patterns[@]})) || { + echo "error: remark reported no Markdown extensions" >&2 + exit 1 +} + +# Untracked files are in scope so a new doc is linted before it is staged; +# --exclude-standard keeps build output out. +files=() +while IFS= read -r -d '' file; do + [[ -f "$file" ]] || continue + files+=("$file") + mkdir -p "$mirror/$(dirname "$file")" + cp "$file" "$mirror/$file" +done < <(git ls-files -z --cached --others --exclude-standard -- "${patterns[@]}") +((${#files[@]})) || exit 0 + +# The config rides along so every .remarkignore pattern resolves against the +# mirror root exactly as it does here, and node_modules is where the plugins +# named by .remarkrc.mjs come from. +cp .remarkrc.mjs .remarkignore "$mirror/" +ln -s "$PWD/node_modules" "$mirror/node_modules" + +status=0 +(cd "$mirror" && bun remark . --quiet --frail --output) || status=$? + +stale=() +for file in "${files[@]}"; do + cmp -s "$file" "$mirror/$file" || stale+=("$file") +done + +if ((${#stale[@]})); then + echo "error: these files are not formatted, run 'just fix':" >&2 + printf ' %s\n' "${stale[@]}" >&2 + status=1 +fi + +exit "$status" diff --git a/sh/obs/check.sh b/sh/obs/check.sh new file mode 100755 index 00000000000..b8b651b41b5 --- /dev/null +++ b/sh/obs/check.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Lint formatting. Skips a tool silently if it isn't on $PATH (matches repo convention). +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" + +if grep -Fq '"MOQ_VERSION"' CMakePresets.json; then + echo "CMakePresets.json must not pin MOQ_VERSION" >&2 + exit 1 +fi +# The headers `just obs compile` type-checks against are a second copy of +# the OBS version this spec downloads, so a bump here has to reach flake.nix +# or the two gates check different libobs. +spec_obs=$(jq -r '.dependencies["obs-studio"].version // ""' buildspec.json) +flake_obs=$(sed -n '/pname = "libobs-headers"/,/}/p' ../../flake.nix | sed -n 's/.*version = "\([^"]*\)".*/\1/p') +# Both are pattern matches against files this recipe doesn't own the shape +# of. An empty parse compares equal to an empty parse, so the guard would +# pass by reporting nothing rather than by finding the versions in sync. +if [ -z "$spec_obs" ] || [ -z "$flake_obs" ]; then + echo "couldn't read the obs-studio version: '$spec_obs' from buildspec.json, '$flake_obs' from flake.nix" >&2 + exit 1 +fi +if [ "$spec_obs" != "$flake_obs" ]; then + echo "obs-studio is $spec_obs in buildspec.json but $flake_obs in flake.nix (libobs-headers)" >&2 + exit 1 +fi +# And a third: `just obs ci` links against nixpkgs' obs-studio, the one OBS +# this repo doesn't pick. Without this the released macOS/Windows binaries +# could be a whole libobs API apart from what CI ever compiles. Patch +# releases carry no API change, so only the major.minor has to agree; that +# also keeps the guard quiet until flake.lock actually moves OBS, which is +# the change that opens the gap and so the one that should fail. +# OBS_LINKED_VERSION comes from the dev shell (see flake.nix), so this leg +# is skipped outside it, where there is no nixpkgs obs-studio to compare. +# MOQ_STRICT turns that skip into an error for the same reason it turns a +# missing tool into one: in CI a leg that checks nothing still reports green. +if [ -z "${OBS_LINKED_VERSION:-}" ]; then + if [ -n "${MOQ_STRICT:-}" ]; then + echo "MOQ_STRICT is set but OBS_LINKED_VERSION is unset; run inside 'nix develop', which exports it" >&2 + exit 1 + fi +else + spec_api=$(echo "$spec_obs" | cut -d. -f1,2) + linked_api=$(echo "$OBS_LINKED_VERSION" | cut -d. -f1,2) + if [ "$spec_api" != "$linked_api" ]; then + echo "obs-studio is $spec_obs in buildspec.json but $OBS_LINKED_VERSION in nixpkgs, which is what 'just obs ci' links" >&2 + echo "bump buildspec.json and flake.nix's libobs-headers to $OBS_LINKED_VERSION, or pin nixpkgs back" >&2 + exit 1 + fi +fi +if command -v cmake >/dev/null; then + build_dir=$(mktemp -d) + trap 'rm -rf "$build_dir"' EXIT + if cmake -S . -B "$build_dir" -DMOQ_LOCAL= >"$build_dir/configure.log" 2>&1; then + echo "CMake accepted a release build without MOQ_VERSION" >&2 + exit 1 + fi + if ! grep -Fq "MOQ_VERSION is required when MOQ_LOCAL does not contain rs/libmoq" "$build_dir/configure.log"; then + cat "$build_dir/configure.log" >&2 + exit 1 + fi +fi +if command -v clang-format >/dev/null; then + git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format --dry-run --Werror +fi +if command -v gersemi >/dev/null; then + gersemi --check CMakeLists.txt cmake +fi diff --git a/sh/obs/compile.sh b/sh/obs/compile.sh new file mode 100755 index 00000000000..6253cc15860 --- /dev/null +++ b/sh/obs/compile.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Type-check every plugin source and unit test, without linking or an obs-deps +# download. +# +# This is the gate to run in a worktree. `just obs build` needs the +# multi-hundred-MB obs-deps bundle (macOS/Windows) and a full cargo build, per +# worktree; this needs neither, because the dev shell carries all three header +# sets: libobs pinned to the same OBS release buildspec.json downloads (see +# `obs-headers` in flake.nix), Qt6, and ffmpeg. CI links the real thing on Linux +# (.github/workflows/obs.yml). +# +# It compiles the Qt sources too, which the CMake build only does when +# ENABLE_QT and ENABLE_FRONTEND_API are on, so the definitions they gate on are +# set here to match. +# +# test/ is in scope because each test file defines the libmoq entry points the +# plugin calls, so a signature that drifts from the generated moq.h is a +# conflicting C declaration. Catching that needs only headers, which is why it +# belongs here: `just obs ci` finds the same drift, but only where obs.yml's +# path filter reaches, and `just obs test` is manual. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$here/../../cpp/obs" + +# Never skips, for the reason `test` doesn't: a compile gate that reports +# success without compiling is worse than no gate. Everything it needs is in +# the dev shell, so the fix is always to enter it. +cxx="${CXX:-c++}" +if ! command -v "$cxx" >/dev/null; then + echo "no C++ compiler at '$cxx'" >&2 + exit 1 +fi +for pkgs in "Qt6Widgets Qt6Gui Qt6Core" "libavcodec libavutil libswscale libswresample"; do + # shellcheck disable=SC2086 + if ! pkg-config --exists $pkgs; then + echo "missing headers for: $pkgs" >&2 + echo "run inside 'nix develop', which supplies Qt6 and ffmpeg on every platform" >&2 + exit 1 + fi +done + +# Assign first so a failure in `_includes` still aborts under `set -e`, +# then split on newlines only: an include path may contain spaces. +includes_raw=$("$here/includes.sh") +includes=() +while IFS= read -r flag; do includes+=("$flag"); done <<<"$includes_raw" + +qt=$(pkg-config --cflags Qt6Widgets Qt6Gui Qt6Core) +ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale libswresample) + +# MOQ_VERSION_STRING only reaches a label in the dock, so any value +# type-checks the same; CMake stamps the real libmoq version. +status=0 +for source in src/*.cpp test/*.cpp; do + # shellcheck disable=SC2086 + # -Wno-unused-command-line-argument: the nix compiler wrapper injects + # link flags, which a syntax-only run reports as unused, once per file. + # $qt and $ffmpeg stay unquoted on purpose: pkg-config hands back one + # space-separated string, and splitting it is the intended reading. + if ! "$cxx" -std=c++17 -fsyntax-only -Wno-unused-command-line-argument \ + "${includes[@]}" $qt $ffmpeg \ + -DMOQ_FRONTEND_ENABLED -DMOQ_VERSION_STRING='"0.0.0"' "$source"; then + status=1 + fi +done +exit $status diff --git a/sh/obs/fix.sh b/sh/obs/fix.sh new file mode 100755 index 00000000000..46cf8d27f86 --- /dev/null +++ b/sh/obs/fix.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Auto-fix formatting. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" + +if command -v clang-format >/dev/null; then + git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format -i +fi +if command -v gersemi >/dev/null; then + gersemi --in-place CMakeLists.txt cmake +fi diff --git a/sh/obs/includes.sh b/sh/obs/includes.sh new file mode 100755 index 00000000000..4456acfecaf --- /dev/null +++ b/sh/obs/includes.sh @@ -0,0 +1,76 @@ +#!/usr/bin/env bash +# Print the -I flags for libobs and libmoq, shared by compile.sh and unit.sh. +# +# moq.h is generated by cbindgen from rs/libmoq/src, so regenerate rather than +# just locating it: a stale header is how a plugin call to a since-changed +# libmoq function type-checks anyway. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" + +# The dev shell's pinned headers first, so every platform checks against the +# same libobs. Then the obs-deps framework (macOS/Windows), the OBS sources +# CMake unpacks beside it, and a system install (Linux without nix). +obs_include="" +for candidate in \ + "${OBS_INCLUDE_DIR:-}" \ + .deps/Frameworks/libobs.framework/Versions/A/Headers \ + .deps/obs-studio-*/libobs; do + if [ -n "$candidate" ] && [ -f "$candidate/obs.h" ]; then + obs_include="$candidate" + break + fi +done +if [ -z "$obs_include" ] && command -v pkg-config >/dev/null && pkg-config --exists libobs; then + obs_include=$(pkg-config --variable=includedir libobs)/obs +fi +if [ -z "$obs_include" ]; then + echo "libobs headers not found; run inside 'nix develop' (or set OBS_INCLUDE_DIR)" >&2 + exit 1 +fi + +# obs.h reaches as of OBS 32, which used to be vendored +# under libobs/util/ and so came along with the headers above. The dev shell +# carries it (see flake.nix); the obs-deps bundle ships its own copy beside +# libobs, which is how the macOS and Windows builds get it. +simde_include="" +if command -v pkg-config >/dev/null && pkg-config --exists simde; then + simde_include=$(pkg-config --variable=includedir simde) +else + for candidate in .deps/obs-deps-*/include; do + if [ -f "$candidate/simde/x86/sse2.h" ]; then + simde_include="$candidate" + break + fi + done +fi +if [ -z "$simde_include" ]; then + echo "simde headers not found; run inside 'nix develop', which supplies them" >&2 + exit 1 +fi + +(cd ../.. && cargo check --locked --quiet -p libmoq) + +# Ask cargo where its build script ran rather than reconstructing the path. +# CARGO_TARGET_DIR, a `--target` triple and a custom profile each move the +# generated header, and guessing wrong doesn't fail: it type-checks against +# whatever stale copy the default directory still holds. Replays the check +# above from cache, so it costs a process rather than a compile. +out_dir=$(cd ../.. && cargo check --locked -p libmoq --message-format=json | + jq -r 'select(.reason == "build-script-executed") | select(.package_id | tostring | test("libmoq")) | .out_dir' | + tail -1) +if [ -z "$out_dir" ]; then + echo "cargo reported no build script output for libmoq" >&2 + exit 1 +fi +# //build/libmoq-/out, and the header sits beside the +# profile directory. rs/libmoq/build.rs derives it from OUT_DIR the same way. +moq_include="$(dirname "$(dirname "$(dirname "$(dirname "$out_dir")")")")/include" +if [ ! -f "$moq_include/moq.h" ]; then + echo "$moq_include/moq.h missing after 'cargo check --locked -p libmoq'" >&2 + exit 1 +fi + +# One flag per line, path glued to the -I, so a caller can rebuild the +# arguments without splitting on the spaces in a path. +printf -- '-I%s\n' src "$obs_include" "$simde_include" "$moq_include" diff --git a/sh/obs/preset.sh b/sh/obs/preset.sh new file mode 100755 index 00000000000..c6bd950dc62 --- /dev/null +++ b/sh/obs/preset.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Detect the CMake preset for the current platform (or use the override). +# Pass `ci` as the second argument for the warnings-as-errors variant, whose +# name upstream spells differently on each platform. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" +override=${1:-} +variant=${2:-} + +if [[ -n "$override" ]]; then + echo "$override" +elif [[ "$OSTYPE" == "darwin"* ]]; then + [[ "$variant" == "ci" ]] && echo "macos-ci" || echo "macos" +elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + [[ "$variant" == "ci" ]] && echo "ubuntu-ci-x86_64" || echo "ubuntu-x86_64" +elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then + [[ "$variant" == "ci" ]] && echo "windows-ci-x64" || echo "windows-x64" +else + echo "Unknown platform: $OSTYPE" >&2 + exit 1 +fi diff --git a/sh/obs/test.sh b/sh/obs/test.sh new file mode 100755 index 00000000000..45033fbcfff --- /dev/null +++ b/sh/obs/test.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Unit-test the plugin sources against stubbed libobs/libmoq/ffmpeg, under +# ThreadSanitizer. +# +# Manual, like `just rs macos` and `just rs windows`, because ThreadSanitizer +# needs its own build. `just obs ci` runs the same tests without it, so a +# regression these assertions catch still turns PR CI red; the sanitizer is what +# adds the races on top. Run this whenever you touch src/, especially the session +# callback plumbing, whose orderings the build can't check. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$here/../../cpp/obs" + +# Unlike the lint recipes, this one never skips: a test gate that reports +# success without running anything is worse than no gate. +cxx="${CXX:-c++}" +unsupported="'$cxx' cannot build and run -fsanitize=thread binaries. These tests need a +Clang or GCC whose ThreadSanitizer runtime works on this host: set CXX, or on Windows run +them from WSL, since neither MSVC nor Clang on Windows implements ThreadSanitizer. +They also run without the sanitizer as part of 'just obs ci'." +if ! command -v "$cxx" >/dev/null; then + echo "no C++ compiler at '$cxx'." >&2 + echo "$unsupported" >&2 + exit 1 +fi + +probe=$(mktemp -d) +trap 'rm -rf "$probe"' EXIT + +# Run the probe, don't just link it: a mismatch between the compiler's TSan +# runtime and the host kernel (macOS is where this shows up) links fine and +# then segfaults before main, which is an unreadable way for the real test +# to fail. +# The subshell keeps bash's own "Segmentation fault" job message out of the +# output, so the explanation below is all the reader gets. +if ! printf 'int main(){}\n' | "$cxx" -x c++ -fsanitize=thread -o "$probe/probe" - >/dev/null 2>&1 || + ! ("$probe/probe" >/dev/null 2>&1) 2>/dev/null; then + echo "$unsupported" >&2 + exit 1 +fi + +"$here/unit.sh" tsan diff --git a/sh/obs/unit.sh b/sh/obs/unit.sh new file mode 100755 index 00000000000..54b775baace --- /dev/null +++ b/sh/obs/unit.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Compile and run the unit tests: sh/obs/unit.sh [plain|tsan] +# +# Shared by `just obs test` (ThreadSanitizer) and `just obs ci` (plain), so +# both run the same assertions. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$here/../../cpp/obs" +mode=${1:-plain} + +cxx="${CXX:-c++}" +if ! command -v "$cxx" >/dev/null; then + echo "no C++ compiler at '$cxx'" >&2 + exit 1 +fi +# moq-source.cpp decodes with ffmpeg, so its test needs the headers even +# though it stubs every function out of them. +if ! pkg-config --exists libavcodec libavutil libswscale; then + echo "missing ffmpeg headers" >&2 + echo "run inside 'nix develop', which supplies ffmpeg on every platform" >&2 + exit 1 +fi + +includes_raw=$("$here/includes.sh") +includes=() +while IFS= read -r flag; do includes+=("$flag"); done <<<"$includes_raw" + +ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale) + +flags=(-std=c++17 -g -O0 -pthread) +if [ "$mode" = "tsan" ]; then + flags+=(-fsanitize=thread) +fi + +out=$(mktemp -d) +trap 'rm -rf "$out"' EXIT + +# One binary per source under test: each test file defines its own libobs and +# libmoq stubs, so two of them can't share a link. Header-only helpers +# (quality defaults) build as a third binary with no plugin source. +for name in moq-output moq-source; do + # shellcheck disable=SC2086 + # $ffmpeg stays unquoted on purpose: pkg-config hands back one + # space-separated string, and splitting it is the intended reading. + "$cxx" "${flags[@]}" "${includes[@]}" $ffmpeg \ + -o "$out/$name-test" "test/$name-test.cpp" "src/$name.cpp" + TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" +done +for name in moq-quality-defaults moq-dock-stop moq-error moq-encoder-latency moq-dial moq-spark; do + "$cxx" "${flags[@]}" "${includes[@]}" \ + -o "$out/$name-test" "test/$name-test.cpp" + TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" +done diff --git a/sh/rs/c-tests.sh b/sh/rs/c-tests.sh new file mode 100755 index 00000000000..249ee6c581b --- /dev/null +++ b/sh/rs/c-tests.sh @@ -0,0 +1,60 @@ +#!/usr/bin/env bash +# Compile and run libmoq's C fixtures (`rs/libmoq/c-tests/*.c`) against +# libmoq.a, linked the way an embedder does: an external `cc` against the +# generated `moq.h`, plus the native libraries from `rs/libmoq/native-libs` +# that cargo can't inject into a link it doesn't drive. Unix only. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +cc="${CC:-cc}" +if ! command -v "$cc" >/dev/null; then + echo "no C compiler at '$cc'" >&2 + exit 1 +fi + +cargo build --locked -p libmoq + +# Ask cargo where it put the staticlib rather than assuming target/debug: +# CARGO_TARGET_DIR and a configured build target (`target//debug`) +# both move it, and guessing wrong would link a stale copy or nothing. +# Replays the build above from cache, so it costs a process, not a compile. +lib=$(cargo build --locked -p libmoq --message-format=json | + jq -r 'select(.reason == "compiler-artifact") | .filenames[] | select(endswith("/libmoq.a"))' | + tail -1) +if [ -z "$lib" ]; then + echo "cargo reported no libmoq.a for libmoq" >&2 + exit 1 +fi +# The header sits beside the profile directory: rs/libmoq/build.rs writes it +# to `/include`, one level above `//libmoq.a`. +profile=$(dirname "$lib") +include="$(dirname "$profile")/include" +if [ ! -f "$include/moq.h" ]; then + echo "$include/moq.h missing after 'cargo build --locked -p libmoq'" >&2 + exit 1 +fi + +# Same list build.rs (moq.pc), CMakeLists.txt, and test/interop/interop.sh read: +# `framework:Foo` is a linker framework flag, anything else a plain library. +case "$(uname -s)" in + Darwin) native_libs=rs/libmoq/native-libs/apple.txt ;; + *) native_libs=rs/libmoq/native-libs/linux.txt ;; +esac +libs=() +while read -r entry; do + case "$entry" in + '' | '#'*) continue ;; + framework:*) libs+=(-framework "${entry#framework:}") ;; + *) libs+=("-l$entry") ;; + esac +done <"$native_libs" + +out=$(mktemp -d) +trap 'rm -rf "$out"' EXIT +for source in rs/libmoq/c-tests/*.c; do + bin="$out/$(basename "$source" .c)" + "$cc" "$source" -I"$include" -L"$profile" -lmoq "${libs[@]}" -o "$bin" + echo "running $source" + "$bin" +done diff --git a/sh/rs/fuzz.sh b/sh/rs/fuzz.sh new file mode 100755 index 00000000000..5c77f75d07c --- /dev/null +++ b/sh/rs/fuzz.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Coverage-guided fuzzing of the moq-net wire codecs: sh/rs/fuzz.sh TARGET [ARGS...] +# +# TARGET is lite, ietf, varint, or path; ARGS pass to `cargo fuzz run`, so +# `just rs fuzz lite -- -max_total_time=300` bounds a run. See +# rs/moq-net/fuzz/README.md. +# +# Seeds are regenerated first so the corpus follows the dispatch rather than a +# stale run, and libFuzzer writes what it discovers to the FIRST corpus +# directory, which is why the generated seeds are passed second and stay +# read-only. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +target=${1:?usage: sh/rs/fuzz.sh TARGET [ARGS...]} +shift + +if ! command -v cargo-fuzz >/dev/null 2>&1; then + echo "rs: cargo-fuzz is missing: cargo install --locked cargo-fuzz" >&2 + exit 1 +fi + +if ! rustup run nightly rustc --version >/dev/null 2>&1; then + echo "rs: the nightly toolchain is missing: rustup toolchain install nightly" >&2 + exit 1 +fi + +cargo run --locked -q -p moq-net --features fuzz --example fuzz-seeds -- rs/moq-net/fuzz/seeds +mkdir -p rs/moq-net/fuzz/corpus/"$target" + +# Nightly by PATH rather than by `cargo +nightly`: the dev shell puts a +# Nix-provided cargo ahead of the rustup shim, so the `+` form is not +# understood, and cargo-fuzz shells out to a bare `cargo` and `rustc` anyway. +# Both have to be the nightly ones or the sanitizer flags are rejected. +sysroot=$(rustup run nightly rustc --print sysroot) +export PATH="$sysroot/bin:$PATH" + +cargo fuzz run --fuzz-dir rs/moq-net/fuzz "$target" \ + rs/moq-net/fuzz/corpus/"$target" rs/moq-net/fuzz/seeds/"$target" "$@" diff --git a/rs/scripts/media-features.sh b/sh/rs/media-features.sh similarity index 100% rename from rs/scripts/media-features.sh rename to sh/rs/media-features.sh diff --git a/rs/scripts/package-binary.sh b/sh/rs/package-binary.sh similarity index 99% rename from rs/scripts/package-binary.sh rename to sh/rs/package-binary.sh index 114b9e269fb..23fcde18c40 100755 --- a/rs/scripts/package-binary.sh +++ b/sh/rs/package-binary.sh @@ -16,7 +16,7 @@ set -euo pipefail # produces an extensionless executable beside the archive. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -RS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +RS_DIR="$(cd "$SCRIPT_DIR/../../rs" && pwd)" WORKSPACE_DIR="$(cd "$RS_DIR/.." && pwd)" CRATE="" diff --git a/rs/scripts/package-nfpm.sh b/sh/rs/package-nfpm.sh similarity index 100% rename from rs/scripts/package-nfpm.sh rename to sh/rs/package-nfpm.sh diff --git a/rs/scripts/package-rename.test.py b/sh/rs/package-rename.test.py similarity index 98% rename from rs/scripts/package-rename.test.py rename to sh/rs/package-rename.test.py index d4222ce00fd..484dda146aa 100644 --- a/rs/scripts/package-rename.test.py +++ b/sh/rs/package-rename.test.py @@ -27,7 +27,7 @@ def test_packages(self): for config, packager in (("nfpm", "deb"), ("transition", "deb"), ("nfpm", "rpm")): output = scratch / f"{name}-{config}.{packager}" subprocess.run([ - "bash", "rs/scripts/package-nfpm.sh", + "bash", "sh/rs/package-nfpm.sh", f"packaging/{old}/{config}.yaml", packager, str(output), ], env=env, cwd=ROOT, check=True) outputs[config, packager] = output diff --git a/rs/scripts/package-windows.sh b/sh/rs/package-windows.sh similarity index 98% rename from rs/scripts/package-windows.sh rename to sh/rs/package-windows.sh index 33658a71d51..19aee9b6a9b 100755 --- a/rs/scripts/package-windows.sh +++ b/sh/rs/package-windows.sh @@ -15,7 +15,7 @@ set -euo pipefail # releases, so wingetcreate can carry it forward unchanged. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -RS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +RS_DIR="$(cd "$SCRIPT_DIR/../../rs" && pwd)" WORKSPACE_DIR="$(cd "$RS_DIR/.." && pwd)" CRATE="" diff --git a/sh/rs/package.sh b/sh/rs/package.sh new file mode 100755 index 00000000000..970d7c8ef7f --- /dev/null +++ b/sh/rs/package.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Build a .deb or .rpm for moq-relay or moq-cli locally: sh/rs/package.sh CRATE deb|rpm +# +# nfpm comes from the dev shell. An .rpm built this way links against the +# host's glibc; CI builds in an AlmaLinux 9 container for broad compatibility. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +crate=${1:?usage: sh/rs/package.sh CRATE deb|rpm} +packager=${2:?usage: sh/rs/package.sh CRATE deb|rpm} + +case "$crate" in + moq-relay) bin=moq-relay ;; + moq-cli) bin=moq ;; + *) + echo "Unknown crate: $crate (use moq-relay or moq-cli)" >&2 + exit 1 + ;; +esac +case "$packager" in + deb) + if command -v dpkg >/dev/null 2>&1; then + arch=$(dpkg --print-architecture) + else + case "$(uname -m)" in + x86_64) arch=amd64 ;; + aarch64 | arm64) arch=arm64 ;; + *) + echo "Cannot infer deb arch from host $(uname -m)" >&2 + exit 1 + ;; + esac + fi + ;; + rpm) arch=$(uname -m) ;; + *) + echo "Unknown packager: $packager (use deb or rpm)" >&2 + exit 1 + ;; +esac +version=$(grep -m1 '^version' "rs/$crate/Cargo.toml" | sed 's/.*"\(.*\)".*/\1/') +cargo build --locked --release -p "$crate" +mkdir -p dist +VERSION="$version" ARCH="$arch" BINARY_PATH="target/release/$bin" \ + sh/rs/package-nfpm.sh "packaging/$crate/nfpm.yaml" "$packager" dist/ +if [[ "$packager" == deb && -f "packaging/$crate/transition.yaml" ]]; then + VERSION="$version" ARCH="$arch" \ + sh/rs/package-nfpm.sh "packaging/$crate/transition.yaml" deb dist/ +fi +ls -1 dist/ diff --git a/rs/scripts/scrub-macho.sh b/sh/rs/scrub-macho.sh similarity index 100% rename from rs/scripts/scrub-macho.sh rename to sh/rs/scrub-macho.sh diff --git a/sh/rs/select.sh b/sh/rs/select.sh new file mode 100755 index 00000000000..1724524b4db --- /dev/null +++ b/sh/rs/select.sh @@ -0,0 +1,136 @@ +#!/usr/bin/env bash +# Run `just rs check|fix|test` on the crates a diff touches, plus their dependents. +# +# Usage: sh/rs/select.sh check|fix|test LISTFILE|--all +# +# LISTFILE holds the changed paths, one per line. `--all` is the whole +# workspace, as is a change to anything every crate is built or tested by. +set -euo pipefail + +usage="usage: sh/rs/select.sh check|fix|test LISTFILE|--all" +action=${1:?$usage} +list=${2:?$usage} + +cd "$(git rev-parse --show-toplevel)" + +# Print `ALL`, nothing (no crate affected), or cargo package ids. +select_packages() { + if [[ "$list" == --all ]] || grep -qE '^(Cargo\.(toml|lock)|rust-toolchain\.toml|rs/justfile|sh/rs/select\.sh|\.config/nextest\.toml)$' "$list"; then + echo ALL + return + fi + + grep -q '^rs/' "$list" || return 0 + + # `--no-deps` keeps this to workspace members and off the network. + local metadata + metadata=$(cargo metadata --format-version 1 --no-deps) + + # A seed is the first path segment under `rs/`, so only a crate whose + # manifest sits directly at `rs//Cargo.toml` can be one. That drops + # `moq-net-fuzz` (at `rs/moq-net/fuzz`), which a moq-net diff would + # otherwise select as a dependent and compile libFuzzer for. The separator + # is normalized first so Windows paths split too. + metadata=$(jq '.packages |= map((.manifest_path |= gsub("\\\\"; "/")) | select(.manifest_path | split("/")[-3] == "rs"))' <<<"$metadata") + + # A changed path is a seed only because every crate directory is named + # after its crate. Under-selecting checks nothing, so fall back to all. + if jq -e '.packages[] | select((.manifest_path | split("/")[-2]) != .name)' <<<"$metadata" >/dev/null; then + echo "rs: a crate directory no longer matches its crate name; selecting everything." >&2 + echo ALL + return + fi + + local seeds edges selected + seeds=$(sed -n 's|^rs/\([^/]*\)/.*|\1|p' "$list" | sort -u) + edges=$(jq -r '.packages[] | .name as $name | .dependencies[] | "\($name) \(.name)"' <<<"$metadata") + + # A crate is rebuilt when anything it depends on changed, so walk the edges + # backwards until the selection stops growing. Test the value, not the key: + # awk creates a key on every `want[dep[i]]` read. The id lookup below drops + # names that are not workspace crates, such as a seed from `rs/CLAUDE.md`. + selected=$(awk -v seeds="$seeds" ' + BEGIN { split(seeds, s, "\n"); for (i in s) want[s[i]] = 1 } + { pkg[NR] = $1; dep[NR] = $2 } + END { + do { + grew = 0 + for (i = 1; i <= NR; i++) + if (want[dep[i]] && !want[pkg[i]]) { want[pkg[i]] = 1; grew = 1 } + } while (grew) + for (name in want) if (want[name]) print name + } + ' <<<"$edges" | sort -u) + + [[ -n "$selected" ]] || return 0 + + # Cargo's own package ids, not bare names: a workspace crate that is also + # pulled from crates.io (kio) makes `--package kio` ambiguous, and cargo + # refuses before compiling anything. + jq -r --arg selected "$selected" ' + ($selected | split("\n")) as $want + | .packages[] + | select(.name as $name | $want | index($name) != null) + | "--package \(.id)" + ' <<<"$metadata" | sort -u | tr '\n' ' ' +} + +packages=$(select_packages) + +# Ids are `path+file://#` for a directory named after its crate. +names=$(tr ' ' '\n' <<<"$packages" | sed -n 's|.*/\([^/#]*\)#.*|\1|p' | sort -u | tr '\n' ' ') + +# True when the selection includes a crate matching the pattern. +wants() { + [[ "$packages" == ALL ]] || grep -qwE "$1" <<<"$names" +} + +case "$packages" in + "") + echo "rs: no crates affected; skipping." + exit 0 + ;; + ALL) echo "rs: selecting the workspace" ;; + *) echo "rs: selecting $names" ;; +esac + +# `moq-net-fuzz` needs a nightly toolchain; only `just rs fuzz` compiles it. +if [[ "$packages" == ALL ]]; then + flags=(--workspace --exclude moq-net-fuzz) +else + # Space-separated `--package ` pairs; ids contain no spaces. + # shellcheck disable=SC2206 + flags=($packages) +fi + +case "$action" in + check) + just rs check "${flags[@]}" + # Workspace feature unification hides a broken moq-tokio feature set + # whenever any dependent enables a transport. + if wants moq-tokio; then just rs tokio-features; fi + # moq-wasm's crate root is `#![cfg(target_arch = "wasm32")]`, so only a + # wasm32 pass checks it. moq-mux and moq-ffi ride along in that pass, and + # nothing depends on moq-ffi, so each has to reach it on its own. + if wants '(moq-wasm|moq-mux|moq-ffi)'; then just rs wasm; fi + # Device code behind the off-by-default `capture` feature. + if wants '(moq-video|moq-audio)'; then just rs capture; fi + # The relay's io_uring listener, off the default feature set. + if wants moq-relay; then just rs uring-check; fi + ;; + fix) + just rs fix "${flags[@]}" + if wants '(moq-wasm|moq-mux|moq-ffi)'; then just rs wasm-fix; fi + ;; + test) + # A selection can hold nothing testable (moq-wasm has no host tests), and + # nextest exits 4 on that; the whole workspace finding none really is wrong. + [[ "$packages" == ALL ]] || flags+=(--no-tests=pass) + just rs test "${flags[@]}" + if wants moq-relay; then just rs relay-minimal; fi + ;; + *) + echo "$usage" >&2 + exit 2 + ;; +esac diff --git a/rs/scripts/stats-docs.py b/sh/rs/stats-docs.py similarity index 100% rename from rs/scripts/stats-docs.py rename to sh/rs/stats-docs.py diff --git a/sh/rs/tokio-features.sh b/sh/rs/tokio-features.sh new file mode 100755 index 00000000000..b80de4f8e83 --- /dev/null +++ b/sh/rs/tokio-features.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Compile moq-tokio by itself at the feature extremes and with each crypto +# provider. Its default-feature build is already part of the ordinary clippy +# pass; this catches what workspace feature unification hides. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +cargo clippy --locked -p moq-tokio --no-default-features -- -D warnings +cargo clippy --locked -p moq-tokio --all-features -- -D warnings +cargo clippy --locked -p moq-tokio --no-default-features --features noq,aws-lc-rs -- -D warnings +cargo clippy --locked -p moq-tokio --no-default-features --features noq,ring -- -D warnings + +for feature in aws-lc-rs ring tcp uds websocket; do + cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings +done + +# shellcheck disable=SC2016 # the backticks are literal, as rustc prints them +want='a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' +if output=$(cargo check --locked -p moq-tokio --no-default-features --features noq 2>&1); then + echo "rs: moq-tokio with only noq should reject its missing crypto provider" >&2 + exit 1 +fi +if ! grep -qF "$want" <<<"$output"; then + echo "rs: moq-tokio with only noq should fail on: $want" >&2 + echo "$output" | grep -E '^error' >&2 || true + exit 1 +fi diff --git a/sh/rs/vulkan-cuda.sh b/sh/rs/vulkan-cuda.sh new file mode 100755 index 00000000000..bcdc12944c0 --- /dev/null +++ b/sh/rs/vulkan-cuda.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Exercise the Vulkan producer -> CUDA image contract, the GPU color conversion +# and resize, and NVENC encoding of the result on Linux/NVIDIA hardware. The +# driver directory is added explicitly because the Nix shell's dynamic loader +# path does not include Ubuntu's host driver directory. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +driver=$(/usr/sbin/ldconfig -p | awk '/libcuda\.so\.1/{print $NF; exit}') +if [[ -z "$driver" ]]; then + echo "libcuda.so.1 is not installed" >&2 + exit 1 +fi +driver_dir=$(dirname "$driver") +export LD_LIBRARY_PATH="$driver_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +cargo nextest run --locked -p moq-video --run-ignored only -E 'test(/^frame::(vulkan|cuda)::tests::vulkan_cuda_/)' diff --git a/sh/shell.sh b/sh/shell.sh new file mode 100755 index 00000000000..fb6ae022c8b --- /dev/null +++ b/sh/shell.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Lint or format every tracked shell script: sh/shell.sh check|fix +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +scripts=() +while IFS= read -r -d '' file; do + if git --literal-pathspecs ls-files --error-unmatch -- "$file" >/dev/null 2>&1; then + scripts+=("$file") + fi +done < <(shfmt -f=0 .) +((${#scripts[@]})) || exit 0 + +case "${1:-}" in + check) + shfmt --diff "${scripts[@]}" + shellcheck "${scripts[@]}" + ;; + fix) + shfmt --write "${scripts[@]}" + ;; + *) + echo "usage: sh/shell.sh check|fix" >&2 + exit 2 + ;; +esac diff --git a/swift/scripts/check.sh b/sh/swift/check.sh similarity index 98% rename from swift/scripts/check.sh rename to sh/swift/check.sh index afe96444647..bcacf802112 100755 --- a/swift/scripts/check.sh +++ b/sh/swift/check.sh @@ -8,7 +8,7 @@ set -euo pipefail # `cargo`. Intended for `just swift check`. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SWIFT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SWIFT_DIR="$(cd "$SCRIPT_DIR/../../swift" && pwd)" WORKSPACE_DIR="$(cd "$SWIFT_DIR/.." && pwd)" if ! command -v swift >/dev/null 2>&1; then diff --git a/swift/scripts/package-ffi.sh b/sh/swift/package-ffi.sh similarity index 98% rename from swift/scripts/package-ffi.sh rename to sh/swift/package-ffi.sh index 5958c213ae1..8166429c734 100755 --- a/swift/scripts/package-ffi.sh +++ b/sh/swift/package-ffi.sh @@ -12,7 +12,7 @@ set -euo pipefail # Only macOS hosts can run this (xcodebuild is required). # # Usage: -# swift/scripts/package-ffi.sh --version 0.0.0-dev --lib-dir dist --output dist +# sh/swift/package-ffi.sh --version 0.0.0-dev --lib-dir dist --output dist # # --version Version baked into Package.swift (the moq-ffi crate version). # --lib-dir Directory containing per-target moq-ffi outputs. @@ -24,7 +24,7 @@ set -euo pipefail # override when publishing from a fork. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SWIFT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SWIFT_DIR="$(cd "$SCRIPT_DIR/../../swift" && pwd)" WORKSPACE_DIR="$(cd "$SWIFT_DIR/.." && pwd)" VERSION="" diff --git a/swift/scripts/package.sh b/sh/swift/package.sh similarity index 98% rename from swift/scripts/package.sh rename to sh/swift/package.sh index e1d40527192..4d281925ee1 100755 --- a/swift/scripts/package.sh +++ b/sh/swift/package.sh @@ -12,7 +12,7 @@ set -euo pipefail # package depends on at .upToNextMinor(from: ). # # Usage: -# swift/scripts/package.sh --version 0.3.0 --output dist +# sh/swift/package.sh --version 0.3.0 --output dist # # --version Wrapper version (from swift/VERSION). Names the tarball only; # the SPM version comes from the mirror's git tag. @@ -21,7 +21,7 @@ set -euo pipefail # --output Destination directory for the .tar.gz. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SWIFT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SWIFT_DIR="$(cd "$SCRIPT_DIR/../../swift" && pwd)" WORKSPACE_DIR="$(cd "$SWIFT_DIR/.." && pwd)" VERSION="" diff --git a/swift/scripts/publish-ffi.sh b/sh/swift/publish-ffi.sh similarity index 100% rename from swift/scripts/publish-ffi.sh rename to sh/swift/publish-ffi.sh diff --git a/swift/scripts/publish.sh b/sh/swift/publish.sh similarity index 100% rename from swift/scripts/publish.sh rename to sh/swift/publish.sh diff --git a/swift/scripts/verify-ffi.sh b/sh/swift/verify-ffi.sh similarity index 96% rename from swift/scripts/verify-ffi.sh rename to sh/swift/verify-ffi.sh index 65c8f60d851..886fb75c6d9 100755 --- a/swift/scripts/verify-ffi.sh +++ b/sh/swift/verify-ffi.sh @@ -13,8 +13,8 @@ set -euo pipefail # never reaches consumers. # # Usage: -# swift/scripts/verify-ffi.sh --staged-dir -# swift/scripts/verify-ffi.sh --tarball +# sh/swift/verify-ffi.sh --staged-dir +# sh/swift/verify-ffi.sh --tarball # # Exactly one of --staged-dir / --tarball must be passed. diff --git a/swift/scripts/verify.sh b/sh/swift/verify.sh similarity index 96% rename from swift/scripts/verify.sh rename to sh/swift/verify.sh index 865495798d9..6df5a561b59 100755 --- a/swift/scripts/verify.sh +++ b/sh/swift/verify.sh @@ -13,8 +13,8 @@ set -euo pipefail # already published (true on a normal release; a correct failure otherwise). # # Usage: -# swift/scripts/verify.sh --staged-dir -# swift/scripts/verify.sh --tarball +# sh/swift/verify.sh --staged-dir +# sh/swift/verify.sh --tarball # # Exactly one of --staged-dir / --tarball must be passed. diff --git a/swift/.gitignore b/swift/.gitignore index 17554c75de8..2a28e6b5cc6 100644 --- a/swift/.gitignore +++ b/swift/.gitignore @@ -3,7 +3,7 @@ .swiftpm/ Package.resolved -# Generated by swift/scripts/{check,package}.sh — never check these in. +# Generated by sh/swift/{check,package}.sh — never check these in. Sources/MoqFFI/Generated.swift MoqFFI.xcframework/ diff --git a/swift/.spi.yml b/swift/.spi.yml index 0259deae9cc..9fb2ec6db2c 100644 --- a/swift/.spi.yml +++ b/swift/.spi.yml @@ -3,7 +3,7 @@ # served at https://swiftpackageindex.com/moq-dev/moq-swift/documentation. # # This file is copied into the published moq-dev/moq-swift mirror by -# swift/scripts/package.sh, since SPI reads it from the package root. +# sh/swift/package.sh, since SPI reads it from the package root. version: 1 builder: configs: diff --git a/swift/Package.swift b/swift/Package.swift index 97256a74c93..e06be8154a6 100644 --- a/swift/Package.swift +++ b/swift/Package.swift @@ -1,9 +1,9 @@ // swift-tools-version:5.9 -// Local-development manifest. swift/scripts/check.sh builds a single-slice +// Local-development manifest. sh/swift/check.sh builds a single-slice // MoqFFI.xcframework next to this file so the path-based binaryTarget below // resolves locally without a network fetch. // -// The released form (URL + checksum) is generated by swift/scripts/package.sh +// The released form (URL + checksum) is generated by sh/swift/package.sh // from Package.swift.template at release time. Edit the template, not this // file, when changing the released manifest. diff --git a/swift/Package.swift.template b/swift/Package.swift.template index 0cdfbf06060..f6339a07a8d 100644 --- a/swift/Package.swift.template +++ b/swift/Package.swift.template @@ -1,7 +1,7 @@ // swift-tools-version:5.9 // Released manifest for the ergonomic wrapper at moq-dev/moq-swift. The // source-of-truth template lives at swift/Package.swift.template in -// moq-dev/moq; swift/scripts/package.sh substitutes the moq-ffi version pin +// moq-dev/moq; sh/swift/package.sh substitutes the moq-ffi version pin // (REPLACE_FFI_VERSION) at release time. // // The wrapper versions independently of the bindings (see swift/VERSION). The diff --git a/swift/README.md b/swift/README.md index d6c17792132..b22d5a89105 100644 --- a/swift/README.md +++ b/swift/README.md @@ -82,7 +82,7 @@ is an `AsyncSequence` of the decoded type. Pass matching `compression` on both s ## Local development -`swift/scripts/check.sh` builds `moq-ffi` for the host, regenerates the UniFFI Swift bindings, builds a single-slice `MoqFFI.xcframework`, and runs `swift test`. Requires macOS with `xcodebuild` and `swift` on `$PATH`. Run via `just swift check`; skips cleanly on non-macOS hosts. +`sh/swift/check.sh` builds `moq-ffi` for the host, regenerates the UniFFI Swift bindings, builds a single-slice `MoqFFI.xcframework`, and runs `swift test`. Requires macOS with `xcodebuild` and `swift` on `$PATH`. Run via `just swift check`; skips cleanly on non-macOS hosts. Local development uses one **monolithic** `Package.swift` containing both the `Moq` and `MoqFFI` targets plus the path-based XCFramework, so `swift test` and Xcode work against a single package. The split into two packages exists only in the released artifacts, assembled from the two templates below at release time. Because the FFI module is named `MoqFFI` in both layouts, the wrapper sources (`import MoqFFI`) compile identically either way. @@ -98,7 +98,6 @@ swift/ Moq/ Ergonomic wrapper (Client, Server, Origin, Broadcast, Track, Media, Audio, …) MoqFFI/ UniFFI-generated swift (populated by check.sh/package-ffi.sh, gitignored) Tests/MoqTests/ Smoke tests - scripts/ check.sh, package{,-ffi}.sh, verify{,-ffi}.sh, publish{,-ffi}.sh ``` Edit the templates when changing a released manifest; never copy the monolithic dev-mode form into the release path. @@ -117,8 +116,8 @@ To release a new wrapper version: bump `swift/VERSION` in a PR. On merge, `relea To dry-run a publish locally against a staged tarball: ```bash -BUILD_VERSION= ./swift/scripts/publish.sh --dry-run # wrapper -> moq-swift -BUILD_VERSION= ./swift/scripts/publish-ffi.sh --dry-run # bindings -> moq-swift-ffi +BUILD_VERSION= ./sh/swift/publish.sh --dry-run # wrapper -> moq-swift +BUILD_VERSION= ./sh/swift/publish-ffi.sh --dry-run # bindings -> moq-swift-ffi ``` No Apple Developer account or App Store Connect setup needed. diff --git a/swift/ffi/Package.swift.template b/swift/ffi/Package.swift.template index d66fb3d01b6..55588ffc092 100644 --- a/swift/ffi/Package.swift.template +++ b/swift/ffi/Package.swift.template @@ -1,7 +1,7 @@ // swift-tools-version:5.9 // Released manifest for the raw UniFFI bindings package at moq-dev/moq-swift-ffi. // The source-of-truth template lives at swift/ffi/Package.swift.template in -// moq-dev/moq; swift/scripts/package-ffi.sh substitutes the xcframework URL and +// moq-dev/moq; sh/swift/package-ffi.sh substitutes the xcframework URL and // SHA-256 (REPLACE_URL / REPLACE_CHECKSUM) at release time. // // Lockstep with the moq-ffi Rust crate: each moq-ffi-v* tag publishes a matching diff --git a/swift/justfile b/swift/justfile index 83b585910dc..acb2b150297 100644 --- a/swift/justfile +++ b/swift/justfile @@ -9,29 +9,13 @@ default: just check # Skips cleanly on non-macOS hosts, which is why swift.yml runs it on a Mac. -# -# This runs the tests too, so there is no separate `just swift test`. Takes an -# optional newline-separated list of changed files and skips when none are -# Swift-scoped; `just swift check` (no FILES) always runs. +# This runs the tests too, so there is no separate `just swift test`. # Build moq-ffi for the host, regenerate uniffi bindings, run swift test. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '^(swift/|rs/moq-ffi/)' <<< "$FILES"; then - echo "swift: no Swift changes; skipping." - exit 0 - fi - bash scripts/check.sh +check: + bash ../sh/swift/check.sh # Assemble the released wrapper package from Package.swift.template. - # Used by .github/workflows/release-swift-lib.yml. package *args: - bash scripts/package.sh {{ args }} - -# Remove SwiftPM build output plus the XCFramework and generated bindings - -# that scripts/check.sh lays down (see swift/.gitignore). -clean: - rm -rf .build .swiftpm Package.resolved Sources/MoqFFI/Generated.swift MoqFFI.xcframework + bash ../sh/swift/package.sh {{ args }} diff --git a/test/README.md b/test/README.md index 5c4f994c433..94d044a7736 100644 --- a/test/README.md +++ b/test/README.md @@ -84,25 +84,3 @@ with the `rm -rf` the run prints; nothing expires it for you. In CI the harness writes under `MOQ_TEST_RUNS`, and `interop.yml` and `wasm.yml` upload that directory as a short-lived artifact when the job fails. - -## Worktrees - -`just worktree` reports what a checkout can actually do before anything is built: -its base and how stale that base is, and whether the Git metadata is reachable for -fetch, branch creation, and rebase. A linked worktree keeps its shared metadata -under `--git-common-dir`, inside the main repository, so write access to the -source tree does not imply any of the three. - -```bash -just worktree # report, change nothing -just worktree setup # fetch, set the branch upstream, record the base SHA -``` - -`setup` never resets, rebases, or cleans, so it is safe to run against a checkout -with work in progress. - -The upstream is the only place the base survives, so `setup BASE` fails when it -cannot be written -- a detached HEAD has no branch to hang it on, and the shared -config may be read-only. Left as a warning, `just check` would go on scoping -against `origin/main` while setup reported success. Without a `BASE`, that -fallback is what would have been written anyway, so it warns instead. diff --git a/test/interop/README.md b/test/interop/README.md index e41a0d7179d..e2641b7aa75 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -29,7 +29,7 @@ player survive the publication lifecycle. See [Media QA](#media-qa). |---|---|---|---| | Rust | `rs/moq-relay` + `rs/moq-cli` | `cargo build` | publish (video) + subscribe | | Python | `py/moq-rs` (+ `rs/moq-ffi`, import `moq`) | `just py build` (maturin editable into `.venv`) | publish (video + audio) + subscribe | -| Go | `go/wrapper` (+ `rs/moq-ffi`, import `moq-go/moq`) | `go/scripts/stage.sh` (uniffi-bindgen-go) + `go build` | publish (video + audio) + subscribe | +| Go | `go/wrapper` (+ `rs/moq-ffi`, import `moq-go/moq`) | `sh/go/stage.sh` (uniffi-bindgen-go) + `go build` | publish (video + audio) + subscribe | | Browser | `js/watch` + `js/publish` | `vite build` + headless Chromium (Playwright) | publish (video + audio) + rendered playback | | Native JS | `js/net` + `js/hang` + the npm `@moq/web-transport` polyfill | `node` (tsx) and `bun` | subscribe | | C | `rs/libmoq` | `cargo build -p libmoq` + `cc` | subscribe | @@ -40,7 +40,7 @@ The browser, native JS, C, and GStreamer clients subscribe only by choice intentionally minimal, and `moqsink` publishing needs request-pad muxing this client doesn't drive). Rust, Python, Go, and the browser publish. -The Go client builds against the modules `go/scripts/stage.sh` assembles from +The Go client builds against the modules `sh/go/stage.sh` assembles from this checkout: `moq-ffi` compiled for the host, bindings regenerated with `uniffi-bindgen-go`, and the `go/wrapper` module wired to them by a `replace`. That is the same staging `just go check` uses, so this cell covers the Go diff --git a/test/interop/clients/go/go.mod b/test/interop/clients/go/go.mod index a57ce62b534..bd1ae2e2045 100644 --- a/test/interop/clients/go/go.mod +++ b/test/interop/clients/go/go.mod @@ -1,6 +1,6 @@ // Not a published module. test/interop/interop.sh builds this out of a scratch copy // and rewrites the require below into a `replace` pointing at the wrapper that -// go/scripts/stage.sh assembles from this checkout, so nothing resolves from the +// sh/go/stage.sh assembles from this checkout, so nothing resolves from the // module proxy. The placeholder version keeps the committed file honest: there // is no such tag, and there is no go.sum because every dependency is local. module moq.dev/interop diff --git a/test/interop/clients/js-native/subscribe.ts b/test/interop/clients/js-native/subscribe.ts index 61df7fdeac4..1cdfcd8e673 100644 --- a/test/interop/clients/js-native/subscribe.ts +++ b/test/interop/clients/js-native/subscribe.ts @@ -40,6 +40,9 @@ if (role !== "subscribe" || !url || !broadcast || !Number.isFinite(timeoutMs) || process.exit(2); } +// Group resets that only mean content was skipped at the live edge. +const GAPS: Moq.StreamCode[] = [Moq.StreamCode.TooFarBehind, Moq.StreamCode.Old, Moq.StreamCode.Evicted]; + async function run(): Promise { const origin = new Moq.Origin.Producer(); const connection = await Moq.Connection.connect({ url: new URL(url as string), consume: origin }); @@ -72,7 +75,15 @@ async function run(): Promise { const group = await video.recvGroup(); if (!group) break; for (;;) { - const frame = await group.readFrame(); + let frame: Moq.Group.Frame | undefined; + try { + frame = await group.readFrame(); + } catch (err) { + // A live subscriber can be handed a group the relay then drops for a newer one; + // that is a gap, not a failure, so move on to the next group. + if (!(err instanceof Moq.Error.Stream && GAPS.includes(err.code))) throw err; + break; + } if (!frame) break; total += frame.payload.byteLength; if (total > 0) { diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 713d4e6865b..13a50032c80 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -80,6 +80,14 @@ while [[ $# -gt 0 ]]; do MEDIA=1 shift ;; + # The full matrix. --timeout 30 gives headless Chromium cold-start + # headroom; flags after it still override. + --all) + PUBLISHERS="rust,python,go,js" + SUBSCRIBERS="rust,python,go,js,js-native-node,js-native-bun,c,gst" + TIMEOUT=30 + shift + ;; *) echo "unknown arg: $1" >&2 exit 2 @@ -240,7 +248,7 @@ prepare_js() { fi } -# Stage the Go modules from this checkout (go/scripts/stage.sh builds moq-ffi for +# Stage the Go modules from this checkout (sh/go/stage.sh builds moq-ffi for # the host, regenerates the bindings, and wires the wrapper to them by replace), # then build the interop client against that exact tree. The client is copied to a # scratch dir first so the committed go.mod keeps its placeholder require; every @@ -256,8 +264,8 @@ prepare_go() { } echo "building go client (workspace moq-go via uniffi-bindgen-go)..." local staged ffi_pkg wrapper_pkg src="$HARNESS_RUN/go-client" - if ! staged=$(bash "$WORKSPACE/go/scripts/stage.sh" 2>"$HARNESS_RUN/go-stage.log"); then - mark_broken go "go/scripts/stage.sh failed" + if ! staged=$(bash "$WORKSPACE/sh/go/stage.sh" 2>"$HARNESS_RUN/go-stage.log"); then + mark_broken go "sh/go/stage.sh failed" sed 's/^/ /' "$HARNESS_RUN/go-stage.log" >&2 || true return fi diff --git a/test/justfile b/test/justfile index fb0ecbf1686..ed94847ff84 100644 --- a/test/justfile +++ b/test/justfile @@ -3,89 +3,37 @@ # Tests that span more than one language. Per-language unit tests live in each # language's own justfile; this module aggregates them. # -# Shebang recipes read their arguments as $1..$n. -set positional-arguments - # Two cross-language interop tests exist: # - https://github.com/moq-dev/smoke installs each client from its PUBLIC # package registry to catch packaging breakage in a release. # - `just test interop` (interop/, below) builds every client from THIS checkout # to catch interop regressions before anything is published. -# Fall back to the root justfile so `just js test` (etc.) resolve from here. +# Fall back to the root justfile so `just rs test` resolves from here. set fallback set working-directory := '.' -# `interop` reads its args as `$1`/`$@`; the `positional-arguments` setting above -# is what lets a shebang recipe see them. # The `check` sibling: same diff scoping, same BASE resolution, and what CI runs -# after `just check`. Only js/rs/py appear here. The kt, swift, and go wrappers -# build, regenerate bindings, and test in a single script, so their tests run as -# part of `just check` and there is nothing left to dispatch. -# -# `just _changed` resolves through `set fallback` to the root justfile, which is -# also where it runs, so the paths it prints stay repo-relative. -# -# BASE has to be passed as `just test default origin/dev`. This is a module, so -# a bare `just test origin/dev` makes just look for a recipe by that name rather -# than reaching the default recipe's parameter. +# after `just check`. Only js/rs/py have tests to scope; the kt, swift, and go +# wrappers test inside their `check`. BASE has to be passed as `just test +# default origin/dev`: a bare `just test origin/dev` names a recipe. # Run unit tests for what the branch changed since BASE, plus its dependents. default $BASE="": - #!/usr/bin/env bash - set -euo pipefail - - files=$(just _changed "$BASE") - - # Mirrors the root `check`: too long for argv means test everything instead. - if [[ "$files" == ALL ]]; then - just all - exit 0 - fi - - just _tools "$files" - - # Mirrors the root `check`: the dispatch below lives in these two files and - # matches no language scope, so a change to either has to run everything or - # it goes untested. - if grep -qE '^(justfile|test/justfile)$' <<< "$files"; then - echo "test: root orchestration changed; testing everything." >&2 - just all - exit 0 - fi - - # An empty list means "force-run" to the per-lang recipes, which is the - # wrong semantic here, so don't dispatch at all. - if [[ -n "$files" ]]; then - just js test "$files" - just rs test-changed "$files" - just py test "$files" - else - echo "test: nothing changed." - fi + ../sh/dispatch.sh test "$BASE" # Run every language's unit tests, unscoped. all: - just js test - just rs test --workspace --exclude moq-net-fuzz - just py test + ../sh/dispatch.sh test --all # Cross-language media interop test, built from this checkout. Stands up a # relay and runs the publisher x subscriber matrix. Default: rust only (fast # sanity check). `interop --all` runs the full matrix: rust + python + go + # browser publish; everyone subscribes (incl. native node/bun JS, the C libmoq -# client, and the GStreamer moqsrc plugin). --timeout 30 gives headless Chromium -# cold-start headroom. Other flags pass through, e.g. +# client, and the GStreamer moqsrc plugin). Other flags pass through, e.g. # `just test interop --publishers rust,python --subscribers rust,c`. interop *args: - #!/usr/bin/env bash - set -euo pipefail - if [[ "${1:-}" == --all ]]; then - shift - ./interop/interop.sh --publishers rust,python,go,js --subscribers rust,python,go,js,js-native-node,js-native-bun,c,gst --timeout 30 "$@" - else - ./interop/interop.sh "$@" - fi + ./interop/interop.sh {{ args }} # Negative control: no publisher, every subscriber must time out (proves the diff --git a/test/wasm/README.md b/test/wasm/README.md index a1eb599b497..e9ab1301de7 100644 --- a/test/wasm/README.md +++ b/test/wasm/README.md @@ -21,7 +21,7 @@ just test wasm just test wasm --timeout 60 ``` -Everything is built from this checkout: `just wasm` for the bindings, `cargo +Everything is built from this checkout: `just js wasm` for the bindings, `cargo build -p moq-relay` for the relay. Each relay reserves its own port (see [the harness contract](../README.md)); `WASM_PORT` pins the first one instead. `WASM_PROFILE` picks the relay's cargo profile, and `RELAY_BIN` points at a diff --git a/test/wasm/run.sh b/test/wasm/run.sh index e2e649743e5..71ed51ba538 100755 --- a/test/wasm/run.sh +++ b/test/wasm/run.sh @@ -92,7 +92,7 @@ command -v wasm-bindgen >/dev/null 2>&1 || { # ── build ─────────────────────────────────────────────────────────────────── echo "building @moq/wasm..." -(cd "$WORKSPACE" && just wasm) +(cd "$WORKSPACE" && just js wasm) echo "building moq-relay ($PROFILE)..." flag=() From c5000beb3c27218dd28d11aa1d9f3209949c883c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Thu, 24 Sep 2026 23:44:54 -0700 Subject: [PATCH 03/18] chore(ci): every workflow step runs a just recipe (#4098) Co-authored-by: Claude Opus 5.5 --- .github/actions/just/action.yml | 11 +++++++ .github/justfile | 29 ++++++++++++++++- .github/workflows/alert.yml | 4 ++- .github/workflows/android.yml | 7 +++-- .github/workflows/apt-repo.yml | 5 +-- .github/workflows/check.yml | 3 -- .github/workflows/docker.yml | 4 ++- .github/workflows/libmoq.yml | 26 ++++++++++------ .github/workflows/moq-cli.yml | 34 ++++++++++++-------- .github/workflows/moq-gst.yml | 26 ++++++++++------ .github/workflows/moq-relay.yml | 32 +++++++++++-------- .github/workflows/nightly.yml | 15 ++------- .github/workflows/release-brew.yml | 6 ++-- .github/workflows/release-dart-ffi.yml | 18 ++++++++--- .github/workflows/release-dart.yml | 6 ++-- .github/workflows/release-go-ffi.yml | 24 +++++++++++---- .github/workflows/release-go.yml | 10 ++++-- .github/workflows/release-kt-ffi.yml | 16 +++++++--- .github/workflows/release-kt-lib.yml | 14 ++++++--- .github/workflows/release-py-ffi.yml | 4 ++- .github/workflows/release-py.yml | 10 +++--- .github/workflows/release-swift-ffi.yml | 30 +++++++++++++----- .github/workflows/release-swift-lib.yml | 24 ++++++++++----- .github/workflows/rpm-repo.yml | 5 +-- .github/workflows/swift.yml | 4 ++- CONTRIBUTING.md | 4 +++ cpp/obs/justfile | 9 ++++++ dart/justfile | 9 ++++++ go/ffi/README.md | 2 +- go/justfile | 33 +++++++++++++++++--- js/justfile | 5 +++ kt/justfile | 11 ++++--- quest/m1/tooling/README.md | 1 - quest/m1/tooling/release-binary.md | 4 --- quest/m1/tooling/workflows-call-just.md | 41 ------------------------- rs/justfile | 35 +++++++++++++++++++++ sh/gh/check-runs.ts | 26 ++++++++++++++++ swift/justfile | 38 +++++++++++++++++++++-- 38 files changed, 409 insertions(+), 176 deletions(-) create mode 100644 .github/actions/just/action.yml delete mode 100644 quest/m1/tooling/workflows-call-just.md create mode 100644 sh/gh/check-runs.ts diff --git a/.github/actions/just/action.yml b/.github/actions/just/action.yml new file mode 100644 index 00000000000..f52ffe41e37 --- /dev/null +++ b/.github/actions/just/action.yml @@ -0,0 +1,11 @@ +name: Install just +description: Install the pinned just for jobs outside the nix dev shell, which already provides it. + +runs: + using: composite + steps: + # A checksum-verified prebuilt, so it costs seconds rather than a + # `cargo install`. Keep the version in step with nixpkgs' just in flake.lock. + - uses: taiki-e/install-action@9983c65e42da123ff25d1f78505eb6de315aa172 # v2.87.20 + with: + tool: just@1.58.0 diff --git a/.github/justfile b/.github/justfile index 8a3fb9ff678..1bccfc53380 100644 --- a/.github/justfile +++ b/.github/justfile @@ -1,9 +1,10 @@ set working-directory := '..' -# Lint GitHub Actions workflows and their alert coverage. +# Lint GitHub Actions workflows, their alert coverage, and that every step runs a recipe. check: actionlint sh/gh/alert.sh check-coverage + bun sh/gh/check-runs.ts # Tests of the release tooling itself; nightly, not on every pull request. test: @@ -11,3 +12,29 @@ test: sh/gh/release.test.sh python3 sh/rs/package-rename.test.py sh/go/publish-wrapper.test.sh + bash .claude/hooks/direnv.test.sh + +# Release workflow helpers, e.g. `just gh release parse-version moq-cli`. +[positional-arguments] +release *args: + sh/gh/release.sh "$@" + +# Post a failed run to Discord, or check that alert.yml covers every workflow. +[positional-arguments] +alert *args: + sh/gh/alert.sh "$@" + +# Render a Homebrew formula template for a release. +[positional-arguments] +formula *args: + sh/gh/render-formula.sh "$@" + +# Copy a rendered formula into a Homebrew tap checkout, migrating renamed formulae. +[positional-arguments] +install-formula *args: + python3 sh/gh/install-formula.py "$@" + +# Trigger the apt-repo and rpm-repo workflows for a release tag. +[positional-arguments] +trigger-repo-publish *args: + sh/gh/trigger-repo-publish.sh "$@" diff --git a/.github/workflows/alert.yml b/.github/workflows/alert.yml index 78d9c481259..006dfad7f7a 100644 --- a/.github/workflows/alert.yml +++ b/.github/workflows/alert.yml @@ -78,6 +78,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Post to Discord env: DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} @@ -88,4 +90,4 @@ jobs: RUN_REF: ${{ github.event.workflow_run.head_branch }} RUN_TITLE: ${{ github.event.workflow_run.display_title }} RUN_ACTOR: ${{ github.event.workflow_run.triggering_actor.login }} - run: sh/gh/alert.sh discord + run: just gh alert discord diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 9b8e9b4d854..44c6e807a4d 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -50,7 +50,6 @@ concurrency: env: CARGO_NDK_VERSION: 4.1.2 - JUST_VERSION: 1.52.0 jobs: android: @@ -74,8 +73,10 @@ jobs: # No Rust cache: this workflow only runs on pull requests, which may not # save one, so it would have nothing to restore. - - name: Install just and cargo-ndk - run: cargo install --locked "just@$JUST_VERSION" "cargo-ndk@$CARGO_NDK_VERSION" + - uses: ./.github/actions/just + + - name: Install cargo-ndk + run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" - name: Check run: just rs android diff --git a/.github/workflows/apt-repo.yml b/.github/workflows/apt-repo.yml index ff790e1583b..5e8905ed21c 100644 --- a/.github/workflows/apt-repo.yml +++ b/.github/workflows/apt-repo.yml @@ -34,6 +34,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -53,10 +55,9 @@ jobs: - name: Publish to R2 shell: bash env: - ARTIFACTS_DIR: artifacts R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} SIGNING_KEY: ${{ secrets.SIGNING_KEY }} SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} - run: ./infra/apt/publish.sh + run: just infra apt publish "$GITHUB_WORKSPACE/artifacts" diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 4b26c72456a..5cc62c44b7d 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -73,9 +73,6 @@ jobs: # Locally an absent formatter means less gets checked; here it would mean # a green run that checked nothing, which is the failure mode worth # spending a preflight on. - - name: Test direnv hook - run: bash .claude/hooks/direnv.test.sh - - name: Check run: nix develop --command just ci check env: diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 9f2ba00f14b..5fcbc152e44 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,6 +39,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - id: parse env: CRATE: ${{ inputs.package }} @@ -58,7 +60,7 @@ jobs: *) package="$crate" ;; esac echo "package=${package}" >> "$GITHUB_OUTPUT" - sh/gh/release.sh parse-version "$crate" + just gh release parse-version "$crate" build: name: Build ${{ matrix.platform }} diff --git a/.github/workflows/libmoq.yml b/.github/workflows/libmoq.yml index 3ad2daf514e..b74274da813 100644 --- a/.github/workflows/libmoq.yml +++ b/.github/workflows/libmoq.yml @@ -40,6 +40,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Install Nix if: matrix.use_nix uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main @@ -55,12 +57,12 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version libmoq + run: just gh release parse-version libmoq - name: Build and package shell: bash run: | - ./rs/libmoq/build.sh \ + just rs package-libmoq \ --target ${{ matrix.target }} \ --version ${{ steps.parse.outputs.version }} \ --output dist @@ -85,13 +87,15 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version libmoq + run: just gh release parse-version libmoq - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag libmoq + run: just gh release prev-tag libmoq - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -105,7 +109,7 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "libmoq v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts # The OBS plugin (cpp/obs) statically links libmoq, so it ships with every # libmoq release: rebuild it against the release we just published (no @@ -137,10 +141,12 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version libmoq + run: just gh release parse-version libmoq # Native Xcode / Visual Studio; libobs/Qt6/ffmpeg come from the obs-deps # bundle and libmoq from the release we just published (--libmoq-release). @@ -150,7 +156,7 @@ jobs: TARGET: ${{ matrix.target }} VERSION: ${{ steps.parse.outputs.version }} run: | - ./cpp/obs/build.sh --target "$TARGET" --libmoq-release "$VERSION" --output dist + just obs package --target "$TARGET" --libmoq-release "$VERSION" --output dist - name: Upload artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -171,9 +177,11 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version libmoq + run: just gh release parse-version libmoq # Only the obs-moq archives: the libmoq-* artifacts from the build job # belong to the libmoq release, not this one. @@ -208,4 +216,4 @@ jobs: RELEASE_TAG: "obs-moq-v${{ steps.parse.outputs.version }}" RELEASE_TITLE: "obs-moq v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts diff --git a/.github/workflows/moq-cli.yml b/.github/workflows/moq-cli.yml index 9e829309910..3975e32663c 100644 --- a/.github/workflows/moq-cli.yml +++ b/.github/workflows/moq-cli.yml @@ -39,6 +39,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -47,7 +49,7 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-cli + run: just gh release parse-version moq-cli - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -64,7 +66,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} TARGET: ${{ matrix.target }} run: | - ./sh/rs/package-binary.sh \ + just rs package-binary \ --crate moq-cli \ --bin moq \ --binary "target/${TARGET}/release/moq" \ @@ -80,8 +82,8 @@ jobs: ARCH: ${{ matrix.deb-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq run: | - sh/rs/package-nfpm.sh packaging/moq-cli/nfpm.yaml deb dist/ - sh/rs/package-nfpm.sh packaging/moq-cli/transition.yaml deb dist/ + just rs package-nfpm packaging/moq-cli/nfpm.yaml deb dist/ + just rs package-nfpm packaging/moq-cli/transition.yaml deb dist/ - name: Package .rpm shell: bash @@ -89,7 +91,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.rpm-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq - run: sh/rs/package-nfpm.sh packaging/moq-cli/nfpm.yaml rpm dist/ + run: just rs package-nfpm packaging/moq-cli/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -114,6 +116,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: determinate: false @@ -121,12 +125,12 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-cli + run: just gh release parse-version moq-cli - name: Build and package tarball shell: bash run: | - ./sh/rs/package-binary.sh \ + just rs package-binary \ --crate moq-cli \ --bin moq \ --target ${{ matrix.target }} \ @@ -154,6 +158,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -169,12 +175,12 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-cli + run: just gh release parse-version moq-cli - name: Build and package zip shell: bash run: | - ./sh/rs/package-windows.sh \ + just rs package-windows \ --crate moq-cli \ --bin moq \ --target ${{ matrix.target }} \ @@ -202,13 +208,15 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-cli + run: just gh release parse-version moq-cli - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag moq-cli + run: just gh release prev-tag moq-cli - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -222,10 +230,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-cli v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: sh/gh/trigger-repo-publish.sh + run: just gh trigger-repo-publish diff --git a/.github/workflows/moq-gst.yml b/.github/workflows/moq-gst.yml index 139d066c0fc..dd0a3218e51 100644 --- a/.github/workflows/moq-gst.yml +++ b/.github/workflows/moq-gst.yml @@ -34,6 +34,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: determinate: false @@ -41,7 +43,7 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-gst + run: just gh release parse-version moq-gst - name: Build and package (tarball) shell: bash @@ -49,7 +51,7 @@ jobs: TARGET: ${{ matrix.target }} VERSION: ${{ steps.parse.outputs.version }} run: | - ./rs/moq-gst/build.sh \ + just rs package-gst \ --target "$TARGET" \ --version "$VERSION" \ --output dist @@ -80,7 +82,7 @@ jobs: run: | set -euo pipefail tar -xzf "dist/moq-gst-v${VERSION}-${TARGET}.tar.gz" -C dist/ - ./rs/moq-gst/smoke.sh "dist/moq-gst-v${VERSION}-${TARGET}/lib/gstreamer-1.0" + just rs gst-smoke "dist/moq-gst-v${VERSION}-${TARGET}/lib/gstreamer-1.0" - name: Upload artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -112,6 +114,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -123,7 +127,7 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-gst + run: just gh release parse-version moq-gst - name: Install GStreamer dev libraries shell: bash @@ -160,7 +164,7 @@ jobs: PLUGIN_DIR: /usr/lib/${{ matrix.deb-multiarch }}/gstreamer-1.0 run: | mkdir -p dist - sh/rs/package-nfpm.sh packaging/moq-gst/nfpm.yaml deb dist/ + just rs package-nfpm packaging/moq-gst/nfpm.yaml deb dist/ - name: Package .rpm shell: bash @@ -172,7 +176,7 @@ jobs: PLUGIN_DIR: /usr/lib64/gstreamer-1.0 run: | mkdir -p dist - sh/rs/package-nfpm.sh packaging/moq-gst/nfpm.yaml rpm dist/ + just rs package-nfpm packaging/moq-gst/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -195,13 +199,15 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-gst + run: just gh release parse-version moq-gst - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag moq-gst + run: just gh release prev-tag moq-gst - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -215,10 +221,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-gst v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: sh/gh/trigger-repo-publish.sh + run: just gh trigger-repo-publish diff --git a/.github/workflows/moq-relay.yml b/.github/workflows/moq-relay.yml index 96496d97319..b9ac806f969 100644 --- a/.github/workflows/moq-relay.yml +++ b/.github/workflows/moq-relay.yml @@ -42,6 +42,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -50,7 +52,7 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-relay + run: just gh release parse-version moq-relay - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -67,7 +69,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} TARGET: ${{ matrix.target }} run: | - ./sh/rs/package-binary.sh \ + just rs package-binary \ --crate moq-relay \ --bin moq-relay \ --binary "target/${TARGET}/release/moq-relay" \ @@ -82,7 +84,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.deb-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: sh/rs/package-nfpm.sh packaging/moq-relay/nfpm.yaml deb dist/ + run: just rs package-nfpm packaging/moq-relay/nfpm.yaml deb dist/ - name: Package .rpm shell: bash @@ -90,7 +92,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} ARCH: ${{ matrix.rpm-arch }} BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: sh/rs/package-nfpm.sh packaging/moq-relay/nfpm.yaml rpm dist/ + run: just rs package-nfpm packaging/moq-relay/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -115,6 +117,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: determinate: false @@ -122,12 +126,12 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-relay + run: just gh release parse-version moq-relay - name: Build and package tarball shell: bash run: | - ./sh/rs/package-binary.sh \ + just rs package-binary \ --crate moq-relay \ --target ${{ matrix.target }} \ --version ${{ steps.parse.outputs.version }} \ @@ -154,6 +158,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -169,12 +175,12 @@ jobs: - name: Parse version id: parse shell: bash - run: sh/gh/release.sh parse-version moq-relay + run: just gh release parse-version moq-relay - name: Build and package zip shell: bash run: | - ./sh/rs/package-windows.sh \ + just rs package-windows \ --crate moq-relay \ --target ${{ matrix.target }} \ --version ${{ steps.parse.outputs.version }} \ @@ -201,13 +207,15 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-relay + run: just gh release parse-version moq-relay - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag moq-relay + run: just gh release prev-tag moq-relay - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -221,10 +229,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-relay v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: sh/gh/trigger-repo-publish.sh + run: just gh trigger-repo-publish diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 431d89002e8..3341c6d0405 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -73,9 +73,7 @@ jobs: # over independent observers of the new live broadcast map. - name: JS origin broadcast map benchmark if: ${{ !cancelled() }} - run: | - nix develop --command bun install --frozen-lockfile - nix develop --command bun js/net/bench/broadcasts.ts + run: nix develop --command just js bench - name: Audit if: ${{ !cancelled() }} @@ -286,11 +284,7 @@ jobs: choco install nasm -y --no-progress "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - # Pinned: `--locked` fixes just's own dependencies, not which version of - # just cargo selects. - - name: Install just - shell: bash - run: cargo install --locked just@1.52.0 + - uses: ./.github/actions/just - name: Check shell: bash @@ -323,10 +317,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - # Pinned: `--locked` fixes just's own dependencies, not which version of - # just cargo selects. - - name: Install just - run: cargo install --locked just@1.52.0 + - uses: ./.github/actions/just - name: Check run: just rs macos diff --git a/.github/workflows/release-brew.yml b/.github/workflows/release-brew.yml index fb06fe444f1..5e8594de054 100644 --- a/.github/workflows/release-brew.yml +++ b/.github/workflows/release-brew.yml @@ -41,6 +41,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse crate and version id: parse env: @@ -98,7 +100,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} run: | mkdir -p rendered - ./sh/gh/render-formula.sh \ + just gh formula \ --template ".github/homebrew/Formula/${CRATE}.rb.tmpl" \ --version "$VERSION" \ --release-dir release-assets \ @@ -156,7 +158,7 @@ jobs: EOF fi - python3 ../sh/gh/install-formula.py "$CRATE" "../rendered/${CRATE}.rb" . + just gh install-formula "$CRATE" "rendered/${CRATE}.rb" homebrew-tap if git diff --quiet && [[ -z "$(git status --porcelain)" ]]; then echo "No changes to commit (formula already up to date)." diff --git a/.github/workflows/release-dart-ffi.yml b/.github/workflows/release-dart-ffi.yml index 59f6c7a61c0..2744516380a 100644 --- a/.github/workflows/release-dart-ffi.yml +++ b/.github/workflows/release-dart-ffi.yml @@ -46,9 +46,11 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-ffi + run: just gh release parse-version moq-ffi build: name: Build Native Asset (${{ matrix.target }}) @@ -87,6 +89,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -114,7 +118,7 @@ jobs: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --target "$BUILD_TARGET" \ --version "$BUILD_VERSION" \ --output dist \ @@ -151,13 +155,15 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - uses: dart-lang/setup-dart@6afc89df92d6eb3834022f73cd65adc8cdfcb92d # v1.8.1 - name: Stage and check package env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./sh/dart/package.sh \ + just dart package \ --package moq_ffi \ --version "$BUILD_VERSION" \ --output release-out @@ -186,9 +192,11 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag moq-ffi + run: just gh release prev-tag moq-ffi - name: Download Native Assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -204,7 +212,7 @@ jobs: RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} RELEASE_CHECKSUMS: SHA256SUMS-dart - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts publish: name: Publish moq_ffi to pub.dev diff --git a/.github/workflows/release-dart.yml b/.github/workflows/release-dart.yml index 4df8f205f43..3578ae386c2 100644 --- a/.github/workflows/release-dart.yml +++ b/.github/workflows/release-dart.yml @@ -37,6 +37,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - uses: dart-lang/setup-dart@6afc89df92d6eb3834022f73cd65adc8cdfcb92d # v1.8.1 - name: Resolve version @@ -47,7 +49,7 @@ jobs: if [[ "$EVENT_NAME" == "pull_request" ]]; then VERSION=$(sed -n 's/^version: //p' dart/moq/pubspec.yaml | head -1) else - sh/gh/release.sh parse-version moq-dart + just gh release parse-version moq-dart exit 0 fi echo "version=$VERSION" >> "$GITHUB_OUTPUT" @@ -69,7 +71,7 @@ jobs: env: BUILD_VERSION: ${{ steps.version.outputs.version }} run: | - ./sh/dart/package.sh \ + just dart package \ --package moq \ --version "$BUILD_VERSION" \ --output release-out diff --git a/.github/workflows/release-go-ffi.yml b/.github/workflows/release-go-ffi.yml index d1d8477d387..2039f200b31 100644 --- a/.github/workflows/release-go-ffi.yml +++ b/.github/workflows/release-go-ffi.yml @@ -51,9 +51,11 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-ffi + run: just gh release parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -78,6 +80,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -103,7 +107,7 @@ jobs: BUILD_TARGET: ${{ matrix.target }} BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --target "$BUILD_TARGET" \ --version "$BUILD_VERSION" \ --output dist @@ -125,6 +129,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -144,7 +150,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --bindings-only \ --version "$BUILD_VERSION" \ --output dist @@ -165,6 +171,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download per-target libs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -208,7 +216,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./sh/go/package-ffi.sh \ + just go package-ffi \ --version "$BUILD_VERSION" \ --source-dir go/ffi \ --lib-dir libs \ @@ -232,6 +240,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -256,7 +266,7 @@ jobs: GO_FFI_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./sh/go/publish-ffi.sh + run: just go publish-ffi publish-dry-run: name: Publish dry-run @@ -269,6 +279,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -278,4 +290,4 @@ jobs: - name: Dry-run publish to mirror env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: ./sh/go/publish-ffi.sh --dry-run + run: just go publish-ffi --dry-run diff --git a/.github/workflows/release-go.yml b/.github/workflows/release-go.yml index 92300e5f131..bb817841acd 100644 --- a/.github/workflows/release-go.yml +++ b/.github/workflows/release-go.yml @@ -66,6 +66,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: "1.23" @@ -105,7 +107,7 @@ jobs: if: github.event_name == 'push' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" + run: just gh release ffi-unreleased "$FFI_VERSION" # Skip on unreleased ffi changes, and when the pinned ffi's mirror tag is # missing (release-plz merged but Release Go FFI hasn't published yet): @@ -160,7 +162,7 @@ jobs: LINE: ${{ steps.line.outputs.line }} FFI_VERSION: ${{ steps.ffi.outputs.version }} run: | - ./sh/go/package-wrapper.sh --line "$LINE" --ffi-version "$FFI_VERSION" --output release-out + just go package-wrapper --line "$LINE" --ffi-version "$FFI_VERSION" --output release-out - name: Upload wrapper module if: steps.gate.outputs.skip != 'true' @@ -180,6 +182,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -203,4 +207,4 @@ jobs: GO_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./sh/go/publish-wrapper.sh + run: just go publish-wrapper diff --git a/.github/workflows/release-kt-ffi.yml b/.github/workflows/release-kt-ffi.yml index 31a6979587a..207e70f35bf 100644 --- a/.github/workflows/release-kt-ffi.yml +++ b/.github/workflows/release-kt-ffi.yml @@ -37,9 +37,11 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-ffi + run: just gh release parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -72,6 +74,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -98,7 +102,7 @@ jobs: BUILD_TARGET: ${{ matrix.target }} BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --target "$BUILD_TARGET" \ --version "$BUILD_VERSION" \ --output dist @@ -120,6 +124,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -130,7 +136,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --bindings-only \ --version "$BUILD_VERSION" \ --output dist @@ -160,6 +166,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: distribution: temurin @@ -199,7 +207,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./sh/kt/package.sh \ + just kt package \ --version "$BUILD_VERSION" \ --lib-dir libs \ --bindings-dir bindings \ diff --git a/.github/workflows/release-kt-lib.yml b/.github/workflows/release-kt-lib.yml index 1bec21db752..ce0334622e8 100644 --- a/.github/workflows/release-kt-lib.yml +++ b/.github/workflows/release-kt-lib.yml @@ -75,6 +75,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Read moq.version id: version run: | @@ -103,18 +105,18 @@ jobs: - name: Check for unreleased FFI changes id: ffi-diff if: github.event_name != 'workflow_run' - run: sh/gh/release.sh ffi-unreleased "${{ steps.ffi.outputs.version }}" + run: just gh release ffi-unreleased "${{ steps.ffi.outputs.version }}" - name: Check wrapper publication id: wrapper - run: sh/gh/release.sh maven-exists dev.moq moq "${{ steps.version.outputs.version }}" + run: just gh release maven-exists dev.moq moq "${{ steps.version.outputs.version }}" # The bindings this checkout generates must be on Maven Central, or the # wrapper's POM range resolves to an older moq-ffi without them. - name: Check bindings publication id: bindings if: github.event_name != 'workflow_run' && steps.wrapper.outputs.exists != 'true' && steps.ffi-diff.outputs.unreleased != 'true' - run: sh/gh/release.sh maven-exists dev.moq moq-ffi "${{ steps.ffi.outputs.version }}" + run: just gh release maven-exists dev.moq moq-ffi "${{ steps.ffi.outputs.version }}" - name: Decide publish id: decide @@ -167,6 +169,8 @@ jobs: ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -178,7 +182,7 @@ jobs: env: BUILD_VERSION: ${{ needs.check-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh --bindings-only --version "$BUILD_VERSION" --output dist + just rs package-ffi --bindings-only --version "$BUILD_VERSION" --output dist mkdir -p kt/moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi/moq cp dist/bindings/kotlin/uniffi/moq/moq.kt \ kt/moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi/moq/moq.kt @@ -202,7 +206,7 @@ jobs: - name: Recheck wrapper publication id: recheck if: github.event_name != 'pull_request' - run: sh/gh/release.sh maven-exists dev.moq moq "${{ needs.check-version.outputs.version }}" + run: just gh release maven-exists dev.moq moq "${{ needs.check-version.outputs.version }}" - name: Publish to Maven Central if: github.event_name != 'pull_request' && steps.recheck.outputs.exists != 'true' diff --git a/.github/workflows/release-py-ffi.yml b/.github/workflows/release-py-ffi.yml index 2e1fa1bb49e..678abd24388 100644 --- a/.github/workflows/release-py-ffi.yml +++ b/.github/workflows/release-py-ffi.yml @@ -34,9 +34,11 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-ffi + run: just gh release parse-version moq-ffi # Maturin reads the wheel version from rs/moq-ffi/Cargo.toml (via # `dynamic = ["version"]` in pyproject.toml). If a tag is pushed by diff --git a/.github/workflows/release-py.yml b/.github/workflows/release-py.yml index 67a2741e8bf..fdb4717c7b3 100644 --- a/.github/workflows/release-py.yml +++ b/.github/workflows/release-py.yml @@ -56,9 +56,11 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Read version id: version - run: sh/gh/release.sh read-version py/moq-rs/pyproject.toml + run: just gh release read-version py/moq-rs/pyproject.toml - name: Resolve FFI version id: ffi @@ -78,14 +80,14 @@ jobs: if: github.event_name == 'push' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" + run: just gh release ffi-unreleased "$FFI_VERSION" - name: Check FFI on PyPI id: ffi-pypi if: github.event_name == 'push' && steps.ffi-gate.outputs.unreleased != 'true' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: sh/gh/release.sh pypi-exists moq-ffi "$FFI_VERSION" + run: just gh release pypi-exists moq-ffi "$FFI_VERSION" - name: Unpublished FFI notice if: github.event_name == 'push' && steps.ffi-gate.outputs.unreleased != 'true' && steps.ffi-pypi.outputs.exists != 'true' @@ -100,7 +102,7 @@ jobs: id: pypi env: VERSION: ${{ steps.version.outputs.version }} - run: sh/gh/release.sh pypi-exists moq-rs "$VERSION" + run: just gh release pypi-exists moq-rs "$VERSION" - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: diff --git a/.github/workflows/release-swift-ffi.yml b/.github/workflows/release-swift-ffi.yml index 966a1b86c35..0ab0eaa0564 100644 --- a/.github/workflows/release-swift-ffi.yml +++ b/.github/workflows/release-swift-ffi.yml @@ -34,9 +34,11 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: sh/gh/release.sh parse-version moq-ffi + run: just gh release parse-version moq-ffi build: name: Build moq-ffi (${{ matrix.target }}) @@ -56,6 +58,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -72,7 +76,7 @@ jobs: # Set iOS deployment target to avoid ___chkstk_darwin linker errors. IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --target "$BUILD_TARGET" \ --version "$BUILD_VERSION" \ --output dist @@ -93,6 +97,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -103,7 +109,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh \ + just rs package-ffi \ --bindings-only \ --version "$BUILD_VERSION" \ --output dist @@ -124,6 +130,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download per-target libs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -149,7 +157,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./sh/swift/package-ffi.sh \ + just swift package-ffi \ --version "$BUILD_VERSION" \ --lib-dir libs \ --bindings-dir bindings \ @@ -181,9 +189,11 @@ jobs: # up GH_TOKEN directly, so don't persist a contents-write token. persist-credentials: false + - uses: ./.github/actions/just + - name: Find previous tag id: prev_tag - run: sh/gh/release.sh prev-tag moq-ffi + run: just gh release prev-tag moq-ffi - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -197,7 +207,7 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: sh/gh/release.sh create artifacts + run: just gh release create artifacts verify: # Gate the mirror push on actually being able to resolve the staged package @@ -213,6 +223,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download staged package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -223,7 +235,7 @@ jobs: env: BUILD_VERSION: ${{ needs.parse-version.outputs.version }} run: | - ./sh/swift/verify-ffi.sh --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" + just swift verify-ffi --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" publish: name: Publish to Swift Package mirror @@ -235,6 +247,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -259,4 +273,4 @@ jobs: SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./sh/swift/publish-ffi.sh + run: just swift publish-ffi diff --git a/.github/workflows/release-swift-lib.yml b/.github/workflows/release-swift-lib.yml index 6361566a574..2684aa4f595 100644 --- a/.github/workflows/release-swift-lib.yml +++ b/.github/workflows/release-swift-lib.yml @@ -79,6 +79,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Read versions id: version env: @@ -101,13 +103,13 @@ jobs: if: github.event_name != 'workflow_run' env: FFI_VERSION: ${{ steps.version.outputs.ffi_version }} - run: sh/gh/release.sh ffi-unreleased "$FFI_VERSION" + run: just gh release ffi-unreleased "$FFI_VERSION" - name: Check FFI mirror tag id: ffi-gate env: FFI_VERSION: ${{ steps.version.outputs.ffi_version }} - run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" + run: just gh release git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" - name: Unpublished FFI notice if: steps.ffi-diff.outputs.unreleased != 'true' && steps.ffi-gate.outputs.exists != 'true' && github.event_name != 'pull_request' @@ -121,12 +123,12 @@ jobs: id: gate env: VERSION: ${{ steps.version.outputs.version }} - run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift "$VERSION" + run: just gh release git-tag-exists moq-dev/moq-swift "$VERSION" - name: Package env: BUILD_VERSION: ${{ steps.version.outputs.version }} - run: ./sh/swift/package.sh --version "$BUILD_VERSION" --output release-out + run: just swift package --version "$BUILD_VERSION" --output release-out - name: Upload Swift package uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -149,6 +151,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Check FFI mirror tag id: ffi if: needs.build.outputs.publish == 'true' || (github.event_name == 'pull_request' && needs.build.outputs.ffi_unreleased != 'true') @@ -158,7 +162,7 @@ jobs: # rather than a hard failure, so the first bootstrap (before # moq-swift-ffi exists) skips the cross-package resolve instead of # failing the job. publish.sh stays idempotent regardless. - run: sh/gh/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" || echo "exists=false" >> "$GITHUB_OUTPUT" + run: just gh release git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" || echo "exists=false" >> "$GITHUB_OUTPUT" - name: Download staged package if: steps.ffi.outputs.exists == 'true' @@ -171,7 +175,7 @@ jobs: if: steps.ffi.outputs.exists == 'true' env: BUILD_VERSION: ${{ needs.build.outputs.version }} - run: ./sh/swift/verify.sh --tarball "artifacts/moq-${BUILD_VERSION}-swift.tar.gz" + run: just swift verify --tarball "artifacts/moq-${BUILD_VERSION}-swift.tar.gz" - name: Skip notice if: needs.build.outputs.ffi_unreleased == 'true' @@ -194,6 +198,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -218,7 +224,7 @@ jobs: SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./sh/swift/publish.sh + run: just swift publish publish-dry-run: name: Publish dry-run @@ -231,6 +237,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -240,4 +248,4 @@ jobs: - name: Dry-run publish to mirror env: BUILD_VERSION: ${{ needs.build.outputs.version }} - run: ./sh/swift/publish.sh --dry-run + run: just swift publish --dry-run diff --git a/.github/workflows/rpm-repo.yml b/.github/workflows/rpm-repo.yml index 4c7df9dd844..93dc8c4a943 100644 --- a/.github/workflows/rpm-repo.yml +++ b/.github/workflows/rpm-repo.yml @@ -32,6 +32,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -51,10 +53,9 @@ jobs: - name: Publish to R2 shell: bash env: - ARTIFACTS_DIR: artifacts R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} SIGNING_KEY: ${{ secrets.SIGNING_KEY }} SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} - run: ./infra/rpm/publish.sh + run: just infra rpm publish "$GITHUB_WORKSPACE/artifacts" diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index 23fd44fbf30..9ba45fe5cd9 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -55,6 +55,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -83,4 +85,4 @@ jobs: done - name: Swift check - run: ./sh/swift/check.sh + run: just swift check diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 10c8135fdb7..ccccac57573 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -45,6 +45,10 @@ For each finding: - If you don't agree with it, reply to the finding and move on. - If it's a relatively easy improvement, fix it and push. Update the summary if needed. +# CI + +Workflow steps run `just` recipes, never a script path; `just gh check` enforces it. + # Follow-ups If you encounter issues, or findings that are out of scope, create follow-up quests. diff --git a/cpp/obs/justfile b/cpp/obs/justfile index eed0d373444..8039fc603fb 100644 --- a/cpp/obs/justfile +++ b/cpp/obs/justfile @@ -57,3 +57,12 @@ check: # Auto-fix formatting. fix: ../../sh/obs/fix.sh + +# Runs in the caller's directory, so a relative `--output` means what it would +# to the script itself. + +# Build and package the obs-moq plugin for release. +[no-cd] +[positional-arguments] +package *args: + {{ quote(justfile_directory() / "cpp/obs/build.sh") }} "$@" diff --git a/dart/justfile b/dart/justfile index 4387f113c81..5c2fc803ffc 100644 --- a/dart/justfile +++ b/dart/justfile @@ -11,3 +11,12 @@ fix: # Regenerate the raw UniFFI Dart bindings. generate: ../sh/dart/generate.sh + +# `package` runs in the caller's directory, so a relative `--output` means +# what it would to the script itself. + +# Package a Dart library for pub.dev. +[no-cd] +[positional-arguments] +package *args: + {{ quote(justfile_directory() / "sh/dart/package.sh") }} "$@" diff --git a/go/ffi/README.md b/go/ffi/README.md index c45b1e70264..308836ae30e 100644 --- a/go/ffi/README.md +++ b/go/ffi/README.md @@ -53,4 +53,4 @@ Compiled binaries never live in the source tree. `sh/go/check.sh` stages into th ## Release -The `release-go-ffi.yml` workflow fires on every `moq-ffi-v*` tag, builds per-target static libraries, runs `uniffi-bindgen-go`, calls `sh/go/package-ffi.sh` to assemble the module, and `sh/go/publish-ffi.sh` to push the result to `moq-dev/moq-go-ffi` with a bare-semver tag (e.g. `v0.2.18`) lockstep with the crate. Go's module proxy picks up the new tag automatically. +The `release-go-ffi.yml` workflow fires on every `moq-ffi-v*` tag, builds per-target static libraries, runs `uniffi-bindgen-go`, calls `just go package-ffi` to assemble the module, and `just go publish-ffi` to push the result to `moq-dev/moq-go-ffi` with a bare-semver tag (e.g. `v0.2.18`) lockstep with the crate. Go's module proxy picks up the new tag automatically. diff --git a/go/justfile b/go/justfile index ad26a9875d9..b4fa2d0d07c 100644 --- a/go/justfile +++ b/go/justfile @@ -16,12 +16,35 @@ default: check: bash ../sh/go/check.sh -# Stage the in-tree go/ffi source + per-target moq-ffi libs + generated -# bindings into the moq.dev/moq-ffi module ready for publish. +# Release recipes run in the caller's directory, so a relative `--output` or +# `--tarball` means what it would to the script itself. + +# Stage the in-tree go/ffi source, per-target moq-ffi libs, and generated +# bindings into the moq.dev/moq-ffi module. + +# Stage the moq.dev/moq-ffi module for publish. +[no-cd] +[positional-arguments] package-ffi *args: - bash ../sh/go/package-ffi.sh {{ args }} + {{ quote(justfile_directory() / "sh/go/package-ffi.sh") }} "$@" # Stage the in-tree go/wrapper source into the moq.dev/moq module, rewriting its -# moq.dev/moq-ffi require to the target ffi version, ready for publish. +# moq.dev/moq-ffi require to the target ffi version. + +# Stage the moq.dev/moq wrapper module for publish. +[no-cd] +[positional-arguments] package-wrapper *args: - bash ../sh/go/package-wrapper.sh {{ args }} + {{ quote(justfile_directory() / "sh/go/package-wrapper.sh") }} "$@" + +# Push the staged moq-go-ffi module to the moq-dev/moq-go-ffi mirror. +[no-cd] +[positional-arguments] +publish-ffi *args: + {{ quote(justfile_directory() / "sh/go/publish-ffi.sh") }} "$@" + +# Push the staged moq-go wrapper module to the moq-dev/moq-go mirror. +[no-cd] +[positional-arguments] +publish-wrapper *args: + {{ quote(justfile_directory() / "sh/go/publish-wrapper.sh") }} "$@" diff --git a/js/justfile b/js/justfile index 3cc52a9e22a..812fa6d9c97 100644 --- a/js/justfile +++ b/js/justfile @@ -43,6 +43,11 @@ test: bun test common/deps.test.ts bun run --filter='*' --elide-lines=0 test +# Benchmark re-pricing the origin broadcast map as routes and observers grow. +bench: + bun install --frozen-lockfile + bun net/bench/broadcasts.ts + # Build all packages. build: bun run --filter='*' build diff --git a/kt/justfile b/kt/justfile index af419fb33e0..2a3994710a6 100644 --- a/kt/justfile +++ b/kt/justfile @@ -21,8 +21,11 @@ check: generate: bash ../sh/kt/generate.sh -# Assemble the KMP module from per-target moq-ffi binaries + bindings. -# Used by .github/workflows/release-kt-ffi.yml; see kt/README.md for the -# expected --lib-dir layout. +# Runs in the caller's directory, so a relative `--lib-dir` or `--output` means +# what it would to the script itself. See kt/README.md for the --lib-dir layout. + +# Assemble the KMP module from per-target moq-ffi binaries and bindings. +[no-cd] +[positional-arguments] package *args: - bash ../sh/kt/package.sh {{ args }} + {{ quote(justfile_directory() / "sh/kt/package.sh") }} "$@" diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index ae2b81172bd..e01a88ebc77 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -19,6 +19,5 @@ requires the one before it, so they land as one line of pull requests. ## Quests -- [Workflows call just](/quest/m1/tooling/workflows-call-just.md) - no workflow `run:` step names a `.sh`; every script a workflow needs has a recipe - [Binary release workflow](/quest/m1/tooling/release-binary.md) - moq-cli and moq-relay share one reusable workflow behind two thin callers - [FFI release workflow](/quest/m1/tooling/release-ffi.md) - the five `release-*-ffi.yml` share the moq-ffi target matrix and artifact staging diff --git a/quest/m1/tooling/release-binary.md b/quest/m1/tooling/release-binary.md index 95eecf91f16..4a1dcdf0c55 100644 --- a/quest/m1/tooling/release-binary.md +++ b/quest/m1/tooling/release-binary.md @@ -25,7 +25,3 @@ and workflow name, so `alert.yml`, `release-brew.yml`, and - Verify with `just gh check` and by diffing the rendered job list of each caller against its predecessor with `gh workflow view`. The next tagged release is the end-to-end check; say so in the PR. - -## Required - -- [Workflows call just](/quest/m1/tooling/workflows-call-just.md) - the reusable workflow is written in the recipe style from the start diff --git a/quest/m1/tooling/workflows-call-just.md b/quest/m1/tooling/workflows-call-just.md deleted file mode 100644 index a38fcc5217d..00000000000 --- a/quest/m1/tooling/workflows-call-just.md +++ /dev/null @@ -1,41 +0,0 @@ -# [M] Every workflow step runs a recipe - -## Goal - -No `run:` line in `.github/workflows` names a `.sh` file. Every script a -workflow needs has a `just` recipe, so the menu is the whole command surface -and a script can move or change its arguments without touching twenty -workflows. The same recipes work locally for a dry run. - -## Plan - -Today the workflows call scripts directly at roughly ninety sites: -`sh/gh/release.sh` (54 calls as a helper library: `parse-version`, -`prev-tag`, `create`, registry existence checks), `sh/rs/package-nfpm.sh`, -`sh/rs/package-binary.sh`, `sh/rs/package-windows.sh`, -`sh/gh/trigger-repo-publish.sh`, `sh/gh/render-formula.sh`, `sh/gh/alert.sh`, -the swift `package`, `package-ffi`, `publish`, `publish-ffi`, `verify`, -`verify-ffi`, and `check` scripts, the go `package-ffi`, `package-wrapper`, -`publish-ffi`, `publish-wrapper` scripts, `sh/dart/package.sh`, -`sh/kt/package.sh`, and `infra/*/publish.sh`. - -- Add pass-through recipes where none exist, named by role under the area - module: `just gh release ...`, `just gh alert ...`, - `just gh formula ...`, `just gh trigger-repo-publish ...`, - `just rs package-binary ...`, `just rs package-nfpm ...`, - `just rs package-windows ...`, `just swift publish|verify|package-ffi|...`, - `just go publish-ffi|publish-wrapper`, `just dart package`, - `just infra apt publish` and `just infra rpm publish` (the recipes exist). - Existing wrappers (`kt package`, `swift package`, `go package-ffi`, - `go package-wrapper`, `py package`) are used as they are. -- Rewrite every `run:` step to the recipe. Where a workflow runs outside the - nix dev shell (Windows and macOS runners, AlmaLinux containers), `just` is - already installed or installable the way nightly.yml does it; keep that - pattern rather than falling back to a path. -- Record the rule in the CI section of `CONTRIBUTING.md` in one line, and - have `just gh check` enforce it: fail on any workflow `run:` line matching - `\.sh\b`. -- Verify with `actionlint` via `just gh check`, and with a manual - `workflow_dispatch` of apt-repo.yml, rpm-repo.yml, and release-winget.yml - where the inputs allow a dry run. The tag-triggered release workflows are - verified by the next release; say so in the PR. diff --git a/rs/justfile b/rs/justfile index 7b5919eec11..71c272ae43c 100644 --- a/rs/justfile +++ b/rs/justfile @@ -236,3 +236,38 @@ release: # Build a .deb or .rpm locally: `just rs package moq-relay deb`. package crate packager: sh/rs/package.sh {{ crate }} {{ packager }} + +# Build and package a workspace binary as a release tarball. +[positional-arguments] +package-binary *args: + sh/rs/package-binary.sh "$@" + +# Build and package a workspace binary as a Windows .zip. +[positional-arguments] +package-windows *args: + sh/rs/package-windows.sh "$@" + +# Render an nfpm config from the environment and build a .deb or .rpm: CONFIG PACKAGER OUTPUT. +[positional-arguments] +package-nfpm *args: + sh/rs/package-nfpm.sh "$@" + +# Build and package the moq-ffi native libraries, or only its bindings. +[positional-arguments] +package-ffi *args: + rs/moq-ffi/build.sh "$@" + +# Build and package libmoq. +[positional-arguments] +package-libmoq *args: + rs/libmoq/build.sh "$@" + +# Build and package the moq-gst GStreamer plugin. +[positional-arguments] +package-gst *args: + rs/moq-gst/build.sh "$@" + +# Load a packaged moq-gst plugin into the host's GStreamer and find its elements. +[positional-arguments] +gst-smoke *args: + rs/moq-gst/smoke.sh "$@" diff --git a/sh/gh/check-runs.ts b/sh/gh/check-runs.ts new file mode 100644 index 00000000000..c979f47b73b --- /dev/null +++ b/sh/gh/check-runs.ts @@ -0,0 +1,26 @@ +// Fail when a workflow `run:` step names a script file. Workflows call `just` +// recipes, so a script can move or change its arguments without touching them, +// and the same command works locally. +import { Glob } from "bun"; + +type Step = { run?: string }; +type Job = { steps?: Step[] }; +type Workflow = { jobs?: Record }; + +const violations: string[] = []; +for await (const path of new Glob(".github/workflows/*.{yml,yaml}").scan({ dot: true })) { + const workflow = Bun.YAML.parse(await Bun.file(path).text()) as Workflow; + for (const [name, job] of Object.entries(workflow.jobs ?? {})) { + for (const step of job.steps ?? []) { + for (const line of step.run?.split("\n") ?? []) { + if (/\.(sh|py|ts|js|mjs)\b/.test(line)) violations.push(`${path} (${name}): ${line.trim()}`); + } + } + } +} + +if (violations.length > 0) { + console.error("workflow steps must run a `just` recipe, not a script:"); + for (const violation of violations.sort()) console.error(` ${violation}`); + process.exit(1); +} diff --git a/swift/justfile b/swift/justfile index acb2b150297..075ae923142 100644 --- a/swift/justfile +++ b/swift/justfile @@ -15,7 +15,41 @@ default: check: bash ../sh/swift/check.sh +# Release recipes run in the caller's directory, so a relative `--output` or +# `--tarball` means what it would to the script itself. + # Assemble the released wrapper package from Package.swift.template. -# Used by .github/workflows/release-swift-lib.yml. +[no-cd] +[positional-arguments] package *args: - bash ../sh/swift/package.sh {{ args }} + {{ quote(justfile_directory() / "sh/swift/package.sh") }} "$@" + +# Assemble the MoqFFI package from per-target static libs and the generated bindings. +[no-cd] +[positional-arguments] +package-ffi *args: + {{ quote(justfile_directory() / "sh/swift/package-ffi.sh") }} "$@" + +# Build a throwaway consumer of a staged wrapper package. +[no-cd] +[positional-arguments] +verify *args: + {{ quote(justfile_directory() / "sh/swift/verify.sh") }} "$@" + +# Build a throwaway consumer of a staged MoqFFI package. +[no-cd] +[positional-arguments] +verify-ffi *args: + {{ quote(justfile_directory() / "sh/swift/verify-ffi.sh") }} "$@" + +# Push the staged wrapper package to the moq-dev/moq-swift mirror. +[no-cd] +[positional-arguments] +publish *args: + {{ quote(justfile_directory() / "sh/swift/publish.sh") }} "$@" + +# Push the staged MoqFFI package to the moq-dev/moq-swift-ffi mirror. +[no-cd] +[positional-arguments] +publish-ffi *args: + {{ quote(justfile_directory() / "sh/swift/publish-ffi.sh") }} "$@" From 39348893659958376d898f8074d71b40e1c52d67 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 15:29:26 -0700 Subject: [PATCH 04/18] chore(ci): moq-cli and moq-relay share one reusable release workflow (#4174) Co-authored-by: Claude Opus 5.5 --- .github/workflows/moq-cli.yml | 220 +---------------------- .github/workflows/moq-relay.yml | 219 +---------------------- .github/workflows/release-binary.yml | 253 +++++++++++++++++++++++++++ quest/m1/moq-installer.md | 2 - quest/m1/tooling/README.md | 1 - quest/m1/tooling/release-binary.md | 27 --- quest/m1/tooling/release-ffi.md | 13 +- sh/gh/alert.sh | 11 +- 8 files changed, 273 insertions(+), 473 deletions(-) create mode 100644 .github/workflows/release-binary.yml delete mode 100644 quest/m1/tooling/release-binary.md diff --git a/.github/workflows/moq-cli.yml b/.github/workflows/moq-cli.yml index 3975e32663c..0258e0ca072 100644 --- a/.github/workflows/moq-cli.yml +++ b/.github/workflows/moq-cli.yml @@ -17,223 +17,11 @@ permissions: contents: read jobs: - build: - name: Build (${{ matrix.target }}) - runs-on: ${{ matrix.runs-on }} - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-unknown-linux-gnu - runs-on: ubuntu-22.04 - deb-arch: amd64 - rpm-arch: x86_64 - - target: aarch64-unknown-linux-gnu - runs-on: ubuntu-22.04-arm - deb-arch: arm64 - rpm-arch: aarch64 - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-cli - - - name: Set up packaging tools - uses: ./.github/actions/setup-packaging - - - name: Build (release, glibc 2.34) - shell: bash - # Pin to glibc 2.34 so one binary covers Ubuntu 22.04+ (.deb) - # and AlmaLinux/RHEL/Rocky 9+ (.rpm). - run: cargo zigbuild --locked --release --target ${{ matrix.target }}.2.34 -p moq-cli - - - name: Package release assets - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - TARGET: ${{ matrix.target }} - run: | - just rs package-binary \ - --crate moq-cli \ - --bin moq \ - --binary "target/${TARGET}/release/moq" \ - --bare \ - --version "$VERSION" \ - --target "$TARGET" \ - --output dist - - - name: Package .deb (Ubuntu 22.04 toolchain) - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.deb-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq - run: | - just rs package-nfpm packaging/moq-cli/nfpm.yaml deb dist/ - just rs package-nfpm packaging/moq-cli/transition.yaml deb dist/ - - - name: Package .rpm - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.rpm-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq - run: just rs package-nfpm packaging/moq-cli/nfpm.yaml rpm dist/ - - - name: Upload artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-cli-${{ matrix.target }} - path: dist/* - - build-tarball-macos: - name: Build macOS tarball (${{ matrix.target }}) - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - os: macos-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: ./.github/actions/just - - - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - with: - determinate: false - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-cli - - - name: Build and package tarball - shell: bash - run: | - just rs package-binary \ - --crate moq-cli \ - --bin moq \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-cli-${{ matrix.target }} - path: dist/* - - build-windows: - name: Build Windows zip (${{ matrix.target }}) - runs-on: windows-latest - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-pc-windows-msvc - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - # aws-lc-rs assembles its x86_64 crypto with NASM on Windows. - - name: Install NASM - shell: pwsh - run: | - choco install nasm -y --no-progress - "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-cli - - - name: Build and package zip - shell: bash - run: | - just rs package-windows \ - --crate moq-cli \ - --bin moq \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-cli-${{ matrix.target }} - path: dist/* - release: - name: Release - needs: [build, build-tarball-macos, build-windows] - if: github.ref_type == 'tag' - runs-on: ubuntu-latest + uses: ./.github/workflows/release-binary.yml + with: + crate: moq-cli + bin: moq permissions: contents: write actions: write - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-cli - - - name: Find previous tag - id: prev_tag - run: just gh release prev-tag moq-cli - - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - path: artifacts - merge-multiple: true - - - name: Create or update release - env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-cli v${{ steps.parse.outputs.version }}" - RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: just gh release create artifacts - - - name: Trigger apt/rpm repo publish - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ github.ref_name }} - run: just gh trigger-repo-publish diff --git a/.github/workflows/moq-relay.yml b/.github/workflows/moq-relay.yml index b9ac806f969..cfb4767b530 100644 --- a/.github/workflows/moq-relay.yml +++ b/.github/workflows/moq-relay.yml @@ -17,222 +17,11 @@ permissions: contents: read jobs: - build: - name: Build (${{ matrix.target }}) - runs-on: ${{ matrix.runs-on }} - - strategy: - fail-fast: false - matrix: - include: - # Native runners. Each binary is built on the same arch it ships - # for, so the .deb/.rpm linkage matches the distro's glibc on the - # oldest LTS we support (Ubuntu 22.04 / AlmaLinux 9). - - target: x86_64-unknown-linux-gnu - runs-on: ubuntu-22.04 - deb-arch: amd64 - rpm-arch: x86_64 - - target: aarch64-unknown-linux-gnu - runs-on: ubuntu-22.04-arm - deb-arch: arm64 - rpm-arch: aarch64 - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-relay - - - name: Set up packaging tools - uses: ./.github/actions/setup-packaging - - - name: Build (release, glibc 2.34) - shell: bash - # Pin to glibc 2.34 so one binary covers Ubuntu 22.04+ (.deb) - # and AlmaLinux/RHEL/Rocky 9+ (.rpm). - run: cargo zigbuild --locked --release --target ${{ matrix.target }}.2.34 -p moq-relay - - - name: Package release assets - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - TARGET: ${{ matrix.target }} - run: | - just rs package-binary \ - --crate moq-relay \ - --bin moq-relay \ - --binary "target/${TARGET}/release/moq-relay" \ - --bare \ - --version "$VERSION" \ - --target "$TARGET" \ - --output dist - - - name: Package .deb (Ubuntu 22.04 toolchain, glibc 2.35) - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.deb-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: just rs package-nfpm packaging/moq-relay/nfpm.yaml deb dist/ - - - name: Package .rpm - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.rpm-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: just rs package-nfpm packaging/moq-relay/nfpm.yaml rpm dist/ - - - name: Upload artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-relay-${{ matrix.target }} - path: dist/* - - build-tarball-macos: - name: Build macOS tarball (${{ matrix.target }}) - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - os: macos-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: ./.github/actions/just - - - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - with: - determinate: false - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-relay - - - name: Build and package tarball - shell: bash - run: | - just rs package-binary \ - --crate moq-relay \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-relay-${{ matrix.target }} - path: dist/* - - build-windows: - name: Build Windows zip (${{ matrix.target }}) - runs-on: windows-latest - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-pc-windows-msvc - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - # aws-lc-rs assembles its x86_64 crypto with NASM on Windows. - - name: Install NASM - shell: pwsh - run: | - choco install nasm -y --no-progress - "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Parse version - id: parse - shell: bash - run: just gh release parse-version moq-relay - - - name: Build and package zip - shell: bash - run: | - just rs package-windows \ - --crate moq-relay \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-relay-${{ matrix.target }} - path: dist/* - release: - name: Release - needs: [build, build-tarball-macos, build-windows] - if: github.ref_type == 'tag' - runs-on: ubuntu-latest + uses: ./.github/workflows/release-binary.yml + with: + crate: moq-relay + bin: moq-relay permissions: contents: write actions: write - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-relay - - - name: Find previous tag - id: prev_tag - run: just gh release prev-tag moq-relay - - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - path: artifacts - merge-multiple: true - - - name: Create or update release - env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-relay v${{ steps.parse.outputs.version }}" - RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: just gh release create artifacts - - - name: Trigger apt/rpm repo publish - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ github.ref_name }} - run: just gh trigger-repo-publish diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml new file mode 100644 index 00000000000..56b518f5a23 --- /dev/null +++ b/.github/workflows/release-binary.yml @@ -0,0 +1,253 @@ +name: release-binary + +# Builds, packages, and (on a tag) releases one workspace binary. moq-cli.yml +# and moq-relay.yml call it; they keep the tag trigger and the workflow name +# that alert.yml, release-brew.yml, and release-winget.yml match on. + +on: + workflow_call: + inputs: + crate: + description: "Cargo package to build, e.g. moq-cli" + required: true + type: string + bin: + description: "Executable the package ships, e.g. moq" + required: true + type: string + +permissions: + contents: read + +env: + CRATE: ${{ inputs.crate }} + BIN: ${{ inputs.bin }} + +jobs: + build: + name: Build (${{ matrix.target }}) + runs-on: ${{ matrix.runs-on }} + + strategy: + fail-fast: false + matrix: + include: + # Native runners. Each binary is built on the same arch it ships + # for, so the .deb/.rpm linkage matches the distro's glibc on the + # oldest LTS we support (Ubuntu 22.04 / AlmaLinux 9). + - target: x86_64-unknown-linux-gnu + runs-on: ubuntu-22.04 + deb-arch: amd64 + rpm-arch: x86_64 + - target: aarch64-unknown-linux-gnu + runs-on: ubuntu-22.04-arm + deb-arch: arm64 + rpm-arch: aarch64 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + with: + targets: ${{ matrix.target }} + + - name: Parse version + id: parse + shell: bash + run: just gh release parse-version "$CRATE" + + - name: Set up packaging tools + uses: ./.github/actions/setup-packaging + + - name: Build (release, glibc 2.34) + shell: bash + # Pin to glibc 2.34 so one binary covers Ubuntu 22.04+ (.deb) + # and AlmaLinux/RHEL/Rocky 9+ (.rpm). + run: cargo zigbuild --locked --release --target ${{ matrix.target }}.2.34 -p "$CRATE" + + - name: Package release assets + shell: bash + env: + VERSION: ${{ steps.parse.outputs.version }} + TARGET: ${{ matrix.target }} + run: | + just rs package-binary \ + --crate "$CRATE" \ + --bin "$BIN" \ + --binary "target/${TARGET}/release/${BIN}" \ + --bare \ + --version "$VERSION" \ + --target "$TARGET" \ + --output dist + + # A transition.yaml beside nfpm.yaml is a deb-only package that keeps + # apt upgrades following a renamed package. + - name: Package .deb (Ubuntu 22.04 toolchain) + shell: bash + env: + VERSION: ${{ steps.parse.outputs.version }} + ARCH: ${{ matrix.deb-arch }} + BINARY_PATH: target/${{ matrix.target }}/release/${{ inputs.bin }} + run: | + just rs package-nfpm "packaging/$CRATE/nfpm.yaml" deb dist/ + if [[ -f "packaging/$CRATE/transition.yaml" ]]; then + just rs package-nfpm "packaging/$CRATE/transition.yaml" deb dist/ + fi + + - name: Package .rpm + shell: bash + env: + VERSION: ${{ steps.parse.outputs.version }} + ARCH: ${{ matrix.rpm-arch }} + BINARY_PATH: target/${{ matrix.target }}/release/${{ inputs.bin }} + run: just rs package-nfpm "packaging/$CRATE/nfpm.yaml" rpm dist/ + + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.crate }}-${{ matrix.target }} + path: dist/* + + build-tarball-macos: + name: Build macOS tarball (${{ matrix.target }}) + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + os: macos-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: ./.github/actions/just + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + + - name: Parse version + id: parse + shell: bash + run: just gh release parse-version "$CRATE" + + - name: Build and package tarball + shell: bash + run: | + just rs package-binary \ + --crate "$CRATE" \ + --bin "$BIN" \ + --target ${{ matrix.target }} \ + --version ${{ steps.parse.outputs.version }} \ + --output dist + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.crate }}-${{ matrix.target }} + path: dist/* + + build-windows: + name: Build Windows zip (${{ matrix.target }}) + runs-on: windows-latest + + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-pc-windows-msvc + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + with: + targets: ${{ matrix.target }} + + # aws-lc-rs assembles its x86_64 crypto with NASM on Windows. + - name: Install NASM + shell: pwsh + run: | + choco install nasm -y --no-progress + "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Parse version + id: parse + shell: bash + run: just gh release parse-version "$CRATE" + + - name: Build and package zip + shell: bash + run: | + just rs package-windows \ + --crate "$CRATE" \ + --bin "$BIN" \ + --target ${{ matrix.target }} \ + --version ${{ steps.parse.outputs.version }} \ + --output dist + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.crate }}-${{ matrix.target }} + path: dist/* + + release: + name: Release + needs: [build, build-tarball-macos, build-windows] + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + contents: write + actions: write + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Parse version + id: parse + run: just gh release parse-version "$CRATE" + + - name: Find previous tag + id: prev_tag + run: just gh release prev-tag "$CRATE" + + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + path: artifacts + merge-multiple: true + + - name: Create or update release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} + RELEASE_TITLE: "${{ inputs.crate }} v${{ steps.parse.outputs.version }}" + RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} + run: just gh release create artifacts + + - name: Trigger apt/rpm repo publish + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: just gh trigger-repo-publish diff --git a/quest/m1/moq-installer.md b/quest/m1/moq-installer.md index 78b023d5145..6dfcd4072e2 100644 --- a/quest/m1/moq-installer.md +++ b/quest/m1/moq-installer.md @@ -76,8 +76,6 @@ service setup, Windows support, or new release targets. ## Related -- [Binary release workflow](/quest/m1/tooling/release-binary.md) - reuse its - artifacts without requiring workflow consolidation - [`moq relay`](/quest/m1/moq-relay-subcommand.md) - relay functionality joins the same executable independently of its installation method - [Install URL](/quest/m1/moq-install-url.md) - exposes this installer through diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index e01a88ebc77..c210608dc1d 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -19,5 +19,4 @@ requires the one before it, so they land as one line of pull requests. ## Quests -- [Binary release workflow](/quest/m1/tooling/release-binary.md) - moq-cli and moq-relay share one reusable workflow behind two thin callers - [FFI release workflow](/quest/m1/tooling/release-ffi.md) - the five `release-*-ffi.yml` share the moq-ffi target matrix and artifact staging diff --git a/quest/m1/tooling/release-binary.md b/quest/m1/tooling/release-binary.md deleted file mode 100644 index 4a1dcdf0c55..00000000000 --- a/quest/m1/tooling/release-binary.md +++ /dev/null @@ -1,27 +0,0 @@ -# [M] One reusable workflow for the binary releases - -## Goal - -`moq-cli.yml` and `moq-relay.yml` are two copies of the same workflow (226 -and 225 lines) with the binary name swapped. They become one reusable -`release-binary.yml` taking `crate` and `bin` inputs, and two callers of a -dozen lines each that keep their tag trigger -and workflow name, so `alert.yml`, `release-brew.yml`, and -`release-winget.yml` keep matching on the names they match today. - -## Plan - -- `release-binary.yml` on `workflow_call` with inputs `crate` (the cargo - package) and `bin` (the executable name; `moq-cli` ships `moq`). It holds the Linux native-runner matrix, - the macOS tarball job, the Windows job, the `.deb` and `.rpm` packaging, the - release creation, and the repo-publish trigger, all as `just` recipes per - the preceding quest. -- Each caller keeps `name:` and `on.push.tags` exactly as they are and does - nothing but `uses: ./.github/workflows/release-binary.yml` with the two - inputs and the secrets it forwards. `workflow_run` consumers trigger off - the caller's name, so nothing downstream changes. -- The one text difference between the three today (a comment and a step name - about the glibc floor) belongs in the reusable file. -- Verify with `just gh check` and by diffing the rendered job list of each - caller against its predecessor with `gh workflow view`. The next tagged - release is the end-to-end check; say so in the PR. diff --git a/quest/m1/tooling/release-ffi.md b/quest/m1/tooling/release-ffi.md index 1627f981c8a..58ac9efe08b 100644 --- a/quest/m1/tooling/release-ffi.md +++ b/quest/m1/tooling/release-ffi.md @@ -24,12 +24,11 @@ only its packaging and publish steps. read the version from `rs/moq-ffi/Cargo.toml` instead. The reusable workflow keeps that fallback (an input or a ref check), or those PR runs die before building. -- Callers keep their `name:` and triggers, as the binary quest did, so - `alert.yml` and the `workflow_run` chains (`release-go.yml`, - `release-py.yml`, `release-swift-lib.yml`) are untouched. +- Callers keep their `name:` and triggers, as `moq-cli.yml` and + `moq-relay.yml` do over `release-binary.yml`, so `alert.yml` and the + `workflow_run` chains (`release-go.yml`, `release-py.yml`, + `release-swift-lib.yml`) are untouched. A calling job must grant every + permission a called job raises to, and `alert.sh` already skips a + `workflow_call`-only file. - Verify with `just gh check`; the next tagged `moq-ffi-v*` release is the end-to-end check. - -## Required - -- [Binary release workflow](/quest/m1/tooling/release-binary.md) - proves the reusable-plus-callers pattern before it is applied to five files diff --git a/sh/gh/alert.sh b/sh/gh/alert.sh index cfdec039c48..ee95b2d7e39 100755 --- a/sh/gh/alert.sh +++ b/sh/gh/alert.sh @@ -88,7 +88,7 @@ check_coverage() { return 1 } -# Names of workflows carrying at least one non-pull_request trigger. +# Names of workflows with a trigger that starts a run outside a pull request. # # The YAML is parsed rather than pattern-matched. GitHub accepts `on:` as a # block map, block sequence, flow sequence or bare scalar, any of which may wrap @@ -113,9 +113,10 @@ non_pr_workflow_names() { printf "%s\n" "${files[@]}" | bun -e ' const files = (await Bun.stdin.text()).split("\n").filter(Boolean); -// Both report their failure as a check on the PR itself, so alert.yml skips -// them at runtime and a workflow triggered only by these needs no entry. -const PR_EVENTS = new Set(["pull_request", "pull_request_target"]); +// The PR events report their failure as a check on the PR itself, and +// workflow_call runs inside the run that called it, so a workflow triggered +// only by these needs no entry. +const UNWATCHED = new Set(["pull_request", "pull_request_target", "workflow_call"]); const names = []; for (const file of files) { const doc = Bun.YAML.parse(await Bun.file(file).text()); @@ -132,7 +133,7 @@ for (const file of files) { console.error("alert.sh: cannot read the on: value of " + file); process.exit(2); } - if (!triggers.some((t) => !PR_EVENTS.has(t))) continue; + if (!triggers.some((t) => !UNWATCHED.has(t))) continue; const name = doc.name; if (typeof name !== "string" || name.trim() === "") { From e1997df5866c9a578a65ff02644619fbaf85c953 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 15:43:18 -0700 Subject: [PATCH 05/18] chore(ci): the moq-ffi releases share one reusable build workflow (#4185) Co-authored-by: Claude Opus 5.5 --- .github/workflows/release-dart-ffi.yml | 166 ++++++++-------------- .github/workflows/release-ffi.yml | 174 ++++++++++++++++++++++++ .github/workflows/release-go-ffi.yml | 157 +++------------------ .github/workflows/release-kt-ffi.yml | 148 +++----------------- .github/workflows/release-swift-ffi.yml | 118 ++-------------- quest/m1/tooling/README.md | 9 +- quest/m1/tooling/release-ffi.md | 34 ----- 7 files changed, 288 insertions(+), 518 deletions(-) create mode 100644 .github/workflows/release-ffi.yml delete mode 100644 quest/m1/tooling/release-ffi.md diff --git a/.github/workflows/release-dart-ffi.yml b/.github/workflows/release-dart-ffi.yml index 2744516380a..3b887b22c8f 100644 --- a/.github/workflows/release-dart-ffi.yml +++ b/.github/workflows/release-dart-ffi.yml @@ -15,6 +15,7 @@ on: pull_request: paths: - ".github/workflows/release-dart-ffi.yml" + - ".github/workflows/release-ffi.yml" - "dart/moq_ffi/**" - "sh/dart/**" - "rs/moq-ffi/**" @@ -31,123 +32,71 @@ concurrency: group: release-dart-ffi-${{ github.ref_type == 'tag' && 'publish' || github.ref }} cancel-in-progress: ${{ github.ref_type != 'tag' }} -env: - CARGO_NDK_VERSION: 4.1.2 - jobs: - parse-version: - name: Parse version + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: dart + # The raw bindings are checked in under dart/moq_ffi, so none here. + bindings: false + default-features: false + targets: >- + [ + "aarch64-linux-android", + "armv7-linux-androideabi", + "x86_64-linux-android", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-sim", + "x86_64-apple-ios", + "x86_64-pc-windows-msvc" + ] + + # hook/build.dart downloads moq-ffi--- and its + # .sha256 from the release, so each target's shared library is renamed flat. + assets: + name: Stage Native Assets + needs: [ffi] runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-ffi - - build: - name: Build Native Asset (${{ matrix.target }}) - needs: [parse-version] - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-linux-android - os: ubuntu-latest - - target: armv7-linux-androideabi - os: ubuntu-latest - - target: x86_64-linux-android - os: ubuntu-latest - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-apple-darwin - os: macos-latest - - target: x86_64-apple-darwin - os: macos-latest - - target: aarch64-apple-ios - os: macos-latest - - target: aarch64-apple-ios-sim - os: macos-latest - - target: x86_64-apple-ios - os: macos-latest - - target: x86_64-pc-windows-msvc - os: windows-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 + - name: Download per-target libs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - save-if: ${{ github.event_name != 'pull_request' }} + path: libs-raw + pattern: dart-lib-* - - name: Install cross-compilation tools (Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get update - sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - - - name: Install cargo-ndk (Android) - if: contains(matrix.target, 'android') - run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" - - - name: Build and stage asset - shell: bash + - name: Stage env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | - just rs package-ffi \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist \ - --no-default-features - - case "$BUILD_TARGET" in - *-apple-ios* | *-apple-darwin*) LIBRARY=libmoq_ffi.dylib ;; - *-windows-*) LIBRARY=moq_ffi.dll ;; - *) LIBRARY=libmoq_ffi.so ;; - esac - - NAME="moq-ffi-$BUILD_VERSION-$BUILD_TARGET-$LIBRARY" mkdir -p assets - cp "dist/moq-ffi-$BUILD_VERSION-$BUILD_TARGET/lib/$LIBRARY" "assets/$NAME" - if command -v sha256sum >/dev/null 2>&1; then - sha256sum "assets/$NAME" > "assets/$NAME.sha256" - else - shasum -a 256 "assets/$NAME" > "assets/$NAME.sha256" - fi - - - name: Upload Native Asset + for dir in libs-raw/dart-lib-*; do + target="${dir#libs-raw/dart-lib-}" + case "$target" in + *-apple-ios* | *-apple-darwin*) library=libmoq_ffi.dylib ;; + *-windows-*) library=moq_ffi.dll ;; + *) library=libmoq_ffi.so ;; + esac + name="moq-ffi-$BUILD_VERSION-$target-$library" + cp "$dir/$library" "assets/$name" + (cd assets && sha256sum "$name" > "$name.sha256") + done + ls -la assets + + - name: Upload Native Assets uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: dart-asset-${{ matrix.target }} + name: dart-assets path: assets/* package: name: Check pub package - needs: [parse-version] + needs: [ffi] runs-on: ubuntu-latest steps: @@ -161,7 +110,7 @@ jobs: - name: Stage and check package env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | just dart package \ --package moq_ffi \ @@ -180,7 +129,7 @@ jobs: release: name: Attach Native Assets - needs: [parse-version, build] + needs: [ffi, assets] if: github.ref_type == 'tag' runs-on: ubuntu-latest permissions: @@ -201,22 +150,21 @@ jobs: - name: Download Native Assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: + name: dart-assets path: artifacts - pattern: dart-asset-* - merge-multiple: true - name: Create or update release env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" + RELEASE_TITLE: "moq-ffi v${{ needs.ffi.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} RELEASE_CHECKSUMS: SHA256SUMS-dart run: just gh release create artifacts publish: name: Publish moq_ffi to pub.dev - needs: [parse-version, package, release] + needs: [ffi, package, release] if: github.ref_type == 'tag' runs-on: ubuntu-latest environment: @@ -235,5 +183,5 @@ jobs: - name: Publish env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: cd "package/moq_ffi-$BUILD_VERSION" && dart pub publish --force diff --git a/.github/workflows/release-ffi.yml b/.github/workflows/release-ffi.yml new file mode 100644 index 00000000000..26eaf7fa90a --- /dev/null +++ b/.github/workflows/release-ffi.yml @@ -0,0 +1,174 @@ +name: release-ffi + +# Builds rs/moq-ffi for each requested target and generates its UniFFI +# bindings. The release-{dart,go,kt,swift}-ffi.yml callers package and publish; +# they keep the moq-ffi-v* tag trigger and the workflow name that alert.yml and +# the workflow_run chains match on. +# +# Uploads, all prefixed by `language` so callers sharing one nightly run never +# collide: +# -lib- the target's native libraries (the lib/ directory) +# -bindings dist/bindings//, when `bindings` is set + +on: + workflow_call: + inputs: + language: + description: "Binding language and artifact prefix, e.g. kotlin" + required: true + type: string + targets: + description: "JSON array of Rust targets to build" + required: true + type: string + bindings: + description: "Generate and upload the bindings for `language`" + type: boolean + default: true + default-features: + description: "Build moq-ffi with its default features (audio and video)" + type: boolean + default: true + outputs: + version: + description: "moq-ffi version, from the tag or (off a tag) rs/moq-ffi/Cargo.toml" + value: ${{ jobs.parse-version.outputs.version }} + +permissions: + contents: read + +# Pinned rather than "latest": `--locked` only fixes the tool's own +# dependencies, not which version of the tool cargo selects. +env: + CARGO_NDK_VERSION: 4.1.2 + +jobs: + parse-version: + name: Parse version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.parse.outputs.version }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Parse version + id: parse + run: just gh release parse-version moq-ffi + + build: + name: Build moq-ffi (${{ matrix.target }}) + needs: [parse-version] + runs-on: ${{ contains(matrix.target, '-apple-') && 'macos-latest' || contains(matrix.target, '-windows-') && 'windows-latest' || 'ubuntu-latest' }} + + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(inputs.targets) }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + with: + targets: ${{ matrix.target }} + + - name: Rust cache + uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 + with: + # Don't let untrusted PR builds populate caches reused by trusted runs. + save-if: ${{ github.event_name != 'pull_request' }} + + - name: Install cross-compilation tools (Linux ARM64) + if: matrix.target == 'aarch64-unknown-linux-gnu' + run: | + sudo apt-get update + # g++ as well as gcc: moq-video's openh264 fallback is C++, and + # cc-rs looks for aarch64-linux-gnu-g++ by name. + sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + + - name: Install cargo-ndk (Android) + if: contains(matrix.target, 'android') + run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" + + - name: Build + shell: bash + env: + BUILD_TARGET: ${{ matrix.target }} + BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + DEFAULT_FEATURES: ${{ inputs.default-features }} + # Set iOS deployment target to avoid ___chkstk_darwin linker errors. + IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} + run: | + args=(--target "$BUILD_TARGET" --version "$BUILD_VERSION" --output dist) + if [[ "$DEFAULT_FEATURES" == false ]]; then + args+=(--no-default-features) + fi + just rs package-ffi "${args[@]}" + + - name: Upload lib + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.language }}-lib-${{ matrix.target }} + # Only the lib/ contents, so a caller's download is /. + path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ + + bindings: + name: Generate bindings + needs: [parse-version] + if: inputs.bindings + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + - name: Rust cache + uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 + with: + # Don't let untrusted PR builds populate caches reused by trusted runs. + save-if: ${{ github.event_name != 'pull_request' }} + + # Go uses a third-party generator; build.sh skips Go when it is absent. + - name: Install uniffi-bindgen-go + if: inputs.language == 'go' + env: + # A fork until NordSecurity ships a uniffi 0.32 generator; see flake.nix. + UNIFFI_BINDGEN_GO_REPO: https://github.com/kixelated/uniffi-bindgen-go + UNIFFI_BINDGEN_GO_REV: v0.9.0+v0.32.0 + run: | + cargo install --locked uniffi-bindgen-go \ + --git "$UNIFFI_BINDGEN_GO_REPO" \ + --rev "$UNIFFI_BINDGEN_GO_REV" + + - name: Generate bindings + env: + BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + run: | + just rs package-ffi \ + --bindings-only \ + --version "$BUILD_VERSION" \ + --output dist + + - name: Upload bindings + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.language }}-bindings + path: dist/bindings/${{ inputs.language }}/ + # A language build.sh does not generate would otherwise upload nothing. + if-no-files-found: error diff --git a/.github/workflows/release-go-ffi.yml b/.github/workflows/release-go-ffi.yml index 2039f200b31..e35e940046b 100644 --- a/.github/workflows/release-go-ffi.yml +++ b/.github/workflows/release-go-ffi.yml @@ -16,6 +16,7 @@ on: pull_request: paths: - ".github/workflows/release-go-ffi.yml" + - ".github/workflows/release-ffi.yml" - "sh/gh/release.sh" - "go/ffi/**" - "sh/go/package-ffi.sh" @@ -33,141 +34,27 @@ concurrency: group: release-go-ffi-${{ github.ref_type == 'tag' && 'publish' || github.ref }} cancel-in-progress: ${{ github.ref_type != 'tag' }} -env: - # A fork until NordSecurity ships a uniffi 0.32 generator; see flake.nix. - # The revision is v0.9.0+v0.32.0. - UNIFFI_BINDGEN_GO_REPO: https://github.com/kixelated/uniffi-bindgen-go - UNIFFI_BINDGEN_GO_REV: v0.9.0+v0.32.0 - jobs: - parse-version: - name: Parse version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-ffi - - build: - name: Build moq-ffi (${{ matrix.target }}) - needs: [parse-version] - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-apple-darwin - os: macos-latest - - target: x86_64-pc-windows-msvc - os: windows-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - with: - # Don't let untrusted PR builds populate caches reused by trusted runs. - save-if: ${{ github.event_name != 'pull_request' }} - - - name: Install cross-compilation tools (Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get update - # g++ as well as gcc: moq-video's openh264 fallback is C++, and - # cc-rs looks for aarch64-linux-gnu-g++ by name. - sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - - - name: Build - shell: bash - env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - just rs package-ffi \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload lib - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: go-lib-${{ matrix.target }} - # package.sh expects /; upload just lib/ contents. - path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ - - bindings: - name: Generate bindings - needs: [parse-version] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - with: - # Don't let untrusted PR builds populate caches reused by trusted runs. - save-if: ${{ github.event_name != 'pull_request' }} - - - name: Install uniffi-bindgen-go - run: | - cargo install --locked uniffi-bindgen-go \ - --git "$UNIFFI_BINDGEN_GO_REPO" \ - --rev "$UNIFFI_BINDGEN_GO_REV" - - - name: Generate bindings - env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - just rs package-ffi \ - --bindings-only \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload bindings - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: go-bindings - path: dist/bindings/go/ + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: go + targets: >- + [ + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc" + ] package: name: Package - needs: [parse-version, build, bindings] + needs: [ffi] runs-on: ubuntu-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -214,7 +101,7 @@ jobs: - name: Package env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | just go package-ffi \ --version "$BUILD_VERSION" \ @@ -231,12 +118,12 @@ jobs: publish: name: Publish to Go module mirror - needs: [package, parse-version] + needs: [ffi, package] runs-on: ubuntu-latest if: github.ref_type == 'tag' steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -261,7 +148,7 @@ jobs: - name: Publish to moq-dev/moq-go-ffi mirror env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} # Token is minted fresh per run, expires in 1 hour, never at rest. GO_FFI_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot @@ -270,12 +157,12 @@ jobs: publish-dry-run: name: Publish dry-run - needs: [package, parse-version] + needs: [ffi, package] runs-on: ubuntu-latest if: github.ref_type != 'tag' steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false @@ -289,5 +176,5 @@ jobs: - name: Dry-run publish to mirror env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: just go publish-ffi --dry-run diff --git a/.github/workflows/release-kt-ffi.yml b/.github/workflows/release-kt-ffi.yml index 207e70f35bf..0ce2867dafd 100644 --- a/.github/workflows/release-kt-ffi.yml +++ b/.github/workflows/release-kt-ffi.yml @@ -20,137 +20,27 @@ concurrency: group: release-kt-ffi cancel-in-progress: false -# Pinned rather than "latest": `--locked` only fixes the tool's own -# dependencies, not which version of the tool cargo selects. -env: - CARGO_NDK_VERSION: 4.1.2 - jobs: - parse-version: - name: Parse version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-ffi - - build: - name: Build moq-ffi (${{ matrix.target }}) - needs: [parse-version] - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - # Android - - target: aarch64-linux-android - os: ubuntu-latest - - target: armv7-linux-androideabi - os: ubuntu-latest - - target: x86_64-linux-android - os: ubuntu-latest - # Desktop JVM - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-apple-darwin - os: macos-latest - - target: x86_64-pc-windows-msvc - os: windows-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - - - name: Install cross-compilation tools (Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get update - # g++ as well as gcc: moq-video's openh264 fallback is C++, and - # cc-rs looks for aarch64-linux-gnu-g++ by name. - sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - - - name: Install cargo-ndk (Android) - if: contains(matrix.target, 'android') - run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" - - - name: Build - shell: bash - env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - just rs package-ffi \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload lib - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: kotlin-lib-${{ matrix.target }} - # Upload only the lib/ contents; package.sh expects /. - path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ - - bindings: - name: Generate bindings - needs: [parse-version] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - - - name: Generate bindings - env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - just rs package-ffi \ - --bindings-only \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload bindings - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: kotlin-bindings - # Upload only the kotlin subdir. - path: dist/bindings/kotlin/ + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: kotlin + # Android, then the desktop JVM. + targets: >- + [ + "aarch64-linux-android", + "armv7-linux-androideabi", + "x86_64-linux-android", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc" + ] package: name: Package and publish - needs: [parse-version, build, bindings] + needs: [ffi] runs-on: ubuntu-latest env: # Picked up by com.vanniktech.maven.publish via Gradle project properties. @@ -205,7 +95,7 @@ jobs: - name: Stage native libs + bindings into the gradle module env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | just kt package \ --version "$BUILD_VERSION" \ @@ -216,7 +106,7 @@ jobs: - name: Publish to Maven Central if: github.ref_type == 'tag' env: - MOQFFI_VERSION: ${{ needs.parse-version.outputs.version }} + MOQFFI_VERSION: ${{ needs.ffi.outputs.version }} ORG_GRADLE_PROJECT_signingInMemoryKey: ${{ secrets.SIGNING_KEY }} ORG_GRADLE_PROJECT_signingInMemoryKeyPassword: ${{ secrets.SIGNING_PASSWORD }} run: | diff --git a/.github/workflows/release-swift-ffi.yml b/.github/workflows/release-swift-ffi.yml index 0ab0eaa0564..afa2fbf175e 100644 --- a/.github/workflows/release-swift-ffi.yml +++ b/.github/workflows/release-swift-ffi.yml @@ -23,106 +23,16 @@ concurrency: cancel-in-progress: false jobs: - parse-version: - name: Parse version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Parse version - id: parse - run: just gh release parse-version moq-ffi - - build: - name: Build moq-ffi (${{ matrix.target }}) - needs: [parse-version] - runs-on: macos-latest - - strategy: - fail-fast: false - matrix: - target: - - aarch64-apple-ios - - aarch64-apple-ios-sim - - aarch64-apple-darwin - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - - - name: Build - shell: bash - env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - # Set iOS deployment target to avoid ___chkstk_darwin linker errors. - IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} - run: | - just rs package-ffi \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload lib - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: swift-lib-${{ matrix.target }} - path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ - - bindings: - name: Generate bindings - needs: [parse-version] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - uses: ./.github/actions/just - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - - - name: Rust cache - uses: Swatinem/rust-cache@f0d9c3887740aee45f6153b24b3a6b815192ec16 # v2 - - - name: Generate bindings - env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - just rs package-ffi \ - --bindings-only \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload bindings - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: swift-bindings - path: dist/bindings/swift/ + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: swift + targets: '["aarch64-apple-ios", "aarch64-apple-ios-sim", "aarch64-apple-darwin"]' package: name: Package - needs: [parse-version, build, bindings] + needs: [ffi] runs-on: macos-latest steps: @@ -155,7 +65,7 @@ jobs: - name: Package env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | just swift package-ffi \ --version "$BUILD_VERSION" \ @@ -175,7 +85,7 @@ jobs: # points at this asset, so it must exist before any consumer resolves the # SPM tag. name: Attach assets to GitHub release - needs: [package, parse-version] + needs: [ffi, package] if: github.ref_type == 'tag' runs-on: ubuntu-latest permissions: @@ -205,7 +115,7 @@ jobs: env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" + RELEASE_TITLE: "moq-ffi v${{ needs.ffi.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} run: just gh release create artifacts @@ -215,7 +125,7 @@ jobs: # fine but SPM cannot resolve (e.g. a path-based binaryTarget slipping in # where a URL+checksum is expected). name: Verify staged package resolves - needs: [release, parse-version] + needs: [ffi, release] runs-on: macos-latest steps: @@ -233,13 +143,13 @@ jobs: - name: Resolve and build smoke consumer env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | just swift verify-ffi --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" publish: name: Publish to Swift Package mirror - needs: [verify, parse-version] + needs: [ffi, verify] runs-on: ubuntu-latest steps: @@ -268,7 +178,7 @@ jobs: - name: Publish to moq-dev/moq-swift-ffi mirror env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} # Token is minted fresh per run, expires in 1 hour, never at rest. SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index c210608dc1d..2cfdc224821 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -1,4 +1,4 @@ -# Tooling: thin justfiles and CI that calls them +# [XS] Tooling: thin justfiles and CI that calls them ## Goal @@ -14,9 +14,4 @@ runs a recipe rather than a script path. `just` stays as the entry point because the vocabulary (`just check`, `just fix`) is in every doc, skill, and workflow. Its cost was -self-inflicted: logic inside recipes. The quests below are ordered and each -requires the one before it, so they land as one line of pull requests. - -## Quests - -- [FFI release workflow](/quest/m1/tooling/release-ffi.md) - the five `release-*-ffi.yml` share the moq-ffi target matrix and artifact staging +self-inflicted: logic inside recipes. diff --git a/quest/m1/tooling/release-ffi.md b/quest/m1/tooling/release-ffi.md deleted file mode 100644 index 58ac9efe08b..00000000000 --- a/quest/m1/tooling/release-ffi.md +++ /dev/null @@ -1,34 +0,0 @@ -# [M] One reusable workflow for the moq-ffi releases - -## Goal - -`release-dart-ffi.yml`, `release-go-ffi.yml`, `release-kt-ffi.yml`, -`release-swift-ffi.yml`, and `release-py-ffi.yml` share half to two thirds of -their lines: the moq-ffi cross-target build matrix and the artifact staging. -That shared half becomes one reusable `release-ffi.yml`; each language keeps -only its packaging and publish steps. - -## Plan - -- Measure the shared lines first (`comm` over the sorted files shows 74 to - 176 common lines per pair) and confirm the matrix targets agree. Where a - language needs a different target set (iOS simulator for swift, Android - ABIs for kt and dart), make the target list an input rather than forcing - one matrix. -- `release-ffi.yml` on `workflow_call`: builds `rs/moq-ffi` for the requested - targets, uploads one artifact per target, and outputs the version parsed - from the tag. Callers download the artifacts and run their `just - package` and publish recipes. -- `release-dart-ffi.yml` and `release-go-ffi.yml` also run on `pull_request`, - where `GITHUB_REF` is not a `moq-ffi-v*` tag and `parse-version` fails; they - read the version from `rs/moq-ffi/Cargo.toml` instead. The reusable - workflow keeps that fallback (an input or a ref check), or those PR runs die - before building. -- Callers keep their `name:` and triggers, as `moq-cli.yml` and - `moq-relay.yml` do over `release-binary.yml`, so `alert.yml` and the - `workflow_run` chains (`release-go.yml`, `release-py.yml`, - `release-swift-lib.yml`) are untouched. A calling job must grant every - permission a called job raises to, and `alert.sh` already skips a - `workflow_call`-only file. -- Verify with `just gh check`; the next tagged `moq-ffi-v*` release is the - end-to-end check. From 3b76b6460b86f9b4a64c8d9f4b876cd0b0befa0c Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Fri, 25 Sep 2026 15:58:34 -0700 Subject: [PATCH 06/18] quest(tooling): scoped checks refuse a host toolchain (#4176) Co-authored-by: Claude Opus 5.5 Co-authored-by: Grok 4.7 --- quest/m1/tooling/README.md | 6 +++++- quest/m1/tooling/nix-guard.md | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 quest/m1/tooling/nix-guard.md diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index 2cfdc224821..b29d374fa90 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -1,4 +1,4 @@ -# [XS] Tooling: thin justfiles and CI that calls them +# Tooling: thin justfiles and CI that calls them ## Goal @@ -15,3 +15,7 @@ runs a recipe rather than a script path. `just` stays as the entry point because the vocabulary (`just check`, `just fix`) is in every doc, skill, and workflow. Its cost was self-inflicted: logic inside recipes. + +## Quests + +- [Nix guard](/quest/m1/tooling/nix-guard.md) - the scoped check, fix, and test refuse to run outside the Nix dev shell unless told to diff --git a/quest/m1/tooling/nix-guard.md b/quest/m1/tooling/nix-guard.md new file mode 100644 index 00000000000..e9ab5934bc3 --- /dev/null +++ b/quest/m1/tooling/nix-guard.md @@ -0,0 +1,29 @@ +# [S] Scoped checks refuse a host toolchain + +## Goal + +`just check`, `just fix`, and `just ci test` stop at once with a clear "run +inside `nix develop`" message when they run outside the Nix dev shell, instead +of failing deep in a build on a host toolchain difference. The repository +already requires the dev shell; this makes the requirement loud. Set +`MOQ_ALLOW_HOST=1` to keep a deliberate host run possible. + +## Plan + +- The trigger: `moq-tokio --all-features` builds jemalloc, whose configure + adds `-Werror` to its `strerror_r` probes, so any host warning (likely + `_FORTIFY_SOURCE` at `-O0`) makes it report "cannot determine return type of + strerror_r". The dev shell disables fortify hardening, which is why CI + passes. Reproduce it once outside the shell and confirm the cause from + jemalloc's `config.log` before relying on it. +- Put the guard in `sh/dispatch.sh`, the one place the scoped verbs resolve, + next to the existing `MOQ_STRICT` missing-tool check. The verbs are `check`, + `fix`, `ci-check`, and `ci-test`. `just test` is the cross-language module + and does not reach this script, so it stays unguarded. How to detect the dev + shell (`IN_NIX_SHELL`, a variable the flake sets, or a toolchain probe) is + the implementer's call; prefer whatever direnv and `nix develop --command` + both set. +- CI already runs inside the dev shell, so it is unaffected. Verify that a + direnv-loaded shell passes the guard. + +Public API: none. Wire: none. From c53f1a85ea681450d733b62011f8ca3f66abd8de Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Sat, 26 Sep 2026 20:16:01 -0700 Subject: [PATCH 07/18] chore(just): scoped checks refuse a host toolchain (#4269) Co-authored-by: Claude Opus 5.5 --- doc/setup/dev.md | 3 ++- flake.nix | 4 ++++ quest/m1/README.md | 2 +- quest/m1/tooling/README.md | 13 +++++++++---- quest/m1/tooling/demo-scripts.md | 30 ++++++++++++++++++++++++++++++ quest/m1/tooling/nix-guard.md | 29 ----------------------------- sh/dispatch.sh | 10 ++++++++++ 7 files changed, 56 insertions(+), 35 deletions(-) create mode 100644 quest/m1/tooling/demo-scripts.md delete mode 100644 quest/m1/tooling/nix-guard.md diff --git a/doc/setup/dev.md b/doc/setup/dev.md index b3d9fb5b1eb..d0217449029 100644 --- a/doc/setup/dev.md +++ b/doc/setup/dev.md @@ -7,7 +7,8 @@ description: Build, test, and debug the MoQ repository The repository uses [Just](https://github.com/casey/just) as its command runner. Run commands inside the Nix dev shell (`nix develop`) so your tools -match CI. +match CI. `just check` and `just fix` refuse to run outside it; set +`MOQ_ALLOW_HOST=1` to use the host toolchain anyway. | Command | Purpose | | --- | --- | diff --git a/flake.nix b/flake.nix index 312db411973..5993015f34c 100644 --- a/flake.nix +++ b/flake.nix @@ -505,6 +505,10 @@ ''; env = { + # What sh/dispatch.sh checks before a scoped `just check` or `just + # fix`. IN_NIX_SHELL would also pass inside another project's shell. + MOQ_DEV_SHELL = "1"; + # Where `just obs compile` and `just obs test` look for libobs. Set # on every platform so the plugin type-checks against the pinned OBS # release everywhere, rather than whatever the host happens to have. diff --git a/quest/m1/README.md b/quest/m1/README.md index 93cb6d34588..8faef6fb7a1 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -45,7 +45,7 @@ transport, benchmark tooling); worktrees isolate commits, not semantics. - [JavaScript FETCH](/quest/m1/js-fetch.md) - generic on-demand group serving and IETF FETCH for browser publishers - [Archive](/quest/m1/archive/README.md) - record selected tracks to any object_store and replay them over FETCH or derived HLS, on the catalog and store the release ships - [Wildcard](/quest/m1/wildcard/README.md) - a relay resolves subscriptions against advertised prefixes, a service claims the prefix it could serve and refuses the rest instead of enumerating broadcasts, and the browser player treats a covering claim as availability -- [Tooling](/quest/m1/tooling/README.md) - justfiles become a one-line menu over `sh/`, one impact map scopes CI, and every workflow step runs a recipe +- [Tooling](/quest/m1/tooling/README.md) - justfiles become a thin menu over scripts in `sh/`, one impact map scopes CI, and every workflow step runs a recipe - [Path patterns](/quest/m1/path-patterns.md) - one matcher for every predicate over broadcast paths: tokens, origins, interest - [In-band auth](/quest/m1/auth/README.md) - a session tells its peer what it may publish and subscribe to, unions tokens presented in band, and fails loud on an out-of-scope publish - [Decoded frame ownership](/quest/m1/decoded-frames.md) - retain moq-video Frames across bindings, with native views or CPU conversion as needed diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index b29d374fa90..627e81c2d0c 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -6,9 +6,11 @@ A month of merges left the command surface bloated: 873 lines of root `justfile`, seven self-tests of the tooling itself on every pull request, three metadata guards, recipes nothing calls, and release workflows that are the same file with a binary name swapped. The result is a `just` tree that is -a menu, not a language: every recipe is one line, every script lives under -`sh/`, the diff is resolved once by one impact map, and every workflow step -runs a recipe rather than a script path. +a menu, not a language: recipes stay thin, and any non-trivial logic +(conditionals, loops, traps, background jobs) lives in a script under `sh/` +or the package's own script directory, which the recipe and CI both call. The +diff is resolved once by one impact map, and every workflow step runs a recipe +rather than a script path. ## Plan @@ -16,6 +18,9 @@ runs a recipe rather than a script path. fix`) is in every doc, skill, and workflow. Its cost was self-inflicted: logic inside recipes. +A recipe that runs a short fixed sequence of commands is thin and stays +inline; line count is not the test. + ## Quests -- [Nix guard](/quest/m1/tooling/nix-guard.md) - the scoped check, fix, and test refuse to run outside the Nix dev shell unless told to +- [Demo scripts](/quest/m1/tooling/demo-scripts.md) - the demo justfiles' inline bash moves into scripts, the last logic left inside recipes diff --git a/quest/m1/tooling/demo-scripts.md b/quest/m1/tooling/demo-scripts.md new file mode 100644 index 00000000000..b2e721fd281 --- /dev/null +++ b/quest/m1/tooling/demo-scripts.md @@ -0,0 +1,30 @@ +# [S] Demo recipes call scripts + +## Goal + +The demo justfiles hold the last inline bash in the `just` tree. Move each +recipe body with real logic into a script the recipe calls, so every recipe +is a thin command sequence: + +- `demo/pub`: `hls` (a background ffmpeg, playlist polling, and a cleanup + trap) and `serve-hls`. +- `demo/justfile`: `default` (port probing and the `concurrently` launch), + `port`, and `wait`. +- `demo/relay`: `ca`, `cert`, `key`, and `token`, the idempotent certificate + and token generation. +- `demo/pub` and `demo/boy`: `sync`, the same R2 upload loop with a different + bucket, which becomes one script taking the bucket. `download` in both, and + the argument check in `demo/pub`'s `clock`, go with them. + +Behavior stays the same, including Windows Git Bash, where `port` falls back +without `lsof`. + +## Plan + +- Put the scripts under `sh/demo/`, matching the `sh//` layout of the + other modules, and run `shellcheck` and `shfmt` on them through the existing + `shell` lint. +- Short command sequences (`just rs features`, `just py check`, the `ffmpeg` + pipelines) are thin already and stay inline. + +Public API: none. Wire: none. diff --git a/quest/m1/tooling/nix-guard.md b/quest/m1/tooling/nix-guard.md deleted file mode 100644 index e9ab5934bc3..00000000000 --- a/quest/m1/tooling/nix-guard.md +++ /dev/null @@ -1,29 +0,0 @@ -# [S] Scoped checks refuse a host toolchain - -## Goal - -`just check`, `just fix`, and `just ci test` stop at once with a clear "run -inside `nix develop`" message when they run outside the Nix dev shell, instead -of failing deep in a build on a host toolchain difference. The repository -already requires the dev shell; this makes the requirement loud. Set -`MOQ_ALLOW_HOST=1` to keep a deliberate host run possible. - -## Plan - -- The trigger: `moq-tokio --all-features` builds jemalloc, whose configure - adds `-Werror` to its `strerror_r` probes, so any host warning (likely - `_FORTIFY_SOURCE` at `-O0`) makes it report "cannot determine return type of - strerror_r". The dev shell disables fortify hardening, which is why CI - passes. Reproduce it once outside the shell and confirm the cause from - jemalloc's `config.log` before relying on it. -- Put the guard in `sh/dispatch.sh`, the one place the scoped verbs resolve, - next to the existing `MOQ_STRICT` missing-tool check. The verbs are `check`, - `fix`, `ci-check`, and `ci-test`. `just test` is the cross-language module - and does not reach this script, so it stays unguarded. How to detect the dev - shell (`IN_NIX_SHELL`, a variable the flake sets, or a toolchain probe) is - the implementer's call; prefer whatever direnv and `nix develop --command` - both set. -- CI already runs inside the dev shell, so it is unaffected. Verify that a - direnv-loaded shell passes the guard. - -Public API: none. Wire: none. diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 034944872fa..cac54e9ca1a 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -21,6 +21,16 @@ case "$action" in ;; esac +# A host toolchain fails deep in a build instead: its gcc keeps the +# _FORTIFY_SOURCE the dev shell disables, which breaks jemalloc's -Werror +# configure probes. This flake's dev shell sets MOQ_DEV_SHELL. Windows has no +# Nix, so it always runs on the host. +if [[ -z "${MOQ_DEV_SHELL:-}" && -z "${MOQ_ALLOW_HOST:-}" && "$OSTYPE" != msys && "$OSTYPE" != cygwin ]]; then + echo "error: run inside the Nix dev shell ('nix develop' or direnv) so tools match CI" >&2 + echo " or set MOQ_ALLOW_HOST=1 to use the host toolchain anyway" >&2 + exit 1 +fi + cd "$(git rev-parse --show-toplevel)" changed=$(mktemp) From 5e83656b338040a0f8fbcac1647bc865a2900a9b Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 16:16:58 -0700 Subject: [PATCH 08/18] quest: point moq-installer at the tooling line for the binary release workflow Co-Authored-By: Claude Opus 5.5 --- quest/m1/moq-installer.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/quest/m1/moq-installer.md b/quest/m1/moq-installer.md index 6dfcd4072e2..449cdd99807 100644 --- a/quest/m1/moq-installer.md +++ b/quest/m1/moq-installer.md @@ -76,6 +76,8 @@ service setup, Windows support, or new release targets. ## Related +- [Tooling](/quest/m1/tooling/README.md) - its shared binary release + workflow builds the artifacts this installer downloads - [`moq relay`](/quest/m1/moq-relay-subcommand.md) - relay functionality joins the same executable independently of its installation method - [Install URL](/quest/m1/moq-install-url.md) - exposes this installer through From 32a69f99b9fc1d3b4349b6843bcd837db5e36a64 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Mon, 28 Sep 2026 17:38:57 -0700 Subject: [PATCH 09/18] chore(quest): scope check.yml from the impact map Co-Authored-By: Claude Opus 5.5 --- quest/m1/tooling/README.md | 1 + quest/m1/tooling/check-scope.md | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 quest/m1/tooling/check-scope.md diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index b8e485a6257..3820b828aa6 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -25,3 +25,4 @@ inline; line count is not the test. - [Demo scripts](/quest/m1/tooling/demo-scripts.md) - the demo justfiles' inline bash moves into scripts, the last logic left inside recipes - [Forward arguments](/quest/m1/tooling/forward-args.md) - recipes pass a quoted nextest filterset through intact instead of re-splitting it +- [Check scope](/quest/m1/tooling/check-scope.md) - check.yml's build-or-skip decision comes from sh/dispatch.sh instead of an inline diff grep diff --git a/quest/m1/tooling/check-scope.md b/quest/m1/tooling/check-scope.md new file mode 100644 index 00000000000..0b92cec0b9a --- /dev/null +++ b/quest/m1/tooling/check-scope.md @@ -0,0 +1,9 @@ +# [S] check.yml scopes from the impact map + +## Goal + +`.github/workflows/check.yml` decides whether a PR needs the build and test jobs from `sh/dispatch.sh`, the line's single impact map. The two inline "Scope" steps that grep the diff for quest-only and doc-only changes go away, so there is one place that maps changed paths to work. + +## Plan + +Main's #4407 added the inline Scope steps (a quest, `.claude/`, or root-markdown-only diff skips the build) after this line had already moved CI scoping into `sh/dispatch.sh`. Teach the impact map that those paths need no build, have the Scope steps call it (or drop them if the dispatcher already makes the jobs no-ops), and keep the skipped-but-required checks reporting success so branch protection still passes on quest-only PRs. `just gh check` must keep enforcing that workflow steps run recipes. From 8c18b66656c89bcc4a9e4e0e62ca52c790eb9ac1 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 13:30:56 -0700 Subject: [PATCH 10/18] chore(quest): claim forward-args Co-Authored-By: Claude Opus 5.5 From 11c16fbe8cb15889ef52f8fadd158a82b1d3ce92 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 13:43:07 -0700 Subject: [PATCH 11/18] fix(just): forward recipe arguments without re-splitting them Recipes that interpolated {{ args }} let just join the arguments before the shell parsed them, so a quoted nextest filterset such as -E 'test(a) | test(b)' became a shell syntax error. Every forwarding recipe in rs/ and test/ now uses [positional-arguments] and "$@". Closes #4342 Co-Authored-By: Claude Opus 5.5 --- quest/m1/tooling/README.md | 1 - quest/m1/tooling/forward-args.md | 22 --------- rs/justfile | 77 ++++++++++++++++++++------------ test/justfile | 24 ++++++---- 4 files changed, 65 insertions(+), 59 deletions(-) delete mode 100644 quest/m1/tooling/forward-args.md diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index 3820b828aa6..3cf8cf0a80a 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -24,5 +24,4 @@ inline; line count is not the test. ## Required - [Demo scripts](/quest/m1/tooling/demo-scripts.md) - the demo justfiles' inline bash moves into scripts, the last logic left inside recipes -- [Forward arguments](/quest/m1/tooling/forward-args.md) - recipes pass a quoted nextest filterset through intact instead of re-splitting it - [Check scope](/quest/m1/tooling/check-scope.md) - check.yml's build-or-skip decision comes from sh/dispatch.sh instead of an inline diff grep diff --git a/quest/m1/tooling/forward-args.md b/quest/m1/tooling/forward-args.md deleted file mode 100644 index beffa8b556d..00000000000 --- a/quest/m1/tooling/forward-args.md +++ /dev/null @@ -1,22 +0,0 @@ -# [S] Recipes forward arguments intact - -## Goal - -`just rs test`, `check-test`, and `capture-test` interpolate `{{ args }}`, so -just joins the arguments before the shell parses them and a quoted nextest -filterset such as `-E 'test(a) | test(b)'` becomes a shell syntax error -(#4342). Recipes that forward arguments pass each one through unchanged. - -## Plan - -- Recipes that forward arguments use `[positional-arguments]` and `"$@"` - instead of `{{ args }}`, starting with the three above and covering the - other forwarding recipes in `rs/justfile`. -- No self-test for it: the line deleted the tooling's self-tests from every - pull request, and this adds none back. - -Public API: none. Wire: none. - -## Closes - -- [#4342](https://github.com/moq-dev/moq/issues/4342) - close this issue when the quest finishes diff --git a/rs/justfile b/rs/justfile index bdf0c898961..3c5a5320455 100644 --- a/rs/justfile +++ b/rs/justfile @@ -27,9 +27,10 @@ default: # moq-gst, and moq-wasm; pass `--workspace` for all of it. # Compile, lint, format-check, doc-check, and verify dependency hygiene. +[positional-arguments] check *args: - cargo clippy --locked --all-targets {{ args }} -- -D warnings - just rs _lint {{ args }} + cargo clippy --locked --all-targets "$@" -- -D warnings + just rs _lint "$@" # `check` with clippy replaced by the `test` build, whose clippy-driver reports # the same findings. `build.warnings` turns them into errors without entering @@ -37,21 +38,24 @@ check *args: # selection can hold nothing testable (moq-wasm has no host tests). # Compile, lint, and test in one build, plus everything else `check` verifies. +[positional-arguments] check-test *args: - CARGO_BUILD_WARNINGS=deny RUSTC_WORKSPACE_WRAPPER={{ require("clippy-driver") }} cargo nextest run --locked --all-targets --no-tests=pass {{ args }} - just rs _lint {{ args }} + CARGO_BUILD_WARNINGS=deny RUSTC_WORKSPACE_WRAPPER={{ require("clippy-driver") }} cargo nextest run --locked --all-targets --no-tests=pass "$@" + just rs _lint "$@" # Everything `check` verifies besides clippy. +[positional-arguments] _lint *args: python3 sh/rs/stats-docs.py cargo fmt --all --check - RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps {{ args }} + RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps "$@" cargo shear cargo sort --workspace --check --no-format # Auto-fix clippy/format/shear/sort. +[positional-arguments] fix *args: - cargo clippy --locked --fix --allow-staged --allow-dirty --all-targets {{ args }} + cargo clippy --locked --fix --allow-staged --allow-dirty --all-targets "$@" cargo fmt --all cargo shear --fix cargo sort --workspace --no-format @@ -83,37 +87,43 @@ tokio-features: # `moq-cli/capture` turn on the device and render code. # Compile the whole workspace on a Windows host. Must run ON Windows. +[positional-arguments] windows *args: - cargo check --locked --workspace --exclude moq-gst {{ no_fuzz }} --all-targets --features "moq-cli/play moq-cli/capture" {{ args }} + cargo check --locked --workspace --exclude moq-gst {{ no_fuzz }} --all-targets --features "moq-cli/play moq-cli/capture" "$@" # Compile and test the io_uring relay path, which is off the default feature set. # Linux-only, and the tests below the 6.12 kernel floor skip themselves. +[positional-arguments] uring *args: - just rs uring-check {{ args }} - cargo nextest run --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets {{ args }} - cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring,moq-relay/qlog --all-targets {{ args }} -- -D warnings - cargo nextest run --locked -p moq-uring --features qlog --all-targets {{ args }} + just rs uring-check "$@" + cargo nextest run --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets "$@" + cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring,moq-relay/qlog --all-targets "$@" -- -D warnings + cargo nextest run --locked -p moq-uring --features qlog --all-targets "$@" # Compile the io_uring relay path, off the default feature set. +[positional-arguments] uring-check *args: - cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets {{ args }} -- -D warnings + cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets "$@" -- -D warnings # The Apple-gated code in moq-video, moq-audio, and moq-sock, which # platform.yml compiles on a Mac runner alongside `windows`. # Compile the whole workspace on a macOS host. Must run ON macOS. +[positional-arguments] macos *args: - cargo check --locked --workspace --exclude moq-gst {{ no_fuzz }} --all-targets --features "moq-cli/play moq-cli/capture" {{ args }} + cargo check --locked --workspace --exclude moq-gst {{ no_fuzz }} --all-targets --features "moq-cli/play moq-cli/capture" "$@" # Compile and lint the capture feature, which the default check skips. +[positional-arguments] capture *args: - cargo clippy --locked -p moq-video -p moq-audio --all-targets --features moq-video/capture,moq-audio/capture {{ args }} -- -D warnings + cargo clippy --locked -p moq-video -p moq-audio --all-targets --features moq-video/capture,moq-audio/capture "$@" -- -D warnings # The capture tests drive synthetic devices, so they need no hardware. # Test the capture feature, which the default test pass skips. +[positional-arguments] capture-test *args: - just rs test -p moq-video -p moq-audio --features moq-video/capture,moq-audio/capture {{ args }} + just rs test -p moq-video -p moq-audio --features moq-video/capture,moq-audio/capture "$@" # moq-video's MediaCodec code is `#[cfg(target_os = "android")]`. Needs an NDK, # which the dev shell lacks, so android.yml runs it outside Nix. `check` rather @@ -121,8 +131,9 @@ capture-test *args: # capture backend. # Compile the Android backends, which the host-target check skips. +[positional-arguments] android *args: - cargo ndk -t arm64-v8a --platform 26 check --locked -p moq-video --all-targets {{ args }} + cargo ndk -t arm64-v8a --platform 26 check --locked -p moq-video --all-targets "$@" # moq-wasm's crate root is `#![cfg(target_arch = "wasm32")]`, so only a wasm32 # build sees the code. Not to be confused with `just js wasm`, which builds the @@ -131,12 +142,14 @@ android *args: # features unify only within one cargo invocation. # Compile and lint the browser/WASM bindings, which the host-target check skips. +[positional-arguments] wasm *args: - cargo clippy --locked -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib {{ args }} -- -D warnings + cargo clippy --locked -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib "$@" -- -D warnings # Auto-fix the browser/WASM bindings. +[positional-arguments] wasm-fix *args: - cargo clippy --locked --fix --allow-staged --allow-dirty -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib {{ args }} + cargo clippy --locked --fix --allow-staged --allow-dirty -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib "$@" cargo fmt -p moq-wasm -p moq-mux -p moq-ffi # Audit dependencies against deny.toml. Nightly: advisories land without a diff. @@ -184,8 +197,9 @@ media-features: # artifact cache, which is why neither is set any other way. # Run the test suite, reporting clippy findings from the same compile. +[positional-arguments] test *args: - RUSTC_WORKSPACE_WRAPPER="$(command -v clippy-driver || true)" cargo nextest run --locked --all-targets {{ args }} + RUSTC_WORKSPACE_WRAPPER="$(command -v clippy-driver || true)" cargo nextest run --locked --all-targets "$@" # Exercise Vulkan -> CUDA -> NVENC on Linux/NVIDIA hardware. vulkan-cuda: @@ -195,12 +209,14 @@ vulkan-cuda: # `rs/moq-cli/src/play/fake.rs`), so this needs no speaker or display. # Test moq-cli's `play` feature, which default features leave out. +[positional-arguments] play *args: - just rs test -p moq-cli --features play {{ args }} + just rs test -p moq-cli --features play "$@" # Compile and run the `/// ```` examples, which nextest skips. +[positional-arguments] doctest *args: - cargo test --locked --doc {{ args }} + cargo test --locked --doc "$@" # `--cfg loom` swaps kio's primitives for loom's, so it rebuilds the world and # stays out of `check`. `--release` because a model check runs the body once @@ -210,36 +226,41 @@ doctest *args: # `cfg(not(loom))`. # Permutation-test the concurrent handoffs with loom. +[positional-arguments] loom *args: - RUSTFLAGS="--cfg loom" cargo test --locked --release -p kio --lib loom:: {{ args }} - RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test loom {{ args }} + RUSTFLAGS="--cfg loom" cargo test --locked --release -p kio --lib loom:: "$@" + RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test loom "$@" # Needs nightly and cargo-fuzz; CI replays the corpus on stable through # moq-net's own tests instead. See rs/moq-net/fuzz/README.md. # Fuzz the moq-net wire codecs: `just rs fuzz lite -- -max_total_time=300`. +[positional-arguments] fuzz target *args: - sh/rs/fuzz.sh {{ target }} {{ args }} + sh/rs/fuzz.sh "$@" # Build the default workspace members. build: cargo build --locked # Run the comparable epoll and io_uring UDP matrices. +[positional-arguments] bench-udp *args: - cargo bench -p moq-uring --bench udp_tokio -- {{ args }} - cargo bench -p moq-uring --bench udp_uring -- {{ args }} + cargo bench -p moq-uring --bench udp_tokio -- "$@" + cargo bench -p moq-uring --bench udp_uring -- "$@" # Measure a noq echo through the io_uring worker, ablating receive # batching, GRO, and GSO one at a time. Linux 6.12+ (skips loudly otherwise). +[positional-arguments] bench-echo *args: - cargo bench -p moq-uring --bench echo_noq -- {{ args }} + cargo bench -p moq-uring --bench echo_noq -- "$@" # Measure a full moq-lite session (publisher to subscriber) through the # io_uring worker, same ablations and per-iteration unit as bench-echo. # Linux 6.12+ (skips loudly otherwise). +[positional-arguments] bench-session *args: - cargo bench -p moq-uring --bench session_lite -- {{ args }} + cargo bench -p moq-uring --bench session_lite -- "$@" # Create release PRs and publish crates via release-plz. release: diff --git a/test/justfile b/test/justfile index 2ca3b7fbfa6..4d47d0b4b9b 100644 --- a/test/justfile +++ b/test/justfile @@ -32,14 +32,16 @@ harness: # browser publish; everyone subscribes (incl. native node/bun JS, the C libmoq # client, and the GStreamer moqsrc plugin). Other flags pass through, e.g. # `just test interop --publishers rust,python --subscribers rust,c`. +[positional-arguments] interop *args: - ./interop/interop.sh {{ args }} + ./interop/interop.sh "$@" # Negative control: no publisher, every subscriber must time out (proves the # harness can actually report failure). +[positional-arguments] interop-negative *args: - ./interop/interop.sh --negative {{ args }} + ./interop/interop.sh --negative "$@" # Browser-to-browser media QA against a deterministic fixture: presented frame progress, audible # tone, audio/video skew, and the publication lifecycle (cold start, pause, rejoin, detach, @@ -47,8 +49,9 @@ interop-negative *args: # shifted tone, and a leaked session; each must fail the assertion it names. --timeout 30 gives # headless Chromium cold-start headroom. +[positional-arguments] media *args: - ./interop/interop.sh --media --timeout 30 {{ args }} + ./interop/interop.sh --media --timeout 30 "$@" # Disrupts a real relay over real QUIC three ways: cancel a stalled reader, kill # the relay mid-group, and republish an interrupted publisher's name. Ordinary @@ -57,16 +60,18 @@ media *args: # no-publisher negative control. See drill/README.md. # Transport and lifecycle failure drills. +[positional-arguments] drill *args: - just rs test -p moq-relay --test drills {{ args }} + just rs test -p moq-relay --test drills "$@" # Removes one recovery behavior at a time in a disposable copy of the tree and # requires the drill covering it to fail, for the reason the mutation names. The # checkout is never modified. `--list` shows the mutations. # Prove the drills fail without the fixes they cover. +[positional-arguments] drill-sensitivity *args: - ./drill/sensitivity.sh {{ args }} + ./drill/sensitivity.sh "$@" # Round-trips a PCR-paced TS through a relay and checks the subscriber's `export # ts` output with TSDuck plus a custom analyzer (PCR jitter/repetition, @@ -77,21 +82,24 @@ drill-sensitivity *args: # ts/README.md. # Check the subscriber's MPEG-TS output for IRD compliance. +[positional-arguments] ts *args: - ./ts/run.sh {{ args }} + ./ts/run.sh "$@" # Round-trip the EIT fixtures: the sparse schedule must survive and # pending-version sections must be dropped at import. +[positional-arguments] ts-eit *args: - ./ts/eit-roundtrip.sh {{ args }} + ./ts/eit-roundtrip.sh "$@" # Runs the @moq/wasm bindings in headless Chromium against a real relay, one per # protocol flavour. The crate is `#![cfg(target_arch = "wasm32")]`, so nothing in # `just check` or `just rs wasm` gets past compiling it. See wasm/README.md. # Browser coverage for the @moq/wasm bindings. +[positional-arguments] wasm *args: - ./wasm/run.sh {{ args }} + ./wasm/run.sh "$@" # Cross-language subscription termination over in-memory transports. bare-fin: From 82d96177a0fdb845233026a63f1b28ace69d3ee5 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:05:22 -0700 Subject: [PATCH 12/18] quest(tooling/check-scope): claim From 74dc6dfc8143439058f11f7051c7201bf4e10514 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:05:29 -0700 Subject: [PATCH 13/18] quest(tooling/demo-scripts): claim Co-Authored-By: Claude Opus 5.5 From a533d36aab177a789b587f382fdd500117f35036 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:28:28 -0700 Subject: [PATCH 14/18] refactor(demo): move recipe logic into sh/demo scripts The demo justfiles held the last inline bash in the just tree. Recipes with real logic (HLS generation, port probing, cert and token generation, R2 sync, downloads) now call scripts under sh/demo/, linted by the shell lint. Forwarding recipes in demo/pub and demo/sub use positional arguments so quoted arguments survive, and clock validates its action with a just argument pattern. Co-Authored-By: Claude Opus 5.5 --- demo/boy/justfile | 18 +--- demo/justfile | 51 +-------- demo/pub/justfile | 171 ++++++------------------------- demo/relay/justfile | 53 +--------- demo/sub/justfile | 4 +- quest/m1/tooling/README.md | 1 - quest/m1/tooling/demo-scripts.md | 30 ------ sh/demo/ca.sh | 15 +++ sh/demo/cert.sh | 20 ++++ sh/demo/download.sh | 15 +++ sh/demo/hls.sh | 70 +++++++++++++ sh/demo/key.sh | 11 ++ sh/demo/port.sh | 18 ++++ sh/demo/run.sh | 24 +++++ sh/demo/serve-hls.sh | 43 ++++++++ sh/demo/sync.sh | 16 +++ sh/demo/token.sh | 19 ++++ sh/demo/wait.sh | 14 +++ 18 files changed, 314 insertions(+), 279 deletions(-) delete mode 100644 quest/m1/tooling/demo-scripts.md create mode 100755 sh/demo/ca.sh create mode 100755 sh/demo/cert.sh create mode 100755 sh/demo/download.sh create mode 100755 sh/demo/hls.sh create mode 100755 sh/demo/key.sh create mode 100755 sh/demo/port.sh create mode 100755 sh/demo/run.sh create mode 100755 sh/demo/serve-hls.sh create mode 100755 sh/demo/sync.sh create mode 100755 sh/demo/token.sh create mode 100755 sh/demo/wait.sh diff --git a/demo/boy/justfile b/demo/boy/justfile index 9312d15bd55..5837865a5e9 100644 --- a/demo/boy/justfile +++ b/demo/boy/justfile @@ -11,13 +11,9 @@ default: "just wait && just web" # Download the given ROM from our R2 bucket. +[positional-arguments] download rom: - #!/usr/bin/env bash - if [ ! -f "rom/{{ rom }}" ]; then - mkdir -p rom - curl -fSL -o "rom/{{ rom }}.tmp" "https://rom.moq.dev/{{ rom }}" - mv "rom/{{ rom }}.tmp" "rom/{{ rom }}" - fi + @../../sh/demo/download.sh "https://rom.moq.dev/$1" "rom/$1" # Big2Small (puzzle): https://mdsteele.itch.io/big2small b2s: @@ -70,15 +66,9 @@ upload file: bun wrangler r2 object put "rom-moq-dev/{{ file }}" --file "rom/{{ file }}" --remote # Sync all ROMs to R2 +[positional-arguments] sync dir="rom": - #!/usr/bin/env bash - set -euo pipefail - for file in "{{ dir }}"/*; do - [ -e "$file" ] || continue - key=$(basename "$file") - echo "Uploading $key..." - bun wrangler r2 object put "rom-moq-dev/$key" --file "$file" --remote - done + ../../sh/demo/sync.sh rom-moq-dev "$@" # List files in the R2 bucket list: diff --git a/demo/justfile b/demo/justfile index b6128734659..08b359d9ffa 100644 --- a/demo/justfile +++ b/demo/justfile @@ -2,58 +2,17 @@ set fallback # Run the web demo (default). # -# Picks the first free port at/after 4443 (both UDP for QUIC and TCP for HTTP) -# so multiple worktrees can run `just dev` without colliding. The relay reads - -# the port from MOQ_LISTEN/MOQ_WEB_HTTP_LISTEN, overriding localhost.toml. +# Picks the first free port at/after 4443 so multiple worktrees can run +# `just dev` without colliding; see sh/demo/run.sh. default: - #!/usr/bin/env bash - set -euo pipefail bun install - - port=$(just port) - if [ "$port" != "4443" ]; then - echo ">>> Port 4443 is busy, using $port instead" >&2 - fi - - base="http://localhost:$port" - - # Scope the bind env vars to the relay only. The unified `moq` binary also - # reads MOQ_LISTEN, so exporting it globally would make the `pub` client - # try to start a server (and fail without a TLS cert). - bun run concurrently --kill-others --names rly,bbb,web --prefix-colors auto \ - "MOQ_LISTEN='[::]:$port' MOQ_WEB_HTTP_LISTEN='[::]:$port' just relay" \ - "just wait $base/certificate.sha256 && just pub bbb $base" \ - "just wait $base/certificate.sha256 && just web serve $base" - -# Find the first free port at/after `start` (checked on both TCP and UDP). -# `lsof` isn't available everywhere (e.g. Git Bash on Windows); without it we -# can't probe, so fall back to `start` and let the relay surface a bind error if - -# it's actually taken. -[private] -port start="4443": - #!/usr/bin/env bash - set -euo pipefail - port={{ start }} - if command -v lsof >/dev/null 2>&1; then - while lsof -nP -iTCP:"$port" -sTCP:LISTEN -t >/dev/null 2>&1 \ - || lsof -nP -iUDP:"$port" -t >/dev/null 2>&1; do - port=$((port + 1)) - done - fi - echo "$port" + ../sh/demo/run.sh # Wait for the localhost relay to be ready by repeatedly requesting the cert +[positional-arguments] [private] wait url="http://localhost:4443/certificate.sha256": - #!/usr/bin/env bash - set -euo pipefail - for i in $(seq 1 600); do - curl -sf "{{ url }}" > /dev/null 2>&1 && exit 0 - sleep 1 - done - exit 1 + ../sh/demo/wait.sh "$@" # Throttle UDP traffic for testing (macOS only, requires sudo). throttle: diff --git a/demo/pub/justfile b/demo/pub/justfile index fb780f5e73e..16cb04aea6e 100644 --- a/demo/pub/justfile +++ b/demo/pub/justfile @@ -1,4 +1,5 @@ set fallback +set shell := ["bash", "-euo", "pipefail", "-c"] # --- R2 bucket management (vid.moq.dev) --- @@ -18,15 +19,9 @@ upload file: bun wrangler r2 object put "video-moq-dev/{{ file }}" --file "media/{{ file }}" --remote # Sync all files in a local directory to R2 +[positional-arguments] sync dir="media": - #!/usr/bin/env bash - set -euo pipefail - for file in "{{ dir }}"/*; do - [ -e "$file" ] || continue - key=$(basename "$file") - echo "Uploading $key..." - bun wrangler r2 object put "video-moq-dev/$key" --file "$file" --remote - done + ../../sh/demo/sync.sh video-moq-dev "$@" # List files in the bucket list: @@ -36,6 +31,7 @@ list: # --- Publishing --- # Publish Big Buck Bunny to a relay server. +[positional-arguments] bbb url='http://localhost:4443' *args: just download bbb just download bbb-sd @@ -44,7 +40,7 @@ bbb url='http://localhost:4443' *args: -stream_loop -1 -re -i media/bbb.mp4 \ -stream_loop -1 -re -i media/bbb-sd.mp4 \ -map 0 -map 1:v -c copy -f mpegts -pes_payload_size 0 - \ - | cargo run --bin moq -- {{ args }} --connect "{{ url }}" --broadcast bbb.hang import ts + | cargo run --bin moq -- "${@:2}" --connect "{{ url }}" --broadcast bbb.hang import ts # Encode the video-only SD rendition, preserving source timestamps and loop duration. encode-bbb-sd: @@ -58,19 +54,22 @@ check-bbb: python3 bbb.py check # Publish Tears of Steel to a relay server. +[positional-arguments] tos url='http://localhost:4443' *args: just download tos - just ts tos "{{ url }}" {{ args }} + just ts tos "$@" # Publish a video using ffmpeg (CMAF / fMP4) to a relay server. +[positional-arguments] cmaf name url='http://localhost:4443' *args: cargo build --bin moq - just ffmpeg-cmaf "media/{{ name }}.mp4" | cargo run --bin moq -- {{ args }} --connect "{{ url }}" --broadcast "{{ name }}.hang" import fmp4 + just ffmpeg-cmaf "media/{{ name }}.mp4" | cargo run --bin moq -- "${@:3}" --connect "{{ url }}" --broadcast "{{ name }}.hang" import fmp4 # Publish a video using ffmpeg (MPEG-TS) to a relay server. +[positional-arguments] ts name url='http://localhost:4443' *args: cargo build --bin moq - just ffmpeg-ts "media/{{ name }}.mp4" | cargo run --bin moq -- {{ args }} --connect "{{ url }}" --broadcast "{{ name }}.hang" import ts + just ffmpeg-ts "media/{{ name }}.mp4" | cargo run --bin moq -- "${@:3}" --connect "{{ url }}" --broadcast "{{ name }}.hang" import ts # Publish a video via iroh. iroh name url prefix="": @@ -79,82 +78,13 @@ iroh name url prefix="": just ffmpeg-ts "media/{{ name }}.mp4" | cargo run --bin moq -- --iroh-enabled --connect "{{ url }}" --broadcast "{{ prefix }}{{ name }}.hang" import ts # Generate and ingest an HLS stream from a video file. +[positional-arguments] hls name relay="http://localhost:4443": - #!/usr/bin/env bash - set -euo pipefail - - just download "{{ name }}" - - INPUT="media/{{ name }}.mp4" - OUT_DIR="media/{{ name }}" - - rm -rf "$OUT_DIR" - mkdir -p "$OUT_DIR" - - echo ">>> Generating HLS stream to disk (1280x720 + 256x144)..." - - ffmpeg -hide_banner -loglevel warning -re -stream_loop -1 -i "$INPUT" \ - -filter_complex "\ - [0:v]split=2[v0][v1]; \ - [v0]scale=-2:720[v720]; \ - [v1]scale=-2:144[v144]" \ - -map "[v720]" -map "[v144]" -map 0:a:0 \ - -r 25 -preset veryfast -g 50 -keyint_min 50 -sc_threshold 0 \ - -c:v:0 libx264 -profile:v:0 high -level:v:0 4.1 -pix_fmt:v:0 yuv420p -tag:v:0 avc1 \ - -b:v:0 4M -maxrate:v:0 4.4M -bufsize:v:0 8M \ - -c:v:1 libx264 -profile:v:1 high -level:v:1 4.1 -pix_fmt:v:1 yuv420p -tag:v:1 avc1 \ - -b:v:1 300k -maxrate:v:1 330k -bufsize:v:1 600k \ - -c:a aac -b:a 128k \ - -f hls -hls_time 2 -hls_list_size 6 \ - -hls_flags independent_segments+delete_segments \ - -hls_segment_type fmp4 \ - -master_pl_name master.m3u8 \ - -var_stream_map "v:0,agroup:audio,name:720 v:1,agroup:audio,name:144 a:0,agroup:audio,name:audio" \ - -hls_segment_filename "$OUT_DIR/v%v/segment_%09d.m4s" \ - "$OUT_DIR/v%v/stream.m3u8" & - - FFMPEG_PID=$! - - echo ">>> Waiting for HLS playlist generation..." - for i in {1..30}; do - if [ -f "$OUT_DIR/master.m3u8" ]; then break; fi - sleep 0.5 - done - - if [ ! -f "$OUT_DIR/master.m3u8" ]; then - kill $FFMPEG_PID 2>/dev/null || true - echo "Error: master.m3u8 not generated in time" - exit 1 - fi - - echo ">>> Waiting for variant playlists..." - sleep 2 - for i in {1..20}; do - if [ -f "$OUT_DIR/v0/stream.m3u8" ] || [ -f "$OUT_DIR/v720/stream.m3u8" ] || [ -f "$OUT_DIR/v144/stream.m3u8" ] || [ -f "$OUT_DIR/vaudio/stream.m3u8" ]; then - break - fi - sleep 0.5 - done - - CLEANUP_CALLED=false - cleanup() { - if [ "$CLEANUP_CALLED" = "true" ]; then return; fi - CLEANUP_CALLED=true - echo "Shutting down..." - kill $FFMPEG_PID 2>/dev/null || true - sleep 0.5 - kill -9 $FFMPEG_PID 2>/dev/null || true - } - trap cleanup SIGINT SIGTERM EXIT - - echo ">>> Importing HLS via moq-cli" - cargo run --bin moq -- --connect "{{ relay }}" --broadcast "{{ name }}.hang" import hls "$OUT_DIR/master.m3u8" - EXIT_CODE=$? - - cleanup - exit $EXIT_CODE + just download "$1" + ../../sh/demo/hls.sh "$@" # Publish a video using H.264 Annex B format. +[positional-arguments] h264 name url="http://localhost:4443" *args: just download "{{ name }}" cargo build --bin moq @@ -165,68 +95,31 @@ h264 name url="http://localhost:4443" *args: -c:v copy -an \ -bsf:v h264_mp4toannexb \ -f h264 \ - - | cargo run --bin moq -- {{ args }} --connect "{{ url }}" --broadcast "{{ name }}.hang" import avc3 + - | cargo run --bin moq -- "${@:3}" --connect "{{ url }}" --broadcast "{{ name }}.hang" import avc3 # Publish a video using ffmpeg directly from moq to the localhost. +[positional-arguments] serve name *args: just download "{{ name }}" cargo build --bin moq - just ffmpeg-ts "media/{{ name }}.mp4" | cargo run --bin moq -- {{ args }} --listen "[::]:4443" --listen-tls-generate "localhost" --auth-public "**" --broadcast "{{ name }}.hang" import ts + just ffmpeg-ts "media/{{ name }}.mp4" | cargo run --bin moq -- "${@:2}" --listen "[::]:4443" --listen-tls-generate "localhost" --auth-public "**" --broadcast "{{ name }}.hang" import ts # Generate and serve an HLS stream from a video for testing. +[positional-arguments] serve-hls name port="8000": - #!/usr/bin/env bash - set -euo pipefail - - just download "{{ name }}" - - INPUT="media/{{ name }}.mp4" - OUT_DIR="media/{{ name }}" - - rm -rf "$OUT_DIR" - mkdir -p "$OUT_DIR" - - echo ">>> Starting HLS stream generation..." - echo ">>> Master playlist: http://localhost:{{ port }}/master.m3u8" - - cleanup() { - echo "Shutting down..." - kill $(jobs -p) 2>/dev/null || true - exit 0 - } - trap cleanup SIGINT SIGTERM - - ffmpeg -loglevel warning -re -stream_loop -1 -i "$INPUT" \ - -map 0:v:0 -map 0:v:0 -map 0:a:0 \ - -r 25 -preset veryfast -g 50 -keyint_min 50 -sc_threshold 0 \ - -c:v:0 libx264 -profile:v:0 high -level:v:0 4.1 -pix_fmt:v:0 yuv420p -tag:v:0 avc1 -bsf:v:0 dump_extra -b:v:0 4M -vf:0 "scale=1920:-2" \ - -c:v:1 libx264 -profile:v:1 high -level:v:1 4.1 -pix_fmt:v:1 yuv420p -tag:v:1 avc1 -bsf:v:1 dump_extra -b:v:1 300k -vf:1 "scale=256:-2" \ - -c:a aac -b:a 128k \ - -f hls \ - -hls_time 2 -hls_list_size 12 \ - -hls_flags independent_segments+delete_segments \ - -hls_segment_type fmp4 \ - -master_pl_name master.m3u8 \ - -var_stream_map "v:0,agroup:audio v:1,agroup:audio a:0,agroup:audio" \ - -hls_segment_filename "$OUT_DIR/v%v/segment_%09d.m4s" \ - "$OUT_DIR/v%v/stream.m3u8" & - - sleep 2 - echo ">>> HTTP server: http://localhost:{{ port }}/" - cd "$OUT_DIR" && python3 -m http.server {{ port }} + just download "$1" + ../../sh/demo/serve-hls.sh "$@" # Publish the clock broadcast. +[arg("action", pattern="publish|subscribe")] +[positional-arguments] clock action url="http://localhost:4443" *args: - @if [ "{{ action }}" != "publish" ] && [ "{{ action }}" != "subscribe" ]; then \ - echo "Error: action must be 'publish' or 'subscribe', got '{{ action }}'" >&2; \ - exit 1; \ - fi - - cargo run -p moq-tokio --example clock -- --connect "{{ url }}" --broadcast "clock" {{ args }} {{ action }} + cargo run -p moq-tokio --example clock -- --connect "{{ url }}" --broadcast "clock" "${@:3}" "{{ action }}" # --- GStreamer --- # Publish a video using GStreamer to a relay server. +[positional-arguments] gst name url='http://localhost:4443' *args: just download "{{ name }}" cargo build -p moq-gst @@ -235,34 +128,34 @@ gst name url='http://localhost:4443' *args: gst-launch-1.0 -v -e multifilesrc location="media/{{ name }}.mp4" loop=true ! parsebin name=parse \ parse. ! queue ! identity sync=true ! mux.sink_0 \ parse. ! queue ! identity sync=true ! mux.sink_1 \ - moqsink name=mux url="{{ url }}" broadcast="{{ name }}.hang" {{ args }} + moqsink name=mux url="{{ url }}" broadcast="{{ name }}.hang" "${@:3}" # --- Private helpers --- # Download a test video (already fragmented on vid.moq.dev). +[positional-arguments] [private] download name: - @if [ ! -f "media/{{ name }}.mp4" ]; then \ - mkdir -p media; \ - curl -fsSL "https://vid.moq.dev/{{ name }}.mp4" -o "media/{{ name }}.mp4"; \ - fi + @../../sh/demo/download.sh "https://vid.moq.dev/$1.mp4" "media/$1.mp4" # Convert an h264 input file to CMAF (fmp4) format to stdout, with one fragment per sample. # -frag_duration rather than frag_every_frame: the latter repeats a fragment's tfdt when it # interleaves audio and video, landing two samples on one timestamp. +[positional-arguments] [private] ffmpeg-cmaf input output='-' *args: ffmpeg -hide_banner -v quiet \ -stream_loop -1 -re \ -i "{{ input }}" \ -c copy \ - -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 {{ args }} {{ output }} + -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 "${@:3}" "{{ output }}" # Convert an h264 input file to MPEG-TS format to stdout, with one PES per audio frame. +[positional-arguments] [private] ffmpeg-ts input output='-' *args: ffmpeg -hide_banner -v quiet \ -stream_loop -1 -re \ -i "{{ input }}" \ -c copy \ - -f mpegts -pes_payload_size 0 {{ args }} {{ output }} + -f mpegts -pes_payload_size 0 "${@:3}" "{{ output }}" diff --git a/demo/relay/justfile b/demo/relay/justfile index c0e33a31971..db04682e600 100644 --- a/demo/relay/justfile +++ b/demo/relay/justfile @@ -34,60 +34,17 @@ leaf1: (cert "leaf1") # Generate a self-signed CA ca: - #!/usr/bin/env bash - set -euo pipefail - umask 077 - - [ -f ca.pem ] && [ -f ca.key ] && exit 0 - rm -f ca.pem ca.key - printf '[req]\ndistinguished_name = req_dn\n[req_dn]\n' > ca.cnf - export OPENSSL_CONF="ca.cnf" - openssl req -x509 -sha256 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ - -days 365 -subj "/CN=moq cluster CA" \ - -keyout ca.key -out ca.pem + ../../sh/demo/ca.sh # Generate a self-signed certificate using the root CA. - -# Chrome requires a <=14 day validity for self-signed certs. +[positional-arguments] cert name: ca - #!/usr/bin/env bash - set -euo pipefail - umask 077 - - export OPENSSL_CONF="ca.cnf" - openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ - -subj "/CN={{ name }}" \ - -keyout "{{ name }}.key" -out "{{ name }}.csr" - openssl x509 -req -sha256 -in "{{ name }}.csr" \ - -CA ca.pem -CAkey ca.key -CAcreateserial \ - -days 14 -extfile <(printf "subjectAltName=DNS:localhost\n") \ - -out "{{ name }}.crt" - rm "{{ name }}.csr" + ../../sh/demo/cert.sh "$@" # Generate a random secret key for authentication. key: - #!/usr/bin/env bash - set -euo pipefail - umask 077 - - [ -f root.jwk ] && exit 0 - rm -f *.jwt - cargo run --bin moq -- auth generate --out root.jwk + ../../sh/demo/key.sh # Generate authentication tokens for local development. token: key - #!/usr/bin/env bash - set -euo pipefail - umask 077 - - if [ ! -f demo-web.jwt ]; then - cargo run --quiet --bin moq -- auth sign --key root.jwk \ - --root demo --subscribe '**' --publish 'me/**' \ - > demo-web.jwt - fi - - if [ ! -f demo-cli.jwt ]; then - cargo run --quiet --bin moq -- auth sign --key root.jwk \ - --root demo --publish '**' \ - > demo-cli.jwt - fi + ../../sh/demo/token.sh diff --git a/demo/sub/justfile b/demo/sub/justfile index e7ec69ed855..824fd6674a2 100644 --- a/demo/sub/justfile +++ b/demo/sub/justfile @@ -1,6 +1,8 @@ set fallback +set shell := ["bash", "-euo", "pipefail", "-c"] # Subscribe to a broadcast using GStreamer and render to the screen. +[positional-arguments] gst name url='http://localhost:4443' *args: cargo build -p moq-gst @@ -8,6 +10,6 @@ gst name url='http://localhost:4443' *args: # by name so a bare `! decodebin3` can't latch onto the audio pad and leave the # video sink empty; route audio to its own sink so it doesn't stall the session. GST_PLUGIN_PATH_1_0="${PWD}/../../target/debug${GST_PLUGIN_PATH_1_0:+:$GST_PLUGIN_PATH_1_0}" \ - gst-launch-1.0 -v -e moqsrc name=s url="{{ url }}" broadcast="{{ name }}" {{ args }} \ + gst-launch-1.0 -v -e moqsrc name=s url="{{ url }}" broadcast="{{ name }}" "${@:3}" \ s.video_0 ! queue ! decodebin3 ! videoconvert ! autovideosink \ s.audio_0 ! queue ! decodebin3 ! audioconvert ! autoaudiosink diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index 3820b828aa6..28e81dc194c 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -23,6 +23,5 @@ inline; line count is not the test. ## Required -- [Demo scripts](/quest/m1/tooling/demo-scripts.md) - the demo justfiles' inline bash moves into scripts, the last logic left inside recipes - [Forward arguments](/quest/m1/tooling/forward-args.md) - recipes pass a quoted nextest filterset through intact instead of re-splitting it - [Check scope](/quest/m1/tooling/check-scope.md) - check.yml's build-or-skip decision comes from sh/dispatch.sh instead of an inline diff grep diff --git a/quest/m1/tooling/demo-scripts.md b/quest/m1/tooling/demo-scripts.md deleted file mode 100644 index b2e721fd281..00000000000 --- a/quest/m1/tooling/demo-scripts.md +++ /dev/null @@ -1,30 +0,0 @@ -# [S] Demo recipes call scripts - -## Goal - -The demo justfiles hold the last inline bash in the `just` tree. Move each -recipe body with real logic into a script the recipe calls, so every recipe -is a thin command sequence: - -- `demo/pub`: `hls` (a background ffmpeg, playlist polling, and a cleanup - trap) and `serve-hls`. -- `demo/justfile`: `default` (port probing and the `concurrently` launch), - `port`, and `wait`. -- `demo/relay`: `ca`, `cert`, `key`, and `token`, the idempotent certificate - and token generation. -- `demo/pub` and `demo/boy`: `sync`, the same R2 upload loop with a different - bucket, which becomes one script taking the bucket. `download` in both, and - the argument check in `demo/pub`'s `clock`, go with them. - -Behavior stays the same, including Windows Git Bash, where `port` falls back -without `lsof`. - -## Plan - -- Put the scripts under `sh/demo/`, matching the `sh//` layout of the - other modules, and run `shellcheck` and `shfmt` on them through the existing - `shell` lint. -- Short command sequences (`just rs features`, `just py check`, the `ffmpeg` - pipelines) are thin already and stay inline. - -Public API: none. Wire: none. diff --git a/sh/demo/ca.sh b/sh/demo/ca.sh new file mode 100755 index 00000000000..01cfbd0c057 --- /dev/null +++ b/sh/demo/ca.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Generate the relay cluster's self-signed CA (ca.pem, ca.key, ca.cnf) in the +# current directory, unless it already exists. +# +# Usage: sh/demo/ca.sh, from demo/relay. +set -euo pipefail +umask 077 + +[ -f ca.pem ] && [ -f ca.key ] && exit 0 +rm -f ca.pem ca.key +printf '[req]\ndistinguished_name = req_dn\n[req_dn]\n' >ca.cnf +export OPENSSL_CONF="ca.cnf" +openssl req -x509 -sha256 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ + -days 365 -subj "/CN=moq cluster CA" \ + -keyout ca.key -out ca.pem diff --git a/sh/demo/cert.sh b/sh/demo/cert.sh new file mode 100755 index 00000000000..916ece90f61 --- /dev/null +++ b/sh/demo/cert.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Issue NAME.crt and NAME.key for localhost from the CA that sh/demo/ca.sh +# generates in the current directory. Chrome requires a <=14 day validity for +# self-signed certs. +# +# Usage: sh/demo/cert.sh NAME, from demo/relay. +set -euo pipefail +umask 077 + +name=${1:?usage: sh/demo/cert.sh NAME} + +export OPENSSL_CONF="ca.cnf" +openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ + -subj "/CN=$name" \ + -keyout "$name.key" -out "$name.csr" +openssl x509 -req -sha256 -in "$name.csr" \ + -CA ca.pem -CAkey ca.key -CAcreateserial \ + -days 14 -extfile <(printf "subjectAltName=DNS:localhost\n") \ + -out "$name.crt" +rm "$name.csr" diff --git a/sh/demo/download.sh b/sh/demo/download.sh new file mode 100755 index 00000000000..3b1a4f57a51 --- /dev/null +++ b/sh/demo/download.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Download URL to FILE unless FILE already exists. The download lands in a +# temporary file first, so an interrupted one is retried instead of cached. +# +# Usage: sh/demo/download.sh URL FILE +set -euo pipefail + +usage="usage: sh/demo/download.sh URL FILE" +url=${1:?$usage} +file=${2:?$usage} + +[ -f "$file" ] && exit 0 +mkdir -p "$(dirname "$file")" +curl -fsSL -o "$file.tmp" "$url" +mv "$file.tmp" "$file" diff --git a/sh/demo/hls.sh b/sh/demo/hls.sh new file mode 100755 index 00000000000..c203866bbdf --- /dev/null +++ b/sh/demo/hls.sh @@ -0,0 +1,70 @@ +#!/usr/bin/env bash +# Encode media/NAME.mp4 into a looping HLS stream (1280x720 + 256x144 + audio) +# under media/NAME/, and import it into RELAY as NAME.hang. +# +# Usage: sh/demo/hls.sh NAME RELAY, from demo/pub. +set -euo pipefail + +usage="usage: sh/demo/hls.sh NAME RELAY" +name=${1:?$usage} +relay=${2:?$usage} + +input="media/$name.mp4" +out="media/$name" + +rm -rf "$out" +mkdir -p "$out" + +echo ">>> Generating HLS stream to disk (1280x720 + 256x144)..." + +ffmpeg -hide_banner -loglevel warning -re -stream_loop -1 -i "$input" \ + -filter_complex "[0:v]split=2[v0][v1]; [v0]scale=-2:720[v720]; [v1]scale=-2:144[v144]" \ + -map "[v720]" -map "[v144]" -map 0:a:0 \ + -r 25 -preset veryfast -g 50 -keyint_min 50 -sc_threshold 0 \ + -c:v:0 libx264 -profile:v:0 high -level:v:0 4.1 -pix_fmt:v:0 yuv420p -tag:v:0 avc1 \ + -b:v:0 4M -maxrate:v:0 4.4M -bufsize:v:0 8M \ + -c:v:1 libx264 -profile:v:1 high -level:v:1 4.1 -pix_fmt:v:1 yuv420p -tag:v:1 avc1 \ + -b:v:1 300k -maxrate:v:1 330k -bufsize:v:1 600k \ + -c:a aac -b:a 128k \ + -f hls -hls_time 2 -hls_list_size 6 \ + -hls_flags independent_segments+delete_segments \ + -hls_segment_type fmp4 \ + -master_pl_name master.m3u8 \ + -var_stream_map "v:0,agroup:audio,name:720 v:1,agroup:audio,name:144 a:0,agroup:audio,name:audio" \ + -hls_segment_filename "$out/v%v/segment_%09d.m4s" \ + "$out/v%v/stream.m3u8" & +ffmpeg=$! + +cleaned=false +cleanup() { + if [ "$cleaned" = true ]; then return; fi + cleaned=true + echo "Shutting down..." + kill "$ffmpeg" 2>/dev/null || true + sleep 0.5 + kill -9 "$ffmpeg" 2>/dev/null || true +} +trap cleanup INT TERM EXIT + +echo ">>> Waiting for HLS playlist generation..." +for _ in {1..30}; do + if [ -f "$out/master.m3u8" ]; then break; fi + sleep 0.5 +done + +if [ ! -f "$out/master.m3u8" ]; then + echo "Error: master.m3u8 not generated in time" >&2 + exit 1 +fi + +echo ">>> Waiting for variant playlists..." +sleep 2 +for _ in {1..20}; do + if [ -f "$out/v720/stream.m3u8" ] || [ -f "$out/v144/stream.m3u8" ] || [ -f "$out/vaudio/stream.m3u8" ]; then + break + fi + sleep 0.5 +done + +echo ">>> Importing HLS via moq-cli" +cargo run --bin moq -- --connect "$relay" --broadcast "$name.hang" import hls "$out/master.m3u8" diff --git a/sh/demo/key.sh b/sh/demo/key.sh new file mode 100755 index 00000000000..1c199dafcbd --- /dev/null +++ b/sh/demo/key.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Generate the relay's signing key (root.jwk) in the current directory, unless +# it already exists. A new key invalidates every token, so they are removed. +# +# Usage: sh/demo/key.sh, from demo/relay. +set -euo pipefail +umask 077 + +[ -f root.jwk ] && exit 0 +rm -f ./*.jwt +cargo run --bin moq -- auth generate --out root.jwk diff --git a/sh/demo/port.sh b/sh/demo/port.sh new file mode 100755 index 00000000000..814314a7038 --- /dev/null +++ b/sh/demo/port.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Print the first port at/after START that is free on both TCP and UDP. +# +# `lsof` isn't available everywhere (e.g. Git Bash on Windows); without it we +# can't probe, so print START and let the relay surface a bind error if it's +# actually taken. +# +# Usage: sh/demo/port.sh START +set -euo pipefail + +port=${1:?usage: sh/demo/port.sh START} +if command -v lsof >/dev/null 2>&1; then + while lsof -nP -iTCP:"$port" -sTCP:LISTEN -t >/dev/null 2>&1 || + lsof -nP -iUDP:"$port" -t >/dev/null 2>&1; do + port=$((port + 1)) + done +fi +echo "$port" diff --git a/sh/demo/run.sh b/sh/demo/run.sh new file mode 100755 index 00000000000..af9d85555ba --- /dev/null +++ b/sh/demo/run.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Run the web demo: a localhost relay, Big Buck Bunny, and the web player. +# +# Picks the first free port at/after 4443 (both UDP for QUIC and TCP for HTTP) +# so multiple worktrees can run `just dev` without colliding. The relay reads +# the port from MOQ_LISTEN/MOQ_WEB_HTTP_LISTEN, overriding localhost.toml. +# +# Usage: sh/demo/run.sh, from demo/ so `just` resolves the demo recipes. +set -euo pipefail + +port=$("$(dirname "$0")/port.sh" 4443) +if [ "$port" != "4443" ]; then + echo ">>> Port 4443 is busy, using $port instead" >&2 +fi + +base="http://localhost:$port" + +# Scope the bind env vars to the relay only. The unified `moq` binary also +# reads MOQ_LISTEN, so exporting it globally would make the `pub` client +# try to start a server (and fail without a TLS cert). +bun run concurrently --kill-others --names rly,bbb,web --prefix-colors auto \ + "MOQ_LISTEN='[::]:$port' MOQ_WEB_HTTP_LISTEN='[::]:$port' just relay" \ + "just wait $base/certificate.sha256 && just pub bbb $base" \ + "just wait $base/certificate.sha256 && just web serve $base" diff --git a/sh/demo/serve-hls.sh b/sh/demo/serve-hls.sh new file mode 100755 index 00000000000..bf5257bc11a --- /dev/null +++ b/sh/demo/serve-hls.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Encode media/NAME.mp4 into a looping HLS stream under media/NAME/ and serve +# it over HTTP on PORT, for testing HLS import. +# +# Usage: sh/demo/serve-hls.sh NAME PORT, from demo/pub. +set -euo pipefail + +usage="usage: sh/demo/serve-hls.sh NAME PORT" +name=${1:?$usage} +port=${2:?$usage} + +input="media/$name.mp4" +out="media/$name" + +rm -rf "$out" +mkdir -p "$out" + +echo ">>> Starting HLS stream generation..." +echo ">>> Master playlist: http://localhost:$port/master.m3u8" + +ffmpeg -loglevel warning -re -stream_loop -1 -i "$input" \ + -map 0:v:0 -map 0:v:0 -map 0:a:0 \ + -r 25 -preset veryfast -g 50 -keyint_min 50 -sc_threshold 0 \ + -c:v:0 libx264 -profile:v:0 high -level:v:0 4.1 -pix_fmt:v:0 yuv420p -tag:v:0 avc1 -bsf:v:0 dump_extra -b:v:0 4M -vf:0 "scale=1920:-2" \ + -c:v:1 libx264 -profile:v:1 high -level:v:1 4.1 -pix_fmt:v:1 yuv420p -tag:v:1 avc1 -bsf:v:1 dump_extra -b:v:1 300k -vf:1 "scale=256:-2" \ + -c:a aac -b:a 128k \ + -f hls \ + -hls_time 2 -hls_list_size 12 \ + -hls_flags independent_segments+delete_segments \ + -hls_segment_type fmp4 \ + -master_pl_name master.m3u8 \ + -var_stream_map "v:0,agroup:audio v:1,agroup:audio a:0,agroup:audio" \ + -hls_segment_filename "$out/v%v/segment_%09d.m4s" \ + "$out/v%v/stream.m3u8" & +ffmpeg=$! + +trap 'echo "Shutting down..."; exit 0' INT TERM +trap 'kill "$ffmpeg" 2>/dev/null || true' EXIT + +sleep 2 +echo ">>> HTTP server: http://localhost:$port/" +cd "$out" +python3 -m http.server "$port" diff --git a/sh/demo/sync.sh b/sh/demo/sync.sh new file mode 100755 index 00000000000..af5f2fb8860 --- /dev/null +++ b/sh/demo/sync.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Upload every file in DIR to the R2 bucket BUCKET, keyed by its file name. +# +# Usage: sh/demo/sync.sh BUCKET DIR +set -euo pipefail + +usage="usage: sh/demo/sync.sh BUCKET DIR" +bucket=${1:?$usage} +dir=${2:?$usage} + +for file in "$dir"/*; do + [ -e "$file" ] || continue + key=$(basename "$file") + echo "Uploading $key..." + bun wrangler r2 object put "$bucket/$key" --file "$file" --remote +done diff --git a/sh/demo/token.sh b/sh/demo/token.sh new file mode 100755 index 00000000000..e85ebc7bace --- /dev/null +++ b/sh/demo/token.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Sign the local development tokens (demo-web.jwt, demo-cli.jwt) with root.jwk +# in the current directory, skipping any that already exist. +# +# Usage: sh/demo/token.sh, from demo/relay. +set -euo pipefail +umask 077 + +if [ ! -f demo-web.jwt ]; then + cargo run --quiet --bin moq -- auth sign --key root.jwk \ + --root demo --subscribe '**' --publish 'me/**' \ + >demo-web.jwt +fi + +if [ ! -f demo-cli.jwt ]; then + cargo run --quiet --bin moq -- auth sign --key root.jwk \ + --root demo --publish '**' \ + >demo-cli.jwt +fi diff --git a/sh/demo/wait.sh b/sh/demo/wait.sh new file mode 100755 index 00000000000..e556127c3a3 --- /dev/null +++ b/sh/demo/wait.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Wait up to ten minutes for URL to respond, e.g. a relay's certificate +# fingerprint once it is listening. +# +# Usage: sh/demo/wait.sh URL +set -euo pipefail + +url=${1:?usage: sh/demo/wait.sh URL} +for _ in $(seq 1 600); do + curl -sf "$url" >/dev/null 2>&1 && exit 0 + sleep 1 +done +echo "timed out waiting for $url" >&2 +exit 1 From fefa63cbeef1fa1984c3738e6d05d4847bd6061a Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 15:30:57 -0700 Subject: [PATCH 15/18] ci: scope check.yml from the dispatch impact map check.yml's inline Scope steps kept a second path map. sh/dispatch.sh now answers scope-check/scope-test (just ci-scope) from the one impact map, and both jobs gate their build setup on it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/check.yml | 39 +++++++++++++--------------- justfile | 4 +++ quest/m1/tooling/README.md | 1 - quest/m1/tooling/check-scope.md | 9 ------- sh/dispatch.sh | 45 ++++++++++++++++++++++++--------- 5 files changed, 54 insertions(+), 44 deletions(-) delete mode 100644 quest/m1/tooling/check-scope.md diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 52d79d7b98c..5a8a125f9fe 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -39,19 +39,17 @@ jobs: # Full history so `just ci check` can diff against origin/$GITHUB_BASE_REF. fetch-depth: 0 - # Quests, agent config, and root Markdown compile nothing, so a diff of only - # those skips the disk cleanup and Rust cache that exist for the build. + - uses: ./.github/actions/just + + # sh/dispatch.sh's impact map decides. A diff that reaches only lints, + # such as a quest-only one, skips the disk cleanup and Rust cache that + # exist for the build. - name: Scope id: scope - run: | - base=$(git merge-base "origin/$GITHUB_BASE_REF" HEAD) - files=$(git diff --name-only "$base") - if grep -qvE '^(quest/|\.claude/|[^/]+\.md$)' <<< "$files"; then - echo build=true >> "$GITHUB_OUTPUT" - fi + run: just ci-scope check >> "$GITHUB_OUTPUT" - name: Free disk space - if: steps.scope.outputs.build + if: steps.scope.outputs.build == 'true' uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main with: tool-cache: false @@ -75,7 +73,7 @@ jobs: # scope such an entry to the PR's own branch anyway, where no later PR # could read it while it ate the repository's 10 GB budget. - name: Rust cache - if: steps.scope.outputs.build + if: steps.scope.outputs.build == 'true' uses: ./.github/actions/rust-cache # The same recipe a developer runs locally. It diffs against @@ -105,25 +103,22 @@ jobs: # Full history so `just ci test` can diff against origin/$GITHUB_BASE_REF. fetch-depth: 0 - # Same scope as the `check` job. No test covers those files either, so the - # job skips everything and reports green instead of spending a minute of + - uses: ./.github/actions/just + + # With no test in scope, the job skips everything and still reports + # green, so the required check passes instead of spending a minute of # setup to run nothing. - name: Scope id: scope - run: | - base=$(git merge-base "origin/$GITHUB_BASE_REF" HEAD) - files=$(git diff --name-only "$base") - if grep -qvE '^(quest/|\.claude/|[^/]+\.md$)' <<< "$files"; then - echo build=true >> "$GITHUB_OUTPUT" - fi + run: just ci-scope test >> "$GITHUB_OUTPUT" - name: Free disk space - if: steps.scope.outputs.build + if: steps.scope.outputs.build == 'true' uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main with: tool-cache: false - - if: steps.scope.outputs.build + - if: steps.scope.outputs.build == 'true' uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: determinate: false @@ -139,13 +134,13 @@ jobs: # Restore only, same as the `check` job above. - name: Rust cache - if: steps.scope.outputs.build + if: steps.scope.outputs.build == 'true' uses: ./.github/actions/rust-cache # NEXTEST_PROFILE picks up the longer hang timeout in .config/nextest.toml; # without it a runner under load could trip the local one. - name: Test - if: steps.scope.outputs.build + if: steps.scope.outputs.build == 'true' run: nix develop --command just ci test env: MOQ_STRICT: 1 diff --git a/justfile b/justfile index c20feb7f2d6..6cf89744ace 100644 --- a/justfile +++ b/justfile @@ -72,6 +72,10 @@ check $BASE="": ci $JOB $BASE="": sh/dispatch.sh "ci-$JOB" "$BASE" +# Print `build=true` when CI job JOB has more than lints in scope, so it sets up the build. +ci-scope $JOB $BASE="": + sh/dispatch.sh "scope-$JOB" "$BASE" + # Auto-fix lint and formatting for what the branch changed since BASE. fix $BASE="": sh/dispatch.sh fix "$BASE" diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md index 3820b828aa6..b8e485a6257 100644 --- a/quest/m1/tooling/README.md +++ b/quest/m1/tooling/README.md @@ -25,4 +25,3 @@ inline; line count is not the test. - [Demo scripts](/quest/m1/tooling/demo-scripts.md) - the demo justfiles' inline bash moves into scripts, the last logic left inside recipes - [Forward arguments](/quest/m1/tooling/forward-args.md) - recipes pass a quoted nextest filterset through intact instead of re-splitting it -- [Check scope](/quest/m1/tooling/check-scope.md) - check.yml's build-or-skip decision comes from sh/dispatch.sh instead of an inline diff grep diff --git a/quest/m1/tooling/check-scope.md b/quest/m1/tooling/check-scope.md deleted file mode 100644 index 0b92cec0b9a..00000000000 --- a/quest/m1/tooling/check-scope.md +++ /dev/null @@ -1,9 +0,0 @@ -# [S] check.yml scopes from the impact map - -## Goal - -`.github/workflows/check.yml` decides whether a PR needs the build and test jobs from `sh/dispatch.sh`, the line's single impact map. The two inline "Scope" steps that grep the diff for quest-only and doc-only changes go away, so there is one place that maps changed paths to work. - -## Plan - -Main's #4407 added the inline Scope steps (a quest, `.claude/`, or root-markdown-only diff skips the build) after this line had already moved CI scoping into `sh/dispatch.sh`. Teach the impact map that those paths need no build, have the Scope steps call it (or drop them if the dispatcher already makes the jobs no-ops), and keep the skipped-but-required checks reporting success so branch protection still passes on quest-only PRs. `just gh check` must keep enforcing that workflow steps run recipes. diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 87a8c271bcd..18a0324c2fb 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Run `just check`, `just fix`, or `just ci check|test` over what the branch -# changed. +# changed, or tell CI whether a job needs its build setup (`just ci-scope`). # -# Usage: sh/dispatch.sh check|fix|ci-check|ci-test [BASE|--all] +# Usage: sh/dispatch.sh check|fix|ci-check|ci-test|scope-check|scope-test [BASE|--all] # # The branch diff is resolved once and matched against the impact map below, # the one place that says which paths put which module in scope. `check` is @@ -10,11 +10,11 @@ # so there is no second definition of "checked". set -euo pipefail -usage="usage: sh/dispatch.sh check|fix|ci-check|ci-test [BASE|--all]" +usage="usage: sh/dispatch.sh check|fix|ci-check|ci-test|scope-check|scope-test [BASE|--all]" action=${1:?$usage} base=${2:-} case "$action" in - check | fix | ci-check | ci-test) ;; + check | fix | ci-check | ci-test | scope-check | scope-test) ;; *) echo "$usage" >&2 exit 2 @@ -24,8 +24,9 @@ esac # A host toolchain fails deep in a build instead: its gcc keeps the # _FORTIFY_SOURCE the dev shell disables, which breaks jemalloc's -Werror # configure probes. This flake's dev shell sets MOQ_DEV_SHELL. Windows has no -# Nix, so it always runs on the host. -if [[ -z "${MOQ_DEV_SHELL:-}" && -z "${MOQ_ALLOW_HOST:-}" && "$OSTYPE" != msys && "$OSTYPE" != cygwin ]]; then +# Nix, so it always runs on the host. A scope query only reads git, and CI asks +# it before installing Nix. +if [[ "$action" != scope-* && -z "${MOQ_DEV_SHELL:-}" && -z "${MOQ_ALLOW_HOST:-}" && "$OSTYPE" != msys && "$OSTYPE" != cygwin ]]; then echo "error: run inside the Nix dev shell ('nix develop' or direnv) so tools match CI" >&2 echo " or set MOQ_ALLOW_HOST=1 to use the host toolchain anyway" >&2 exit 1 @@ -139,19 +140,39 @@ declare -A tools=( [gh]='actionlint bun' ) +# Modules that lint without compiling. A CI job whose scope holds nothing else +# skips its build setup: the disk cleanup, the Rust cache, and for `test`, Nix +# itself. A new module gets the build setup until it is listed here. +lint_only=(quest markdown shell toml nix justfile gh) + case "$action" in - check | ci-check) modules=(js workers drafts rs bench quest drill py kt swift go dart obs_compile obs flake markdown shell toml nix justfile gh) ;; + check | ci-check | scope-check) modules=(js workers drafts rs bench quest drill py kt swift go dart obs_compile obs flake markdown shell toml nix justfile gh) ;; fix) modules=(js rs py dart obs markdown shell toml nix justfile) ;; - ci-test) modules=(js rs py) ;; + ci-test | scope-test) modules=(js rs py) ;; esac -selected=() -missing=() +in_scope=() for module in "${modules[@]}"; do pattern=${scope[$module]} - if [[ -z "$all" && -n "$pattern" ]] && ! grep -qE "$pattern" "$changed"; then - continue + if [[ -n "$all" || -z "$pattern" ]] || grep -qE "$pattern" "$changed"; then + in_scope+=("$module") fi +done + +# key=value, which CI appends to $GITHUB_OUTPUT. +if [[ "$action" == scope-* ]]; then + build=false + for module in "${in_scope[@]}"; do + [[ " ${lint_only[*]} " == *" $module "* ]] || build=true + done + echo "$action: in scope: ${in_scope[*]:-nothing}" >&2 + echo "build=$build" + exit 0 +fi + +selected=() +missing=() +for module in "${in_scope[@]}"; do absent=() for tool in ${tools[$module]}; do command -v "$tool" >/dev/null 2>&1 || absent+=("$tool") From 657e0dc4e1c35846f5a45d1c2851754c808a2c8d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 16:25:38 -0700 Subject: [PATCH 16/18] ci: drop check.yml's Scope steps; always set up The impact map in sh/dispatch.sh already makes `just ci check|test` run only what a diff reaches, so a second path map in the workflow is not worth the minute of setup it saves on quest-only PRs. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/check.yml | 31 +++++-------------------- justfile | 4 ---- sh/dispatch.sh | 45 ++++++++++--------------------------- 3 files changed, 17 insertions(+), 63 deletions(-) diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5a8a125f9fe..834ac0fac63 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -39,17 +39,7 @@ jobs: # Full history so `just ci check` can diff against origin/$GITHUB_BASE_REF. fetch-depth: 0 - - uses: ./.github/actions/just - - # sh/dispatch.sh's impact map decides. A diff that reaches only lints, - # such as a quest-only one, skips the disk cleanup and Rust cache that - # exist for the build. - - name: Scope - id: scope - run: just ci-scope check >> "$GITHUB_OUTPUT" - - name: Free disk space - if: steps.scope.outputs.build == 'true' uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main with: tool-cache: false @@ -73,12 +63,14 @@ jobs: # scope such an entry to the PR's own branch anyway, where no later PR # could read it while it ate the repository's 10 GB budget. - name: Rust cache - if: steps.scope.outputs.build == 'true' uses: ./.github/actions/rust-cache # The same recipe a developer runs locally. It diffs against # origin/$GITHUB_BASE_REF and touches only the packages this PR changed - # plus their dependents, so a one-crate PR compiles one crate. + # plus their dependents, so a one-crate PR compiles one crate and a + # quest-only PR runs only the lints. sh/dispatch.sh's impact map is the one + # place that decides; the setup above always runs rather than keep a second + # path map here. # # MOQ_STRICT turns every "tool missing, skipping" guard into an error. # Locally an absent formatter means less gets checked; here it would mean @@ -103,23 +95,12 @@ jobs: # Full history so `just ci test` can diff against origin/$GITHUB_BASE_REF. fetch-depth: 0 - - uses: ./.github/actions/just - - # With no test in scope, the job skips everything and still reports - # green, so the required check passes instead of spending a minute of - # setup to run nothing. - - name: Scope - id: scope - run: just ci-scope test >> "$GITHUB_OUTPUT" - - name: Free disk space - if: steps.scope.outputs.build == 'true' uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main with: tool-cache: false - - if: steps.scope.outputs.build == 'true' - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: determinate: false # Trust the flake's cachix substituter. `nixConfig` in flake.nix is @@ -134,13 +115,11 @@ jobs: # Restore only, same as the `check` job above. - name: Rust cache - if: steps.scope.outputs.build == 'true' uses: ./.github/actions/rust-cache # NEXTEST_PROFILE picks up the longer hang timeout in .config/nextest.toml; # without it a runner under load could trip the local one. - name: Test - if: steps.scope.outputs.build == 'true' run: nix develop --command just ci test env: MOQ_STRICT: 1 diff --git a/justfile b/justfile index 6cf89744ace..c20feb7f2d6 100644 --- a/justfile +++ b/justfile @@ -72,10 +72,6 @@ check $BASE="": ci $JOB $BASE="": sh/dispatch.sh "ci-$JOB" "$BASE" -# Print `build=true` when CI job JOB has more than lints in scope, so it sets up the build. -ci-scope $JOB $BASE="": - sh/dispatch.sh "scope-$JOB" "$BASE" - # Auto-fix lint and formatting for what the branch changed since BASE. fix $BASE="": sh/dispatch.sh fix "$BASE" diff --git a/sh/dispatch.sh b/sh/dispatch.sh index 18a0324c2fb..87a8c271bcd 100755 --- a/sh/dispatch.sh +++ b/sh/dispatch.sh @@ -1,8 +1,8 @@ #!/usr/bin/env bash # Run `just check`, `just fix`, or `just ci check|test` over what the branch -# changed, or tell CI whether a job needs its build setup (`just ci-scope`). +# changed. # -# Usage: sh/dispatch.sh check|fix|ci-check|ci-test|scope-check|scope-test [BASE|--all] +# Usage: sh/dispatch.sh check|fix|ci-check|ci-test [BASE|--all] # # The branch diff is resolved once and matched against the impact map below, # the one place that says which paths put which module in scope. `check` is @@ -10,11 +10,11 @@ # so there is no second definition of "checked". set -euo pipefail -usage="usage: sh/dispatch.sh check|fix|ci-check|ci-test|scope-check|scope-test [BASE|--all]" +usage="usage: sh/dispatch.sh check|fix|ci-check|ci-test [BASE|--all]" action=${1:?$usage} base=${2:-} case "$action" in - check | fix | ci-check | ci-test | scope-check | scope-test) ;; + check | fix | ci-check | ci-test) ;; *) echo "$usage" >&2 exit 2 @@ -24,9 +24,8 @@ esac # A host toolchain fails deep in a build instead: its gcc keeps the # _FORTIFY_SOURCE the dev shell disables, which breaks jemalloc's -Werror # configure probes. This flake's dev shell sets MOQ_DEV_SHELL. Windows has no -# Nix, so it always runs on the host. A scope query only reads git, and CI asks -# it before installing Nix. -if [[ "$action" != scope-* && -z "${MOQ_DEV_SHELL:-}" && -z "${MOQ_ALLOW_HOST:-}" && "$OSTYPE" != msys && "$OSTYPE" != cygwin ]]; then +# Nix, so it always runs on the host. +if [[ -z "${MOQ_DEV_SHELL:-}" && -z "${MOQ_ALLOW_HOST:-}" && "$OSTYPE" != msys && "$OSTYPE" != cygwin ]]; then echo "error: run inside the Nix dev shell ('nix develop' or direnv) so tools match CI" >&2 echo " or set MOQ_ALLOW_HOST=1 to use the host toolchain anyway" >&2 exit 1 @@ -140,39 +139,19 @@ declare -A tools=( [gh]='actionlint bun' ) -# Modules that lint without compiling. A CI job whose scope holds nothing else -# skips its build setup: the disk cleanup, the Rust cache, and for `test`, Nix -# itself. A new module gets the build setup until it is listed here. -lint_only=(quest markdown shell toml nix justfile gh) - case "$action" in - check | ci-check | scope-check) modules=(js workers drafts rs bench quest drill py kt swift go dart obs_compile obs flake markdown shell toml nix justfile gh) ;; + check | ci-check) modules=(js workers drafts rs bench quest drill py kt swift go dart obs_compile obs flake markdown shell toml nix justfile gh) ;; fix) modules=(js rs py dart obs markdown shell toml nix justfile) ;; - ci-test | scope-test) modules=(js rs py) ;; + ci-test) modules=(js rs py) ;; esac -in_scope=() +selected=() +missing=() for module in "${modules[@]}"; do pattern=${scope[$module]} - if [[ -n "$all" || -z "$pattern" ]] || grep -qE "$pattern" "$changed"; then - in_scope+=("$module") + if [[ -z "$all" && -n "$pattern" ]] && ! grep -qE "$pattern" "$changed"; then + continue fi -done - -# key=value, which CI appends to $GITHUB_OUTPUT. -if [[ "$action" == scope-* ]]; then - build=false - for module in "${in_scope[@]}"; do - [[ " ${lint_only[*]} " == *" $module "* ]] || build=true - done - echo "$action: in scope: ${in_scope[*]:-nothing}" >&2 - echo "build=$build" - exit 0 -fi - -selected=() -missing=() -for module in "${in_scope[@]}"; do absent=() for tool in ${tools[$module]}; do command -v "$tool" >/dev/null 2>&1 || absent+=("$tool") From 23882d8b3b4b15ef2d25bdbabd9a2af4e7d24516 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Tue, 29 Sep 2026 17:06:23 -0700 Subject: [PATCH 17/18] fix(demo): a signal ends hls.sh instead of resuming its waits Co-Authored-By: Claude Opus 5.5 --- sh/demo/hls.sh | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/sh/demo/hls.sh b/sh/demo/hls.sh index c203866bbdf..f2c15aa1f6d 100755 --- a/sh/demo/hls.sh +++ b/sh/demo/hls.sh @@ -35,16 +35,15 @@ ffmpeg -hide_banner -loglevel warning -re -stream_loop -1 -i "$input" \ "$out/v%v/stream.m3u8" & ffmpeg=$! -cleaned=false cleanup() { - if [ "$cleaned" = true ]; then return; fi - cleaned=true echo "Shutting down..." kill "$ffmpeg" 2>/dev/null || true sleep 0.5 kill -9 "$ffmpeg" 2>/dev/null || true } -trap cleanup INT TERM EXIT +trap cleanup EXIT +# A signal exits, so the EXIT trap cleans up instead of the script resuming its waits. +trap 'exit 130' INT TERM echo ">>> Waiting for HLS playlist generation..." for _ in {1..30}; do From 5ba7bee58435f9cef1f3e9a8f981f8c371da2301 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 30 Sep 2026 07:55:05 -0700 Subject: [PATCH 18/18] ci: main's new steps run through recipes Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 2 +- js/justfile | 2 +- sh/rs/select.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c409d56be4d..6f8720356cb 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -99,7 +99,7 @@ jobs: # has to keep running, and its numbers are the baseline for the generated codec. - name: JS varint benchmark if: ${{ !cancelled() }} - run: nix develop --command bun js/net/bench/varint.ts + run: nix develop --command just js bench varint # Fails if publishing a group costs more as the track retains more groups, # which means the latency guard or the cache went back to scanning them all. diff --git a/js/justfile b/js/justfile index 58a3446af20..c83f51134e2 100644 --- a/js/justfile +++ b/js/justfile @@ -43,7 +43,7 @@ test: bun test common/declarations.test.ts common/deps.test.ts bun run --filter='*' --elide-lines=0 test -# Run one net benchmark: broadcasts, forward, reader, frames, or track. +# Run one net benchmark: broadcasts, forward, reader, frames, track, or varint. bench name: bun install --frozen-lockfile bun net/bench/{{ name }}.ts diff --git a/sh/rs/select.sh b/sh/rs/select.sh index d5b1c7e51ce..28bf44afa05 100755 --- a/sh/rs/select.sh +++ b/sh/rs/select.sh @@ -168,7 +168,7 @@ case "$action" in # render code in moq-video, moq-audio, and the cli. Only when moq-cli is # selected, since cargo rejects a feature of a package outside the # selection; a change to anything they reach selects moq-cli anyway. - if wants moq-cli; then flags+=(--features moq-cli/play,moq-cli/capture); fi + if wants moq-cli; then flags+=(--features "moq-cli/play moq-cli/capture"); fi cargo check --locked "${flags[@]}" --all-targets ;; *)