diff --git a/.github/actions/just/action.yml b/.github/actions/just/action.yml new file mode 100644 index 00000000000..a41f055151a --- /dev/null +++ b/.github/actions/just/action.yml @@ -0,0 +1,11 @@ +name: Install just +description: Install the pinned just for jobs outside the nix dev shell, which already provides it. + +runs: + using: composite + steps: + # A checksum-verified prebuilt, so it costs seconds rather than a + # `cargo install`. Keep the version in step with nixpkgs' just in flake.lock. + - uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 + with: + tool: just@1.58.0 diff --git a/.github/justfile b/.github/justfile index f341da77296..1bccfc53380 100644 --- a/.github/justfile +++ b/.github/justfile @@ -1,13 +1,40 @@ -set fallback +set working-directory := '..' -# actionlint is skipped when absent locally; the root `_tools` preflight makes -# it a hard requirement under MOQ_STRICT, so CI can't skip it. - -# Lint GitHub Actions workflows. +# Lint GitHub Actions workflows, their alert coverage, and that every step runs a recipe. check: - python3 {{ source_directory() }}/scripts/install-formula.test.py - {{ source_directory() }}/scripts/release.test.sh - @if command -v nfpm >/dev/null 2>&1 && command -v envsubst >/dev/null 2>&1 && command -v dpkg-deb >/dev/null 2>&1 && command -v rpm >/dev/null 2>&1; then python3 {{ source_directory() }}/../rs/scripts/package-rename.test.py; fi - @if command -v actionlint >/dev/null 2>&1; then actionlint; fi - {{ source_directory() }}/scripts/alert.sh check-coverage - @if command -v rustc >/dev/null 2>&1; then {{ source_directory() }}/scripts/package-binary.test.sh; fi + actionlint + sh/gh/alert.sh check-coverage + bun sh/gh/check-runs.ts + +# Tests of the release tooling itself; nightly, not on every pull request. +test: + python3 sh/gh/install-formula.test.py + sh/gh/release.test.sh + python3 sh/rs/package-rename.test.py + sh/go/publish-wrapper.test.sh + bash .claude/hooks/direnv.test.sh + +# Release workflow helpers, e.g. `just gh release parse-version moq-cli`. +[positional-arguments] +release *args: + sh/gh/release.sh "$@" + +# Post a failed run to Discord, or check that alert.yml covers every workflow. +[positional-arguments] +alert *args: + sh/gh/alert.sh "$@" + +# Render a Homebrew formula template for a release. +[positional-arguments] +formula *args: + sh/gh/render-formula.sh "$@" + +# Copy a rendered formula into a Homebrew tap checkout, migrating renamed formulae. +[positional-arguments] +install-formula *args: + python3 sh/gh/install-formula.py "$@" + +# Trigger the apt-repo and rpm-repo workflows for a release tag. +[positional-arguments] +trigger-repo-publish *args: + sh/gh/trigger-repo-publish.sh "$@" diff --git a/.github/scripts/package-binary.test.sh b/.github/scripts/package-binary.test.sh deleted file mode 100755 index b1390552f15..00000000000 --- a/.github/scripts/package-binary.test.sh +++ /dev/null @@ -1,72 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKSPACE_DIR="$(cd "$SCRIPT_DIR/../.." && pwd)" - -tmp="$(mktemp -d)" -trap 'rm -rf "$tmp"' EXIT - -binary="$tmp/moq-relay" -printf '#!/usr/bin/env sh\necho test\n' >"$binary" -chmod 0755 "$binary" -target="$(rustc -vV | awk '/^host:/ {print $2}')" - -# This test covers archive staging, not the macOS Mach-O rewrite. -mkdir "$tmp/bin" -printf '#!/usr/bin/env sh\nprintf "Linux\\n"\n' >"$tmp/bin/uname" -chmod 0755 "$tmp/bin/uname" - -PATH="$tmp/bin:$PATH" "$WORKSPACE_DIR/rs/scripts/package-binary.sh" \ - --crate moq-relay \ - --bin moq-relay \ - --binary "$binary" \ - --bare \ - --version 0.14.13 \ - --target "$target" \ - --output "$tmp/dist" - -name="moq-relay-v0.14.13-$target" -bare="$tmp/dist/$name" -archive="$tmp/dist/$name.tar.gz" - -[[ -x "$bare" ]] -[[ -f "$archive" ]] - -mkdir "$tmp/extracted" -tar -xzf "$archive" -C "$tmp/extracted" -[[ -x "$tmp/extracted/$name/bin/moq-relay" ]] -[[ -f "$tmp/extracted/$name/LICENSE-MIT" ]] -[[ -f "$tmp/extracted/$name/LICENSE-APACHE" ]] -cmp "$binary" "$bare" -cmp "$binary" "$tmp/extracted/$name/bin/moq-relay" - -echo "release assets package together without path collisions" - -# Without --binary the script builds the flake package named after the binary: -# `.#moq` for the moq-cli crate, since `.#moq-cli` is a stub refusing the old name. -cat >"$tmp/bin/nix" <<'NIX' -#!/usr/bin/env bash -set -euo pipefail -[[ "$1 $3" == "build --out-link" && "$2" == *"#moq" ]] || { - echo "unexpected: nix $*" >&2 - exit 1 -} -mkdir -p "$4/bin" -printf '#!/usr/bin/env sh\necho moq\n' >"$4/bin/moq" -chmod 0755 "$4/bin/moq" -NIX -chmod 0755 "$tmp/bin/nix" - -PATH="$tmp/bin:$PATH" "$WORKSPACE_DIR/rs/scripts/package-binary.sh" \ - --crate moq-cli \ - --bin moq \ - --version 0.12.2 \ - --target "$target" \ - --output "$tmp/dist" - -name="moq-cli-v0.12.2-$target" -tar -xzf "$tmp/dist/$name.tar.gz" -C "$tmp/extracted" -[[ "$("$tmp/extracted/$name/bin/moq")" == moq ]] - -echo "a nix build packages the flake output named after the binary" diff --git a/.github/workflows/alert.yml b/.github/workflows/alert.yml index 9e264da8d30..d0d3f51a29a 100644 --- a/.github/workflows/alert.yml +++ b/.github/workflows/alert.yml @@ -81,6 +81,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Post to Discord env: DISCORD_WEBHOOK: ${{ secrets.DISCORD_WEBHOOK }} @@ -91,4 +93,4 @@ jobs: RUN_REF: ${{ github.event.workflow_run.head_branch }} RUN_TITLE: ${{ github.event.workflow_run.display_title }} RUN_ACTOR: ${{ github.event.workflow_run.triggering_actor.login }} - run: .github/scripts/alert.sh discord + run: just gh alert discord diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 881d1276862..44c6e807a4d 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -50,7 +50,6 @@ concurrency: env: CARGO_NDK_VERSION: 4.1.2 - JUST_VERSION: 1.52.0 jobs: android: @@ -74,15 +73,10 @@ jobs: # No Rust cache: this workflow only runs on pull requests, which may not # save one, so it would have nothing to restore. + - uses: ./.github/actions/just + - name: Install cargo-ndk run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" - # A checksummed release binary, where `cargo install` would spend ~2 minutes - # building just from source. - - name: Install just - uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 - with: - tool: just@${{ env.JUST_VERSION }} - - name: Check run: just rs android diff --git a/.github/workflows/apt-repo.yml b/.github/workflows/apt-repo.yml index ff790e1583b..5e8905ed21c 100644 --- a/.github/workflows/apt-repo.yml +++ b/.github/workflows/apt-repo.yml @@ -34,6 +34,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -53,10 +55,9 @@ jobs: - name: Publish to R2 shell: bash env: - ARTIFACTS_DIR: artifacts R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} SIGNING_KEY: ${{ secrets.SIGNING_KEY }} SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} - run: ./infra/apt/publish.sh + run: just infra apt publish "$GITHUB_WORKSPACE/artifacts" diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 8fa0ed62ea5..834ac0fac63 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -39,19 +39,7 @@ jobs: # Full history so `just ci check` can diff against origin/$GITHUB_BASE_REF. fetch-depth: 0 - # Quests, agent config, and root Markdown compile nothing, so a diff of only - # those skips the disk cleanup and Rust cache that exist for the build. - - name: Scope - id: scope - run: | - base=$(git merge-base "origin/$GITHUB_BASE_REF" HEAD) - files=$(git diff --name-only "$base") - if grep -qvE '^(quest/|\.claude/|[^/]+\.md$)' <<< "$files"; then - echo build=true >> "$GITHUB_OUTPUT" - fi - - name: Free disk space - if: steps.scope.outputs.build uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main with: tool-cache: false @@ -75,20 +63,19 @@ jobs: # scope such an entry to the PR's own branch anyway, where no later PR # could read it while it ate the repository's 10 GB budget. - name: Rust cache - if: steps.scope.outputs.build uses: ./.github/actions/rust-cache # The same recipe a developer runs locally. It diffs against # origin/$GITHUB_BASE_REF and touches only the packages this PR changed - # plus their dependents, so a one-crate PR compiles one crate. + # plus their dependents, so a one-crate PR compiles one crate and a + # quest-only PR runs only the lints. sh/dispatch.sh's impact map is the one + # place that decides; the setup above always runs rather than keep a second + # path map here. # # MOQ_STRICT turns every "tool missing, skipping" guard into an error. # Locally an absent formatter means less gets checked; here it would mean # a green run that checked nothing, which is the failure mode worth # spending a preflight on. - - name: Test direnv hook - run: bash .claude/hooks/direnv.test.sh - - name: Check run: nix develop --command just ci check env: @@ -108,26 +95,12 @@ jobs: # Full history so `just ci test` can diff against origin/$GITHUB_BASE_REF. fetch-depth: 0 - # Same scope as the `check` job. No test covers those files either, so the - # job skips everything and reports green instead of spending a minute of - # setup to run nothing. - - name: Scope - id: scope - run: | - base=$(git merge-base "origin/$GITHUB_BASE_REF" HEAD) - files=$(git diff --name-only "$base") - if grep -qvE '^(quest/|\.claude/|[^/]+\.md$)' <<< "$files"; then - echo build=true >> "$GITHUB_OUTPUT" - fi - - name: Free disk space - if: steps.scope.outputs.build uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main with: tool-cache: false - - if: steps.scope.outputs.build - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: determinate: false # Trust the flake's cachix substituter. `nixConfig` in flake.nix is @@ -142,13 +115,11 @@ jobs: # Restore only, same as the `check` job above. - name: Rust cache - if: steps.scope.outputs.build uses: ./.github/actions/rust-cache # NEXTEST_PROFILE picks up the longer hang timeout in .config/nextest.toml; # without it a runner under load could trip the local one. - name: Test - if: steps.scope.outputs.build run: nix develop --command just ci test env: MOQ_STRICT: 1 diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index fc6748864b7..5fcbc152e44 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -39,6 +39,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - id: parse env: CRATE: ${{ inputs.package }} @@ -58,7 +60,7 @@ jobs: *) package="$crate" ;; esac echo "package=${package}" >> "$GITHUB_OUTPUT" - .github/scripts/release.sh parse-version "$crate" + just gh release parse-version "$crate" build: name: Build ${{ matrix.platform }} diff --git a/.github/workflows/interop.yml b/.github/workflows/interop.yml index cbbe0a43f56..74a22e98bd1 100644 --- a/.github/workflows/interop.yml +++ b/.github/workflows/interop.yml @@ -24,7 +24,7 @@ on: - "test/justfile" - "package.json" # The Go client builds against the modules this script stages. - - "go/scripts/**" + - "sh/go/**" - ".github/workflows/interop.yml" concurrency: diff --git a/.github/workflows/libmoq.yml b/.github/workflows/libmoq.yml index 7726371df53..990cd2e65c8 100644 --- a/.github/workflows/libmoq.yml +++ b/.github/workflows/libmoq.yml @@ -40,6 +40,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Install Nix if: matrix.use_nix uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main @@ -60,12 +62,12 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version libmoq + run: just gh release parse-version libmoq - name: Build and package shell: bash run: | - ./rs/libmoq/build.sh \ + just rs package-libmoq \ --target ${{ matrix.target }} \ --version ${{ steps.parse.outputs.version }} \ --output dist @@ -90,13 +92,15 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: .github/scripts/release.sh parse-version libmoq + run: just gh release parse-version libmoq - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag libmoq + run: just gh release prev-tag libmoq - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -110,7 +114,7 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "libmoq v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: just gh release create artifacts # The OBS plugin (cpp/obs) statically links libmoq, so it ships with every # libmoq release: rebuild it against the release we just published (no @@ -142,10 +146,12 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version libmoq + run: just gh release parse-version libmoq # Native Xcode / Visual Studio; libobs/Qt6/ffmpeg come from the obs-deps # bundle and libmoq from the release we just published (--libmoq-release). @@ -155,7 +161,7 @@ jobs: TARGET: ${{ matrix.target }} VERSION: ${{ steps.parse.outputs.version }} run: | - ./cpp/obs/build.sh --target "$TARGET" --libmoq-release "$VERSION" --output dist + just obs package --target "$TARGET" --libmoq-release "$VERSION" --output dist - name: Upload artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -176,9 +182,11 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: .github/scripts/release.sh parse-version libmoq + run: just gh release parse-version libmoq # Only the obs-moq archives: the libmoq-* artifacts from the build job # belong to the libmoq release, not this one. @@ -213,4 +221,4 @@ jobs: RELEASE_TAG: "obs-moq-v${{ steps.parse.outputs.version }}" RELEASE_TITLE: "obs-moq v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: just gh release create artifacts diff --git a/.github/workflows/moq-cli.yml b/.github/workflows/moq-cli.yml index a7963178e4d..0258e0ca072 100644 --- a/.github/workflows/moq-cli.yml +++ b/.github/workflows/moq-cli.yml @@ -17,220 +17,11 @@ permissions: contents: read jobs: - build: - name: Build (${{ matrix.target }}) - runs-on: ${{ matrix.runs-on }} - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-unknown-linux-gnu - runs-on: ubuntu-22.04 - deb-arch: amd64 - rpm-arch: x86_64 - - target: aarch64-unknown-linux-gnu - runs-on: ubuntu-22.04-arm - deb-arch: arm64 - rpm-arch: aarch64 - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Parse version - id: parse - shell: bash - run: .github/scripts/release.sh parse-version moq-cli - - - name: Set up packaging tools - uses: ./.github/actions/setup-packaging - - - name: Build (release, glibc 2.34) - shell: bash - # Pin to glibc 2.34 so one binary covers Ubuntu 22.04+ (.deb) - # and AlmaLinux/RHEL/Rocky 9+ (.rpm). - run: cargo zigbuild --locked --release --target ${{ matrix.target }}.2.34 -p moq-cli - - - name: Package release assets - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - TARGET: ${{ matrix.target }} - run: | - ./rs/scripts/package-binary.sh \ - --crate moq-cli \ - --bin moq \ - --binary "target/${TARGET}/release/moq" \ - --bare \ - --version "$VERSION" \ - --target "$TARGET" \ - --output dist - - - name: Package .deb (Ubuntu 22.04 toolchain) - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.deb-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq - run: | - rs/scripts/package-nfpm.sh packaging/moq-cli/nfpm.yaml deb dist/ - rs/scripts/package-nfpm.sh packaging/moq-cli/transition.yaml deb dist/ - - - name: Package .rpm - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.rpm-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq - run: rs/scripts/package-nfpm.sh packaging/moq-cli/nfpm.yaml rpm dist/ - - - name: Upload artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-cli-${{ matrix.target }} - path: dist/* - - build-tarball-macos: - name: Build macOS tarball (${{ matrix.target }}) - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - os: macos-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - with: - determinate: false - # Trust the flake's cachix substituter, as check.yml does, so the dev - # shell is substituted rather than built from source. - extra-conf: | - extra-substituters = https://kixelated.cachix.org - extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= - - - name: Parse version - id: parse - shell: bash - run: .github/scripts/release.sh parse-version moq-cli - - - name: Build and package tarball - shell: bash - run: | - ./rs/scripts/package-binary.sh \ - --crate moq-cli \ - --bin moq \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-cli-${{ matrix.target }} - path: dist/* - - build-windows: - name: Build Windows zip (${{ matrix.target }}) - runs-on: windows-latest - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-pc-windows-msvc - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - # aws-lc-rs assembles its x86_64 crypto with NASM on Windows. - - name: Install NASM - shell: pwsh - run: | - choco install nasm -y --no-progress - "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Parse version - id: parse - shell: bash - run: .github/scripts/release.sh parse-version moq-cli - - - name: Build and package zip - shell: bash - run: | - ./rs/scripts/package-windows.sh \ - --crate moq-cli \ - --bin moq \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-cli-${{ matrix.target }} - path: dist/* - release: - name: Release - needs: [build, build-tarball-macos, build-windows] - if: github.ref_type == 'tag' - runs-on: ubuntu-latest + uses: ./.github/workflows/release-binary.yml + with: + crate: moq-cli + bin: moq permissions: contents: write actions: write - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Parse version - id: parse - run: .github/scripts/release.sh parse-version moq-cli - - - name: Find previous tag - id: prev_tag - run: .github/scripts/release.sh prev-tag moq-cli - - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - path: artifacts - merge-multiple: true - - - name: Create or update release - env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-cli v${{ steps.parse.outputs.version }}" - RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts - - - name: Trigger apt/rpm repo publish - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ github.ref_name }} - run: .github/scripts/trigger-repo-publish.sh diff --git a/.github/workflows/moq-gst.yml b/.github/workflows/moq-gst.yml index 228c5ef1511..7ce0e927052 100644 --- a/.github/workflows/moq-gst.yml +++ b/.github/workflows/moq-gst.yml @@ -34,6 +34,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: determinate: false @@ -46,7 +48,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-gst + run: just gh release parse-version moq-gst - name: Build and package (tarball) shell: bash @@ -54,7 +56,7 @@ jobs: TARGET: ${{ matrix.target }} VERSION: ${{ steps.parse.outputs.version }} run: | - ./rs/moq-gst/build.sh \ + just rs package-gst \ --target "$TARGET" \ --version "$VERSION" \ --output dist @@ -85,7 +87,7 @@ jobs: run: | set -euo pipefail tar -xzf "dist/moq-gst-v${VERSION}-${TARGET}.tar.gz" -C dist/ - ./rs/moq-gst/smoke.sh "dist/moq-gst-v${VERSION}-${TARGET}/lib/gstreamer-1.0" + just rs gst-smoke "dist/moq-gst-v${VERSION}-${TARGET}/lib/gstreamer-1.0" - name: Upload artifact uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -117,6 +119,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -128,7 +132,7 @@ jobs: - name: Parse version id: parse shell: bash - run: .github/scripts/release.sh parse-version moq-gst + run: just gh release parse-version moq-gst - name: Install GStreamer dev libraries shell: bash @@ -165,7 +169,7 @@ jobs: PLUGIN_DIR: /usr/lib/${{ matrix.deb-multiarch }}/gstreamer-1.0 run: | mkdir -p dist - rs/scripts/package-nfpm.sh packaging/moq-gst/nfpm.yaml deb dist/ + just rs package-nfpm packaging/moq-gst/nfpm.yaml deb dist/ - name: Package .rpm shell: bash @@ -177,7 +181,7 @@ jobs: PLUGIN_DIR: /usr/lib64/gstreamer-1.0 run: | mkdir -p dist - rs/scripts/package-nfpm.sh packaging/moq-gst/nfpm.yaml rpm dist/ + just rs package-nfpm packaging/moq-gst/nfpm.yaml rpm dist/ - name: Upload artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -200,13 +204,15 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-gst + run: just gh release parse-version moq-gst - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-gst + run: just gh release prev-tag moq-gst - name: Download artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -220,10 +226,10 @@ jobs: RELEASE_TAG: ${{ github.ref_name }} RELEASE_TITLE: "moq-gst v${{ steps.parse.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: just gh release create artifacts - name: Trigger apt/rpm repo publish env: GH_TOKEN: ${{ github.token }} TAG: ${{ github.ref_name }} - run: .github/scripts/trigger-repo-publish.sh + run: just gh trigger-repo-publish diff --git a/.github/workflows/moq-relay.yml b/.github/workflows/moq-relay.yml index 19dfd61601c..cfb4767b530 100644 --- a/.github/workflows/moq-relay.yml +++ b/.github/workflows/moq-relay.yml @@ -17,219 +17,11 @@ permissions: contents: read jobs: - build: - name: Build (${{ matrix.target }}) - runs-on: ${{ matrix.runs-on }} - - strategy: - fail-fast: false - matrix: - include: - # Native runners. Each binary is built on the same arch it ships - # for, so the .deb/.rpm linkage matches the distro's glibc on the - # oldest LTS we support (Ubuntu 22.04 / AlmaLinux 9). - - target: x86_64-unknown-linux-gnu - runs-on: ubuntu-22.04 - deb-arch: amd64 - rpm-arch: x86_64 - - target: aarch64-unknown-linux-gnu - runs-on: ubuntu-22.04-arm - deb-arch: arm64 - rpm-arch: aarch64 - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - - name: Parse version - id: parse - shell: bash - run: .github/scripts/release.sh parse-version moq-relay - - - name: Set up packaging tools - uses: ./.github/actions/setup-packaging - - - name: Build (release, glibc 2.34) - shell: bash - # Pin to glibc 2.34 so one binary covers Ubuntu 22.04+ (.deb) - # and AlmaLinux/RHEL/Rocky 9+ (.rpm). - run: cargo zigbuild --locked --release --target ${{ matrix.target }}.2.34 -p moq-relay - - - name: Package release assets - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - TARGET: ${{ matrix.target }} - run: | - ./rs/scripts/package-binary.sh \ - --crate moq-relay \ - --bin moq-relay \ - --binary "target/${TARGET}/release/moq-relay" \ - --bare \ - --version "$VERSION" \ - --target "$TARGET" \ - --output dist - - - name: Package .deb (Ubuntu 22.04 toolchain, glibc 2.35) - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.deb-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: rs/scripts/package-nfpm.sh packaging/moq-relay/nfpm.yaml deb dist/ - - - name: Package .rpm - shell: bash - env: - VERSION: ${{ steps.parse.outputs.version }} - ARCH: ${{ matrix.rpm-arch }} - BINARY_PATH: target/${{ matrix.target }}/release/moq-relay - run: rs/scripts/package-nfpm.sh packaging/moq-relay/nfpm.yaml rpm dist/ - - - name: Upload artifacts - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-relay-${{ matrix.target }} - path: dist/* - - build-tarball-macos: - name: Build macOS tarball (${{ matrix.target }}) - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-apple-darwin - os: macos-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main - with: - determinate: false - # Trust the flake's cachix substituter, as check.yml does, so the dev - # shell is substituted rather than built from source. - extra-conf: | - extra-substituters = https://kixelated.cachix.org - extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= - - - name: Parse version - id: parse - shell: bash - run: .github/scripts/release.sh parse-version moq-relay - - - name: Build and package tarball - shell: bash - run: | - ./rs/scripts/package-binary.sh \ - --crate moq-relay \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-relay-${{ matrix.target }} - path: dist/* - - build-windows: - name: Build Windows zip (${{ matrix.target }}) - runs-on: windows-latest - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-pc-windows-msvc - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - # aws-lc-rs assembles its x86_64 crypto with NASM on Windows. - - name: Install NASM - shell: pwsh - run: | - choco install nasm -y --no-progress - "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - - - name: Parse version - id: parse - shell: bash - run: .github/scripts/release.sh parse-version moq-relay - - - name: Build and package zip - shell: bash - run: | - ./rs/scripts/package-windows.sh \ - --crate moq-relay \ - --target ${{ matrix.target }} \ - --version ${{ steps.parse.outputs.version }} \ - --output dist - - - name: Upload artifact - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: moq-relay-${{ matrix.target }} - path: dist/* - release: - name: Release - needs: [build, build-tarball-macos, build-windows] - if: github.ref_type == 'tag' - runs-on: ubuntu-latest + uses: ./.github/workflows/release-binary.yml + with: + crate: moq-relay + bin: moq-relay permissions: contents: write actions: write - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - fetch-depth: 0 - persist-credentials: false - - - name: Parse version - id: parse - run: .github/scripts/release.sh parse-version moq-relay - - - name: Find previous tag - id: prev_tag - run: .github/scripts/release.sh prev-tag moq-relay - - - name: Download artifacts - uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 - with: - path: artifacts - merge-multiple: true - - - name: Create or update release - env: - GH_TOKEN: ${{ github.token }} - RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-relay v${{ steps.parse.outputs.version }}" - RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts - - - name: Trigger apt/rpm repo publish - env: - GH_TOKEN: ${{ github.token }} - TAG: ${{ github.ref_name }} - run: .github/scripts/trigger-repo-publish.sh diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4d8ee86658d..f020a65d191 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -75,45 +75,43 @@ jobs: # over independent observers of the new live broadcast map. - name: JS origin broadcast map benchmark if: ${{ !cancelled() }} - run: | - nix develop --command bun install --frozen-lockfile - nix develop --command bun js/net/bench/broadcasts.ts + run: nix develop --command just js bench broadcasts # A route above several interest heads lands once per head, so its re-price # should expose any slope over heads times received routes. - name: JS origin announce forwarding benchmark if: ${{ !cancelled() }} - run: nix develop --command bun js/net/bench/forward.ts + run: nix develop --command just js bench forward # Fails if reading a fragmented frame costs more per byte as the frame grows. - name: JS stream reader benchmark if: ${{ !cancelled() }} - run: nix develop --command bun js/net/bench/reader.ts + run: nix develop --command just js bench reader # Fails if a chunk carrying many small frames costs as much per frame as one # carrying a single frame, on either the lite or moq-transport group stream, # which means decoding went back to a wakeup per frame. - name: JS group frame benchmark if: ${{ !cancelled() }} - run: nix develop --command bun js/net/bench/frames.ts + run: nix develop --command just js bench frames # Times one varint encode and decode per wire format. No threshold: it only # has to keep running, and its numbers are the baseline for the generated codec. - name: JS varint benchmark if: ${{ !cancelled() }} - run: nix develop --command bun js/net/bench/varint.ts + run: nix develop --command just js bench varint # Bytes and time for lite-06 against lite-07 frames, groups, and requests. No # threshold: it only has to keep running. - name: JS lite varint benchmark if: ${{ !cancelled() }} - run: nix develop --command bun js/net/bench/lite-varint.ts + run: nix develop --command just js bench lite-varint # Fails if publishing a group costs more as the track retains more groups, # which means the latency guard or the cache went back to scanning them all. - name: JS track retention benchmark if: ${{ !cancelled() }} - run: nix develop --command bun js/net/bench/track.ts + run: nix develop --command just js bench track # Runs each moq-net Criterion routine once, so a bench that panics (an # unparsable version, a shape that deadlocks) fails here instead of on the @@ -172,7 +170,8 @@ jobs: matrix: # `rs uring` is a separate feature compile (off the default set) and # kernel-gated below 6.12. The embedding tests for io_uring live there. - recipe: ["rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring"] + # `gh test` exercises the release tooling, which no pull request runs. + recipe: ["rs doctest --workspace", "rs loom", "test drill-sensitivity", "rs uring", "gh test"] steps: - name: Free disk space uses: jlumbroso/free-disk-space@ceedf095f4ec1a097402bc6bd80831f2e1a6fde6 # main diff --git a/.github/workflows/obs.yml b/.github/workflows/obs.yml index e4c696d0c06..53e3de4a7db 100644 --- a/.github/workflows/obs.yml +++ b/.github/workflows/obs.yml @@ -66,6 +66,7 @@ on: types: [opened, synchronize, reopened, closed] paths: - "cpp/obs/**" + - "sh/obs/**" - "rs/libmoq/**" - "doc/lib/c/**" - "doc/lib/samples.sh" @@ -159,8 +160,10 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - uses: ./.github/actions/just + - name: Build shell: bash env: TARGET: ${{ matrix.target }} - run: ./cpp/obs/build.sh --target "$TARGET" --output "$RUNNER_TEMP/obs" + run: just obs package --target "$TARGET" --output "$RUNNER_TEMP/obs" diff --git a/.github/workflows/platform.yml b/.github/workflows/platform.yml index 8abd8d483f5..819d731bb63 100644 --- a/.github/workflows/platform.yml +++ b/.github/workflows/platform.yml @@ -34,6 +34,7 @@ on: - "Cargo.lock" - "rust-toolchain.toml" - "justfile" + - "sh/**" - ".github/workflows/platform.yml" pull_request: # `closed` is here only so merging/closing a PR cancels its in-flight run @@ -46,6 +47,7 @@ on: - "Cargo.lock" - "rust-toolchain.toml" - "justfile" + - "sh/**" - ".github/workflows/platform.yml" concurrency: @@ -59,9 +61,6 @@ concurrency: # newest pending run, so queued merges still coalesce. cancel-in-progress: ${{ github.event_name == 'pull_request' }} -env: - JUST_VERSION: 1.52.0 - jobs: windows: name: Windows @@ -87,12 +86,7 @@ jobs: choco install nasm -y --no-progress "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append - # A checksummed release binary, where `cargo install` would spend ~2 minutes - # building just from source. - - name: Install just - uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 - with: - tool: just@${{ env.JUST_VERSION }} + - uses: ./.github/actions/just - name: Check shell: bash @@ -116,10 +110,7 @@ jobs: - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - - name: Install just - uses: taiki-e/install-action@4cef1412cce204788f482e778a0b9187f9626a29 # v2.87.21 - with: - tool: just@${{ env.JUST_VERSION }} + - uses: ./.github/actions/just - name: Check env: diff --git a/.github/workflows/release-binary.yml b/.github/workflows/release-binary.yml new file mode 100644 index 00000000000..88350386f05 --- /dev/null +++ b/.github/workflows/release-binary.yml @@ -0,0 +1,258 @@ +name: release-binary + +# Builds, packages, and (on a tag) releases one workspace binary. moq-cli.yml +# and moq-relay.yml call it; they keep the tag trigger and the workflow name +# that alert.yml, release-brew.yml, and release-winget.yml match on. + +on: + workflow_call: + inputs: + crate: + description: "Cargo package to build, e.g. moq-cli" + required: true + type: string + bin: + description: "Executable the package ships, e.g. moq" + required: true + type: string + +permissions: + contents: read + +env: + CRATE: ${{ inputs.crate }} + BIN: ${{ inputs.bin }} + +jobs: + build: + name: Build (${{ matrix.target }}) + runs-on: ${{ matrix.runs-on }} + + strategy: + fail-fast: false + matrix: + include: + # Native runners. Each binary is built on the same arch it ships + # for, so the .deb/.rpm linkage matches the distro's glibc on the + # oldest LTS we support (Ubuntu 22.04 / AlmaLinux 9). + - target: x86_64-unknown-linux-gnu + runs-on: ubuntu-22.04 + deb-arch: amd64 + rpm-arch: x86_64 + - target: aarch64-unknown-linux-gnu + runs-on: ubuntu-22.04-arm + deb-arch: arm64 + rpm-arch: aarch64 + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + with: + targets: ${{ matrix.target }} + + - name: Parse version + id: parse + shell: bash + run: just gh release parse-version "$CRATE" + + - name: Set up packaging tools + uses: ./.github/actions/setup-packaging + + - name: Build (release, glibc 2.34) + shell: bash + # Pin to glibc 2.34 so one binary covers Ubuntu 22.04+ (.deb) + # and AlmaLinux/RHEL/Rocky 9+ (.rpm). + run: cargo zigbuild --locked --release --target ${{ matrix.target }}.2.34 -p "$CRATE" + + - name: Package release assets + shell: bash + env: + VERSION: ${{ steps.parse.outputs.version }} + TARGET: ${{ matrix.target }} + run: | + just rs package-binary \ + --crate "$CRATE" \ + --bin "$BIN" \ + --binary "target/${TARGET}/release/${BIN}" \ + --bare \ + --version "$VERSION" \ + --target "$TARGET" \ + --output dist + + # A transition.yaml beside nfpm.yaml is a deb-only package that keeps + # apt upgrades following a renamed package. + - name: Package .deb (Ubuntu 22.04 toolchain) + shell: bash + env: + VERSION: ${{ steps.parse.outputs.version }} + ARCH: ${{ matrix.deb-arch }} + BINARY_PATH: target/${{ matrix.target }}/release/${{ inputs.bin }} + run: | + just rs package-nfpm "packaging/$CRATE/nfpm.yaml" deb dist/ + if [[ -f "packaging/$CRATE/transition.yaml" ]]; then + just rs package-nfpm "packaging/$CRATE/transition.yaml" deb dist/ + fi + + - name: Package .rpm + shell: bash + env: + VERSION: ${{ steps.parse.outputs.version }} + ARCH: ${{ matrix.rpm-arch }} + BINARY_PATH: target/${{ matrix.target }}/release/${{ inputs.bin }} + run: just rs package-nfpm "packaging/$CRATE/nfpm.yaml" rpm dist/ + + - name: Upload artifacts + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.crate }}-${{ matrix.target }} + path: dist/* + + build-tarball-macos: + name: Build macOS tarball (${{ matrix.target }}) + runs-on: ${{ matrix.os }} + + strategy: + fail-fast: false + matrix: + include: + - target: aarch64-apple-darwin + os: macos-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: ./.github/actions/just + + - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main + with: + determinate: false + # Trust the flake's cachix substituter, as check.yml does, so the dev + # shell is substituted rather than built from source. + extra-conf: | + extra-substituters = https://kixelated.cachix.org + extra-trusted-public-keys = kixelated.cachix.org-1:CmFcV0lyM6KuVM2m9mih0q4SrAa0XyCsiM7GHrz3KKk= + + - name: Parse version + id: parse + shell: bash + run: just gh release parse-version "$CRATE" + + - name: Build and package tarball + shell: bash + run: | + just rs package-binary \ + --crate "$CRATE" \ + --bin "$BIN" \ + --target ${{ matrix.target }} \ + --version ${{ steps.parse.outputs.version }} \ + --output dist + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.crate }}-${{ matrix.target }} + path: dist/* + + build-windows: + name: Build Windows zip (${{ matrix.target }}) + runs-on: windows-latest + + strategy: + fail-fast: false + matrix: + include: + - target: x86_64-pc-windows-msvc + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + with: + targets: ${{ matrix.target }} + + # aws-lc-rs assembles its x86_64 crypto with NASM on Windows. + - name: Install NASM + shell: pwsh + run: | + choco install nasm -y --no-progress + "C:\Program Files\NASM" | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append + + - name: Parse version + id: parse + shell: bash + run: just gh release parse-version "$CRATE" + + - name: Build and package zip + shell: bash + run: | + just rs package-windows \ + --crate "$CRATE" \ + --bin "$BIN" \ + --target ${{ matrix.target }} \ + --version ${{ steps.parse.outputs.version }} \ + --output dist + + - name: Upload artifact + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.crate }}-${{ matrix.target }} + path: dist/* + + release: + name: Release + needs: [build, build-tarball-macos, build-windows] + if: github.ref_type == 'tag' + runs-on: ubuntu-latest + permissions: + contents: write + actions: write + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + fetch-depth: 0 + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Parse version + id: parse + run: just gh release parse-version "$CRATE" + + - name: Find previous tag + id: prev_tag + run: just gh release prev-tag "$CRATE" + + - name: Download artifacts + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 + with: + path: artifacts + merge-multiple: true + + - name: Create or update release + env: + GH_TOKEN: ${{ github.token }} + RELEASE_TAG: ${{ github.ref_name }} + RELEASE_TITLE: "${{ inputs.crate }} v${{ steps.parse.outputs.version }}" + RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} + run: just gh release create artifacts + + - name: Trigger apt/rpm repo publish + env: + GH_TOKEN: ${{ github.token }} + TAG: ${{ github.ref_name }} + run: just gh trigger-repo-publish diff --git a/.github/workflows/release-brew.yml b/.github/workflows/release-brew.yml index 20fe7211562..5e8594de054 100644 --- a/.github/workflows/release-brew.yml +++ b/.github/workflows/release-brew.yml @@ -41,6 +41,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse crate and version id: parse env: @@ -98,7 +100,7 @@ jobs: VERSION: ${{ steps.parse.outputs.version }} run: | mkdir -p rendered - ./.github/scripts/render-formula.sh \ + just gh formula \ --template ".github/homebrew/Formula/${CRATE}.rb.tmpl" \ --version "$VERSION" \ --release-dir release-assets \ @@ -156,7 +158,7 @@ jobs: EOF fi - python3 ../.github/scripts/install-formula.py "$CRATE" "../rendered/${CRATE}.rb" . + just gh install-formula "$CRATE" "rendered/${CRATE}.rb" homebrew-tap if git diff --quiet && [[ -z "$(git status --porcelain)" ]]; then echo "No changes to commit (formula already up to date)." diff --git a/.github/workflows/release-dart-ffi.yml b/.github/workflows/release-dart-ffi.yml index 73dd54b38af..4e75507115c 100644 --- a/.github/workflows/release-dart-ffi.yml +++ b/.github/workflows/release-dart-ffi.yml @@ -19,8 +19,9 @@ on: pull_request: paths: - ".github/workflows/release-dart-ffi.yml" + - ".github/workflows/release-ffi.yml" - "dart/moq_ffi/**" - - "dart/scripts/**" + - "sh/dart/**" - "rs/moq-ffi/build.sh" permissions: @@ -34,117 +35,71 @@ concurrency: group: release-dart-ffi-${{ github.ref_type == 'tag' && 'publish' || github.ref }} cancel-in-progress: ${{ github.ref_type != 'tag' }} -env: - CARGO_NDK_VERSION: 4.1.2 - jobs: - parse-version: - name: Parse version + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: dart + # The raw bindings are checked in under dart/moq_ffi, so none here. + bindings: false + default-features: false + targets: >- + [ + "aarch64-linux-android", + "armv7-linux-androideabi", + "x86_64-linux-android", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-apple-darwin", + "aarch64-apple-ios", + "aarch64-apple-ios-sim", + "x86_64-apple-ios", + "x86_64-pc-windows-msvc" + ] + + # hook/build.dart downloads moq-ffi--- and its + # .sha256 from the release, so each target's shared library is renamed flat. + assets: + name: Stage Native Assets + needs: [ffi] runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Parse version - id: parse - run: .github/scripts/release.sh parse-version moq-ffi - - build: - name: Build Native Asset (${{ matrix.target }}) - needs: [parse-version] - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: aarch64-linux-android - os: ubuntu-latest - - target: armv7-linux-androideabi - os: ubuntu-latest - - target: x86_64-linux-android - os: ubuntu-latest - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-apple-darwin - os: macos-latest - - target: x86_64-apple-darwin - os: macos-latest - - target: aarch64-apple-ios - os: macos-latest - - target: aarch64-apple-ios-sim - os: macos-latest - - target: x86_64-apple-ios - os: macos-latest - - target: x86_64-pc-windows-msvc - os: windows-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + - name: Download per-target libs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: - targets: ${{ matrix.target }} - - # No Rust cache: its entry would evict the store every pull request - # restores from the 10 GB budget. See cache.yml. - - - name: Install cross-compilation tools (Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get update - sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - - - name: Install cargo-ndk (Android) - if: contains(matrix.target, 'android') - run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" + path: libs-raw + pattern: dart-lib-* - - name: Build and stage asset - shell: bash + - name: Stage env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | - ./rs/moq-ffi/build.sh \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist \ - --no-default-features - - case "$BUILD_TARGET" in - *-apple-ios* | *-apple-darwin*) LIBRARY=libmoq_ffi.dylib ;; - *-windows-*) LIBRARY=moq_ffi.dll ;; - *) LIBRARY=libmoq_ffi.so ;; - esac - - NAME="moq-ffi-$BUILD_VERSION-$BUILD_TARGET-$LIBRARY" mkdir -p assets - cp "dist/moq-ffi-$BUILD_VERSION-$BUILD_TARGET/lib/$LIBRARY" "assets/$NAME" - if command -v sha256sum >/dev/null 2>&1; then - sha256sum "assets/$NAME" > "assets/$NAME.sha256" - else - shasum -a 256 "assets/$NAME" > "assets/$NAME.sha256" - fi - - - name: Upload Native Asset + for dir in libs-raw/dart-lib-*; do + target="${dir#libs-raw/dart-lib-}" + case "$target" in + *-apple-ios* | *-apple-darwin*) library=libmoq_ffi.dylib ;; + *-windows-*) library=moq_ffi.dll ;; + *) library=libmoq_ffi.so ;; + esac + name="moq-ffi-$BUILD_VERSION-$target-$library" + cp "$dir/$library" "assets/$name" + (cd assets && sha256sum "$name" > "$name.sha256") + done + ls -la assets + + - name: Upload Native Assets uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: dart-asset-${{ matrix.target }} + name: dart-assets path: assets/* package: name: Check pub package - needs: [parse-version] + needs: [ffi] runs-on: ubuntu-latest steps: @@ -152,13 +107,15 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - uses: dart-lang/setup-dart@6afc89df92d6eb3834022f73cd65adc8cdfcb92d # v1.8.1 - name: Stage and check package env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | - ./dart/scripts/package.sh \ + just dart package \ --package moq_ffi \ --version "$BUILD_VERSION" \ --output release-out @@ -175,7 +132,7 @@ jobs: release: name: Attach Native Assets - needs: [parse-version, build] + needs: [ffi, assets] if: github.ref_type == 'tag' runs-on: ubuntu-latest permissions: @@ -187,29 +144,30 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-ffi + run: just gh release prev-tag moq-ffi - name: Download Native Assets uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: + name: dart-assets path: artifacts - pattern: dart-asset-* - merge-multiple: true - name: Create or update release env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" + RELEASE_TITLE: "moq-ffi v${{ needs.ffi.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} RELEASE_CHECKSUMS: SHA256SUMS-dart - run: .github/scripts/release.sh create artifacts + run: just gh release create artifacts publish: name: Publish moq_ffi to pub.dev - needs: [parse-version, package, release] + needs: [ffi, package, release] if: github.ref_type == 'tag' runs-on: ubuntu-latest environment: @@ -228,5 +186,5 @@ jobs: - name: Publish env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: cd "package/moq_ffi-$BUILD_VERSION" && dart pub publish --force diff --git a/.github/workflows/release-dart.yml b/.github/workflows/release-dart.yml index 2c099bc3188..3578ae386c2 100644 --- a/.github/workflows/release-dart.yml +++ b/.github/workflows/release-dart.yml @@ -12,7 +12,7 @@ on: paths: - ".github/workflows/release-dart.yml" - "dart/moq/**" - - "dart/scripts/**" + - "sh/dart/**" permissions: contents: read @@ -37,6 +37,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - uses: dart-lang/setup-dart@6afc89df92d6eb3834022f73cd65adc8cdfcb92d # v1.8.1 - name: Resolve version @@ -47,7 +49,7 @@ jobs: if [[ "$EVENT_NAME" == "pull_request" ]]; then VERSION=$(sed -n 's/^version: //p' dart/moq/pubspec.yaml | head -1) else - .github/scripts/release.sh parse-version moq-dart + just gh release parse-version moq-dart exit 0 fi echo "version=$VERSION" >> "$GITHUB_OUTPUT" @@ -69,7 +71,7 @@ jobs: env: BUILD_VERSION: ${{ steps.version.outputs.version }} run: | - ./dart/scripts/package.sh \ + just dart package \ --package moq \ --version "$BUILD_VERSION" \ --output release-out diff --git a/.github/workflows/release-ffi.yml b/.github/workflows/release-ffi.yml new file mode 100644 index 00000000000..ff7cfc85aee --- /dev/null +++ b/.github/workflows/release-ffi.yml @@ -0,0 +1,169 @@ +name: release-ffi + +# Builds rs/moq-ffi for each requested target and generates its UniFFI +# bindings. The release-{dart,go,kt,swift}-ffi.yml callers package and publish; +# they keep the moq-ffi-v* tag trigger and the workflow name that alert.yml and +# the workflow_run chains match on. +# +# Uploads, all prefixed by `language` so callers sharing one nightly run never +# collide: +# -lib- the target's native libraries (the lib/ directory) +# -bindings dist/bindings//, when `bindings` is set + +on: + workflow_call: + inputs: + language: + description: "Binding language and artifact prefix, e.g. kotlin" + required: true + type: string + targets: + description: "JSON array of Rust targets to build" + required: true + type: string + bindings: + description: "Generate and upload the bindings for `language`" + type: boolean + default: true + default-features: + description: "Build moq-ffi with its default features (audio and video)" + type: boolean + default: true + outputs: + version: + description: "moq-ffi version, from the tag or (off a tag) rs/moq-ffi/Cargo.toml" + value: ${{ jobs.parse-version.outputs.version }} + +permissions: + contents: read + +# Pinned rather than "latest": `--locked` only fixes the tool's own +# dependencies, not which version of the tool cargo selects. +env: + CARGO_NDK_VERSION: 4.1.2 + +jobs: + parse-version: + name: Parse version + runs-on: ubuntu-latest + outputs: + version: ${{ steps.parse.outputs.version }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Parse version + id: parse + run: just gh release parse-version moq-ffi + + build: + name: Build moq-ffi (${{ matrix.target }}) + needs: [parse-version] + runs-on: ${{ contains(matrix.target, '-apple-') && 'macos-latest' || contains(matrix.target, '-windows-') && 'windows-latest' || 'ubuntu-latest' }} + + strategy: + fail-fast: false + matrix: + target: ${{ fromJSON(inputs.targets) }} + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + with: + targets: ${{ matrix.target }} + + # No Rust cache: its entry would evict the store every pull request + # restores from the 10 GB budget. See cache.yml. + + - name: Install cross-compilation tools (Linux ARM64) + if: matrix.target == 'aarch64-unknown-linux-gnu' + run: | + sudo apt-get update + # g++ as well as gcc: moq-video's openh264 fallback is C++, and + # cc-rs looks for aarch64-linux-gnu-g++ by name. + sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu + + - name: Install cargo-ndk (Android) + if: contains(matrix.target, 'android') + run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" + + - name: Build + shell: bash + env: + BUILD_TARGET: ${{ matrix.target }} + BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + DEFAULT_FEATURES: ${{ inputs.default-features }} + # Set iOS deployment target to avoid ___chkstk_darwin linker errors. + IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} + run: | + args=(--target "$BUILD_TARGET" --version "$BUILD_VERSION" --output dist) + if [[ "$DEFAULT_FEATURES" == false ]]; then + args+=(--no-default-features) + fi + just rs package-ffi "${args[@]}" + + - name: Upload lib + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.language }}-lib-${{ matrix.target }} + # Only the lib/ contents, so a caller's download is /. + path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ + + bindings: + name: Generate bindings + needs: [parse-version] + if: inputs.bindings + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: ./.github/actions/just + + - name: Install Rust + uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable + + # No Rust cache: its entry would evict the store every pull request + # restores from the 10 GB budget. See cache.yml. + + # Go uses a third-party generator; build.sh skips Go when it is absent. + - name: Install uniffi-bindgen-go + if: inputs.language == 'go' + env: + # A fork until NordSecurity ships a uniffi 0.32 generator; see flake.nix. + # The revision is v0.10.0-kixelated.1+v0.32.0. + UNIFFI_BINDGEN_GO_REPO: https://github.com/kixelated/uniffi-bindgen-go + UNIFFI_BINDGEN_GO_REV: v0.10.0-kixelated.1+v0.32.0 + run: | + cargo install --locked uniffi-bindgen-go \ + --git "$UNIFFI_BINDGEN_GO_REPO" \ + --rev "$UNIFFI_BINDGEN_GO_REV" + + - name: Generate bindings + env: + BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + run: | + just rs package-ffi \ + --bindings-only \ + --version "$BUILD_VERSION" \ + --output dist + + - name: Upload bindings + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 + with: + name: ${{ inputs.language }}-bindings + path: dist/bindings/${{ inputs.language }}/ + # A language build.sh does not generate would otherwise upload nothing. + if-no-files-found: error diff --git a/.github/workflows/release-go-ffi.yml b/.github/workflows/release-go-ffi.yml index 30c3ea56edd..e35e940046b 100644 --- a/.github/workflows/release-go-ffi.yml +++ b/.github/workflows/release-go-ffi.yml @@ -16,10 +16,11 @@ on: pull_request: paths: - ".github/workflows/release-go-ffi.yml" - - ".github/scripts/release.sh" + - ".github/workflows/release-ffi.yml" + - "sh/gh/release.sh" - "go/ffi/**" - - "go/scripts/package-ffi.sh" - - "go/scripts/publish-ffi.sh" + - "sh/go/package-ffi.sh" + - "sh/go/publish-ffi.sh" - "rs/moq-ffi/build.sh" permissions: @@ -33,132 +34,32 @@ concurrency: group: release-go-ffi-${{ github.ref_type == 'tag' && 'publish' || github.ref }} cancel-in-progress: ${{ github.ref_type != 'tag' }} -env: - # A fork until NordSecurity ships a uniffi 0.32 generator; see flake.nix. - # The revision is v0.10.0-kixelated.1+v0.32.0. - UNIFFI_BINDGEN_GO_REPO: https://github.com/kixelated/uniffi-bindgen-go - UNIFFI_BINDGEN_GO_REV: v0.10.0-kixelated.1+v0.32.0 - jobs: - parse-version: - name: Parse version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - persist-credentials: false - - - name: Parse version - id: parse - run: .github/scripts/release.sh parse-version moq-ffi - - build: - name: Build moq-ffi (${{ matrix.target }}) - needs: [parse-version] - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-apple-darwin - os: macos-latest - - target: x86_64-pc-windows-msvc - os: windows-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - persist-credentials: false - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - # No Rust cache: its entry would evict the store every pull request - # restores from the 10 GB budget. See cache.yml. - - - name: Install cross-compilation tools (Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get update - # g++ as well as gcc: moq-video's openh264 fallback is C++, and - # cc-rs looks for aarch64-linux-gnu-g++ by name. - sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - - - name: Build - shell: bash - env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - ./rs/moq-ffi/build.sh \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload lib - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: go-lib-${{ matrix.target }} - # package.sh expects /; upload just lib/ contents. - path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ - - bindings: - name: Generate bindings - needs: [parse-version] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 - with: - persist-credentials: false - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - - # No Rust cache: its entry would evict the store every pull request - # restores from the 10 GB budget. See cache.yml. - - - name: Install uniffi-bindgen-go - run: | - cargo install --locked uniffi-bindgen-go \ - --git "$UNIFFI_BINDGEN_GO_REPO" \ - --rev "$UNIFFI_BINDGEN_GO_REV" - - - name: Generate bindings - env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - ./rs/moq-ffi/build.sh \ - --bindings-only \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload bindings - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: go-bindings - path: dist/bindings/go/ + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: go + targets: >- + [ + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc" + ] package: name: Package - needs: [parse-version, build, bindings] + needs: [ffi] runs-on: ubuntu-latest steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download per-target libs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -200,9 +101,9 @@ jobs: - name: Package env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | - ./go/scripts/package-ffi.sh \ + just go package-ffi \ --version "$BUILD_VERSION" \ --source-dir go/ffi \ --lib-dir libs \ @@ -217,15 +118,17 @@ jobs: publish: name: Publish to Go module mirror - needs: [package, parse-version] + needs: [ffi, package] runs-on: ubuntu-latest if: github.ref_type == 'tag' steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -245,24 +148,26 @@ jobs: - name: Publish to moq-dev/moq-go-ffi mirror env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} # Token is minted fresh per run, expires in 1 hour, never at rest. GO_FFI_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./go/scripts/publish-ffi.sh + run: just go publish-ffi publish-dry-run: name: Publish dry-run - needs: [package, parse-version] + needs: [ffi, package] runs-on: ubuntu-latest if: github.ref_type != 'tag' steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v6 + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -271,5 +176,5 @@ jobs: - name: Dry-run publish to mirror env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: ./go/scripts/publish-ffi.sh --dry-run + BUILD_VERSION: ${{ needs.ffi.outputs.version }} + run: just go publish-ffi --dry-run diff --git a/.github/workflows/release-go.yml b/.github/workflows/release-go.yml index 0f87ed9b743..bb817841acd 100644 --- a/.github/workflows/release-go.yml +++ b/.github/workflows/release-go.yml @@ -66,6 +66,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version: "1.23" @@ -105,7 +107,7 @@ jobs: if: github.event_name == 'push' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: .github/scripts/release.sh ffi-unreleased "$FFI_VERSION" + run: just gh release ffi-unreleased "$FFI_VERSION" # Skip on unreleased ffi changes, and when the pinned ffi's mirror tag is # missing (release-plz merged but Release Go FFI hasn't published yet): @@ -160,7 +162,7 @@ jobs: LINE: ${{ steps.line.outputs.line }} FFI_VERSION: ${{ steps.ffi.outputs.version }} run: | - ./go/scripts/package-wrapper.sh --line "$LINE" --ffi-version "$FFI_VERSION" --output release-out + just go package-wrapper --line "$LINE" --ffi-version "$FFI_VERSION" --output release-out - name: Upload wrapper module if: steps.gate.outputs.skip != 'true' @@ -180,6 +182,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -203,4 +207,4 @@ jobs: GO_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./go/scripts/publish-wrapper.sh + run: just go publish-wrapper diff --git a/.github/workflows/release-kt-ffi.yml b/.github/workflows/release-kt-ffi.yml index 7694fb4b839..0b48e10b929 100644 --- a/.github/workflows/release-kt-ffi.yml +++ b/.github/workflows/release-kt-ffi.yml @@ -20,131 +20,27 @@ concurrency: group: release-kt-ffi cancel-in-progress: false -# Pinned rather than "latest": `--locked` only fixes the tool's own -# dependencies, not which version of the tool cargo selects. -env: - CARGO_NDK_VERSION: 4.1.2 - jobs: - parse-version: - name: Parse version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Parse version - id: parse - run: .github/scripts/release.sh parse-version moq-ffi - - build: - name: Build moq-ffi (${{ matrix.target }}) - needs: [parse-version] - runs-on: ${{ matrix.os }} - - strategy: - fail-fast: false - matrix: - include: - # Android - - target: aarch64-linux-android - os: ubuntu-latest - - target: armv7-linux-androideabi - os: ubuntu-latest - - target: x86_64-linux-android - os: ubuntu-latest - # Desktop JVM - - target: x86_64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-unknown-linux-gnu - os: ubuntu-latest - - target: aarch64-apple-darwin - os: macos-latest - - target: x86_64-pc-windows-msvc - os: windows-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - # No Rust cache: its entry would evict the store every pull request - # restores from the 10 GB budget. See cache.yml. - - - name: Install cross-compilation tools (Linux ARM64) - if: matrix.target == 'aarch64-unknown-linux-gnu' - run: | - sudo apt-get update - # g++ as well as gcc: moq-video's openh264 fallback is C++, and - # cc-rs looks for aarch64-linux-gnu-g++ by name. - sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu - - - name: Install cargo-ndk (Android) - if: contains(matrix.target, 'android') - run: cargo install --locked "cargo-ndk@$CARGO_NDK_VERSION" - - - name: Build - shell: bash - env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - ./rs/moq-ffi/build.sh \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload lib - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: kotlin-lib-${{ matrix.target }} - # Upload only the lib/ contents; package.sh expects /. - path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ - - bindings: - name: Generate bindings - needs: [parse-version] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - - # No Rust cache: its entry would evict the store every pull request - # restores from the 10 GB budget. See cache.yml. - - - name: Generate bindings - env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - ./rs/moq-ffi/build.sh \ - --bindings-only \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload bindings - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: kotlin-bindings - # Upload only the kotlin subdir. - path: dist/bindings/kotlin/ + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: kotlin + # Android, then the desktop JVM. + targets: >- + [ + "aarch64-linux-android", + "armv7-linux-androideabi", + "x86_64-linux-android", + "x86_64-unknown-linux-gnu", + "aarch64-unknown-linux-gnu", + "aarch64-apple-darwin", + "x86_64-pc-windows-msvc" + ] package: name: Package and publish - needs: [parse-version, build, bindings] + needs: [ffi] runs-on: ubuntu-latest env: # Picked up by com.vanniktech.maven.publish via Gradle project properties. @@ -160,6 +56,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - uses: actions/setup-java@de7274f081f381c8f8158605e0321c36c376e2e6 # v6.0.1 with: distribution: temurin @@ -197,9 +95,9 @@ jobs: - name: Stage native libs + bindings into the gradle module env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | - ./kt/scripts/package.sh \ + just kt package \ --version "$BUILD_VERSION" \ --lib-dir libs \ --bindings-dir bindings \ @@ -208,7 +106,7 @@ jobs: - name: Publish to Maven Central if: github.ref_type == 'tag' env: - MOQFFI_VERSION: ${{ needs.parse-version.outputs.version }} + MOQFFI_VERSION: ${{ needs.ffi.outputs.version }} ORG_GRADLE_PROJECT_signingInMemoryKey: ${{ secrets.SIGNING_KEY }} ORG_GRADLE_PROJECT_signingInMemoryKeyPassword: ${{ secrets.SIGNING_PASSWORD }} run: | diff --git a/.github/workflows/release-kt-lib.yml b/.github/workflows/release-kt-lib.yml index 92fcbf64220..f3947ef2ac5 100644 --- a/.github/workflows/release-kt-lib.yml +++ b/.github/workflows/release-kt-lib.yml @@ -30,7 +30,7 @@ on: - "kt/build.gradle.kts" - "kt/settings.gradle.kts" - ".github/workflows/release-kt-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" pull_request: paths: - "kt/moq/**" @@ -38,7 +38,7 @@ on: - "kt/build.gradle.kts" - "kt/settings.gradle.kts" - ".github/workflows/release-kt-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" - "rs/moq-ffi/build.sh" # Manual trigger so a wrapper bump can be published once the bindings it # depends on land on Maven Central (the guard below skips otherwise). @@ -75,6 +75,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Read moq.version id: version run: | @@ -103,18 +105,18 @@ jobs: - name: Check for unreleased FFI changes id: ffi-diff if: github.event_name != 'workflow_run' - run: .github/scripts/release.sh ffi-unreleased "${{ steps.ffi.outputs.version }}" + run: just gh release ffi-unreleased "${{ steps.ffi.outputs.version }}" - name: Check wrapper publication id: wrapper - run: .github/scripts/release.sh maven-exists dev.moq moq "${{ steps.version.outputs.version }}" + run: just gh release maven-exists dev.moq moq "${{ steps.version.outputs.version }}" # The bindings this checkout generates must be on Maven Central, or the # wrapper's POM range resolves to an older moq-ffi without them. - name: Check bindings publication id: bindings if: github.event_name != 'workflow_run' && steps.wrapper.outputs.exists != 'true' && steps.ffi-diff.outputs.unreleased != 'true' - run: .github/scripts/release.sh maven-exists dev.moq moq-ffi "${{ steps.ffi.outputs.version }}" + run: just gh release maven-exists dev.moq moq-ffi "${{ steps.ffi.outputs.version }}" - name: Decide publish id: decide @@ -167,6 +169,8 @@ jobs: ref: ${{ github.event_name == 'workflow_run' && github.event.workflow_run.head_sha || github.sha }} persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable @@ -178,7 +182,7 @@ jobs: env: BUILD_VERSION: ${{ needs.check-version.outputs.version }} run: | - ./rs/moq-ffi/build.sh --bindings-only --version "$BUILD_VERSION" --output dist + just rs package-ffi --bindings-only --version "$BUILD_VERSION" --output dist mkdir -p kt/moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi/moq cp dist/bindings/kotlin/uniffi/moq/moq.kt \ kt/moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi/moq/moq.kt @@ -202,7 +206,7 @@ jobs: - name: Recheck wrapper publication id: recheck if: github.event_name != 'pull_request' - run: .github/scripts/release.sh maven-exists dev.moq moq "${{ needs.check-version.outputs.version }}" + run: just gh release maven-exists dev.moq moq "${{ needs.check-version.outputs.version }}" - name: Publish to Maven Central if: github.event_name != 'pull_request' && steps.recheck.outputs.exists != 'true' diff --git a/.github/workflows/release-py-ffi.yml b/.github/workflows/release-py-ffi.yml index 2a79e7096ee..678abd24388 100644 --- a/.github/workflows/release-py-ffi.yml +++ b/.github/workflows/release-py-ffi.yml @@ -34,9 +34,11 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Parse version id: parse - run: .github/scripts/release.sh parse-version moq-ffi + run: just gh release parse-version moq-ffi # Maturin reads the wheel version from rs/moq-ffi/Cargo.toml (via # `dynamic = ["version"]` in pyproject.toml). If a tag is pushed by diff --git a/.github/workflows/release-py.yml b/.github/workflows/release-py.yml index 8e66b73f188..beb00e7dadf 100644 --- a/.github/workflows/release-py.yml +++ b/.github/workflows/release-py.yml @@ -19,7 +19,7 @@ on: paths: - "py/moq-rs/**" - ".github/workflows/release-py.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" workflow_run: workflows: ["Release Py FFI"] types: @@ -56,9 +56,11 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Read version id: version - run: .github/scripts/release.sh read-version py/moq-rs/pyproject.toml + run: just gh release read-version py/moq-rs/pyproject.toml - name: Resolve FFI version id: ffi @@ -78,14 +80,14 @@ jobs: if: github.event_name == 'push' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: .github/scripts/release.sh ffi-unreleased "$FFI_VERSION" + run: just gh release ffi-unreleased "$FFI_VERSION" - name: Check FFI on PyPI id: ffi-pypi if: github.event_name == 'push' && steps.ffi-gate.outputs.unreleased != 'true' env: FFI_VERSION: ${{ steps.ffi.outputs.version }} - run: .github/scripts/release.sh pypi-exists moq-ffi "$FFI_VERSION" + run: just gh release pypi-exists moq-ffi "$FFI_VERSION" - name: Unpublished FFI notice if: github.event_name == 'push' && steps.ffi-gate.outputs.unreleased != 'true' && steps.ffi-pypi.outputs.exists != 'true' @@ -100,7 +102,7 @@ jobs: id: pypi env: VERSION: ${{ steps.version.outputs.version }} - run: .github/scripts/release.sh pypi-exists moq-rs "$VERSION" + run: just gh release pypi-exists moq-rs "$VERSION" - uses: DeterminateSystems/nix-installer-action@1d87d45818068401a10cf16bdc5f00b24994a83f # main with: diff --git a/.github/workflows/release-swift-ffi.yml b/.github/workflows/release-swift-ffi.yml index bb4c757256c..afa2fbf175e 100644 --- a/.github/workflows/release-swift-ffi.yml +++ b/.github/workflows/release-swift-ffi.yml @@ -23,100 +23,16 @@ concurrency: cancel-in-progress: false jobs: - parse-version: - name: Parse version - runs-on: ubuntu-latest - outputs: - version: ${{ steps.parse.outputs.version }} - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Parse version - id: parse - run: .github/scripts/release.sh parse-version moq-ffi - - build: - name: Build moq-ffi (${{ matrix.target }}) - needs: [parse-version] - runs-on: macos-latest - - strategy: - fail-fast: false - matrix: - target: - - aarch64-apple-ios - - aarch64-apple-ios-sim - - aarch64-apple-darwin - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - with: - targets: ${{ matrix.target }} - - # No Rust cache: its entry would evict the store every pull request - # restores from the 10 GB budget. See cache.yml. - - - name: Build - shell: bash - env: - BUILD_TARGET: ${{ matrix.target }} - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - # Set iOS deployment target to avoid ___chkstk_darwin linker errors. - IPHONEOS_DEPLOYMENT_TARGET: ${{ contains(matrix.target, 'apple-ios') && '16.0' || '' }} - run: | - ./rs/moq-ffi/build.sh \ - --target "$BUILD_TARGET" \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload lib - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: swift-lib-${{ matrix.target }} - path: dist/moq-ffi-${{ needs.parse-version.outputs.version }}-${{ matrix.target }}/lib/ - - bindings: - name: Generate bindings - needs: [parse-version] - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - - name: Install Rust - uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable - - # No Rust cache: its entry would evict the store every pull request - # restores from the 10 GB budget. See cache.yml. - - - name: Generate bindings - env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} - run: | - ./rs/moq-ffi/build.sh \ - --bindings-only \ - --version "$BUILD_VERSION" \ - --output dist - - - name: Upload bindings - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: swift-bindings - path: dist/bindings/swift/ + ffi: + name: moq-ffi + uses: ./.github/workflows/release-ffi.yml + with: + language: swift + targets: '["aarch64-apple-ios", "aarch64-apple-ios-sim", "aarch64-apple-darwin"]' package: name: Package - needs: [parse-version, build, bindings] + needs: [ffi] runs-on: macos-latest steps: @@ -124,6 +40,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download per-target libs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -147,9 +65,9 @@ jobs: - name: Package env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | - ./swift/scripts/package-ffi.sh \ + just swift package-ffi \ --version "$BUILD_VERSION" \ --lib-dir libs \ --bindings-dir bindings \ @@ -167,7 +85,7 @@ jobs: # points at this asset, so it must exist before any consumer resolves the # SPM tag. name: Attach assets to GitHub release - needs: [package, parse-version] + needs: [ffi, package] if: github.ref_type == 'tag' runs-on: ubuntu-latest permissions: @@ -181,9 +99,11 @@ jobs: # up GH_TOKEN directly, so don't persist a contents-write token. persist-credentials: false + - uses: ./.github/actions/just + - name: Find previous tag id: prev_tag - run: .github/scripts/release.sh prev-tag moq-ffi + run: just gh release prev-tag moq-ffi - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 @@ -195,9 +115,9 @@ jobs: env: GH_TOKEN: ${{ github.token }} RELEASE_TAG: ${{ github.ref_name }} - RELEASE_TITLE: "moq-ffi v${{ needs.parse-version.outputs.version }}" + RELEASE_TITLE: "moq-ffi v${{ needs.ffi.outputs.version }}" RELEASE_PREV_TAG: ${{ steps.prev_tag.outputs.tag }} - run: .github/scripts/release.sh create artifacts + run: just gh release create artifacts verify: # Gate the mirror push on actually being able to resolve the staged package @@ -205,7 +125,7 @@ jobs: # fine but SPM cannot resolve (e.g. a path-based binaryTarget slipping in # where a URL+checksum is expected). name: Verify staged package resolves - needs: [release, parse-version] + needs: [ffi, release] runs-on: macos-latest steps: @@ -213,6 +133,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download staged package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -221,13 +143,13 @@ jobs: - name: Resolve and build smoke consumer env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} run: | - ./swift/scripts/verify-ffi.sh --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" + just swift verify-ffi --tarball "artifacts/moq-ffi-${BUILD_VERSION}-swift-ffi.tar.gz" publish: name: Publish to Swift Package mirror - needs: [verify, parse-version] + needs: [ffi, verify] runs-on: ubuntu-latest steps: @@ -235,6 +157,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -254,9 +178,9 @@ jobs: - name: Publish to moq-dev/moq-swift-ffi mirror env: - BUILD_VERSION: ${{ needs.parse-version.outputs.version }} + BUILD_VERSION: ${{ needs.ffi.outputs.version }} # Token is minted fresh per run, expires in 1 hour, never at rest. SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./swift/scripts/publish-ffi.sh + run: just swift publish-ffi diff --git a/.github/workflows/release-swift-lib.yml b/.github/workflows/release-swift-lib.yml index 6c5b07fc5a5..2684aa4f595 100644 --- a/.github/workflows/release-swift-lib.yml +++ b/.github/workflows/release-swift-lib.yml @@ -22,22 +22,22 @@ on: - "swift/Package.swift.template" - "swift/Sources/Moq/**" - "swift/Tests/**" - - "swift/scripts/package.sh" - - "swift/scripts/publish.sh" - - "swift/scripts/verify.sh" + - "sh/swift/package.sh" + - "sh/swift/publish.sh" + - "sh/swift/verify.sh" - ".github/workflows/release-swift-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" pull_request: paths: - "swift/VERSION" - "swift/Package.swift.template" - "swift/Sources/Moq/**" - "swift/Tests/**" - - "swift/scripts/package.sh" - - "swift/scripts/publish.sh" - - "swift/scripts/verify.sh" + - "sh/swift/package.sh" + - "sh/swift/publish.sh" + - "sh/swift/verify.sh" - ".github/workflows/release-swift-lib.yml" - - ".github/scripts/release.sh" + - "sh/gh/release.sh" workflow_run: workflows: ["Release Swift FFI"] types: @@ -79,6 +79,8 @@ jobs: fetch-depth: 0 persist-credentials: false + - uses: ./.github/actions/just + - name: Read versions id: version env: @@ -101,13 +103,13 @@ jobs: if: github.event_name != 'workflow_run' env: FFI_VERSION: ${{ steps.version.outputs.ffi_version }} - run: .github/scripts/release.sh ffi-unreleased "$FFI_VERSION" + run: just gh release ffi-unreleased "$FFI_VERSION" - name: Check FFI mirror tag id: ffi-gate env: FFI_VERSION: ${{ steps.version.outputs.ffi_version }} - run: .github/scripts/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" + run: just gh release git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" - name: Unpublished FFI notice if: steps.ffi-diff.outputs.unreleased != 'true' && steps.ffi-gate.outputs.exists != 'true' && github.event_name != 'pull_request' @@ -121,12 +123,12 @@ jobs: id: gate env: VERSION: ${{ steps.version.outputs.version }} - run: .github/scripts/release.sh git-tag-exists moq-dev/moq-swift "$VERSION" + run: just gh release git-tag-exists moq-dev/moq-swift "$VERSION" - name: Package env: BUILD_VERSION: ${{ steps.version.outputs.version }} - run: ./swift/scripts/package.sh --version "$BUILD_VERSION" --output release-out + run: just swift package --version "$BUILD_VERSION" --output release-out - name: Upload Swift package uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -149,6 +151,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Check FFI mirror tag id: ffi if: needs.build.outputs.publish == 'true' || (github.event_name == 'pull_request' && needs.build.outputs.ffi_unreleased != 'true') @@ -158,7 +162,7 @@ jobs: # rather than a hard failure, so the first bootstrap (before # moq-swift-ffi exists) skips the cross-package resolve instead of # failing the job. publish.sh stays idempotent regardless. - run: .github/scripts/release.sh git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" || echo "exists=false" >> "$GITHUB_OUTPUT" + run: just gh release git-tag-exists moq-dev/moq-swift-ffi "$FFI_VERSION" || echo "exists=false" >> "$GITHUB_OUTPUT" - name: Download staged package if: steps.ffi.outputs.exists == 'true' @@ -171,7 +175,7 @@ jobs: if: steps.ffi.outputs.exists == 'true' env: BUILD_VERSION: ${{ needs.build.outputs.version }} - run: ./swift/scripts/verify.sh --tarball "artifacts/moq-${BUILD_VERSION}-swift.tar.gz" + run: just swift verify --tarball "artifacts/moq-${BUILD_VERSION}-swift.tar.gz" - name: Skip notice if: needs.build.outputs.ffi_unreleased == 'true' @@ -194,6 +198,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Generate moq-bot token uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3 id: token @@ -218,7 +224,7 @@ jobs: SWIFT_MIRROR_TOKEN: ${{ steps.token.outputs.token }} GIT_AUTHOR_NAME: moq-bot GIT_AUTHOR_EMAIL: moq-bot[bot]@users.noreply.github.com - run: ./swift/scripts/publish.sh + run: just swift publish publish-dry-run: name: Publish dry-run @@ -231,6 +237,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Download package uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8 with: @@ -240,4 +248,4 @@ jobs: - name: Dry-run publish to mirror env: BUILD_VERSION: ${{ needs.build.outputs.version }} - run: ./swift/scripts/publish.sh --dry-run + run: just swift publish --dry-run diff --git a/.github/workflows/rpm-repo.yml b/.github/workflows/rpm-repo.yml index 4c7df9dd844..93dc8c4a943 100644 --- a/.github/workflows/rpm-repo.yml +++ b/.github/workflows/rpm-repo.yml @@ -32,6 +32,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Set up packaging tools uses: ./.github/actions/setup-packaging @@ -51,10 +53,9 @@ jobs: - name: Publish to R2 shell: bash env: - ARTIFACTS_DIR: artifacts R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }} R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }} R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }} SIGNING_KEY: ${{ secrets.SIGNING_KEY }} SIGNING_PASSWORD: ${{ secrets.SIGNING_PASSWORD }} - run: ./infra/rpm/publish.sh + run: just infra rpm publish "$GITHUB_WORKSPACE/artifacts" diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml index bc7cdd98b7b..6a806b3ce44 100644 --- a/.github/workflows/swift.yml +++ b/.github/workflows/swift.yml @@ -14,6 +14,7 @@ on: branches: [main] paths: - "swift/**" + - "sh/swift/**" - "doc/lib/swift/**" - "doc/lib/samples.sh" - "rs/moq-ffi/**" @@ -26,6 +27,7 @@ on: types: [opened, synchronize, reopened, closed] paths: - "swift/**" + - "sh/swift/**" - "doc/lib/swift/**" - "doc/lib/samples.sh" - "rs/moq-ffi/**" @@ -53,6 +55,8 @@ jobs: with: persist-credentials: false + - uses: ./.github/actions/just + - name: Install Rust uses: dtolnay/rust-toolchain@29eef336d9b2848a0b548edc03f92a220660cdb8 # stable with: @@ -80,4 +84,4 @@ jobs: done - name: Swift check - run: ./swift/scripts/check.sh + run: just swift check diff --git a/.github/workflows/wasm.yml b/.github/workflows/wasm.yml index d0c8392de3d..5f93aab325c 100644 --- a/.github/workflows/wasm.yml +++ b/.github/workflows/wasm.yml @@ -43,6 +43,7 @@ on: - "test/wasm/**" - "test/lib/**" - "justfile" + - "js/justfile" - "test/justfile" - ".cargo/config.toml" - "package.json" diff --git a/.gitignore b/.gitignore index af3a5c21aca..53171c83752 100644 --- a/.gitignore +++ b/.gitignore @@ -7,9 +7,8 @@ *.local.json # Claude Code scratch space. These are untracked working directories, and -# `just _changed` counts untracked files, so leaving them visible pushes the -# changed-file list past its cap and silently turns every scoped `just check` -# into `check --all`. The worktrees also carry their own biome.jsonc, which the +# `just check` counts untracked files, so leaving them visible would put their +# whole tree in scope. The worktrees also carry their own biome.jsonc, which the # unscoped run then rejects as a nested root config. /.claude/cache/ /.claude/tmp/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 9bf948cca47..03b50aea224 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,10 @@ Wait for Codex to review the final head before merging. Merge only on its thumbs up, or once every Codex finding on the PR is fixed or replied to. Codex skips fork PRs; ask the maintainer to request one. +# CI + +Workflow steps run `just` recipes, never a script path; `just gh check` enforces it. + # Follow-ups If you encounter issues, or findings that are out of scope, create follow-up quests. diff --git a/Cargo.toml b/Cargo.toml index 789ecbbfd04..c27908b5072 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -317,7 +317,7 @@ debug = true # Size-optimized build for the browser/WASM bindings (rs/moq-wasm). Shipping over # the wire, so optimize for size, not speed (~579KB -> ~471KB gzipped vs the -# default release profile). `just wasm` builds with `--profile wasm-release`. +# default release profile). `just js wasm` builds with `--profile wasm-release`. [profile.wasm-release] inherits = "release" opt-level = "z" diff --git a/bench/justfile b/bench/justfile deleted file mode 100755 index b3fa64c3929..00000000000 --- a/bench/justfile +++ /dev/null @@ -1,19 +0,0 @@ -#!/usr/bin/env just --justfile - -set working-directory := '..' - -# Benchmark the current tree, or compare it with a commit. -compare $BASE="": - #!/usr/bin/env bash - exec bench/run.sh "$BASE" - -# Compare the shared Tokio runtime with independent Tokio and io_uring workers. -runtime $ROUNDS="3" $WORKERS="": - #!/usr/bin/env bash - MOQ_BENCH_RUNTIME_ROUNDS="$ROUNDS" MOQ_BENCH_RUNTIME_WORKERS="$WORKERS" \ - exec bench/run.sh --runtime - -# Compile every binary and feature used by the repository-level harness. -check: - cargo check --locked --package moq-relay --package moq-bench \ - --features moq-relay/io-uring diff --git a/cpp/obs/justfile b/cpp/obs/justfile index 8a2778d6cf7..e252eccccba 100644 --- a/cpp/obs/justfile +++ b/cpp/obs/justfile @@ -21,389 +21,48 @@ default: # Configure via CMake presets. MOQ_LOCAL defaults to the repo root inside # CMakeLists.txt (so libmoq builds from rs/libmoq); pass `path=/some/moq` to -# override. We deliberately don't compute the path here: `justfile_directory()` -# resolves to the root justfile when this runs as `just obs setup`, so the - -# relative math would point outside the repo. +# override it. setup preset="" path="": - #!/usr/bin/env bash - set -euo pipefail - PRESET=$(just preset "{{ preset }}") - if [ -n "{{ path }}" ]; then - echo "Configuring with preset: $PRESET and MOQ_LOCAL={{ path }}" - cmake --preset "$PRESET" -DMOQ_LOCAL="{{ path }}" - else - echo "Configuring with preset: $PRESET (MOQ_LOCAL defaults to the repo root)" - cmake --preset "$PRESET" - fi + cmake --preset "$(../../sh/obs/preset.sh '{{ preset }}')" {{ if path == "" { "" } else { "-DMOQ_LOCAL=" + quote(path) } }} # Build via CMake presets. Run `just obs setup` first (it configures + downloads - # deps); not chained here because on macOS setup reconfigures OBS, which is slow. build preset="": - #!/usr/bin/env bash - set -euo pipefail - PRESET=$(just preset "{{ preset }}") - cmake --build --preset "$PRESET" - -# Copy the freshly built plugin into the OBS user plugin dir and launch OBS. - -# Uses the installed OBS (no local OBS build required). macOS only for now. -run: - #!/usr/bin/env bash - set -euo pipefail - if [[ "$OSTYPE" != "darwin"* ]]; then - echo "just run is macOS-only for now; copy build_*/obs-moq.* into your OBS plugin dir manually" >&2 - exit 1 - fi - dest="$HOME/Library/Application Support/obs-studio/plugins" - mkdir -p "$dest" - cp -a build_macos/RelWithDebInfo/obs-moq.plugin "$dest/" - RUST_LOG=debug RUST_BACKTRACE=1 OBS_LOG_LEVEL=debug /Applications/OBS.app/Contents/MacOS/OBS + cmake --build --preset "$(../../sh/obs/preset.sh '{{ preset }}')" # Type-check every plugin source and unit test, without linking or an obs-deps -# download. -# -# This is the gate to run in a worktree. `just obs build` needs the -# multi-hundred-MB obs-deps bundle (macOS/Windows) and a full cargo build, per -# worktree; this needs neither, because the dev shell carries all three header -# sets: libobs pinned to the same OBS release buildspec.json downloads (see -# `obs-headers` in flake.nix), Qt6, and ffmpeg. CI links the real thing on Linux -# (.github/workflows/obs.yml). -# -# It compiles the Qt sources too, which the CMake build only does when -# ENABLE_QT and ENABLE_FRONTEND_API are on, so the definitions they gate on are -# set here to match. -# -# test/ is in scope because each test file defines the libmoq entry points the -# plugin calls, so a signature that drifts from the generated moq.h is a -# conflicting C declaration. Catching that needs only headers, which is why it -# belongs here: `just obs ci` finds the same drift, but only where obs.yml's -# path filter reaches, and `just obs test` is manual. +# download. The gate to run in a worktree. compile: - #!/usr/bin/env bash - set -euo pipefail - # Never skips, for the reason `test` doesn't: a compile gate that reports - # success without compiling is worse than no gate. Everything it needs is in - # the dev shell, so the fix is always to enter it. - cxx="${CXX:-c++}" - if ! command -v "$cxx" >/dev/null; then - echo "no C++ compiler at '$cxx'" >&2 - exit 1 - fi - for pkgs in "Qt6Widgets Qt6Gui Qt6Core" "libavcodec libavutil libswscale libswresample"; do - # shellcheck disable=SC2086 - if ! pkg-config --exists $pkgs; then - echo "missing headers for: $pkgs" >&2 - echo "run inside 'nix develop', which supplies Qt6 and ffmpeg on every platform" >&2 - exit 1 - fi - done - - # Assign first so a failure in `_includes` still aborts under `set -e`, - # then split on newlines only: an include path may contain spaces. - includes_raw=$(just _includes) - includes=() - while IFS= read -r flag; do includes+=("$flag"); done <<< "$includes_raw" - - qt=$(pkg-config --cflags Qt6Widgets Qt6Gui Qt6Core) - ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale libswresample) - - # MOQ_VERSION_STRING only reaches a label in the dock, so any value - # type-checks the same; CMake stamps the real libmoq version. - status=0 - for source in src/*.cpp test/*.cpp; do - # shellcheck disable=SC2086 - # -Wno-unused-command-line-argument: the nix compiler wrapper injects - # link flags, which a syntax-only run reports as unused, once per file. - # $qt and $ffmpeg stay unquoted on purpose: pkg-config hands back one - # space-separated string, and splitting it is the intended reading. - if ! "$cxx" -std=c++17 -fsyntax-only -Wno-unused-command-line-argument \ - "${includes[@]}" $qt $ffmpeg \ - -DMOQ_FRONTEND_ENABLED -DMOQ_VERSION_STRING='"0.0.0"' "$source"; then - status=1 - fi - done - exit $status + ../../sh/obs/compile.sh # Compile and link the plugin the way CI does, against the `.#obs` shell's own # libobs/Qt6/ffmpeg, then run the unit tests. Linux-only in practice: it's the -# one platform where nixpkgs has obs-studio, so nothing has to be downloaded. -# Manual elsewhere. -# -# The tests run here without ThreadSanitizer because this is the only recipe CI -# invokes (.github/workflows/obs.yml). `just obs test` is manual, so on its own -# it can't make a build red; the sanitizer finds more, but only what runs in CI -# gates a merge. -# -# CMAKE_BUILD_TYPE overrides the preset's RelWithDebInfo because the Rust cache -# CI restores holds dev-profile artifacts. A release build of libmoq would -# rebuild its whole dependency tree on every run, and the gate is about whether -# the plugin compiles, which the profile doesn't change. +# one platform where nixpkgs has obs-studio. The Debug build type reuses the +# dev-profile Rust artifacts CI restores, where a release libmoq would rebuild +# its whole dependency tree. ci preset="": - #!/usr/bin/env bash - set -euo pipefail - PRESET=$(just preset "{{ preset }}" ci) - cmake --preset "$PRESET" -DCMAKE_BUILD_TYPE=Debug - cmake --build --preset "$PRESET" - just _unit - -# Print the -I flags for libobs and libmoq, shared by `compile` and `_unit`. -# -# moq.h is generated by cbindgen from rs/libmoq/src, so regenerate rather than -# just locating it: a stale header is how a plugin call to a since-changed -# libmoq function type-checks anyway. -[private] -_includes: - #!/usr/bin/env bash - set -euo pipefail - # The dev shell's pinned headers first, so every platform checks against the - # same libobs. Then the obs-deps framework (macOS/Windows), the OBS sources - # CMake unpacks beside it, and a system install (Linux without nix). - obs_include="" - for candidate in \ - "${OBS_INCLUDE_DIR:-}" \ - .deps/Frameworks/libobs.framework/Versions/A/Headers \ - .deps/obs-studio-*/libobs; do - if [ -n "$candidate" ] && [ -f "$candidate/obs.h" ]; then - obs_include="$candidate" - break - fi - done - if [ -z "$obs_include" ] && command -v pkg-config >/dev/null && pkg-config --exists libobs; then - obs_include=$(pkg-config --variable=includedir libobs)/obs - fi - if [ -z "$obs_include" ]; then - echo "libobs headers not found; run inside 'nix develop' (or set OBS_INCLUDE_DIR)" >&2 - exit 1 - fi - - # obs.h reaches as of OBS 32, which used to be vendored - # under libobs/util/ and so came along with the headers above. The dev shell - # carries it (see flake.nix); the obs-deps bundle ships its own copy beside - # libobs, which is how the macOS and Windows builds get it. - simde_include="" - if command -v pkg-config >/dev/null && pkg-config --exists simde; then - simde_include=$(pkg-config --variable=includedir simde) - else - for candidate in .deps/obs-deps-*/include; do - if [ -f "$candidate/simde/x86/sse2.h" ]; then - simde_include="$candidate" - break - fi - done - fi - if [ -z "$simde_include" ]; then - echo "simde headers not found; run inside 'nix develop', which supplies them" >&2 - exit 1 - fi - - (cd ../.. && cargo check --locked --quiet -p libmoq) - - # rs/libmoq/build.rs writes the header into its OUT_DIR, so ask cargo where - # that is. Replays the check above from cache, so it costs a process rather - # than a compile. - out_dir=$(cd ../.. && cargo check --locked -p libmoq --message-format=json | - jq -r 'select(.reason == "build-script-executed") | select(.package_id | tostring | test("libmoq")) | .out_dir' | - tail -1) - if [ -z "$out_dir" ]; then - echo "cargo reported no build script output for libmoq" >&2 - exit 1 - fi - moq_include="$out_dir/include" - if [ ! -f "$moq_include/moq.h" ]; then - echo "$moq_include/moq.h missing after 'cargo check --locked -p libmoq'" >&2 - exit 1 - fi - - # One flag per line, path glued to the -I, so a caller can rebuild the - # arguments without splitting on the spaces in a path. - printf -- '-I%s\n' src "$obs_include" "$simde_include" "$moq_include" + cmake --preset "$(../../sh/obs/preset.sh '{{ preset }}' ci)" -DCMAKE_BUILD_TYPE=Debug + cmake --build --preset "$(../../sh/obs/preset.sh '{{ preset }}' ci)" + ../../sh/obs/unit.sh # Unit-test the plugin sources against stubbed libobs/libmoq/ffmpeg, under -# ThreadSanitizer. -# -# Manual because ThreadSanitizer needs its own build. `just obs ci` runs the -# same tests without it, so a regression these assertions catch still turns PR -# CI red; the sanitizer is what -# adds the races on top. Run this whenever you touch src/, especially the session -# callback plumbing, whose orderings the build can't check. +# ThreadSanitizer. Manual; `just obs ci` runs them without the sanitizer. test: - #!/usr/bin/env bash - set -euo pipefail - # Unlike the lint recipes, this one never skips: a test gate that reports - # success without running anything is worse than no gate. - cxx="${CXX:-c++}" - unsupported="'$cxx' cannot build and run -fsanitize=thread binaries. These tests need a - Clang or GCC whose ThreadSanitizer runtime works on this host: set CXX, or on Windows run - them from WSL, since neither MSVC nor Clang on Windows implements ThreadSanitizer. - They also run without the sanitizer as part of 'just obs ci'." - if ! command -v "$cxx" >/dev/null; then - echo "no C++ compiler at '$cxx'." >&2 - echo "$unsupported" >&2 - exit 1 - fi - - probe=$(mktemp -d) - trap 'rm -rf "$probe"' EXIT - - # Run the probe, don't just link it: a mismatch between the compiler's TSan - # runtime and the host kernel (macOS is where this shows up) links fine and - # then segfaults before main, which is an unreadable way for the real test - # to fail. - # The subshell keeps bash's own "Segmentation fault" job message out of the - # output, so the explanation below is all the reader gets. - if ! printf 'int main(){}\n' | "$cxx" -x c++ -fsanitize=thread -o "$probe/probe" - >/dev/null 2>&1 \ - || ! ("$probe/probe" >/dev/null 2>&1) 2>/dev/null; then - echo "$unsupported" >&2 - exit 1 - fi - - just _unit tsan + ../../sh/obs/test.sh -# Compile and run the unit tests, plain or under ThreadSanitizer. Shared by -# `test` and `ci` so both run the same assertions. -[private] -_unit mode="plain": - #!/usr/bin/env bash - set -euo pipefail - cxx="${CXX:-c++}" - if ! command -v "$cxx" >/dev/null; then - echo "no C++ compiler at '$cxx'" >&2 - exit 1 - fi - # moq-source.cpp decodes with ffmpeg, so its test needs the headers even - # though it stubs every function out of them. - if ! pkg-config --exists libavcodec libavutil libswscale; then - echo "missing ffmpeg headers" >&2 - echo "run inside 'nix develop', which supplies ffmpeg on every platform" >&2 - exit 1 - fi - - includes_raw=$(just _includes) - includes=() - while IFS= read -r flag; do includes+=("$flag"); done <<< "$includes_raw" - - ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale) - - flags=(-std=c++17 -g -O0 -pthread) - if [ "{{ mode }}" = "tsan" ]; then - flags+=(-fsanitize=thread) - fi - - out=$(mktemp -d) - trap 'rm -rf "$out"' EXIT - - # One binary per source under test: each test file defines its own libobs and - # libmoq stubs, so two of them can't share a link. Header-only helpers - # (quality defaults) build as a third binary with no plugin source. - for name in moq-output moq-source; do - # shellcheck disable=SC2086 - # $ffmpeg stays unquoted on purpose: pkg-config hands back one - # space-separated string, and splitting it is the intended reading. - "$cxx" "${flags[@]}" "${includes[@]}" $ffmpeg \ - -o "$out/$name-test" "test/$name-test.cpp" "src/$name.cpp" - TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" - done - for name in moq-quality-defaults moq-dock-stop moq-error moq-encoder-latency moq-dial moq-spark; do - "$cxx" "${flags[@]}" "${includes[@]}" \ - -o "$out/$name-test" "test/$name-test.cpp" - TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" - done - -# Lint formatting. Skips a tool silently if it isn't on $PATH (matches repo convention). +# Lint formatting and the OBS version pins. check: - #!/usr/bin/env bash - set -euo pipefail - if grep -Fq '"MOQ_VERSION"' CMakePresets.json; then - echo "CMakePresets.json must not pin MOQ_VERSION" >&2 - exit 1 - fi - # The headers `just obs compile` type-checks against are a second copy of - # the OBS version this spec downloads, so a bump here has to reach flake.nix - # or the two gates check different libobs. - spec_obs=$(jq -r '.dependencies["obs-studio"].version // ""' buildspec.json) - flake_obs=$(sed -n '/pname = "libobs-headers"/,/}/p' ../../flake.nix | sed -n 's/.*version = "\([^"]*\)".*/\1/p') - # Both are pattern matches against files this recipe doesn't own the shape - # of. An empty parse compares equal to an empty parse, so the guard would - # pass by reporting nothing rather than by finding the versions in sync. - if [ -z "$spec_obs" ] || [ -z "$flake_obs" ]; then - echo "couldn't read the obs-studio version: '$spec_obs' from buildspec.json, '$flake_obs' from flake.nix" >&2 - exit 1 - fi - if [ "$spec_obs" != "$flake_obs" ]; then - echo "obs-studio is $spec_obs in buildspec.json but $flake_obs in flake.nix (libobs-headers)" >&2 - exit 1 - fi - # And a third: `just obs ci` links against nixpkgs' obs-studio, the one OBS - # this repo doesn't pick. Without this the released macOS/Windows binaries - # could be a whole libobs API apart from what CI ever compiles. Patch - # releases carry no API change, so only the major.minor has to agree; that - # also keeps the guard quiet until flake.lock actually moves OBS, which is - # the change that opens the gap and so the one that should fail. - # OBS_LINKED_VERSION comes from the dev shell (see flake.nix), so this leg - # is skipped outside it, where there is no nixpkgs obs-studio to compare. - # MOQ_STRICT turns that skip into an error for the same reason it turns a - # missing tool into one: in CI a leg that checks nothing still reports green. - if [ -z "${OBS_LINKED_VERSION:-}" ]; then - if [ -n "${MOQ_STRICT:-}" ]; then - echo "MOQ_STRICT is set but OBS_LINKED_VERSION is unset; run inside 'nix develop', which exports it" >&2 - exit 1 - fi - else - spec_api=$(echo "$spec_obs" | cut -d. -f1,2) - linked_api=$(echo "$OBS_LINKED_VERSION" | cut -d. -f1,2) - if [ "$spec_api" != "$linked_api" ]; then - echo "obs-studio is $spec_obs in buildspec.json but $OBS_LINKED_VERSION in nixpkgs, which is what 'just obs ci' links" >&2 - echo "bump buildspec.json and flake.nix's libobs-headers to $OBS_LINKED_VERSION, or pin nixpkgs back" >&2 - exit 1 - fi - fi - if command -v cmake >/dev/null; then - build_dir=$(mktemp -d) - trap 'rm -rf "$build_dir"' EXIT - if cmake -S . -B "$build_dir" -DMOQ_LOCAL= >"$build_dir/configure.log" 2>&1; then - echo "CMake accepted a release build without MOQ_VERSION" >&2 - exit 1 - fi - if ! grep -Fq "MOQ_VERSION is required when MOQ_LOCAL does not contain rs/libmoq" "$build_dir/configure.log"; then - cat "$build_dir/configure.log" >&2 - exit 1 - fi - fi - if command -v clang-format >/dev/null; then - git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format --dry-run --Werror - fi - if command -v gersemi >/dev/null; then - gersemi --check CMakeLists.txt cmake - fi + ../../sh/obs/check.sh # Auto-fix formatting. fix: - #!/usr/bin/env bash - set -euo pipefail - if command -v clang-format >/dev/null; then - git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format -i - fi - if command -v gersemi >/dev/null; then - gersemi --in-place CMakeLists.txt cmake - fi + ../../sh/obs/fix.sh + +# Runs in the caller's directory, so a relative `--output` means what it would +# to the script itself. -# Detect the CMake preset for the current platform (or use the override). -# Pass `ci` as the second argument for the warnings-as-errors variant, whose -# name upstream spells differently on each platform. -preset override="" variant="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "{{ override }}" ]]; then - echo "{{ override }}" - elif [[ "$OSTYPE" == "darwin"* ]]; then - [[ "{{ variant }}" == "ci" ]] && echo "macos-ci" || echo "macos" - elif [[ "$OSTYPE" == "linux-gnu"* ]]; then - [[ "{{ variant }}" == "ci" ]] && echo "ubuntu-ci-x86_64" || echo "ubuntu-x86_64" - elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then - [[ "{{ variant }}" == "ci" ]] && echo "windows-ci-x64" || echo "windows-x64" - else - echo "Unknown platform: $OSTYPE" >&2 - exit 1 - fi +# Build and package the obs-moq plugin for release. +[no-cd] +[positional-arguments] +package *args: + {{ quote(justfile_directory() / "cpp/obs/build.sh") }} "$@" diff --git a/dart/justfile b/dart/justfile index 9823e8cf324..5c2fc803ffc 100644 --- a/dart/justfile +++ b/dart/justfile @@ -1,18 +1,22 @@ #!/usr/bin/env just --justfile # Check the generated bindings and both Dart packages. -check $FILES="": - ./scripts/check.sh "{{ FILES }}" +check: + ../sh/dart/check.sh # Format Dart sources. -fix $FILES="": - ./scripts/check.sh --fix "{{ FILES }}" +fix: + ../sh/dart/check.sh --fix # Regenerate the raw UniFFI Dart bindings. generate: - ./scripts/generate.sh + ../sh/dart/generate.sh -# Remove Dart tool state and build output. -clean: - rm -rf moq/.dart_tool moq_ffi/.dart_tool - rm -rf moq/build moq_ffi/build +# `package` runs in the caller's directory, so a relative `--output` means +# what it would to the script itself. + +# Package a Dart library for pub.dev. +[no-cd] +[positional-arguments] +package *args: + {{ quote(justfile_directory() / "sh/dart/package.sh") }} "$@" diff --git a/dart/moq_ffi/hook/build.dart b/dart/moq_ffi/hook/build.dart index 6916ea46a70..bfb465d6036 100644 --- a/dart/moq_ffi/hook/build.dart +++ b/dart/moq_ffi/hook/build.dart @@ -7,7 +7,7 @@ import 'package:http/http.dart' as http; /// The moq-ffi release whose native assets this package downloads. /// -/// dart/scripts/package.sh rewrites this when staging a release. The sentinel +/// sh/dart/package.sh rewrites this when staging a release. The sentinel /// is never used to download: a checkout builds from source instead, and a /// package built without the injection fails loudly rather than fetching a /// library that does not match these bindings. diff --git a/dart/moq_ffi/pubspec.yaml b/dart/moq_ffi/pubspec.yaml index cf7a8ecadc2..0f53a4f24cb 100644 --- a/dart/moq_ffi/pubspec.yaml +++ b/dart/moq_ffi/pubspec.yaml @@ -1,6 +1,6 @@ name: moq_ffi description: Raw Dart and Flutter bindings for Media over QUIC. -# Overridden by dart/scripts/package.sh from the moq-ffi-v* tag. Committing a +# Overridden by sh/dart/package.sh from the moq-ffi-v* tag. Committing a # real version would go stale the moment release-plz bumps the crate. version: 0.0.0-dev repository: https://github.com/moq-dev/moq/tree/main/dart/moq_ffi diff --git a/demo/boy/justfile b/demo/boy/justfile index 9312d15bd55..5837865a5e9 100644 --- a/demo/boy/justfile +++ b/demo/boy/justfile @@ -11,13 +11,9 @@ default: "just wait && just web" # Download the given ROM from our R2 bucket. +[positional-arguments] download rom: - #!/usr/bin/env bash - if [ ! -f "rom/{{ rom }}" ]; then - mkdir -p rom - curl -fSL -o "rom/{{ rom }}.tmp" "https://rom.moq.dev/{{ rom }}" - mv "rom/{{ rom }}.tmp" "rom/{{ rom }}" - fi + @../../sh/demo/download.sh "https://rom.moq.dev/$1" "rom/$1" # Big2Small (puzzle): https://mdsteele.itch.io/big2small b2s: @@ -70,15 +66,9 @@ upload file: bun wrangler r2 object put "rom-moq-dev/{{ file }}" --file "rom/{{ file }}" --remote # Sync all ROMs to R2 +[positional-arguments] sync dir="rom": - #!/usr/bin/env bash - set -euo pipefail - for file in "{{ dir }}"/*; do - [ -e "$file" ] || continue - key=$(basename "$file") - echo "Uploading $key..." - bun wrangler r2 object put "rom-moq-dev/$key" --file "$file" --remote - done + ../../sh/demo/sync.sh rom-moq-dev "$@" # List files in the R2 bucket list: diff --git a/demo/justfile b/demo/justfile index bfd725f82d1..08b359d9ffa 100644 --- a/demo/justfile +++ b/demo/justfile @@ -1,65 +1,18 @@ set fallback -mod boy -mod pub -mod relay -mod sub -mod web - # Run the web demo (default). # -# Picks the first free port at/after 4443 (both UDP for QUIC and TCP for HTTP) -# so multiple worktrees can run `just dev` without colliding. The relay reads - -# the port from MOQ_LISTEN/MOQ_WEB_HTTP_LISTEN, overriding localhost.toml. +# Picks the first free port at/after 4443 so multiple worktrees can run +# `just dev` without colliding; see sh/demo/run.sh. default: - #!/usr/bin/env bash - set -euo pipefail bun install - - port=$(just port) - if [ "$port" != "4443" ]; then - echo ">>> Port 4443 is busy, using $port instead" >&2 - fi - - base="http://localhost:$port" - - # Scope the bind env vars to the relay only. The unified `moq` binary also - # reads MOQ_LISTEN, so exporting it globally would make the `pub` client - # try to start a server (and fail without a TLS cert). - bun run concurrently --kill-others --names rly,bbb,web --prefix-colors auto \ - "MOQ_LISTEN='[::]:$port' MOQ_WEB_HTTP_LISTEN='[::]:$port' just relay" \ - "just wait $base/certificate.sha256 && just pub bbb $base" \ - "just wait $base/certificate.sha256 && just web serve $base" - -# Find the first free port at/after `start` (checked on both TCP and UDP). -# `lsof` isn't available everywhere (e.g. Git Bash on Windows); without it we -# can't probe, so fall back to `start` and let the relay surface a bind error if - -# it's actually taken. -[private] -port start="4443": - #!/usr/bin/env bash - set -euo pipefail - port={{ start }} - if command -v lsof >/dev/null 2>&1; then - while lsof -nP -iTCP:"$port" -sTCP:LISTEN -t >/dev/null 2>&1 \ - || lsof -nP -iUDP:"$port" -t >/dev/null 2>&1; do - port=$((port + 1)) - done - fi - echo "$port" + ../sh/demo/run.sh # Wait for the localhost relay to be ready by repeatedly requesting the cert +[positional-arguments] [private] wait url="http://localhost:4443/certificate.sha256": - #!/usr/bin/env bash - set -euo pipefail - for i in $(seq 1 600); do - curl -sf "{{ url }}" > /dev/null 2>&1 && exit 0 - sleep 1 - done - exit 1 + ../sh/demo/wait.sh "$@" # Throttle UDP traffic for testing (macOS only, requires sudo). throttle: diff --git a/demo/pub/justfile b/demo/pub/justfile index 2715bd75ccb..f50d1d5447f 100644 --- a/demo/pub/justfile +++ b/demo/pub/justfile @@ -1,4 +1,5 @@ set fallback +set shell := ["bash", "-euo", "pipefail", "-c"] # --- R2 bucket management (vid.moq.dev) --- @@ -18,15 +19,9 @@ upload file: bun wrangler r2 object put "video-moq-dev/{{ file }}" --file "media/{{ file }}" --remote # Sync all files in a local directory to R2 +[positional-arguments] sync dir="media": - #!/usr/bin/env bash - set -euo pipefail - for file in "{{ dir }}"/*; do - [ -e "$file" ] || continue - key=$(basename "$file") - echo "Uploading $key..." - bun wrangler r2 object put "video-moq-dev/$key" --file "$file" --remote - done + ../../sh/demo/sync.sh video-moq-dev "$@" # List files in the bucket list: @@ -36,6 +31,7 @@ list: # --- Publishing --- # Publish Big Buck Bunny to a relay server. +[positional-arguments] bbb url='http://localhost:4443' *args: just download bbb just download bbb-sd @@ -44,7 +40,7 @@ bbb url='http://localhost:4443' *args: -stream_loop -1 -re -i media/bbb.mp4 \ -stream_loop -1 -re -i media/bbb-sd.mp4 \ -map 0 -map 1:v -c copy -f mpegts -pes_payload_size 0 -muxdelay 0 - \ - | cargo run --bin moq -- {{ args }} --connect "{{ url }}" --broadcast bbb.hang import ts + | cargo run --bin moq -- "${@:2}" --connect "{{ url }}" --broadcast bbb.hang import ts # Encode the video-only SD rendition, preserving source timestamps and loop duration. encode-bbb-sd: @@ -58,19 +54,22 @@ check-bbb: python3 bbb.py check # Publish Tears of Steel to a relay server. +[positional-arguments] tos url='http://localhost:4443' *args: just download tos - just ts tos "{{ url }}" {{ args }} + just ts tos "$@" # Publish a video using ffmpeg (CMAF / fMP4) to a relay server. +[positional-arguments] cmaf name url='http://localhost:4443' *args: cargo build --bin moq - just ffmpeg-cmaf "media/{{ name }}.mp4" | cargo run --bin moq -- {{ args }} --connect "{{ url }}" --broadcast "{{ name }}.hang" import fmp4 + just ffmpeg-cmaf "media/{{ name }}.mp4" | cargo run --bin moq -- "${@:3}" --connect "{{ url }}" --broadcast "{{ name }}.hang" import fmp4 # Publish a video using ffmpeg (MPEG-TS) to a relay server. +[positional-arguments] ts name url='http://localhost:4443' *args: cargo build --bin moq - just ffmpeg-ts "media/{{ name }}.mp4" | cargo run --bin moq -- {{ args }} --connect "{{ url }}" --broadcast "{{ name }}.hang" import ts + just ffmpeg-ts "media/{{ name }}.mp4" | cargo run --bin moq -- "${@:3}" --connect "{{ url }}" --broadcast "{{ name }}.hang" import ts # Publish a video via iroh. iroh name url prefix="": @@ -79,82 +78,13 @@ iroh name url prefix="": just ffmpeg-ts "media/{{ name }}.mp4" | cargo run --bin moq -- --iroh-enabled --connect "{{ url }}" --broadcast "{{ prefix }}{{ name }}.hang" import ts # Generate and ingest an HLS stream from a video file. +[positional-arguments] hls name relay="http://localhost:4443": - #!/usr/bin/env bash - set -euo pipefail - - just download "{{ name }}" - - INPUT="media/{{ name }}.mp4" - OUT_DIR="media/{{ name }}" - - rm -rf "$OUT_DIR" - mkdir -p "$OUT_DIR" - - echo ">>> Generating HLS stream to disk (1280x720 + 256x144)..." - - ffmpeg -hide_banner -loglevel warning -re -stream_loop -1 -i "$INPUT" \ - -filter_complex "\ - [0:v]split=2[v0][v1]; \ - [v0]scale=-2:720[v720]; \ - [v1]scale=-2:144[v144]" \ - -map "[v720]" -map "[v144]" -map 0:a:0 \ - -r 25 -preset veryfast -g 50 -keyint_min 50 -sc_threshold 0 \ - -c:v:0 libx264 -profile:v:0 high -level:v:0 4.1 -pix_fmt:v:0 yuv420p -tag:v:0 avc1 \ - -b:v:0 4M -maxrate:v:0 4.4M -bufsize:v:0 8M \ - -c:v:1 libx264 -profile:v:1 high -level:v:1 4.1 -pix_fmt:v:1 yuv420p -tag:v:1 avc1 \ - -b:v:1 300k -maxrate:v:1 330k -bufsize:v:1 600k \ - -c:a aac -b:a 128k \ - -f hls -hls_time 2 -hls_list_size 6 \ - -hls_flags independent_segments+delete_segments \ - -hls_segment_type fmp4 \ - -master_pl_name master.m3u8 \ - -var_stream_map "v:0,agroup:audio,name:720 v:1,agroup:audio,name:144 a:0,agroup:audio,name:audio" \ - -hls_segment_filename "$OUT_DIR/v%v/segment_%09d.m4s" \ - "$OUT_DIR/v%v/stream.m3u8" & - - FFMPEG_PID=$! - - echo ">>> Waiting for HLS playlist generation..." - for i in {1..30}; do - if [ -f "$OUT_DIR/master.m3u8" ]; then break; fi - sleep 0.5 - done - - if [ ! -f "$OUT_DIR/master.m3u8" ]; then - kill $FFMPEG_PID 2>/dev/null || true - echo "Error: master.m3u8 not generated in time" - exit 1 - fi - - echo ">>> Waiting for variant playlists..." - sleep 2 - for i in {1..20}; do - if [ -f "$OUT_DIR/v0/stream.m3u8" ] || [ -f "$OUT_DIR/v720/stream.m3u8" ] || [ -f "$OUT_DIR/v144/stream.m3u8" ] || [ -f "$OUT_DIR/vaudio/stream.m3u8" ]; then - break - fi - sleep 0.5 - done - - CLEANUP_CALLED=false - cleanup() { - if [ "$CLEANUP_CALLED" = "true" ]; then return; fi - CLEANUP_CALLED=true - echo "Shutting down..." - kill $FFMPEG_PID 2>/dev/null || true - sleep 0.5 - kill -9 $FFMPEG_PID 2>/dev/null || true - } - trap cleanup SIGINT SIGTERM EXIT - - echo ">>> Importing HLS via moq-cli" - cargo run --bin moq -- --connect "{{ relay }}" --broadcast "{{ name }}.hang" import hls "$OUT_DIR/master.m3u8" - EXIT_CODE=$? - - cleanup - exit $EXIT_CODE + just download "$1" + ../../sh/demo/hls.sh "$@" # Publish a video using H.264 Annex B format. +[positional-arguments] h264 name url="http://localhost:4443" *args: just download "{{ name }}" cargo build --bin moq @@ -165,68 +95,31 @@ h264 name url="http://localhost:4443" *args: -c:v copy -an \ -bsf:v h264_mp4toannexb \ -f h264 \ - - | cargo run --bin moq -- {{ args }} --connect "{{ url }}" --broadcast "{{ name }}.hang" import avc3 + - | cargo run --bin moq -- "${@:3}" --connect "{{ url }}" --broadcast "{{ name }}.hang" import avc3 # Publish a video using ffmpeg directly from moq to the localhost. +[positional-arguments] serve name *args: just download "{{ name }}" cargo build --bin moq - just ffmpeg-ts "media/{{ name }}.mp4" | cargo run --bin moq -- {{ args }} --listen "[::]:4443" --listen-tls-generate "localhost" --auth-public "**" --broadcast "{{ name }}.hang" import ts + just ffmpeg-ts "media/{{ name }}.mp4" | cargo run --bin moq -- "${@:2}" --listen "[::]:4443" --listen-tls-generate "localhost" --auth-public "**" --broadcast "{{ name }}.hang" import ts # Generate and serve an HLS stream from a video for testing. +[positional-arguments] serve-hls name port="8000": - #!/usr/bin/env bash - set -euo pipefail - - just download "{{ name }}" - - INPUT="media/{{ name }}.mp4" - OUT_DIR="media/{{ name }}" - - rm -rf "$OUT_DIR" - mkdir -p "$OUT_DIR" - - echo ">>> Starting HLS stream generation..." - echo ">>> Master playlist: http://localhost:{{ port }}/master.m3u8" - - cleanup() { - echo "Shutting down..." - kill $(jobs -p) 2>/dev/null || true - exit 0 - } - trap cleanup SIGINT SIGTERM - - ffmpeg -loglevel warning -re -stream_loop -1 -i "$INPUT" \ - -map 0:v:0 -map 0:v:0 -map 0:a:0 \ - -r 25 -preset veryfast -g 50 -keyint_min 50 -sc_threshold 0 \ - -c:v:0 libx264 -profile:v:0 high -level:v:0 4.1 -pix_fmt:v:0 yuv420p -tag:v:0 avc1 -bsf:v:0 dump_extra -b:v:0 4M -vf:0 "scale=1920:-2" \ - -c:v:1 libx264 -profile:v:1 high -level:v:1 4.1 -pix_fmt:v:1 yuv420p -tag:v:1 avc1 -bsf:v:1 dump_extra -b:v:1 300k -vf:1 "scale=256:-2" \ - -c:a aac -b:a 128k \ - -f hls \ - -hls_time 2 -hls_list_size 12 \ - -hls_flags independent_segments+delete_segments \ - -hls_segment_type fmp4 \ - -master_pl_name master.m3u8 \ - -var_stream_map "v:0,agroup:audio v:1,agroup:audio a:0,agroup:audio" \ - -hls_segment_filename "$OUT_DIR/v%v/segment_%09d.m4s" \ - "$OUT_DIR/v%v/stream.m3u8" & - - sleep 2 - echo ">>> HTTP server: http://localhost:{{ port }}/" - cd "$OUT_DIR" && python3 -m http.server {{ port }} + just download "$1" + ../../sh/demo/serve-hls.sh "$@" # Publish the clock broadcast. +[arg("action", pattern="publish|subscribe")] +[positional-arguments] clock action url="http://localhost:4443" *args: - @if [ "{{ action }}" != "publish" ] && [ "{{ action }}" != "subscribe" ]; then \ - echo "Error: action must be 'publish' or 'subscribe', got '{{ action }}'" >&2; \ - exit 1; \ - fi - - cargo run -p moq-tokio --example clock -- --connect "{{ url }}" --broadcast "clock" {{ args }} {{ action }} + cargo run -p moq-tokio --example clock -- --connect "{{ url }}" --broadcast "clock" "${@:3}" "{{ action }}" # --- GStreamer --- # Publish a video using GStreamer to a relay server. +[positional-arguments] gst name url='http://localhost:4443' *args: just download "{{ name }}" cargo build -p moq-gst @@ -235,36 +128,36 @@ gst name url='http://localhost:4443' *args: gst-launch-1.0 -v -e multifilesrc location="media/{{ name }}.mp4" loop=true ! parsebin name=parse \ parse. ! queue ! identity sync=true ! mux.sink_0 \ parse. ! queue ! identity sync=true ! mux.sink_1 \ - moqsink name=mux url="{{ url }}" broadcast="{{ name }}.hang" {{ args }} + moqsink name=mux url="{{ url }}" broadcast="{{ name }}.hang" "${@:3}" # --- Private helpers --- # Download a test video (already fragmented on vid.moq.dev). +[positional-arguments] [private] download name: - @if [ ! -f "media/{{ name }}.mp4" ]; then \ - mkdir -p media; \ - curl -fsSL "https://vid.moq.dev/{{ name }}.mp4" -o "media/{{ name }}.mp4"; \ - fi + @../../sh/demo/download.sh "https://vid.moq.dev/$1.mp4" "media/$1.mp4" # Convert an h264 input file to CMAF (fmp4) format to stdout, with one fragment per sample. # -frag_duration rather than frag_every_frame: the latter repeats a fragment's tfdt when it # interleaves audio and video, landing two samples on one timestamp. +[positional-arguments] [private] ffmpeg-cmaf input output='-' *args: ffmpeg -hide_banner -v quiet \ -stream_loop -1 -re \ -i "{{ input }}" \ -c copy \ - -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 {{ args }} {{ output }} + -f mp4 -movflags cmaf+separate_moof+delay_moov+skip_trailer -frag_duration 1000 "${@:3}" "{{ output }}" # Convert an h264 input file to MPEG-TS format to stdout, with one PES per audio frame. # -pes_payload_size 0 alone still queues audio frames under 170 bytes (silence) for up to # 0.35 s; -muxdelay 0 writes each one as soon as the next arrives. +[positional-arguments] [private] ffmpeg-ts input output='-' *args: ffmpeg -hide_banner -v quiet \ -stream_loop -1 -re \ -i "{{ input }}" \ -c copy \ - -f mpegts -pes_payload_size 0 -muxdelay 0 {{ args }} {{ output }} + -f mpegts -pes_payload_size 0 -muxdelay 0 "${@:3}" "{{ output }}" diff --git a/demo/relay/justfile b/demo/relay/justfile index c0e33a31971..db04682e600 100644 --- a/demo/relay/justfile +++ b/demo/relay/justfile @@ -34,60 +34,17 @@ leaf1: (cert "leaf1") # Generate a self-signed CA ca: - #!/usr/bin/env bash - set -euo pipefail - umask 077 - - [ -f ca.pem ] && [ -f ca.key ] && exit 0 - rm -f ca.pem ca.key - printf '[req]\ndistinguished_name = req_dn\n[req_dn]\n' > ca.cnf - export OPENSSL_CONF="ca.cnf" - openssl req -x509 -sha256 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ - -days 365 -subj "/CN=moq cluster CA" \ - -keyout ca.key -out ca.pem + ../../sh/demo/ca.sh # Generate a self-signed certificate using the root CA. - -# Chrome requires a <=14 day validity for self-signed certs. +[positional-arguments] cert name: ca - #!/usr/bin/env bash - set -euo pipefail - umask 077 - - export OPENSSL_CONF="ca.cnf" - openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ - -subj "/CN={{ name }}" \ - -keyout "{{ name }}.key" -out "{{ name }}.csr" - openssl x509 -req -sha256 -in "{{ name }}.csr" \ - -CA ca.pem -CAkey ca.key -CAcreateserial \ - -days 14 -extfile <(printf "subjectAltName=DNS:localhost\n") \ - -out "{{ name }}.crt" - rm "{{ name }}.csr" + ../../sh/demo/cert.sh "$@" # Generate a random secret key for authentication. key: - #!/usr/bin/env bash - set -euo pipefail - umask 077 - - [ -f root.jwk ] && exit 0 - rm -f *.jwt - cargo run --bin moq -- auth generate --out root.jwk + ../../sh/demo/key.sh # Generate authentication tokens for local development. token: key - #!/usr/bin/env bash - set -euo pipefail - umask 077 - - if [ ! -f demo-web.jwt ]; then - cargo run --quiet --bin moq -- auth sign --key root.jwk \ - --root demo --subscribe '**' --publish 'me/**' \ - > demo-web.jwt - fi - - if [ ! -f demo-cli.jwt ]; then - cargo run --quiet --bin moq -- auth sign --key root.jwk \ - --root demo --publish '**' \ - > demo-cli.jwt - fi + ../../sh/demo/token.sh diff --git a/demo/sub/justfile b/demo/sub/justfile index e7ec69ed855..824fd6674a2 100644 --- a/demo/sub/justfile +++ b/demo/sub/justfile @@ -1,6 +1,8 @@ set fallback +set shell := ["bash", "-euo", "pipefail", "-c"] # Subscribe to a broadcast using GStreamer and render to the screen. +[positional-arguments] gst name url='http://localhost:4443' *args: cargo build -p moq-gst @@ -8,6 +10,6 @@ gst name url='http://localhost:4443' *args: # by name so a bare `! decodebin3` can't latch onto the audio pad and leave the # video sink empty; route audio to its own sink so it doesn't stall the session. GST_PLUGIN_PATH_1_0="${PWD}/../../target/debug${GST_PLUGIN_PATH_1_0:+:$GST_PLUGIN_PATH_1_0}" \ - gst-launch-1.0 -v -e moqsrc name=s url="{{ url }}" broadcast="{{ name }}" {{ args }} \ + gst-launch-1.0 -v -e moqsrc name=s url="{{ url }}" broadcast="{{ name }}" "${@:3}" \ s.video_0 ! queue ! decodebin3 ! videoconvert ! autovideosink \ s.audio_0 ! queue ! decodebin3 ! audioconvert ! autoaudiosink diff --git a/doc/setup/dev.md b/doc/setup/dev.md index d8d35f9ff87..b17faa24049 100644 --- a/doc/setup/dev.md +++ b/doc/setup/dev.md @@ -7,7 +7,8 @@ description: Build, test, and debug the MoQ repository The repository uses [Just](https://github.com/casey/just) as its command runner. Run commands inside the Nix dev shell (`nix develop`) so your tools -match CI. +match CI. `just check` and `just fix` refuse to run outside it; set +`MOQ_ALLOW_HOST=1` to use the host toolchain anyway. | Command | Purpose | | --- | --- | @@ -72,6 +73,11 @@ just fix just check ``` +These diff the branch against its upstream (or `origin/main`) and run only the +modules the diff reaches. The map from paths to modules lives in +`sh/dispatch.sh`. Recipes stay thin: any logic (conditionals, loops, traps) +lives in a script under `sh/`. + See [CONTRIBUTING.md](https://github.com/moq-dev/moq/blob/main/CONTRIBUTING.md) for branch targeting, commit messages, and reviews, and [Agent setup](/setup/agent) if an AI coding agent is doing the work. diff --git a/drafts/justfile b/drafts/justfile index 547465396e7..bdb39c6e4df 100644 --- a/drafts/justfile +++ b/drafts/justfile @@ -5,92 +5,28 @@ # # The toolchain (kramdown-rfc, xml2rfc, mmark) comes from the nix dev shell # (see `draftsDeps` in flake.nix), so `nix develop --command just drafts build` -# renders with the same tool versions everywhere. There is no submodule or -# venv to bootstrap: `kramdown-rfc` turns the markdown into RFC XML, `xml2rfc` -# renders the txt/html. +# renders with the same tool versions everywhere. `kramdown-rfc` turns the +# markdown into RFC XML, `xml2rfc` renders the txt/html. set working-directory := '.' -# Common xml2rfc flags. --allow-local-file-access lets it read the CSS/refs. -xml2rfc := "xml2rfc -q --allow-local-file-access" - # List the draft names (the argument to `build` / `publish`). default: - @for f in draft-*.md; do echo "${f%.md}"; done + @ls draft-*.md | sed 's/\.md$//' -# Render one draft to .txt and .html (local editor's copy, -# gitignored). Renders the -latest docname as-is; use `publish` for a version. +# Render one draft to .txt and .html (local editor's copy, gitignored). build name: - kramdown-rfc --v3 < {{ name }}.md > {{ name }}.xml - {{ xml2rfc }} --text {{ name }}.xml -o {{ name }}.txt - {{ xml2rfc }} --html {{ name }}.xml -o {{ name }}.html - @echo "built {{ name }}.txt and {{ name }}.html" + ../sh/drafts/build.sh {{ name }} # Render every draft. all: - @for f in draft-*.md; do just build "${f%.md}"; done + ../sh/drafts/build.sh # Validate that every draft parses and that its generated XML passes validation. -# -# Both halves are needed. kramdown-rfc catches markdown and frontmatter errors, -# but happily emits XML for a cross-reference that points at no such section; -# xml2rfc is what rejects the dangling IDREF. Checking only the first half lets -# a broken `[text](#anchor)` pass here and then fail at `publish` time, which is -# the worst place to find it. -# -# --preptool stops after validation instead of rendering txt/html, and -N keeps -# it off the network (kramdown-rfc already inlined the references), which is -# what makes this a couple of seconds per draft rather than a minute. check: - bun install --frozen-lockfile --cwd .. - @grep -n "$(printf '\t')" draft-*.md; case $? in \ - 0) echo "tabs render unpredictably in xml2rfc; indent with spaces" >&2; exit 1;; \ - 1) ;; \ - *) exit 1;; \ - esac - @for f in draft-*.md; do \ - echo "checking $f"; \ - kramdown-rfc --v3 < "$f" > "${f%.md}.xml" || exit 1; \ - {{ xml2rfc }} --preptool -N "${f%.md}.xml" -o "${f%.md}.prepped.xml" \ - || { rm -f "${f%.md}.prepped.xml"; exit 1; }; \ - rm -f "${f%.md}.prepped.xml"; \ - done - @for f in moq-e2ee-*.ts; do \ - bun tsc --noEmit --skipLibCheck --target esnext --module esnext --moduleResolution bundler --types bun "$f" || exit 1; \ - bun "$f" || exit 1; \ - done - -# Remove generated files and the reference cache. -clean: - rm -f draft-*.xml draft-*.txt draft-*.html - rm -rf .refcache + ../sh/drafts/check.sh -# Submit a new draft version to the IETF datatracker. The datatracker emails -# the submitter a confirmation link; the submission is not final until that -# link is clicked. For a brand-new draft (-00) set "Replaces" on the -# confirmation page. -# Example: just drafts publish draft-lcurley-moq-lite 05 you@example.com +# Submit a new draft version to the IETF datatracker, e.g. +# `just drafts publish draft-lcurley-moq-lite 05 you@example.com`. publish name version email: - #!/usr/bin/env bash - set -euo pipefail - case "{{ version }}" in - [0-9][0-9]) ;; - *) echo "version must be two digits, e.g. 05" >&2; exit 1 ;; - esac - doc="{{ name }}-{{ version }}" - if [ ! -f "{{ name }}.md" ]; then - echo "no such draft: {{ name }}.md" >&2; exit 1 - fi - echo "Building $doc.xml" - sed "s/{{ name }}-latest/$doc/g" "{{ name }}.md" | kramdown-rfc --v3 > "$doc.xml" - echo "Submitting $doc.xml to the datatracker as {{ email }}" - resp="$(mktemp)" - code="$(curl -sS -o "$resp" -w '%{http_code}' \ - -F "user={{ email }}" -F "xml=@$doc.xml" \ - https://datatracker.ietf.org/api/submission)" - echo "HTTP $code" - cat "$resp"; echo; rm -f "$resp" - case "$code" in - 200|201) echo "Submitted. Check {{ email }} for the confirmation link." ;; - *) echo "Submission failed." >&2; exit 1 ;; - esac + ../sh/drafts/publish.sh {{ name }} {{ version }} {{ email }} diff --git a/flake.nix b/flake.nix index 3529312a1e4..419fce61336 100644 --- a/flake.nix +++ b/flake.nix @@ -135,7 +135,7 @@ # unification would otherwise hide a broken single-crate build. cargo-hack cargo-nextest - # Browser/WASM bindings (rs/moq-wasm -> @moq/wasm via `just wasm`). + # Browser/WASM bindings (rs/moq-wasm -> @moq/wasm via `just js wasm`). # wasm-bindgen-cli must match the `wasm-bindgen` crate version (the # crate is pinned to nixpkgs' CLI version); bump both together. wasm-bindgen-cli @@ -245,10 +245,9 @@ ]; # Linters / formatters used by `just check` and `just fix`, which - # guard each tool with `command -v` so they skip silently when the - # binary isn't on $PATH. CI sets MOQ_STRICT=1, which turns that skip - # into an error (see `_tools` in the root justfile), so this list and - # that one have to stay in step. + # skip a module whose tools aren't on $PATH. CI sets MOQ_STRICT=1, + # which turns that skip into an error (see the tools map in + # sh/dispatch.sh), so this list and that one have to stay in step. lintDeps = with pkgs; [ shellcheck shfmt @@ -277,8 +276,8 @@ # dependency in rs/moq-ffi/Cargo.toml. Five other places name the same # generator version and must be bumped together: the repo and revision # in release-go-ffi.yml, and the `cargo install` line in - # rs/moq-ffi/build.sh, go/ffi/README.md, go/scripts/check.sh, and - # go/scripts/stage.sh. + # rs/moq-ffi/build.sh, go/ffi/README.md, sh/go/check.sh, and + # sh/go/stage.sh. # # This points at a fork rather than NordSecurity because upstream has no # uniffi 0.32 generator: the metadata encoding changed in 0.32 even @@ -531,6 +530,10 @@ ''; env = { + # What sh/dispatch.sh checks before a scoped `just check` or `just + # fix`. IN_NIX_SHELL would also pass inside another project's shell. + MOQ_DEV_SHELL = "1"; + # Where `just obs compile` and `just obs test` look for libobs. Set # on every platform so the plugin type-checks against the pinned OBS # release everywhere, rather than whatever the host happens to have. diff --git a/go/.gitignore b/go/.gitignore index 9d1c2961a27..4e2b41f06ea 100644 --- a/go/.gitignore +++ b/go/.gitignore @@ -1,4 +1,4 @@ -# Generated by scripts/{check,package-ffi}.sh; the main repo stays binary-free. +# Generated by sh/go/{check,package-ffi}.sh; the main repo stays binary-free. ffi/moq/moq.go ffi/moq/moq.h ffi/moq/lib/ diff --git a/go/ffi/README.md b/go/ffi/README.md index 64a12a712ef..6edaf6937e2 100644 --- a/go/ffi/README.md +++ b/go/ffi/README.md @@ -20,9 +20,9 @@ The published module ships prebuilt `libmoq_ffi.a` for `linux/amd64`, `linux/arm ## Local development -`go/scripts/stage.sh` builds `moq-ffi` for the host, runs `uniffi-bindgen-go` to regenerate `moq.go`, and stages this module plus the wrapper into `dist/` with the wrapper wired to the local ffi by a `replace`. It prints the two staged module directories, so anything that needs to build Go against this checkout (`go/scripts/check.sh`, `test/interop`) shares one staging path. +`sh/go/stage.sh` builds `moq-ffi` for the host, runs `uniffi-bindgen-go` to regenerate `moq.go`, and stages this module plus the wrapper into `dist/` with the wrapper wired to the local ffi by a `replace`. It prints the two staged module directories, so anything that needs to build Go against this checkout (`sh/go/check.sh`, `test/interop`) shares one staging path. -`go/scripts/check.sh` stages that way and then runs `go build`/`go vet`/`go test`. It also runs `publish-wrapper.test.sh`, which drives the wrapper publisher against a scratch bare repo (no cargo, no network). Run via `just go check`. Skips cleanly without `cargo`, `go`, or `uniffi-bindgen-go`. +`sh/go/check.sh` stages that way and then runs `go build`/`go vet`/`go test`. Run via `just go check`. Skips cleanly without `cargo`, `go`, or `uniffi-bindgen-go`. The dev shell provides both `go` and `uniffi-bindgen-go`, so `nix develop --command just go check` needs no setup. Without Nix, install `uniffi-bindgen-go` once: @@ -49,8 +49,8 @@ go/ffi/ lib/_/libmoq_ffi.a (gitignored, staged at release time) ``` -Compiled binaries never live in the source tree. `scripts/check.sh` stages into the gitignored `dist/` working dir under the repo root; CI does the equivalent assembly into the [moq-dev/moq-go-ffi](https://github.com/moq-dev/moq-go-ffi) mirror. +Compiled binaries never live in the source tree. `sh/go/check.sh` stages into the gitignored `dist/` working dir under the repo root; CI does the equivalent assembly into the [moq-dev/moq-go-ffi](https://github.com/moq-dev/moq-go-ffi) mirror. ## Release -The `release-go-ffi.yml` workflow fires on every `moq-ffi-v*` tag, builds per-target static libraries, runs `uniffi-bindgen-go`, calls `go/scripts/package-ffi.sh` to assemble the module, and `go/scripts/publish-ffi.sh` to push the result to `moq-dev/moq-go-ffi` with a bare-semver tag (e.g. `v0.2.18`) lockstep with the crate. Go's module proxy picks up the new tag automatically. +The `release-go-ffi.yml` workflow fires on every `moq-ffi-v*` tag, builds per-target static libraries, runs `uniffi-bindgen-go`, calls `just go package-ffi` to assemble the module, and `just go publish-ffi` to push the result to `moq-dev/moq-go-ffi` with a bare-semver tag (e.g. `v0.2.18`) lockstep with the crate. Go's module proxy picks up the new tag automatically. diff --git a/go/ffi/moq/cgo.go b/go/ffi/moq/cgo.go index 8be6f790a68..50dc423b50e 100644 --- a/go/ffi/moq/cgo.go +++ b/go/ffi/moq/cgo.go @@ -3,12 +3,12 @@ // moq.dev/moq wrapper, which builds on this package. // // The exported API is generated from rs/moq-ffi via uniffi-bindgen-go and -// dropped in alongside this file (moq.go) by scripts/package-ffi.sh; the in-tree -// source therefore does not build on its own. Run scripts/check.sh to stage a +// dropped in alongside this file (moq.go) by sh/go/package-ffi.sh; the in-tree +// source therefore does not build on its own. Run sh/go/check.sh to stage a // complete copy into dist/ and exercise it. // // The per-platform static archive is loaded from moq/lib/_/ -// inside the staged module, populated by scripts/package-ffi.sh from the release +// inside the staged module, populated by sh/go/package-ffi.sh from the release // build matrix. package moq diff --git a/go/justfile b/go/justfile index d842e27d103..b4fa2d0d07c 100644 --- a/go/justfile +++ b/go/justfile @@ -10,36 +10,41 @@ default: # Stages both the ffi and wrapper modules into dist/ (wiring the wrapper to the # local ffi via replace). Skips cleanly if cargo, go, or uniffi-bindgen-go is -# missing. -# -# This runs the tests too, so there is no separate `just go test`. Takes an -# optional newline-separated list of changed files and skips when none are -# Go-scoped; `just go check` (no FILES) always runs. +# missing. This runs the tests too, so there is no separate `just go test`. # Build moq-ffi, run uniffi-bindgen-go, then `go build`/`go vet`/`go test`. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '^(go/|rs/moq-ffi/)' <<< "$FILES"; then - echo "go: no Go changes; skipping." - exit 0 - fi - bash scripts/check.sh - -# Stage the in-tree go/ffi source + per-target moq-ffi libs + generated -# bindings into the moq.dev/moq-ffi module ready for publish. +check: + bash ../sh/go/check.sh + +# Release recipes run in the caller's directory, so a relative `--output` or +# `--tarball` means what it would to the script itself. + +# Stage the in-tree go/ffi source, per-target moq-ffi libs, and generated +# bindings into the moq.dev/moq-ffi module. + +# Stage the moq.dev/moq-ffi module for publish. +[no-cd] +[positional-arguments] package-ffi *args: - bash scripts/package-ffi.sh {{ args }} + {{ quote(justfile_directory() / "sh/go/package-ffi.sh") }} "$@" # Stage the in-tree go/wrapper source into the moq.dev/moq module, rewriting its -# moq.dev/moq-ffi require to the target ffi version, ready for publish. -package-wrapper *args: - bash scripts/package-wrapper.sh {{ args }} - -# The paths are the gitignored ones in go/.gitignore. `just go check` stages -# into the workspace dist/ instead, which `just js clean` sweeps, so these only -# show up when uniffi-bindgen-go or `go mod` is pointed at the tree directly. +# moq.dev/moq-ffi require to the target ffi version. -# Remove the generated bindings, native libs, and vendoring from go/. -clean: - rm -rf ffi/moq/moq.go ffi/moq/moq.h ffi/moq/lib ffi/go.sum ffi/vendor wrapper/go.sum wrapper/vendor +# Stage the moq.dev/moq wrapper module for publish. +[no-cd] +[positional-arguments] +package-wrapper *args: + {{ quote(justfile_directory() / "sh/go/package-wrapper.sh") }} "$@" + +# Push the staged moq-go-ffi module to the moq-dev/moq-go-ffi mirror. +[no-cd] +[positional-arguments] +publish-ffi *args: + {{ quote(justfile_directory() / "sh/go/publish-ffi.sh") }} "$@" + +# Push the staged moq-go wrapper module to the moq-dev/moq-go mirror. +[no-cd] +[positional-arguments] +publish-wrapper *args: + {{ quote(justfile_directory() / "sh/go/publish-wrapper.sh") }} "$@" diff --git a/go/wrapper/README.md b/go/wrapper/README.md index dbc9bd15b07..781c8d055e9 100644 --- a/go/wrapper/README.md +++ b/go/wrapper/README.md @@ -121,4 +121,4 @@ The committed `go.mod` carries a `require moq.dev/moq-ffi v0.0.0` **placeholder* ## Local development -Run `just go check`: it builds `moq-ffi` for the host, regenerates the bindings, stages both modules into `dist/` with a `replace` wiring the wrapper to the local ffi, and runs `go build`/`go vet`/`go test`. It also runs `scripts/publish-wrapper.test.sh`, which exercises the publisher's release/no-op/recovery paths against a scratch bare repo standing in for the mirror. See [../ffi/README.md](../ffi/README.md) for the `uniffi-bindgen-go` install. +Run `just go check`: it builds `moq-ffi` for the host, regenerates the bindings, stages both modules into `dist/` with a `replace` wiring the wrapper to the local ffi, and runs `go build`/`go vet`/`go test`. See [../ffi/README.md](../ffi/README.md) for the `uniffi-bindgen-go` install. diff --git a/go/wrapper/go.mod b/go/wrapper/go.mod index e306de284d7..eba975c62c7 100644 --- a/go/wrapper/go.mod +++ b/go/wrapper/go.mod @@ -3,7 +3,7 @@ module moq.dev/moq go 1.23 // The require below is a placeholder. The real version is rewritten at release -// time by go/scripts/package-wrapper.sh to the latest published moq-ffi, and +// time by sh/go/package-wrapper.sh to the latest published moq-ffi, and // `just go check` injects a local `replace` to the freshly-generated bindings. // Do not "fix" this by hand or add a replace directive to the committed file. require moq.dev/moq-ffi v0.0.0 diff --git a/js/common/workers.test.ts b/js/common/workers.test.ts deleted file mode 100644 index 5b11c5e1fba..00000000000 --- a/js/common/workers.test.ts +++ /dev/null @@ -1,45 +0,0 @@ -import { expect, test } from "bun:test"; -import { mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; -import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; - -test("worker checks select changed projects and propagate build failures", () => { - const temp = mkdtempSync(join(tmpdir(), "moq-workers-")); - const log = join(temp, "calls"); - writeFileSync( - join(temp, "bun"), - '#!/bin/sh\nprintf "%s %s\\n" "$PWD" "$*" >> "$WORKER_TEST_LOG"\nif [ "$*" = "run deploy --dry-run" ]; then exit "$WORKER_TEST_EXIT"; fi\n', - { mode: 0o755 }, - ); - try { - for (const project of ["infra/apt", "infra/rpm", "demo/pub"]) { - writeFileSync(log, ""); - const result = Bun.spawnSync(["just", "js", "check", `${project}/bun.lock`], { - cwd: resolve(import.meta.dir, "../.."), - env: { - ...process.env, - PATH: `${temp}:${process.env.PATH}`, - WORKER_TEST_LOG: log, - WORKER_TEST_EXIT: "0", - }, - }); - expect(result.exitCode).toBe(0); - expect(readFileSync(log, "utf8").trim().split("\n")).toEqual([ - `${resolve(import.meta.dir, "../..", project)} install --frozen-lockfile`, - `${resolve(import.meta.dir, "../..", project)} run deploy --dry-run`, - ]); - } - const failed = Bun.spawnSync(["just", "js", "check", "infra/apt/bun.lock"], { - cwd: resolve(import.meta.dir, "../.."), - env: { - ...process.env, - PATH: `${temp}:${process.env.PATH}`, - WORKER_TEST_LOG: log, - WORKER_TEST_EXIT: "1", - }, - }); - expect(failed.exitCode).not.toBe(0); - } finally { - rmSync(temp, { recursive: true, force: true }); - } -}); diff --git a/js/justfile b/js/justfile index 4be426446e8..7118f097734 100644 --- a/js/justfile +++ b/js/justfile @@ -6,108 +6,53 @@ set working-directory := '.' -# Paths a JS check depends on: every bun workspace (they are not all under js/, -# see the root package.json) plus the config shared by all of them. `check`, -# `fix`, and `ci` all skip themselves when a diff misses this. -scope := '^(js/|doc/|drafts/|demo/(boy|web)/|test/interop/clients/js|test/wasm/|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' - # Run all checks by default. default: just check -install: - bun install - -# Takes an optional newline-separated list of changed files and skips if none -# match the JS scope; `just js check` (no FILES) always runs. -# -# FILES is an exported parameter, not `{{ FILES }}`: just interpolates the -# latter into the recipe source, where a filename like `$(...)` would run as a -# command. Git-derived paths are untrusted input, so they stay data. - # `build` runs here rather than as its own gate: the per-package `check` script # is `tsc --noEmit`, which never runs declaration emit, so the errors that only # surface when emitting a `.d.ts` (TS4023 and friends) pass `check` and then # break a JSR/npm publish. It costs about 20s for every package. - +# # `common/deps.ts` runs first and cheaply: bundlers resolve an undeclared # workspace import through hoisting, so nothing else in `check` notices one. +# doc/ is in scope, so the binding pages' stats fields are checked here too. # Lint, type-check, biome-format, and build all packages. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - # These Workers have independent lockfiles outside the Bun workspace. - # Validate the deploy bundle without publishing or requiring credentials. - for project in infra/apt infra/rpm demo/pub; do - if [[ -z "$FILES" ]] || grep -qE "^($project/|js/justfile$)" <<< "$FILES"; then - ( - cd "../$project" - bun install --frozen-lockfile - bun run deploy --dry-run - ) - fi - done - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "js: no JS changes; skipping." - exit 0 - fi - # doc/ is in this scope, so the binding pages are checked here. - python3 ../rs/scripts/stats-docs.py +check: + python3 ../sh/rs/stats-docs.py bun install --frozen-lockfile bun common/deps.ts - if tty -s; then - bun run --filter='*' --elide-lines=0 check - else - bun run --filter='*' check - fi + bun run --filter='*' --elide-lines=0 check bun biome check just build -# Auto-fix lint and formatting issues. Skips on the same terms as `check`. -fix $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "js: no JS changes; skipping." - exit 0 - fi +# Validate the Worker deploy bundles outside the Bun workspace. +workers: + ../sh/js/workers.sh + +# Auto-fix lint and formatting issues. +fix: bun install bun biome check --write -# Run all unit tests. Skips on the same terms as `check`. -test $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "js: no JS changes; skipping tests." - exit 0 - fi +# Run all unit tests. +test: + bun install --frozen-lockfile + bun test common/declarations.test.ts common/deps.test.ts + bun run --filter='*' --elide-lines=0 test + +# Run one net benchmark: broadcasts, forward, reader, frames, track, varint, or lite-varint. +bench name: bun install --frozen-lockfile - bun test common/declarations.test.ts common/deps.test.ts common/workers.test.ts - if tty -s; then - bun run --filter='*' --elide-lines=0 test - else - bun run --filter='*' test - fi + bun net/bench/{{ name }}.ts # Build all packages. build: bun run --filter='*' build -# Remove node_modules and build output. The bun workspace spans the repo - -# (deps hoist to the root, not into js/), so sweep from the workspace root. -clean: - #!/usr/bin/env bash - set -euo pipefail - cd .. - find . -name .claude -prune -o \ - -type d \( -name node_modules -o -name dist -o -name out -o -name pkg \) \ - -prune -exec rm -rf {} + - find . -name .claude -prune -o -type f -name '*.tsbuildinfo' -exec rm -f {} + - -# Pull in new versions and report what's outdated. -update: - bun update - bun outdated +# Build browser/WASM bindings into @moq/wasm using the pinned wasm-bindgen toolchain. +wasm: + cargo build --locked -p moq-wasm --target wasm32-unknown-unknown --profile wasm-release + wasm-bindgen --target web --out-name moq --out-dir wasm/dist "${CARGO_TARGET_DIR:-../target}/wasm32-unknown-unknown/wasm-release/moq_wasm.wasm" diff --git a/js/wasm/.gitignore b/js/wasm/.gitignore index d3d540b01fb..b7322370aeb 100644 --- a/js/wasm/.gitignore +++ b/js/wasm/.gitignore @@ -1,3 +1,3 @@ -# Generated by `just wasm` (wasm-bindgen). The .wasm binary and JS glue are +# Generated by `just js wasm` (wasm-bindgen). The .wasm binary and JS glue are # build artifacts, not source: build them, never commit them. /dist/ diff --git a/js/wasm/README.md b/js/wasm/README.md index a3f2c2f2949..0cb4a6c4686 100644 --- a/js/wasm/README.md +++ b/js/wasm/README.md @@ -32,7 +32,7 @@ calls. Freeing a `Session` also closes it. `dist/` is generated, not committed. Build it from the repo root: ```bash -just wasm +just js wasm ``` That compiles `rs/moq-wasm` for `wasm32-unknown-unknown`, runs `wasm-bindgen` diff --git a/justfile b/justfile index bbba70cb92a..c20feb7f2d6 100644 --- a/justfile +++ b/justfile @@ -1,5 +1,8 @@ #!/usr/bin/env just --justfile # Using Just: https://github.com/casey/just?tab=readme-ov-file#installation +# +# Every recipe is a menu entry: one line that runs a tool, or a script under +# sh/. Logic belongs in the script. set unstable @@ -29,10 +32,6 @@ mod relay 'demo/relay' mod sub 'demo/sub' mod web 'demo/web' -# Byte budget for a changed-file list, sized so it survives being passed as a -# single argv/env string on every hop of the dispatch. See `_changed`. -changed_max := '65536' - # Run the demo by default. default: just demo @@ -43,912 +42,47 @@ dev: # Benchmark the current tree, or compare it with a commit: `just bench origin/main`. bench $BASE="": - #!/usr/bin/env bash - exec just --justfile bench/justfile compare "$BASE" + bench/run.sh "$BASE" # Compare one multi-threaded Tokio runtime with the same number of independent -# Tokio/epoll and io_uring workers. Defaults to every logical CPU. -bench-runtime $ROUNDS="3" $WORKERS="": - #!/usr/bin/env bash - exec just --justfile bench/justfile runtime "$ROUNDS" "$WORKERS" - -# A linked worktree's Git metadata does not live under its own root: the -# per-worktree directory is `--git-dir` and everything shared (objects, remote -# refs, the branch namespace) is under `--git-common-dir`, which for an agent -# checkout is inside the main repository. Write access to the source tree -# therefore says nothing about whether this checkout can fetch, branch, or -# rebase; the answer is a property of those two directories, and finding out by -# running `git fetch` and reading the error is the slow way. -# -# `check` reports; `setup` fetches, points the branch at its base, and records -# the SHA it fetched under the per-worktree Git directory, where `check` reads it -# back to say how stale the recorded base has become. Neither ever resets, -# rebases, cleans, or checks anything out: a dirty tree is someone's work in -# progress, and adopting a checkout must not be able to destroy it. -# -# BASE follows the same rule as the rest of the repo (see `_base`): `main` -# unless the branch's upstream says otherwise. - -# Report a worktree's base, Git metadata access, and state; `setup` also fetches. -worktree ACTION="check" $BASE="": - #!/usr/bin/env bash - set -euo pipefail - - case "{{ ACTION }}" in - check | setup) ;; - *) - echo "usage: just worktree [check|setup] [BASE]" >&2 - exit 2 - ;; - esac - - root=$(git rev-parse --show-toplevel) - git_dir=$(cd "$(git rev-parse --git-dir)" && pwd) - common_dir=$(cd "$(git rev-parse --git-common-dir)" && pwd) - branch=$(git branch --show-current || true) - - # A written probe rather than `[[ -w ]]`: the directory can be readable and - # nominally writable while the sandbox, a read-only mount, or an ACL refuses - # the create, and it is the create that fetch and branch creation need. - access() { - local dir="$1" probe - [[ -d "$dir" ]] || { echo missing; return; } - [[ -r "$dir" ]] || { echo denied; return; } - probe="$dir/.moq-access-probe.$$" - if (umask 077 && : > "$probe") 2>/dev/null; then - rm -f "$probe" - echo write - else - echo read-only - fi - } - - # A directory git will create on demand is only as writable as its nearest - # existing parent, so probe upward instead of reporting it missing. - access_or_parent() { - local dir="$1" parent - while [[ ! -d "$dir" ]]; do - parent=$(dirname "$dir") - [[ "$parent" != "$dir" ]] || { echo missing; return; } - dir="$parent" - done - access "$dir" - } - - base=$(just _base "$BASE") - stamp="$git_dir/moq-base" - # Whichever remote provides the base, not always origin. A base whose first - # segment names no remote (a local branch, a tag) leaves origin. - remote="${base%%/*}" - git remote | grep -qx "$remote" || remote=origin - - objects=$(access "$common_dir/objects") - heads=$(access "$common_dir/refs/heads") - worktree_meta=$(access "$git_dir") - worktree=$(access "$root") - # Tracking is `branch..remote`/`.merge` in the repository config, which - # lives in the common directory alongside the `config.lock` the write needs, - # not in the branch's ref. Probing refs/heads for it would refuse on a - # writable config and, worse, proceed on a read-only one. - config=$(access "$common_dir") - # A fetch writes three places, not one: objects, the remote-tracking refs it - # updates, and FETCH_HEAD in the per-worktree directory. The refs are the - # narrow one: `refs/remotes/` is where `.lock` is created, so - # probing `refs/remotes` stops a directory short and passes a split that git - # then fails on. - tracking=$(access_or_parent "$common_dir/refs/remotes/$remote") - - echo "worktree: $root" - echo "branch: ${branch:-(detached)} $(git rev-parse --short HEAD)" - echo "git-dir: $git_dir ($worktree_meta)" - echo "common-dir: $common_dir ($config)" - echo " fetch needs $common_dir/objects: $objects" - echo " $common_dir/refs/remotes/$remote: $tracking" - echo " $git_dir (FETCH_HEAD): $worktree_meta" - echo " branch needs $common_dir/refs/heads: $heads" - echo " upstream needs $common_dir/config: $config" - echo " rebase needs $git_dir: $worktree_meta" - echo " $root: $worktree" - echo " $common_dir/refs/heads: $heads" - - dirty=$(git status --porcelain | wc -l | tr -d ' ') - echo "dirty: $dirty tracked/untracked path(s)" +# Tokio/epoll and io_uring workers; WORKERS defaults to every logical CPU. - if [[ "{{ ACTION }}" == setup ]]; then - blocked="" - [[ "$objects" == write ]] || blocked="$blocked $common_dir/objects ($objects)" - [[ "$tracking" == write ]] || blocked="$blocked $common_dir/refs/remotes/$remote ($tracking)" - [[ "$worktree_meta" == write ]] || blocked="$blocked $git_dir ($worktree_meta)" - if [[ -n "$blocked" ]]; then - echo "error: cannot fetch;$blocked" >&2 - echo " grant write access to the main repository's Git directory, not just this worktree" >&2 - exit 1 - fi - # The remote resolved above, not always origin: recording a stamp against a - # ref nobody refreshed is worse than recording none, and `just check` would - # scope the branch against a base that has since moved. - git fetch --quiet "$remote" - # Repointing an upstream someone chose would silently change what `just - # check` scopes against, so only three cases write it: no upstream, an - # upstream `_base` discards anyway (the branch's own remote copy, which - # `git push -u` leaves behind and which says nothing about what the branch - # merges into), and a base the caller named on the command line. - upstream=$(git rev-parse --abbrev-ref '@{upstream}' 2> /dev/null || true) - # Two things stop the write, and they fail identically: a detached HEAD has - # no branch to hang an upstream on, and the config it lands in may be - # read-only. Skipping either silently would report a setup that recorded - # the caller's base while `just check` still scoped against origin/main. - blocker="" - if [[ -z "$branch" ]]; then - blocker="HEAD is detached" - elif [[ "$config" != write ]]; then - blocker="$common_dir/config is $config" - fi - if [[ -z "$upstream" ]] || [[ "$upstream" == */"$branch" ]] || [[ -n "$BASE" ]]; then - if [[ -z "$blocker" ]]; then - git branch --set-upstream-to "$base" "$branch" - elif [[ "$base" == origin/main ]]; then - # Nothing is lost: with no upstream `_base` falls back to - # origin/main, which is what this would have written. - echo "warning: cannot record the upstream; $blocker" >&2 - else - # The upstream is the only place this choice survives, so a setup - # that could not write it did not do what it was asked. - echo "error: cannot set the upstream to $base; $blocker" >&2 - echo " the branch would keep scoping against origin/main" >&2 - exit 1 - fi - fi - # Resolved, written, then renamed into place. A redirect straight into the - # stamp truncates it before git runs, so a failure there would leave an - # empty file that reads back as a recorded base that never existed. Each - # setup needs its own temporary file so concurrent runs cannot rename or - # overwrite each other's in-progress stamp. - if [[ "$worktree_meta" == write ]]; then - stamp_tmp=$(mktemp "$git_dir/.moq-base.XXXXXXXX") - if ! git rev-parse "$base" > "$stamp_tmp"; then - rm -f "$stamp_tmp" - exit 1 - fi - mv "$stamp_tmp" "$stamp" - fi - fi - - if ! git rev-parse --verify --quiet "$base^{commit}" > /dev/null; then - echo "base: $base (NOT FETCHED; run 'just worktree setup')" - exit 0 - fi - - head=$(git rev-parse "$base") - echo "base: $base $(git rev-parse --short "$base")" - echo "upstream: $(git rev-parse --abbrev-ref '@{upstream}' 2> /dev/null || echo '(unset)')" - echo "behind: $(git rev-list --count "HEAD..$base") commit(s)" - - recorded="" - [[ -f "$stamp" ]] && recorded=$(cat "$stamp") - - # A stamp that no longer names a commit is worse than none: reporting it would - # abort here on the `rev-parse --short` rather than say what to do about it. - # An interrupted setup, or a base garbage-collected out of the repository. - if [[ -z "$recorded" ]]; then - echo "recorded: (none; run 'just worktree setup')" - elif ! git rev-parse --verify --quiet "$recorded^{commit}" > /dev/null; then - echo "recorded: $recorded (UNKNOWN COMMIT; run 'just worktree setup')" - elif [[ "$recorded" == "$head" ]]; then - echo "recorded: $(git rev-parse --short "$recorded") (current)" - else - echo "recorded: $(git rev-parse --short "$recorded") (STALE; $base has moved since setup)" - fi +# Compare a shared Tokio runtime with independent workers: `just bench-runtime 5 16`. +bench-runtime $ROUNDS="3" $WORKERS="": + MOQ_BENCH_RUNTIME_ROUNDS="$ROUNDS" MOQ_BENCH_RUNTIME_WORKERS="$WORKERS" bench/run.sh --runtime # Install repo-wide tooling. Per-language deps install on first check. install: bun install - cargo install --locked cargo-shear cargo-sort cargo-upgrades cargo-edit cargo-semver-checks release-plz - -# Resolve BASE: arg > $GITHUB_BASE_REF > upstream > origin/main. A branch's -# upstream is the branch it merges into, which is the base a `dev`-targeted -# branch needs. `git push -u` repoints upstream at the branch's own remote copy, -# which would diff HEAD against itself, so ignore that case (see AGENTS.md). -# GITHUB_BASE_REF outranks the upstream because a PR checkout has no upstream -# configured, and the branch being merged into is exactly the base GitHub is -# asking about. -# -# Shared by `_changed` and `worktree`, so a checkout's scope and its reported -# base can never disagree. - -# Print the ref this branch is based on. -[private] -_base $BASE="": - #!/usr/bin/env bash - set -euo pipefail - - base="$BASE" - if [[ -z "$base" && -n "${GITHUB_BASE_REF:-}" ]]; then - base="origin/${GITHUB_BASE_REF}" - fi - if [[ -z "$base" ]]; then - base=$(git rev-parse --abbrev-ref '@{upstream}' 2>/dev/null || true) - if [[ -z "$base" || "$base" == */"$(git branch --show-current)" ]]; then - base="origin/main" - fi - fi - printf '%s\n' "$base" - -# Reports the base it picked on stderr, so a surprising scope is traceable. -# -# LIMIT is the byte budget for the printed list, and exists as a parameter so -# `_changed-test` can force the oversized path without a synthetic 30k-file diff. - -# Print the files this branch changed relative to BASE, one per line, or `ALL`. -[private] -_changed $BASE $LIMIT=changed_max: - #!/usr/bin/env bash - set -euo pipefail - - base=$(just _base "$BASE") - - merge_base=$(git merge-base "$base" HEAD) || { - echo "error: cannot resolve merge-base against $base (is full history fetched?)" >&2 - exit 1 - } - echo "base: $base" >&2 - - # Untracked files count too: a brand new crate or module is the whole change. - files=$({ - git diff --name-only "$merge_base" - git ls-files --others --exclude-standard - } | sort -u) - - # Assigned rather than tested inline, so a failing `_changed-cap` aborts here - # under `set -e`. Inside `[[ ]]` its exit status is discarded, and the caller - # would then scope to a list that never got budgeted -- straight back into the - # E2BIG this exists to prevent. - cap=$(printf '%s' "$files" | just _changed-cap "$LIMIT") - if [[ "$cap" == ALL ]]; then - echo ALL - exit 0 - fi - - # Guarded because a bare printf of an empty list prints a newline, and the - # callers test the result for emptiness to decide whether anything changed. - if [[ -n "$files" ]]; then - printf '%s\n' "$files" - fi - -# Every hop of the dispatch takes the changed-file list as one argument, and -# just exports recipe parameters into the child's environment, so the whole list -# has to fit in a single execve string. Linux caps one string at MAX_ARG_STRLEN -# (32 pages, 131072 bytes) however large ARG_MAX is, so past that each hop dies -# with E2BIG, which just reports as exit code 126 and no mention of the diff. A -# diff that large selects most of the workspace anyway, so the callers widen to -# the unscoped suite, which passes no list at all. -# -# Split out from `_changed` so `_changed-test` can drive the decision with a -# synthetic list, rather than needing the working tree to hold a diff of a -# particular size. -# -# The list arrives on stdin, which is both the only channel that can carry an -# oversized one and the only way to measure it honestly: `${#var}` counts -# CHARACTERS under a UTF-8 locale while execve counts BYTES, so a path set of -# 3-byte characters would read as a third of its real size and sail past a -# budget it actually blows. - -# Print `ALL` when the changed-file list on stdin is too long for one argument. -[private] -_changed-cap $LIMIT=changed_max: - #!/usr/bin/env bash - set -euo pipefail - - [[ "$LIMIT" =~ ^[0-9]+$ ]] || { - echo "changed: not a byte count: $LIMIT" >&2 - exit 2 - } - - bytes=$(wc -c | tr -d '[:space:]') - if ((bytes > LIMIT)); then - echo "changed: $bytes bytes of paths exceeds the $LIMIT budget; selecting everything." >&2 - echo ALL - fi - -# Guards the thing that fails LOUDLY but unhelpfully: past the budget every -# `just` hop dies with "Argument list too long" and exit 126, naming neither the -# diff nor the recipe that could not receive it. Both halves matter -- a budget -# that never trips scopes nothing, and one above what execve accepts still dies. - -# Check that an oversized diff widens to `ALL`, and that the budget fits in argv. -[private] -_changed-test $LIMIT=changed_max: - #!/usr/bin/env bash - set -euo pipefail - - fail() { echo "changed: _changed-test: $1" >&2; exit 1; } - - # Synthetic sizes rather than whatever the working tree happens to hold: a - # clean checkout has no diff at all, and `check --all` runs there (cache.yml - # warms the cache from `main`), so a test keyed on the real list would take - # down the one job allowed to write the shared Rust cache. - [[ "$(printf 'aaaa' | just _changed-cap 3)" == ALL ]] || fail "over budget must print ALL" - [[ -z "$(printf 'aaa' | just _changed-cap 3)" ]] || fail "at budget must print nothing" - [[ -z "$(printf '' | just _changed-cap 3)" ]] || fail "an empty diff must print nothing" - - # The smallest nonempty list there is, against the only budget below it. - [[ "$(printf 'x' | just _changed-cap 0)" == ALL ]] || fail "a 1-byte list must exceed a 0 budget" - - # execve counts bytes, so the budget has to as well. Spelled as raw bytes - # rather than as characters: this is one CJK character, 3 bytes wide, which - # `${#var}` would count as 1 under a UTF-8 locale and let past a 2-byte - # budget it actually blows. - [[ "$(printf '\xe6\x97\xa5' | just _changed-cap 2)" == ALL ]] \ - || fail "the budget must count bytes, not characters" - - # A byte count is the whole input, so anything else is a caller bug, not a - # reason to silently scope to nothing. - ! printf '' | just _changed-cap not-a-number 2> /dev/null || fail "a bad budget must be rejected" - - # ...and that rejection has to reach the caller. A swallowed one would hand - # back an unbudgeted list, which is the failure this whole recipe prevents. - ! just _changed "" not-a-number 2> /dev/null || fail "a rejected budget must fail _changed" - - # Linux caps a single argv/env string at MAX_ARG_STRLEN (32 pages), whatever - # ARG_MAX says, and the list travels as one string. Asserted rather than - # probed because this repo's CI is the Linux host and a dev box may be laxer. - ((LIMIT <= 131072)) || fail "budget $LIMIT exceeds Linux MAX_ARG_STRLEN" - - # The budget still has to survive the hop it was sized for, which the checks - # above cannot show: they never pass a list that big to anything. - payload=$(head -c "$LIMIT" /dev/zero | tr '\0' x) - [[ "$(just _echo "$payload" | wc -c)" -eq $((LIMIT + 1)) ]] \ - || fail "a $LIMIT-byte list does not survive an argv hop" - - # End to end, but only when there is a diff to be oversized: see above. The - # budget is zero rather than one because the real list is whatever the - # checkout holds, and a single one-character root path is a one-byte list - # that a one-byte budget does not exceed. Zero is below every nonempty list. - if [[ -n "$(just _changed "" 100000000)" ]]; then - [[ "$(just _changed "" 0)" == ALL ]] || fail "a diff over budget must print ALL" - fi - - echo "changed: budget ok" - -# Print an argument back, to measure what survives a `just` invocation. -[private] -_echo $VALUE: - @printf '%s\n' "$VALUE" - -# Tools every scope guards with `command -v`, so an incomplete local toolchain -# checks less instead of failing. That trade is wrong in CI, where a skip is -# indistinguishable from a pass, so CI exports MOQ_STRICT=1 and this turns the -# required set into a precondition. Checked up front, and as one list, so a -# missing tool is reported before a long compile rather than after it. -# -# Required per scope, mirroring what `check` actually dispatches for a given -# diff: demanding gradle on a docs-only PR would fail a run that was never going -# to invoke it. Takes the same file list as the dispatch, or `ALL` to require -# everything (`check --all`). -# -# One deliberate absence: swift exists only on macOS, and `swift check` skips -# off-macOS by design; swift.yml is its real gate. - -# Fail when a tool the diff's scopes need is missing. No-op unless MOQ_STRICT. -[private] -_tools $FILES="": - #!/usr/bin/env bash - set -euo pipefail - [[ -n "${MOQ_STRICT:-}" ]] || exit 0 - - scoped() { [[ "$FILES" == ALL ]] || grep -qE "$1" <<< "$FILES"; } - - # `_check-common` runs on every invocation, so its tools are unconditional. - tools=(actionlint bun jq nix nixfmt shellcheck shfmt taplo python3 nfpm dpkg-deb envsubst rpm) - scoped '^(drafts/|doc/\.vitepress/drafts\.ts$)' && tools+=(kramdown-rfc xml2rfc) - scoped '^(bench/|rs/|Cargo\.(toml|lock)$|rust-toolchain\.toml$)' && tools+=(cargo envsubst) - scoped '^(quest/|flake\.lock$)' && tools+=(quest) - scoped '^(py/|pyproject\.toml$|uv\.lock$|rs/moq-ffi/|doc/lib/py/|doc/lib/samples\.sh$)' && tools+=(uv) - scoped '^(kt/|rs/moq-ffi/|doc/lib/kt/|doc/lib/samples\.sh$)' && tools+=(gradle java) - # cargo because `go check` builds moq-ffi for the host, and skips on a - # missing cargo the same way it skips on a missing go. rsync because the - # publish scripts stage the mirror tree with it, so the publisher test skips - # without it, and a skip that keeps `just check` green is what MOQ_STRICT is - # here to prevent. - scoped '^(go/|rs/moq-ffi/)' && tools+=(go uniffi-bindgen-go cargo rsync) - scoped '^(dart/|rs/moq-ffi/)' && tools+=(cargo dart uniffi_bindgen_dart) - # Two obs recipes with two dispatch scopes, so two lines: over-requiring - # would fail a diff that never runs the recipe. `just obs compile` needs - # cargo to regenerate moq.h and pkg-config to locate Qt6 and ffmpeg. Every - # platform: the plugin type-checks against headers, and the dev shell ships - # those even on Darwin, where obs-studio can't build. - scoped '^(cpp/obs/|rs/libmoq/|flake\.nix$)' && tools+=(pkg-config cargo) - # `just obs check` lints with clang-format and gersemi, validates the CMake - # release configuration, and compares the three OBS pins, one of which moves - # on a flake.lock bump alone. - scoped '^(cpp/obs/|flake\.(nix|lock)$)' && tools+=(clang-format gersemi cmake) - - # Scopes overlap (rs/moq-ffi/ is in five of them), so the same tool can land - # in the list twice and be reported missing twice. Splitting on whitespace is - # safe: every entry is a bare command name. - tools=($(printf '%s\n' "${tools[@]}" | sort -u)) - - missing=() - for tool in "${tools[@]}"; do - command -v "$tool" >/dev/null 2>&1 || missing+=("$tool") - done - if ((${#missing[@]})); then - echo "error: MOQ_STRICT is set but these tools are missing: ${missing[*]}" >&2 - echo " run inside 'nix develop', or unset MOQ_STRICT to skip what isn't installed" >&2 - exit 1 - fi + cargo install --locked cargo-shear cargo-sort cargo-semver-checks release-plz # Lints, compiles, and tests only the packages the branch changed plus -# everything depending on them, so several worktrees can build at once. Rust -# compiles once: the test build doubles as the clippy gate (see `rs check-test`). -# `check --all` is the unscoped suite. +# everything depending on them; `check --all` is the unscoped suite. BASE +# defaults to the branch's upstream. Rust compiles once: its test build doubles +# as the clippy gate (see `rs check-test`). # Lint, compile, and test what the branch changed since BASE, plus its dependents. check $BASE="": - just _check "$BASE" true + sh/dispatch.sh check "$BASE" -# CI splits `check` into two parallel jobs, since one runner doing both takes -# about the sum of their times. Both share `check`'s scoping (and CI adds -# MOQ_STRICT=1), so there is no second, drifting definition of "checked". +# CI runs `check` as two parallel jobs, since one runner doing both takes about +# the sum of their times. Same dispatch, with MOQ_STRICT=1. -# Run half of `check` for CI: `check` lints and compiles, `test` runs the tests. +# Run half of `check` for CI: JOB `check` lints and compiles, `test` runs the tests. ci $JOB $BASE="": - #!/usr/bin/env bash - set -euo pipefail - case "$JOB" in - check) just _check "$BASE" false ;; - test) just _test "$BASE" ;; - *) - echo "ci: unknown job '$JOB', want check or test" >&2 - exit 2 - ;; - esac - -# TEST=false is `ci check`: Rust lints through `cargo clippy`, which only -# emits metadata and so finishes sooner than the test build it stands in for. - -# Lint and compile what the branch changed since BASE, and test it when TEST=true. -[private] -_check $BASE $TEST: - #!/usr/bin/env bash - set -euo pipefail - - if [[ "$BASE" == --all ]]; then - files=ALL - else - files=$(just _changed "$BASE") - fi - - # `_changed` says ALL when the list outgrew what argv can carry. The unscoped - # suite is the path that passes no list at all, so it is the one that works. - # The dispatch below lives in this file, which matches no - # language scope, so a PR that rewrites how CI dispatches would otherwise - # validate none of it. Widen to the unscoped suite instead. - if [[ "$files" != ALL ]] && grep -qE '^justfile$' <<< "$files"; then - echo "check: root orchestration changed; checking everything." >&2 - files=ALL - fi - - just _tools "$files" - - if [[ "$files" == ALL ]]; then - just js check - just drafts check - if [[ "$TEST" == true ]]; then - just rs check-test --workspace --exclude moq-net-fuzz - just rs play - else - just rs check --workspace --exclude moq-net-fuzz - fi - just rs tokio-features - just rs media-features - just --justfile bench/justfile check - quest check - just test drill-sensitivity --apply-only - # Not covered by the line above: moq-wasm only exists on the wasm32 target. - just rs wasm - just py check - just kt check - just swift check - just go check - just dart check - just obs check - just obs compile - just _flake - elif [[ -n "$files" ]]; then - just js check "$files" - just rs check-changed "$files" "$TEST" - if echo "$files" | grep -q '^bench/'; then - just --justfile bench/justfile check - fi - # Draft sources render into the doc site and carry their own kramdown-rfc - # plus vector checks; a drafts-only diff would otherwise skip both. - if echo "$files" | grep -qE '^(drafts/|doc/\.vitepress/drafts\.ts$)'; then - just drafts check - fi - # Quest documents form one graph, so validate the whole living tree when - # either a quest or the flake-pinned validator changes. - if echo "$files" | grep -qE '^(quest/|flake\.lock$)'; then - quest check - fi - # The drill mutations patch Rust source, so a Rust change can move the - # code they target. Only nightly runs the drills against them; this - # catches a stale patch in the PR that moved its code. - if echo "$files" | grep -qE '^(rs/|test/drill/)'; then - just test drill-sensitivity --apply-only - fi - just py check "$files" - just kt check "$files" - just swift check "$files" - just go check "$files" - just dart check "$files" - # Type-checking the plugin and its unit tests needs only headers, so it - # runs here rather than waiting for obs.yml to link them on Linux. libmoq - # is in scope because the plugin calls through its generated C header, and - # the tests restate those entry points as stubs, so an ABI change breaks - # both. flake.nix because it owns the libobs headers this compiles - # against -- obs.yml links against nixpkgs' obs-studio instead, so nothing - # else would notice that package going bad. - if echo "$files" | grep -qE '^(cpp/obs/|rs/libmoq/|flake\.nix$)'; then - just obs compile - fi - # flake.nix is in scope because `just obs check` is what compares the OBS - # version pinned there against buildspec.json, and either side can move. - # flake.lock too, because the third OBS the guard compares is nixpkgs' - # obs-studio, the one `just obs ci` links: it moves on a lock bump alone, - # and that bump is the change that opens the gap. - if echo "$files" | grep -qE '^(cpp/obs/|flake\.(nix|lock)$)'; then - just obs check - fi - # Validates flake eval + dev shell build; it no longer compiles the - # workspace, so it's cheap. Gated anyway: a pure doc/JS PR can't - # affect flake eval. - if echo "$files" | grep -qE '(^rs/|^Cargo\.(toml|lock)$|^flake\.lock$|\.nix$)'; then - just _flake - fi - else - echo "check: nothing changed." - fi - - # Rust already ran its tests above, from the build that linted it. The kt, - # swift, go, and dart checks always run theirs, so only js and py remain. - # An empty list means "everything" to the per-language recipes. - if [[ "$TEST" == true && -n "$files" ]]; then - scope=$files - [[ "$scope" == ALL ]] && scope= - just js test "$scope" - just py test "$scope" - fi - - just _check-common - -# `ci test`'s half of `_check`: the same scoping, but only the languages whose -# tests are separate from their check (js, rs, py). - -# Run unit tests for what the branch changed since BASE, plus its dependents. -[private] -_test $BASE: - #!/usr/bin/env bash - set -euo pipefail - - if [[ "$BASE" == --all ]]; then - files=ALL - else - files=$(just _changed "$BASE") - fi - - # Mirrors `_check`: widen to everything when the list outgrew argv or the - # dispatch itself changed. - if [[ "$files" != ALL ]] && grep -qE '^justfile$' <<< "$files"; then - echo "test: root orchestration changed; testing everything." >&2 - files=ALL - fi - - just _tools "$files" - - if [[ "$files" == ALL ]]; then - just js test - just rs test --workspace --exclude moq-net-fuzz - just rs play - just py test - elif [[ -n "$files" ]]; then - just js test "$files" - just rs test-changed "$files" - just py test "$files" - else - echo "test: nothing changed." - fi - -# Skips when nix is absent: the flake is not a precondition for working on the -# repo, and `_tools` already makes it required under MOQ_STRICT. - -# Validate flake evaluation and the dev shell build. -[private] -_flake: - @if command -v nix >/dev/null 2>&1; then nix flake check; fi - -# Repository-wide non-compiling checks shared by scoped and unscoped `check`. -# Optional shell, workflow, TOML, Nix, and justfile lints skip if missing. -# -# `bun install` because remark-cli lives in node_modules and `just js check` is -# where it would otherwise be installed, which a Rust-only diff skips. - -# Run shell checks or formatting over tracked files that exist in the worktree. -[private] -_shell $ACTION: - #!/usr/bin/env bash - set -euo pipefail - - if ! command -v shfmt >/dev/null 2>&1; then - exit 0 - fi - if [[ "$ACTION" == "check" ]] && ! command -v shellcheck >/dev/null 2>&1; then - exit 0 - fi - - scripts_file=$(mktemp) - trap 'rm -f "$scripts_file"' EXIT - shfmt -f=0 . > "$scripts_file" - - scripts=() - while IFS= read -r -d '' file; do - if git --literal-pathspecs ls-files --error-unmatch -- "$file" >/dev/null 2>&1; then - scripts+=("$file") - fi - done < "$scripts_file" - ((${#scripts[@]})) || exit 0 - - case "$ACTION" in - check) - shfmt --diff "${scripts[@]}" - shellcheck "${scripts[@]}" - ;; - fix) - shfmt --write "${scripts[@]}" - ;; - *) - echo "invalid shell action: $ACTION" >&2 - exit 2 - ;; - esac - -# remark-cli has no `--check`. `--frail` raises the exit code on lint messages, -# and only `--output` formats, so a file that is merely misformatted passes both -# ways. Format a scratch mirror and diff that, so the check stays read-only -# while `fix` still writes in place. - -# Run Markdown lints over the worktree, either checking formatting or applying it. -[private] -_markdown $ACTION: - #!/usr/bin/env bash - set -euo pipefail - - case "$ACTION" in - check) ;; - fix) - bun remark . --quiet --output - exit 0 - ;; - *) - echo "invalid markdown action: $ACTION" >&2 - exit 2 - ;; - esac - - mirror=$(mktemp -d) - trap 'rm -rf "$mirror"' EXIT - - extension_list=$(bun -e \ - 'import extensions from "markdown-extensions"; console.log(extensions.join("\n"))') - patterns=() - while IFS= read -r extension; do - [[ -n "$extension" ]] && patterns+=("*.$extension") - done <<< "$extension_list" - ((${#patterns[@]})) || { - echo "error: remark reported no Markdown extensions" >&2 - exit 1 - } - - # Untracked files are in scope so a new doc is linted before it is staged; - # --exclude-standard keeps build output out. - files=() - while IFS= read -r -d '' file; do - [[ -f "$file" ]] || continue - files+=("$file") - mkdir -p "$mirror/$(dirname "$file")" - cp "$file" "$mirror/$file" - done < <(git ls-files -z --cached --others --exclude-standard -- "${patterns[@]}") - ((${#files[@]})) || exit 0 - - # The config rides along so every .remarkignore pattern resolves against the - # mirror root exactly as it does here, and node_modules is where the plugins - # named by .remarkrc.mjs come from. - cp .remarkrc.mjs .remarkignore "$mirror/" - ln -s "$PWD/node_modules" "$mirror/node_modules" - - status=0 - (cd "$mirror" && bun remark . --quiet --frail --output) || status=$? - - stale=() - for file in "${files[@]}"; do - cmp -s "$file" "$mirror/$file" || stale+=("$file") - done - - if ((${#stale[@]})); then - echo "error: these files are not formatted, run 'just fix':" >&2 - printf ' %s\n' "${stale[@]}" >&2 - status=1 - fi - - exit "$status" - -# Check Markdown formatting detection without touching the caller's worktree. -[private] -_markdown-test: - #!/usr/bin/env bash - set -euo pipefail - - fail() { echo "markdown: _markdown-test: $1" >&2; exit 1; } - - repo=$PWD - fixture=$(mktemp -d) - trap 'rm -rf "$fixture"' EXIT - - git -C "$fixture" init --quiet - cp .remarkrc.mjs .remarkignore "$fixture/" - ln -s "$repo/node_modules" "$fixture/node_modules" - for file in dirty.md dirty.markdown; do - printf '# dirty_heading\n' > "$fixture/$file" - git -C "$fixture" add "$file" - done - - original='# dirty_heading' - if just --justfile "$repo/justfile" --working-directory "$fixture" \ - _markdown check > "$fixture/check.log" 2>&1; then - fail "formatter-only drift must fail the check" - fi - for file in dirty.md dirty.markdown; do - [[ "$(<"$fixture/$file")" == "$original" ]] \ - || fail "the check modified $file" - grep -q "$file" "$fixture/check.log" \ - || fail "the check did not name $file" - done - - just --justfile "$repo/justfile" --working-directory "$fixture" _markdown fix - for file in dirty.md dirty.markdown; do - [[ "$(<"$fixture/$file")" == '# dirty\_heading' ]] \ - || fail "the fix did not format $file" - done - just --justfile "$repo/justfile" --working-directory "$fixture" _markdown check - - echo "markdown: check/fix regression ok" - -# Repository-wide lints, shared by scoped and unscoped `check`. -[private] -_check-common: - just _changed-test - bun install --frozen-lockfile - just _markdown-test - just _markdown check - just _shell check - @if command -v taplo >/dev/null 2>&1; then RUST_LOG=error taplo format --check; fi - @if command -v nixfmt >/dev/null 2>&1; then nixfmt --check $(find . -name '*.nix' -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); fi - @for f in $(find . -name justfile -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); do just --fmt --check --justfile "$f"; done - just gh check - -# Scoped exactly like `check`, because `clippy --fix` compiles what it fixes. -# Pass `--all` for the unscoped version. + sh/dispatch.sh "ci-$JOB" "$BASE" # Auto-fix lint and formatting for what the branch changed since BASE. fix $BASE="": - #!/usr/bin/env bash - set -euo pipefail - - if [[ "$BASE" == --all ]]; then - files=ALL - else - files=$(just _changed "$BASE") - fi - - # Mirrors `check`: too long for argv means fix everything instead. - if [[ "$files" == ALL ]]; then - just js fix - just rs fix --workspace --exclude moq-net-fuzz - just py fix - just dart fix - just obs fix - elif [[ -n "$files" ]]; then - just js fix "$files" - just rs fix-changed "$files" - just py fix "$files" - just dart fix "$files" - if echo "$files" | grep -q '^cpp/obs/'; then - just obs fix - fi - else - echo "fix: nothing changed." - fi - - just _fix-common - -# Optional tools skip if missing locally. `bun install` for the same reason as -# `_check-common`. - -# Repository-wide fixes, shared by scoped and unscoped `fix`. -[private] -_fix-common: - bun install - just _markdown fix - just _shell fix - @if command -v taplo >/dev/null 2>&1; then RUST_LOG=error taplo format; fi - @if command -v nixfmt >/dev/null 2>&1; then nixfmt $(find . -name '*.nix' -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); fi - @for f in $(find . -name justfile -not -path './node_modules/*' -not -path './target/*' -not -path './.venv/*' -not -path './.direnv/*'); do just --fmt --justfile "$f"; done + sh/dispatch.sh fix "$BASE" # Build the packages. build: just js build just rs build - if command -v uv &> /dev/null; then just py build; fi - if command -v wasm-bindgen &> /dev/null; then just wasm; fi - -# Build browser/WASM bindings into @moq/wasm using the pinned wasm-bindgen toolchain. -wasm: - cargo build --locked -p moq-wasm --target wasm32-unknown-unknown --profile wasm-release - wasm-bindgen --target web --out-name moq \ - --out-dir js/wasm/dist "${CARGO_TARGET_DIR:-target}/wasm32-unknown-unknown/wasm-release/moq_wasm.wasm" - -# Only this checkout by default. Agent worktrees each carry their own artifacts -# now that the shared target dir is gone, and another agent is usually building -# in one right now: `cargo clean` under a running build fails it, and there is no -# way to tell a finished worktree from a busy one from out here. `just clean all` -# is the explicit opt-in for a machine the caller knows is idle. -# -# Source is never touched either way, dirty or untracked: this deletes build -# output, not work. Nothing here reaches a machine-wide store -- no Nix garbage -# collection, no cargo/bun/uv home cache -- because those are shared with every -# other checkout and rebuilding them costs far more than the space they hold. + just py build + just js wasm # Delete this checkout's build artifacts and caches; `all` includes agent worktrees. clean SCOPE="here": - #!/usr/bin/env bash - set -euo pipefail - - case "{{ SCOPE }}" in - here | all) ;; - *) - echo "usage: just clean [here|all]" >&2 - exit 2 - ;; - esac - - just rs clean - just js clean - just py clean - just kt clean - just swift clean - just go clean - just dart clean - - # Caches not owned by any one language: nix build result, direnv, wrangler. - rm -rf result .direnv - find . -name .claude -prune -o -type d -name .wrangler -prune -exec rm -rf {} + - - # Worktrees don't nest, so this recurses exactly one level. Tolerate stale - # worktrees on branches that predate this recipe. - if [[ "{{ SCOPE }}" == all ]]; then - for wt in .claude/worktrees/*/; do - [ -f "${wt}justfile" ] || continue - echo "==> cleaning ${wt}" - (cd "$wt" && just clean) || echo " (skipped: just clean failed in ${wt})" - done - fi - -# Upgrade any tooling -update: - just js update - just rs update - nix flake update - -# Serve the documentation locally. -doc: - cd doc && bun run dev + sh/clean.sh {{ SCOPE }} diff --git a/kt/.gitignore b/kt/.gitignore index 25c4ac20b35..ae71352dba6 100644 --- a/kt/.gitignore +++ b/kt/.gitignore @@ -4,7 +4,7 @@ build/ local.properties .kotlin/ -# Generated by kt/scripts/{check,package}.sh. Never check these in. +# Generated by sh/kt/{check,package}.sh. Never check these in. moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi/ moq-ffi/src/jvmMain/resources/ moq-ffi/src/androidMain/jniLibs/ diff --git a/kt/README.md b/kt/README.md index 62c509e960f..2afe6524f6d 100644 --- a/kt/README.md +++ b/kt/README.md @@ -89,7 +89,6 @@ kt/ src/ jvmAndAndroidMain/kotlin/dev/moq/ Wrapper sources (Moq, Server, Aliases, Flows, Durations, Errors) jvmAndAndroidTest/ Facade smoke test - scripts/ check.sh, package.sh ``` The native/UniFFI layer stays in a single `dev.moq:moq-ffi` artifact because uniffi-linked libraries can't be split across separately packaged artifacts (the Python `moq-rs` wheel is one umbrella for the same reason). That constraint is about the *native* layer; the pure-Kotlin `dev.moq:moq` wrapper sits cleanly on top of it as its own artifact because it ships no native code. diff --git a/kt/justfile b/kt/justfile index 8acb7ebe9b6..2a3994710a6 100644 --- a/kt/justfile +++ b/kt/justfile @@ -9,42 +9,23 @@ set working-directory := '.' default: just check -# Missing tools are errors so wrapper drift cannot slip past a green check. -# # This runs the tests too, so there is no separate `just kt test`: the script -# builds, generates, and tests as one gradle invocation. Takes an optional -# newline-separated list of changed files and skips when none are Kotlin-scoped; -# `just kt check` (no FILES) always runs. +# builds, generates, and tests as one gradle invocation. # Build moq-ffi, regenerate the bindings, and run :moq-ffi:jvmTest + :moq:jvmTest. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '^(kt/|rs/moq-ffi/|doc/lib/kt/|doc/lib/samples\.sh$)' <<< "$FILES"; then - echo "kt: no Kotlin changes; skipping." - exit 0 - fi - bash scripts/check.sh +check: + bash ../sh/kt/check.sh # Regenerate the checked-in uniffi bindings and host native library without # compiling or testing. For environments that intentionally lack Gradle. generate: - bash scripts/generate.sh + bash ../sh/kt/generate.sh -# Assemble the KMP module from per-target moq-ffi binaries + bindings. -# Used by .github/workflows/release-kt-ffi.yml; see kt/README.md for the +# Runs in the caller's directory, so a relative `--lib-dir` or `--output` means +# what it would to the script itself. See kt/README.md for the --lib-dir layout. -# expected --lib-dir layout. +# Assemble the KMP module from per-target moq-ffi binaries and bindings. +[no-cd] +[positional-arguments] package *args: - bash scripts/package.sh {{ args }} - -# Remove gradle output plus the moq-ffi bindings and native libs that -# scripts/check.sh generates into the source tree (see kt/.gitignore). -clean: - #!/usr/bin/env bash - set -euo pipefail - find . -name .claude -prune -o -type d \( -name build -o -name .gradle -o -name .kotlin \) -prune -exec rm -rf {} + - rm -rf local.properties \ - moq/src/jvmAndAndroidMain/kotlin/uniffi \ - moq/src/jvmMain/resources \ - moq/src/androidMain/jniLibs + {{ quote(justfile_directory() / "sh/kt/package.sh") }} "$@" diff --git a/kt/moq-ffi/build.gradle.kts b/kt/moq-ffi/build.gradle.kts index c5e76a9f0bd..4aa387e97fe 100644 --- a/kt/moq-ffi/build.gradle.kts +++ b/kt/moq-ffi/build.gradle.kts @@ -11,7 +11,7 @@ // ├─ jvmMain JVM-specific: native libs as JAR resources // └─ androidMain Android-specific: native libs in jniLibs // -// Native libraries + bindings are populated by `kt/scripts/package.sh`: +// Native libraries + bindings are populated by `sh/kt/package.sh`: // src/jvmMain/resources/-/ (JNA classpath layout) // src/androidMain/jniLibs//libmoq_ffi.so (Android packaging layout) // src/jvmAndAndroidMain/kotlin/uniffi/moq/moq.kt (uniffi-bindgen output) diff --git a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/docs/Prelude.kt b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/docs/Prelude.kt index 07e047ee815..9f15448a620 100644 --- a/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/docs/Prelude.kt +++ b/kt/moq/src/jvmAndAndroidTest/kotlin/dev/moq/docs/Prelude.kt @@ -1,4 +1,4 @@ -// The inputs the documentation samples leave to the reader. kt/scripts/check.sh +// The inputs the documentation samples leave to the reader. sh/kt/check.sh // compiles every Kotlin sample from the docs beside this file, so a sample that // names a missing symbol or the wrong arity fails the check. Never executed. package dev.moq.docs diff --git a/py/justfile b/py/justfile index b95d0de995c..da3fc8cfbf7 100644 --- a/py/justfile +++ b/py/justfile @@ -10,16 +10,6 @@ set working-directory := '.' -# Paths a Python check depends on. `rs/moq-ffi` is in scope because maturin -# bundles it into the moq-ffi wheel. `check`, `fix`, and `test` all skip -# themselves when a diff misses this; no FILES at all means force-run. The docs -# are in scope because `check` type-checks their samples against the wrapper. -# -# They also skip when uv is absent, matching the kt/swift/go check scripts: an -# incomplete local toolchain should check less rather than fail. The root -# `_tools` preflight makes uv a hard requirement under MOQ_STRICT. -scope := '^(py/|pyproject\.toml$|uv\.lock$|rs/moq-ffi/|doc/lib/py/|doc/lib/samples\.sh$)' - default: just check @@ -40,17 +30,7 @@ _develop: # the Docs after the fact. It mocks the native module, so it costs no Rust build. # Lint, type-check, editable-build, check the doc samples, and render the docs. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "py: no Python changes; skipping." - exit 0 - fi - if ! command -v uv >/dev/null 2>&1; then - echo "py: uv not on PATH, skipping" >&2 - exit 0 - fi +check: uv sync --locked --no-install-workspace uv run --no-sync ruff check . uv run --no-sync ruff format --check . @@ -59,50 +39,18 @@ check $FILES="": just samples just docs -# Samples leave their inputs (`payload`, `pts`) to the reader, so they follow -# the declarations in moq-rs/tests/doc_prelude.py. - # Type-check every Python sample in the docs against the installed wrapper. samples: - #!/usr/bin/env bash - set -euo pipefail - out=$(mktemp -d) - trap 'rm -rf "$out"' EXIT - { - cat moq-rs/tests/doc_prelude.py - bash ../doc/lib/samples.sh python ../doc/lib/py/index.md moq-rs/README.md moq-rs/docs/index.md - } > "$out/samples.py" - echo '{"pythonVersion": "3.10"}' > "$out/pyrightconfig.json" - uv run --no-sync pyright --project "$out" "$out/samples.py" + ../sh/py/samples.sh -# Auto-fix lint and formatting issues. Skips on the same terms as `check`. -fix $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "py: no Python changes; skipping." - exit 0 - fi - if ! command -v uv >/dev/null 2>&1; then - echo "py: uv not on PATH, skipping" >&2 - exit 0 - fi +# Auto-fix lint and formatting issues. +fix: uv sync --no-install-workspace uv run --no-sync ruff check --fix . uv run --no-sync ruff format . -# Run the unit tests. Skips on the same terms as `check`. -test $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '{{ scope }}' <<< "$FILES"; then - echo "py: no Python changes; skipping tests." - exit 0 - fi - if ! command -v uv >/dev/null 2>&1; then - echo "py: uv not on PATH, skipping" >&2 - exit 0 - fi +# Run the unit tests. +test: uv sync --locked --no-install-workspace just _develop uv run --no-sync pytest moq-rs/tests/ moq-ffi/tests/ @@ -114,18 +62,6 @@ build: uv sync --locked --no-install-workspace just _develop -# Remove the virtualenv, release dist, bytecode caches, and the uniffi -# bindings maturin drops in during editable installs. The uv workspace venv - -# lives at the repo root, so reach up for it. -clean: - #!/usr/bin/env bash - set -euo pipefail - rm -rf dist moq-ffi/moq_ffi/_uniffi - rm -rf ../.venv .venv - find . -name .claude -prune -o -type d -name __pycache__ -prune -exec rm -rf {} + - find . -name .claude -prune -o -type f -name '*.pyc' -exec rm -f {} + - # Build the pure-python moq-rs wrapper sdist + wheel into py/dist (for release). # moq-ffi is built separately by maturin (see release-py.yml); the wrapper is diff --git a/quest/m1/README.md b/quest/m1/README.md index b560c4d5831..a758ce5998e 100644 --- a/quest/m1/README.md +++ b/quest/m1/README.md @@ -63,7 +63,6 @@ QUIC studies there on that rule. - [#2991](/quest/m1/2991-net-coalesce-dynamic-tracks-and-preserve-sequences-across.md) - one dynamic producer per track name in both languages, with the sequence namespace surviving a replacement - [JavaScript FETCH](/quest/m1/js-fetch.md) - generic on-demand group serving and IETF FETCH for browser publishers - [Archive](/quest/m1/archive/README.md) - record selected tracks to any object_store and replay them over FETCH or derived HLS; the catalog entry and format may break in place, since no archives exist -- [Tooling](/quest/m1/tooling/README.md) - justfiles become a one-line menu over `sh/`, one impact map scopes CI, and every workflow step runs a recipe - [In-band auth](/quest/m1/auth/README.md) - a session tells its peer what it may publish and subscribe to, unions tokens presented in band, and fails loud on an out-of-scope publish - [Dropped sources](/quest/m1/dropped-sources.md) - track consumers see the producer's real error on every end path, never `Dropped` - [C++ through moq-ffi](/quest/m1/cpp/README.md) - generated C++ over moq-ffi with futures and expected-style errors, shipped as a tarball, vcpkg, and Conan, and adopted by the OBS plugin diff --git a/quest/m1/bench-ci.md b/quest/m1/bench-ci.md index 7376f8e4a5c..79f0081e916 100644 --- a/quest/m1/bench-ci.md +++ b/quest/m1/bench-ci.md @@ -23,8 +23,8 @@ a GitHub App: - PR: one job builds base and head on the same runner and runs only the selected targets, saving and comparing Criterion baselines. Selection is the - changed crates plus their dependents, from the impact map that the - [tooling line](/quest/m1/tooling/README.md) lands. No selected bench + changed crates plus their dependents, from the impact map in + `sh/dispatch.sh`. No selected bench means no job. Extend `bench/run.sh` with a Criterion-only, crate-scoped mode behind a recipe instead of writing a second runner. Hosted runners vary by about 3%, so the comment highlights only changes Criterion calls @@ -43,10 +43,6 @@ a GitHub App: - Document the comment, the trend, and local reproduction in `bench/README.md`. -## Required - -- [Tooling](/quest/m1/tooling/README.md) - owns the diff-to-crate impact map the PR job reuses - ## Related - [Benchmark comparisons](/quest/m1/performance-comparisons.md) - extends the same `bench/run.sh` with repeated paired rounds diff --git a/quest/m1/capture-default.md b/quest/m1/capture-default.md index 8e2264afc87..d27fde791eb 100644 --- a/quest/m1/capture-default.md +++ b/quest/m1/capture-default.md @@ -22,7 +22,7 @@ pre-merge `just check` never runs moq-video's `capture` tests: it depends on [ALSA link](/quest/m1/capture-alsa-link.md). Once libasound loads at runtime, the default costs no system library. - Delete the special capture gate (`rs capture`, `capture-test`, and the - capture branch in `sh/rs/select.sh` on the tooling line) once default + capture branch in `sh/rs/select.sh`) once default `just check` covers it. Keep the platform jobs (`just rs macos`, `just rs windows`). - Update the Cargo feature comments, moq-cli's `capture` feature (it may @@ -34,5 +34,4 @@ tests, and a `default-features = false` consumer (for example ## Required -- [Tooling](/quest/m1/tooling/README.md) - rewrites the capture gate this deletes - [ALSA link](/quest/m1/capture-alsa-link.md) - moq-audio capture can't be on by default while it links libasound at build time diff --git a/quest/m1/doc-samples-go-dart.md b/quest/m1/doc-samples-go-dart.md index 6680e0a0345..50a05e0ec98 100644 --- a/quest/m1/doc-samples-go-dart.md +++ b/quest/m1/doc-samples-go-dart.md @@ -21,7 +21,7 @@ knows neither language. adjusting a sample so it reads naturally and still compiles over growing the extractor. - Wire each into its language's check script, `sh/go/check.sh` and - `sh/dart/check.sh` on the tooling line, the way `sh/kt/check.sh` and + `sh/dart/check.sh`, the way `sh/kt/check.sh` and `sh/py/samples.sh` call `samples.sh`. Add `doc/lib/go/`, `doc/lib/dart/`, and `doc/lib/samples.sh` to the `go` and `dart` patterns of the impact map in `sh/dispatch.sh`, as the `py`, `kt`, and `swift` ones already have. @@ -29,7 +29,3 @@ knows neither language. fail on the doc sample. Public API: none. Wire: none. - -## Required - -- [Tooling](/quest/m1/tooling/README.md) - the `sh//` check scripts and the impact map this extends diff --git a/quest/m1/gpu-ci.md b/quest/m1/gpu-ci.md index e294915c561..b3b0d6175df 100644 --- a/quest/m1/gpu-ci.md +++ b/quest/m1/gpu-ci.md @@ -26,8 +26,7 @@ skipping inside the Nix shell. tests (Android, D3D11, PipeWire). Inside that selection a missing GPU fails the test instead of returning early. Keep the no-driver tests (`missing_driver_errors_instead_of_panicking`) outside it. -- `just rs nvidia`, a one-line recipe over `sh/rs/nvidia.sh` in the tooling - line's `sh//` layout: symlink only those three libraries (by +- `just rs nvidia`, a one-line recipe over `sh/rs/nvidia.sh`: symlink only those three libraries (by soname) from `/usr/lib/x86_64-linux-gnu` into a private directory, put that on `LD_LIBRARY_PATH`, and run that selection. Fail when a library is missing instead of skipping. `just rs vulkan-cuda` (`sh/rs/vulkan-cuda.sh`) puts the @@ -52,7 +51,6 @@ Public API: none. Wire: none. ## Required -- [Tooling](/quest/m1/tooling/README.md) - the `sh/` script layout and recipe-only workflows this follows - A self-hosted runner is registered for moq-dev/moq on the maintainer's host, with the NVIDIA driver ## Related diff --git a/quest/m1/merge-queue.md b/quest/m1/merge-queue.md index a90b27b0d40..308d0f59bf3 100644 --- a/quest/m1/merge-queue.md +++ b/quest/m1/merge-queue.md @@ -29,11 +29,4 @@ and there is no merge queue, so nothing re-runs them on the combined tree. act, after the workflow change lands on `main`. Hand it over with the settings to use rather than changing it. -Decided in the 2026-09-30 audit: no longer waits on the tooling line; the -queue needs only `just ci $JOB $BASE`, which `main` already has. - Public API: none. Wire: none. - -## Related - -- [Tooling](/quest/m1/tooling/README.md) - later changes to the `just ci` entry point the queue runs diff --git a/quest/m1/performance-comparisons.md b/quest/m1/performance-comparisons.md index 9512e1b2cbb..e6e5a01f8a6 100644 --- a/quest/m1/performance-comparisons.md +++ b/quest/m1/performance-comparisons.md @@ -39,10 +39,6 @@ rather than creating another benchmark runner. samples. Validate an unchanged-revision A/A run and a deliberately degraded fixture; keep normal machine timing informational rather than a flaky CI gate. -## Required - -- [Tooling](/quest/m1/tooling/README.md) - the recipe and script layout `just bench` runs under - ## Related - [Windowed latency](/quest/m1/3126-moq-bench-every-readme-example-fails-to-parse-and.md) - owns histogram/window semantics diff --git a/quest/m1/serve-hls-renditions.md b/quest/m1/serve-hls-renditions.md index 3e86c953bc0..36c0094f433 100644 --- a/quest/m1/serve-hls-renditions.md +++ b/quest/m1/serve-hls-renditions.md @@ -10,7 +10,6 @@ options aren't per-stream in ffmpeg, so the last filter wins. Use per-stream filters (`-filter:v:0`, `-filter:v:1`), scaled to 720p and 144p to match `hls`. Scaling the 720p source up to 1080p couldn't encode in -real time on a dev machine. The recipe body moves to `sh/demo/serve-hls.sh` -when the [tooling line](/quest/m1/tooling/README.md) lands; fix it wherever it -lives. Check it by fetching `master.m3u8` and comparing the two renditions' +real time on a dev machine. The recipe body lives in `sh/demo/serve-hls.sh`. +Check it by fetching `master.m3u8` and comparing the two renditions' `RESOLUTION` attributes. diff --git a/quest/m1/size-report.md b/quest/m1/size-report.md index d034a049f46..526a5cf4f14 100644 --- a/quest/m1/size-report.md +++ b/quest/m1/size-report.md @@ -39,8 +39,8 @@ Decided in planning: visible without a local rebuild. `cargo bloat` needs symbols, so build that pass with `CARGO_PROFILE_RELEASE_STRIP=none` and report stripped sizes separately. -- Following the tooling questline, the work lives in a `sh/` script behind a - `just` recipe, and `nightly.yml` calls the recipe. +- The work lives in a `sh/` script behind a `just` recipe, and `nightly.yml` + calls the recipe. Considered and declined (do not re-ask): diff --git a/quest/m1/tooling/README.md b/quest/m1/tooling/README.md deleted file mode 100644 index 5f3055bfc07..00000000000 --- a/quest/m1/tooling/README.md +++ /dev/null @@ -1,25 +0,0 @@ -# Tooling: thin justfiles and CI that calls them - -## Goal - -A month of merges left the command surface bloated: 873 lines of root -`justfile`, seven self-tests of the tooling itself on every pull request, -three metadata guards, recipes nothing calls, and release workflows that are -the same file with a binary name swapped. The result is a `just` tree that is -a menu, not a language: every recipe is one line, every script lives under -`sh/`, the diff is resolved once by one impact map, and every workflow step -runs a recipe rather than a script path. - -## Plan - -`just` stays as the entry point because the vocabulary (`just check`, `just -fix`) is in every doc, skill, and workflow. Its cost was -self-inflicted: logic inside recipes. The quests below are ordered and each -requires the one before it, so they land as one line of pull requests. - -## Required - -- [Thin justfiles](/quest/m1/tooling/justfiles.md) - recipe bodies move to `sh/`, one impact map scopes check/fix/test, self-tests and guards are deleted -- [Workflows call just](/quest/m1/tooling/workflows-call-just.md) - no workflow `run:` step names a `.sh`; every script a workflow needs has a recipe -- [Binary release workflow](/quest/m1/tooling/release-binary.md) - moq-cli and moq-relay share one reusable workflow behind two thin callers -- [FFI release workflow](/quest/m1/tooling/release-ffi.md) - the five `release-*-ffi.yml` share the moq-ffi target matrix and artifact staging diff --git a/quest/m1/tooling/justfiles.md b/quest/m1/tooling/justfiles.md deleted file mode 100644 index 429dedb4325..00000000000 --- a/quest/m1/tooling/justfiles.md +++ /dev/null @@ -1,102 +0,0 @@ -# [L] Thin justfiles with one impact map - -## Goal - -Every justfile is a menu: each recipe is a single line that runs a tool or a -script under `sh/`, `just --list` shows each recipe once, and no recipe body -is bash. The scoped `check`, `fix`, and `test` resolve the branch diff once, -in one impact map, instead of a scope regex per language justfile plus a -mirror of them in the strict-tools preflight. Nothing that tests the tooling -itself runs on a pull request. A recipe that no workflow, doc, skill, or other -recipe invokes is gone, except the hand-run demo and infra ops recipes, which -stay as they are. - -## Plan - -Layout: - -- New root `sh/`. Root helpers sit at the top (`sh/dispatch.sh`, - `sh/markdown.sh`, `sh/shell.sh`, `sh/clean.sh`); per-area tooling moves - under `sh//`: `rs/scripts` to `sh/rs`, `.github/scripts` to `sh/gh`, - `go/scripts` to `sh/go`, `kt/scripts` to `sh/kt`, `swift/scripts` to - `sh/swift`, `dart/scripts` to `sh/dart`, and the OBS recipe bodies to - `sh/obs`. Workflow references to moved paths are updated mechanically here; - the next quest replaces them with recipes. -- Package build scripts that nix, CMake, gradle, and the dart build hook - point at stay put: `rs/*/build.sh`, `rs/moq-gst/{package,scrub,smoke}.sh`, - `cpp/obs/build.sh`, `infra/*/publish.sh`. The harness under `test/` is - harness code, not tooling; it stays too. -- `shfmt -f .` already enumerates `sh/`, so the shell lint covers everything - the justfiles stop containing. - -Dispatch: - -- `just check [BASE]`, `just fix [BASE]`, and `just ci check|test [BASE]` are - one line each: `sh/dispatch.sh check|fix|ci-check|ci-test "$BASE"`. The script resolves BASE - (arg, `GITHUB_BASE_REF`, upstream, `origin/main`), lists changed files into - a temp file, and applies one impact map: which language modules run, which - tools `MOQ_STRICT` demands, and whether the root orchestration changed and - everything runs. The map lives in one place; the `scope` variables in the - js, py, kt, swift, go, and dart justfiles and the regex table in `_tools` - are deleted with it. -- Per-language `check`, `fix`, and `test` lose their `$FILES` parameter and - skip logic. Only rs still receives the list: `just rs check-changed - ` (and `fix-changed`, `test-changed`) hand the path to - `sh/rs/select.sh`, which does the crate selection today spread over - `_select`, `_names`, and `_wants-wasm`. -- Passing a path instead of the list kills the argv budget: - `changed_max`, `_changed-cap`, `_changed-test`, `_echo`, and the E2BIG - commentary go. -- `check --all`, `fix --all`, and `ci check|test --all` keep their names; cache.yml, - nightly.yml, and the docs call them. - -Delete: - -- Self-tests: `_changed-test`, `_markdown-test`, `_select-test`, - `_doc-names-test`, `_publish-test`, `sh/rs/package-nfpm.test.sh`, - `sh/gh/package-binary.test.sh`. -- Guards: `_doc-names` and `doc-names.jq`, and the publish lower-bound - check. This is a deliberate trade: the guards were incidents' worth of - bash plus their self-tests on every pull request. Where each surfaces - instead: a doc-directory collision shows up as an intermittent `failed to - remove directory` in the `cargo doc` step of whichever pull request selects - both crates, not deterministically (resolve with `doc = false` as before); - a stale internal lower bound fails release-plz's publish. `_fuzz-lock` is - already gone: #3543 folded the fuzz harness into the workspace so the root - lockfile and `--locked` cover it. `alert.sh check-coverage` stays: it is a - lint of alert.yml, not of a recipe. -- The `worktree` recipe and its 170 lines, plus the Worktrees section of - `test/README.md`; keep `_base` folded into `sh/dispatch.sh`. -- `rs bump` and `rs semver`. `rs release` stays for release-rs.yml. -- The `mod` lines in `demo/justfile`: `just pub`, `just relay`, `just boy`, - `just sub`, `just web` are the only spelling, and `--list` stops showing - each twice. `just demo` remains the default recipe and `just dev` its - documented alias. - -Rename: - -- Root `wasm` becomes `js wasm`: it emits `js/wasm/dist`, so it is a JS - package build. `rs wasm` stays the compile gate and `test wasm` the browser - run. Update wasm.yml, root `build`, `test/wasm/run.sh` (which runs `just - wasm` to build the package under test), and the four doc references. - -Keep, moved into scripts unchanged in behavior: the remark mirror-and-diff in -`sh/markdown.sh` (remark-cli has no check mode and the lint presets are -wanted), `_shell`, `_flake` as its one-liner, `clean`, the rs `package` and -`fuzz` bodies, and the OBS `compile`, `_includes`, `_unit`, `test`, `check`, -and `preset` bodies. - -Arguments reach tools intact: `just rs test`, `check-test`, and -`capture-test` interpolate `{{ args }}` today, so a quoted nextest filterset -becomes a shell syntax error (#4342). Recipes that forward arguments use -`[positional-arguments]` and `"$@"`. No self-test for it, per the rule above. - -Docs: `doc/setup/dev.md`, `CONTRIBUTING.md`, `test/README.md`, and the -`AGENTS.md` mentions of `just wasm` follow the survivors. Verify with `just -check`, `just ci test`, and `just check --all`, and confirm every recipe name -check.yml, cache.yml, nightly.yml, interop.yml, wasm.yml, obs.yml, swift.yml, -and release-*.yml invoke still resolves. - -## Closes - -- [#4342](https://github.com/moq-dev/moq/issues/4342) - close this issue when the quest finishes diff --git a/quest/m1/tooling/release-binary.md b/quest/m1/tooling/release-binary.md deleted file mode 100644 index 95eecf91f16..00000000000 --- a/quest/m1/tooling/release-binary.md +++ /dev/null @@ -1,31 +0,0 @@ -# [M] One reusable workflow for the binary releases - -## Goal - -`moq-cli.yml` and `moq-relay.yml` are two copies of the same workflow (226 -and 225 lines) with the binary name swapped. They become one reusable -`release-binary.yml` taking `crate` and `bin` inputs, and two callers of a -dozen lines each that keep their tag trigger -and workflow name, so `alert.yml`, `release-brew.yml`, and -`release-winget.yml` keep matching on the names they match today. - -## Plan - -- `release-binary.yml` on `workflow_call` with inputs `crate` (the cargo - package) and `bin` (the executable name; `moq-cli` ships `moq`). It holds the Linux native-runner matrix, - the macOS tarball job, the Windows job, the `.deb` and `.rpm` packaging, the - release creation, and the repo-publish trigger, all as `just` recipes per - the preceding quest. -- Each caller keeps `name:` and `on.push.tags` exactly as they are and does - nothing but `uses: ./.github/workflows/release-binary.yml` with the two - inputs and the secrets it forwards. `workflow_run` consumers trigger off - the caller's name, so nothing downstream changes. -- The one text difference between the three today (a comment and a step name - about the glibc floor) belongs in the reusable file. -- Verify with `just gh check` and by diffing the rendered job list of each - caller against its predecessor with `gh workflow view`. The next tagged - release is the end-to-end check; say so in the PR. - -## Required - -- [Workflows call just](/quest/m1/tooling/workflows-call-just.md) - the reusable workflow is written in the recipe style from the start diff --git a/quest/m1/tooling/release-ffi.md b/quest/m1/tooling/release-ffi.md deleted file mode 100644 index 1627f981c8a..00000000000 --- a/quest/m1/tooling/release-ffi.md +++ /dev/null @@ -1,35 +0,0 @@ -# [M] One reusable workflow for the moq-ffi releases - -## Goal - -`release-dart-ffi.yml`, `release-go-ffi.yml`, `release-kt-ffi.yml`, -`release-swift-ffi.yml`, and `release-py-ffi.yml` share half to two thirds of -their lines: the moq-ffi cross-target build matrix and the artifact staging. -That shared half becomes one reusable `release-ffi.yml`; each language keeps -only its packaging and publish steps. - -## Plan - -- Measure the shared lines first (`comm` over the sorted files shows 74 to - 176 common lines per pair) and confirm the matrix targets agree. Where a - language needs a different target set (iOS simulator for swift, Android - ABIs for kt and dart), make the target list an input rather than forcing - one matrix. -- `release-ffi.yml` on `workflow_call`: builds `rs/moq-ffi` for the requested - targets, uploads one artifact per target, and outputs the version parsed - from the tag. Callers download the artifacts and run their `just - package` and publish recipes. -- `release-dart-ffi.yml` and `release-go-ffi.yml` also run on `pull_request`, - where `GITHUB_REF` is not a `moq-ffi-v*` tag and `parse-version` fails; they - read the version from `rs/moq-ffi/Cargo.toml` instead. The reusable - workflow keeps that fallback (an input or a ref check), or those PR runs die - before building. -- Callers keep their `name:` and triggers, as the binary quest did, so - `alert.yml` and the `workflow_run` chains (`release-go.yml`, - `release-py.yml`, `release-swift-lib.yml`) are untouched. -- Verify with `just gh check`; the next tagged `moq-ffi-v*` release is the - end-to-end check. - -## Required - -- [Binary release workflow](/quest/m1/tooling/release-binary.md) - proves the reusable-plus-callers pattern before it is applied to five files diff --git a/quest/m1/tooling/workflows-call-just.md b/quest/m1/tooling/workflows-call-just.md deleted file mode 100644 index 4d6c79c4e8e..00000000000 --- a/quest/m1/tooling/workflows-call-just.md +++ /dev/null @@ -1,45 +0,0 @@ -# [M] Every workflow step runs a recipe - -## Goal - -No `run:` line in `.github/workflows` names a `.sh` file. Every script a -workflow needs has a `just` recipe, so the menu is the whole command surface -and a script can move or change its arguments without touching twenty -workflows. The same recipes work locally for a dry run. - -## Plan - -Today the workflows call scripts directly at roughly ninety sites: -`sh/gh/release.sh` (54 calls as a helper library: `parse-version`, -`prev-tag`, `create`, registry existence checks), `sh/rs/package-nfpm.sh`, -`sh/rs/package-binary.sh`, `sh/rs/package-windows.sh`, -`sh/gh/trigger-repo-publish.sh`, `sh/gh/render-formula.sh`, `sh/gh/alert.sh`, -the swift `package`, `package-ffi`, `publish`, `publish-ffi`, `verify`, -`verify-ffi`, and `check` scripts, the go `package-ffi`, `package-wrapper`, -`publish-ffi`, `publish-wrapper` scripts, `sh/dart/package.sh`, -`sh/kt/package.sh`, and `infra/*/publish.sh`. - -- Add pass-through recipes where none exist, named by role under the area - module: `just gh release ...`, `just gh alert ...`, - `just gh formula ...`, `just gh trigger-repo-publish ...`, - `just rs package-binary ...`, `just rs package-nfpm ...`, - `just rs package-windows ...`, `just swift publish|verify|package-ffi|...`, - `just go publish-ffi|publish-wrapper`, `just dart package`, - `just infra apt publish` and `just infra rpm publish` (the recipes exist). - Existing wrappers (`kt package`, `swift package`, `go package-ffi`, - `go package-wrapper`, `py package`) are used as they are. -- Rewrite every `run:` step to the recipe. Where a workflow runs outside the - nix dev shell (Windows and macOS runners, AlmaLinux containers), `just` is - already installed or installable the way nightly.yml does it; keep that - pattern rather than falling back to a path. -- Record the rule in the CI section of `CONTRIBUTING.md` in one line, and - have `just gh check` enforce it: fail on any workflow `run:` line matching - `\.sh\b`. -- Verify with `actionlint` via `just gh check`, and with a manual - `workflow_dispatch` of apt-repo.yml, rpm-repo.yml, and release-winget.yml - where the inputs allow a dry run. The tag-triggered release workflows are - verified by the next release; say so in the PR. - -## Required - -- [Thin justfiles](/quest/m1/tooling/justfiles.md) - establishes `sh/` and the module recipes the workflows will call diff --git a/quest/m2/moq-installer.md b/quest/m2/moq-installer.md index c5c88937c05..5a95a000dcb 100644 --- a/quest/m2/moq-installer.md +++ b/quest/m2/moq-installer.md @@ -17,6 +17,8 @@ service setup, Windows support, or new release targets. Deferred to m2 in the 2026-09-30 audit: no named consumer; `cargo install`, Nix, Docker, and winget already install `moq`. +- Download the archives `.github/workflows/release-binary.yml` already + publishes; the installer needs no workflow change. - Default to the latest stable `moq-cli` release, with an explicit version option. Resolve that product's tags, not the repository-wide latest release: this repository publishes multiple independently versioned crates. @@ -46,7 +48,5 @@ Nix, Docker, and winget already install `moq`. ## Related -- [Binary release workflow](/quest/m1/tooling/release-binary.md) - reuse its - artifacts without requiring workflow consolidation - [`moq relay`](/quest/m2/moq-relay-subcommand.md) - relay functionality joins the same executable independently of its installation method diff --git a/rs/justfile b/rs/justfile index 8074f2f9d4c..e5e0eea1582 100644 --- a/rs/justfile +++ b/rs/justfile @@ -11,738 +11,145 @@ set working-directory := '..' # `moq-net-fuzz` is a workspace member only so it shares the root lockfile; libFuzzer # needs a nightly toolchain and sanitizer flags, so `just rs fuzz` is the one recipe -# that may compile it. Every `--workspace` below therefore carries this, and a new -# one that forgets it pays a vendored C++ libFuzzer build for nothing. The root -# justfile's `check --all` and `fix --all` spell it out, since a module cannot read this. +# that may compile it. Every `--workspace` below therefore carries this, as does +# sh/rs/select.sh, since it cannot read this. no_fuzz := "--exclude moq-net-fuzz" default: just check -# Default features for the selected workspace packages, here and in CI. The full -# workspace permutations (`--all-features`, `--no-default-features`) each cost a -# separate compile that shares almost nothing with this one, so `features` below -# covers them nightly. `check-changed` makes one scoped exception for moq-tokio, -# whose feature-gated transport surface must compile in isolation to avoid -# workspace feature unification hiding a broken configuration. -# -# No `cargo check` pass: clippy is a superset of it, and the two use different -# rustc wrappers, so running both means compiling the workspace twice. +# Default features for the selected packages. The full workspace permutations +# each cost a separate compile, so `features` covers them nightly. No `cargo +# check` pass: clippy is a superset of it, and the two use different rustc +# wrappers, so running both compiles the workspace twice. # # A bare `just rs check` is default-members, which skips libmoq, moq-ffi, -# moq-gst, and moq-wasm (they need cbindgen, a uniffi build, GStreamer, and a -# wasm32 target respectively, so they stay out of a casual local check). Every -# recipe that means "all of it" therefore passes `--workspace` explicitly, since -# `check` is the only gate now and nothing else would compile them. +# moq-gst, and moq-wasm; pass `--workspace` for all of it. # Compile, lint, format-check, doc-check, and verify dependency hygiene. +[positional-arguments] check *args: - cargo clippy --locked --all-targets {{ args }} -- -D warnings - just rs _lint {{ args }} + cargo clippy --locked --all-targets "$@" -- -D warnings + just rs _lint "$@" -# `check` with the clippy pass replaced by `test`, whose clippy-driver build -# reports the same findings. `build.warnings` turns them into errors without -# entering cargo's fingerprint, so a later plain `test` reuses the artifacts. -# `--no-tests=pass` because a selection can hold nothing testable (see -# `test-changed`). +# `check` with clippy replaced by the `test` build, whose clippy-driver reports +# the same findings. `build.warnings` turns them into errors without entering +# cargo's fingerprint, so a later plain `test` reuses the artifacts. A +# selection can hold nothing testable (moq-wasm has no host tests). # Compile, lint, and test in one build, plus everything else `check` verifies. +[positional-arguments] check-test *args: - #!/usr/bin/env bash - set -euo pipefail - if ! command -v clippy-driver >/dev/null 2>&1; then - echo "rs: check-test needs clippy-driver to lint" >&2 - exit 1 - fi - CARGO_BUILD_WARNINGS=deny just rs test --no-tests=pass {{ args }} - just rs _lint {{ args }} + CARGO_BUILD_WARNINGS=deny RUSTC_WORKSPACE_WRAPPER={{ require("clippy-driver") }} cargo nextest run --locked --all-targets --no-tests=pass "$@" + just rs _lint "$@" # Everything `check` verifies besides clippy. +[positional-arguments] _lint *args: - just rs _select-test - just rs _publish-test - just rs _doc-names-test - # A new MoqConnectionStats field has to reach every binding's doc page. - python3 rs/scripts/stats-docs.py - rs/scripts/package-nfpm.test.sh + python3 sh/rs/stats-docs.py cargo fmt --all --check - just rs _doc-names - RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps {{ args }} + RUSTDOCFLAGS="-D warnings" cargo doc --locked --no-deps "$@" cargo shear cargo sort --workspace --check --no-format -# Two targets whose names differ only by `-` vs `_` render to the same -# `target/doc/`, and a `cargo doc` that selects both races to clean it: the -# collision warning escalates to `error: failed to remove directory` depending on -# which rustdoc process gets there first. That reads as a flake, and because the doc -# pass above is diff-scoped it surfaces on whichever unrelated PR happens to select -# both crates. Resolve a pair by setting `doc = false` on whichever target has no -# published API to render (usually the binary). -# -# Metadata only, so this costs no compilation and can sit in the PR gate. Takes an -# optional metadata file so `_doc-names-test` can feed it synthetic workspaces. -_doc-names metadata="": - #!/usr/bin/env bash - set -euo pipefail - - if [[ -n "{{ metadata }}" ]]; then - collisions=$(jq -r -f rs/scripts/doc-names.jq < "{{ metadata }}") - else - collisions=$(cargo metadata --locked --no-deps --format-version 1 | jq -r -f rs/scripts/doc-names.jq) - fi - - if [[ -n "$collisions" ]]; then - echo "rs: these targets render to the same doc directory:" >&2 - echo "$collisions" >&2 - echo "set doc = false on whichever has no published API." >&2 - exit 1 - fi - -# Both directions of `_doc-names`, since the interesting half is what it must NOT -# report: cargo documents only the library when a binary shares its own package's -# name, so flagging that would reject the conventional src/lib.rs + src/main.rs layout. -_doc-names-test: - #!/usr/bin/env bash - set -euo pipefail - - scratch=$(mktemp -d) - trap 'rm -rf "$scratch"' EXIT - - # Same package, same name: cargo suppresses the bin, so this must pass. - cat > "$scratch/ok.json" <<'JSON' - {"packages":[{"name":"solo","targets":[ - {"name":"solo","kind":["lib"],"doc":true}, - {"name":"solo","kind":["bin"],"doc":true}]}]} - JSON - just rs _doc-names "$scratch/ok.json" - - # Same package, differing only by `-` vs `_`: cargo folds before suppressing, so - # this must pass too. Comparing the names as spelled would report it. - cat > "$scratch/folded.json" <<'JSON' - {"packages":[{"name":"folded","targets":[ - {"name":"foo_bar","kind":["lib"],"doc":true}, - {"name":"foo-bar","kind":["bin"],"doc":true}]}]} - JSON - just rs _doc-names "$scratch/folded.json" - - # Different packages: a real collision, must be reported. - cat > "$scratch/bad.json" <<'JSON' - {"packages":[ - {"name":"a-lib","targets":[{"name":"shared_name","kind":["lib"],"doc":true}]}, - {"name":"b-cli","targets":[{"name":"shared-name","kind":["bin"],"doc":true}]}]} - JSON - if just rs _doc-names "$scratch/bad.json" 2>/dev/null; then - echo "rs: _doc-names missed a cross-package collision" >&2 - exit 1 - fi - - # `doc = false` is how a pair gets resolved, so it must stop being reported. - cat > "$scratch/fixed.json" <<'JSON' - {"packages":[ - {"name":"a-lib","targets":[{"name":"shared_name","kind":["lib"],"doc":true}]}, - {"name":"b-cli","targets":[{"name":"shared-name","kind":["bin"],"doc":false}]}]} - JSON - just rs _doc-names "$scratch/fixed.json" - - echo "rs: _doc-names ok" - # Auto-fix clippy/format/shear/sort. +[positional-arguments] fix *args: - cargo clippy --locked --fix --allow-staged --allow-dirty --all-targets {{ args }} + cargo clippy --locked --fix --allow-staged --allow-dirty --all-targets "$@" cargo fmt --all cargo shear --fix cargo sort --workspace --no-format -# Prints `ALL` when the diff hits something every crate depends on, and nothing -# when no crate is affected. -# -# FILES is an exported parameter, not `{{ FILES }}`: just interpolates the -# latter into the recipe source, where a filename like `$(...)` would run as a -# command. Git-derived paths are untrusted input, so they stay data. - -# Print `--package` flags for the crates a diff touches, plus their dependents. -[private] -_select $FILES: - #!/usr/bin/env bash - set -euo pipefail - - # Workspace-wide inputs affect every crate, so there is nothing to scope to. - # - # `rs/justfile` and the nextest config are in here because they define how - # every crate is checked and tested. They also match `^rs/` below while - # seeding no crate (the seed pattern needs `rs//...`), so without this - # a PR editing only the recipes would compile and test nothing at all, and a - # broken cargo flag would pass CI on a format check alone. - if [[ -z "$FILES" ]] || grep -qE '^(Cargo\.(toml|lock)|rust-toolchain\.toml|rs/justfile|\.config/nextest\.toml)$' <<< "$FILES"; then - echo ALL - exit 0 - fi - - grep -q '^rs/' <<< "$FILES" || exit 0 - - # `--no-deps` keeps this to workspace members and off the network. - metadata=$(cargo metadata --format-version 1 --no-deps) - - # A seed is the first path segment under `rs/`, so only a crate whose manifest - # sits directly at `rs//Cargo.toml` can ever be one. Drop the rest here: - # `moq-net-fuzz` lives at `rs/moq-net/fuzz`, so its seed is already `moq-net`, - # and leaving it in the graph would select it as a dependent of the crate it - # fuzzes and compile libFuzzer on every moq-net diff. - # - # `manifest_path` is a native path, so normalize the separator before splitting - # on it. Windows would otherwise match no segment, drop every package, and - # select nothing at all, where the assertion below fails safe to ALL. - metadata=$(jq '.packages |= map((.manifest_path |= gsub("\\\\"; "/")) | select(.manifest_path | split("/")[-3] == "rs"))' <<< "$metadata") - - # Every remaining crate directory is named after the crate it holds, which is - # what makes a changed path a seed below. Verify it rather than trust it: under- - # selecting silently checks nothing, so fall back to checking everything. - if jq -e '.packages[] | select((.manifest_path | split("/")[-2]) != .name)' <<< "$metadata" > /dev/null; then - echo "rs: a crate directory no longer matches its crate name; selecting everything." >&2 - echo ALL - exit 0 - fi - - seeds=$(sed -n 's|^rs/\([^/]*\)/.*|\1|p' <<< "$FILES" | sort -u) - - # Every workspace member, not just the default ones. `--no-deps` already - # limits this to the workspace. The non-default members (libmoq, moq-ffi, - # moq-gst, moq-wasm) have to participate as both seeds and dependents: they - # used to drop out here because `check` matched a bare `cargo check`, but - # `check` is the only gate now, and the `cargo clippy --workspace` in the - # deleted `just rs ci` is what used to cover them. Without them, a PR - # touching only rs/libmoq selects nothing and compiles nothing. - edges=$(jq -r ' - .packages[] - | .name as $name - | .dependencies[] - | "\($name) \(.name)" - ' <<< "$metadata") - - # A crate has to be rebuilt when anything it depends on changed, so walk the - # edges backwards until the selection stops growing. The seeds go through the - # environment because BSD awk (macOS) rejects a newline in a `-v` value. - selected=$(SEEDS="$seeds" awk ' - BEGIN { split(ENVIRON["SEEDS"], s, "\n"); for (i in s) want[s[i]] = 1 } - { pkg[NR] = $1; dep[NR] = $2 } - END { - do { - grew = 0 - for (i = 1; i <= NR; i++) - if (want[dep[i]] && !want[pkg[i]]) { want[pkg[i]] = 1; grew = 1 } - } while (grew) - # Every wanted crate, not just those that appear as an edge source: a - # crate with no dependencies of its own emits no edge, so keying the - # output on `pkg[i]` would drop it and select nothing for a direct - # change to it. Test the VALUE, not the key -- awk creates a key on - # every `want[dep[i]]` read above, so iterating keys alone would emit - # every crate the workspace mentions. The lookup below drops names that - # are not workspace crates, which is what keeps a seed like the - # non-crate `rs/scripts` from reaching cargo. - for (name in want) if (want[name]) print name - } - ' <<< "$edges" | sort -u) - - [[ -n "$selected" ]] || exit 0 - - # Emit cargo's own package ids, not bare names. A workspace crate we also - # publish collides with its crates.io copy as soon as any dependency pulls - # that copy in (the web-transport-* crates depend on a published `kio`), and - # `--package kio` is then ambiguous: cargo refuses during package selection, - # before compiling anything, so the whole gate fails. Ids from `cargo - # metadata` carry the source, so they are unambiguous by construction. - jq -r --arg selected "$selected" ' - ($selected | split("\n")) as $want - | .packages[] - | select(.name as $name | $want | index($name) != null) - | "--package \(.id)" - ' <<< "$metadata" | sort -u | tr '\n' ' ' - -# Bare crate names for the `--package ` flags `_select` emits, for log lines -# and for the wasm gate's alternation below. Matching names rather than the flag -# spelling keeps those call sites working whatever spec form `_select` uses; the -# previous alternation matched `--package ` literally and silently stopped -# firing the moment the spec form changed. Ids are `path+file://#` -# and `_select` only emits crates whose directory it has verified matches -# their crate name, so the last path segment is the name. - -# True when a selection includes a crate the wasm32 pass covers. One predicate -# rather than the same alternation in each gate, so the test below exercises what -# the gates actually run instead of a copy of it. -[private] -_wants-wasm $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qwE '(moq-wasm|moq-mux|moq-ffi)' <<< "$(just rs _names "$PACKAGES")" - -# True when the selection includes moq-tokio, whose zero-feature build is hidden -# by workspace feature unification whenever any dependent enables a transport. -[private] -_wants-tokio-features $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qw 'moq-tokio' <<< "$(just rs _names "$PACKAGES")" - -# True when the selection includes a crate with a `capture` feature. Selection -# pulls in dependents, so a diff to moq-mux or moq-net lands here too, which is -# how a rename in either reaches the device code that calls it. -[private] -_wants-capture $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qwE '(moq-video|moq-audio)' <<< "$(just rs _names "$PACKAGES")" - -# True when the selection includes moq-cli, whose `play` media tasks are off the -# default feature set. Selection pulls in dependents, so a diff to anything the -# player is built from (moq-audio, moq-video, moq-mux) lands here too. -[private] -_wants-play $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qw 'moq-cli' <<< "$(just rs _names "$PACKAGES")" - -# True when the selection includes moq-relay, whose io_uring listener is off the -# default feature set. Selection pulls in dependents, so a diff to any crate the -# relay is built from (moq-uring, moq-sock, moq-tokio) lands here too. -[private] -_wants-uring $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - grep -qw 'moq-relay' <<< "$(just rs _names "$PACKAGES")" - -# True when the media feature contracts have to run. The contract script is not -# a crate, so `_select` drops it; a diff that only edits the script must still -# reach the gate, or a broken command merges on shell lint alone. -[private] -_wants-media-features $PACKAGES $FILES: - #!/usr/bin/env bash - set -euo pipefail - grep -qwE '(moq-video|moq-audio|moq-transcode)' <<< "$(just rs _names "$PACKAGES")" \ - || grep -qx 'rs/scripts/media-features.sh' <<< "$FILES" +# Like `check`, but only the crates in a changed-file list plus their dependents. +check-changed $LIST: + sh/rs/select.sh check "$LIST" -# Print the crate names behind a list of `--package` flags. -[private] -_names $PACKAGES: - #!/usr/bin/env bash - set -euo pipefail - tr ' ' '\n' <<< "$PACKAGES" | sed -n 's|.*/\([^/#]*\)#.*|\1|p' | sort -u | tr '\n' ' ' +# Like `check-test`, but only the crates in a changed-file list plus their dependents. +check-test-changed $LIST: + sh/rs/select.sh check-test "$LIST" -# Guards the two things about selection that fail SILENTLY rather than loudly: -# a `--package` spec cargo cannot resolve takes the gate down before it compiles -# anything, and a wasm gate that stops matching skips the only pass that ever -# looks at moq-wasm's code (its crate root is `#![cfg(target_arch = "wasm32")]`, -# so the host pass sees an empty crate and reports success). Both have happened. +# Like `fix`, but only the crates in a changed-file list plus their dependents. +fix-changed $LIST: + sh/rs/select.sh fix "$LIST" -# Check that selection emits resolvable specs and still drives the wasm gate. -[private] -_select-test: - #!/usr/bin/env bash - set -euo pipefail - - fail() { echo "rs: _select-test: $1" >&2; exit 1; } - - # `kio` is published and pulled from crates.io by the web-transport-* crates, - # so the workspace copy and the registry copy share a name: a bare name is - # ambiguous and cargo refuses. Every emitted spec must name its source, and - # cargo must accept it. - packages=$(just rs _select "rs/kio/src/waiter.rs") - for spec in $(sed 's/--package //g' <<< "$packages"); do - [[ "$spec" == *"file://"* ]] || fail "expected a source-qualified id, got: $spec" - cargo pkgid --offline "$spec" > /dev/null 2>&1 || fail "cargo cannot resolve: $spec" - done - - # The wasm gate keys on names, so it survives whatever spec form the ids use. - just rs _wants-wasm "$packages" \ - || fail "a kio diff must still drive the wasm gate: $(just rs _names "$packages")" - - # ...and it must still be able to say no, or it is not a gate. - relay=$(just rs _select "rs/moq-relay/src/web.rs") - if just rs _wants-wasm "$relay"; then - fail "a moq-relay diff must not drive it: $(just rs _names "$relay")" - fi - - # The break the capture gate was added for: a moq-mux API change that missed - # its callers behind `capture`. A moq-mux diff has to reach it. - mux=$(just rs _select "rs/moq-mux/src/clock.rs") - just rs _wants-capture "$mux" \ - || fail "a moq-mux diff must drive the capture gate: $(just rs _names "$mux")" - if just rs _wants-capture "$relay"; then - fail "a moq-relay diff must not drive the capture gate: $(just rs _names "$relay")" - fi - - # The break the uring gate was added for: a change that breaks moq-relay's - # io_uring listener, which the default pass above never compiles. A moq-relay - # diff has to reach it, and a diff that cannot affect the relay must not. - just rs _wants-uring "$relay" \ - || fail "a moq-relay diff must drive the uring gate: $(just rs _names "$relay")" - hls=$(just rs _select "rs/moq-hls/src/lib.rs") - if just rs _wants-uring "$hls"; then - fail "a moq-hls diff must not drive the uring gate: $(just rs _names "$hls")" - fi - - tokio=$(just rs _select "rs/moq-tokio/src/lib.rs") - just rs _wants-tokio-features "$tokio" \ - || fail "a moq-tokio diff must drive its feature matrix: $(just rs _names "$tokio")" - if just rs _wants-tokio-features "$relay"; then - fail "a moq-relay-only selection must not drive the moq-tokio feature matrix" - fi - - # A non-crate directory under rs/ seeds a name that is not a package. It must - # select nothing rather than reach cargo as a bogus spec. - [[ -z "$(just rs _select "rs/scripts/package-binary.sh")" ]] \ - || fail "a non-crate seed must select nothing" - - # The contract script is that same kind of seed, so selection stays empty. - # The gate still has to run it: this is the diff that can break the commands. - [[ -z "$(just rs _select "rs/scripts/media-features.sh")" ]] \ - || fail "the media contract script must not select a crate" - just rs _wants-media-features "" "rs/scripts/media-features.sh" \ - || fail "a media-features.sh diff must drive the contract" - if just rs _wants-media-features "" "rs/scripts/package-binary.sh"; then - fail "an unrelated script must not drive the media contract" - fi - video=$(just rs _select "rs/moq-video/src/lib.rs") - just rs _wants-media-features "$video" "" \ - || fail "a moq-video diff must drive the media contract: $(just rs _names "$video")" - just rs _wants-media-features "--package file:///tmp/rs/moq-transcode#0" "" \ - || fail "moq-transcode alone must drive the media contract" - if just rs _wants-media-features "$relay" ""; then - fail "a moq-relay diff must not drive the media contract: $(just rs _names "$relay")" - fi - - # Several seeds select all of them, not just the first. - pair=$(just rs _names "$(just rs _select $'rs/moq-auth/src/lib.rs\nrs/moq-hls/src/lib.rs')") - grep -qw moq-auth <<< "$pair" && grep -qw moq-hls <<< "$pair" \ - || fail "a two-crate diff must select both: $pair" - - # A crate nested inside another crate's directory is covered by its parent's - # seed. Selecting it directly would compile libFuzzer on every moq-net diff, - # and its directory name is not its crate name, so it would also mis-report. - fuzz=$(just rs _names "$(just rs _select "rs/moq-net/fuzz/fuzz_targets/lite.rs")") - grep -qw moq-net <<< "$fuzz" || fail "a fuzz diff must select moq-net: $fuzz" - ! grep -qw fuzz <<< "$fuzz" || fail "a fuzz diff must not select the harness: $fuzz" - - echo "rs: selection ok" - -# Published crates need the current workspace version as the lower bound for -# internal dependencies. The workspace only tests that version set, and a looser -# bound lets package verification combine incompatible older releases. -# -# Internal dev-dependencies must be path-only. `cargo publish` keeps a versioned -# dev-dependency and requires it on crates.io, but release-plz orders publishes -# by normal dependencies only, so a bumped dev-dependency is often not there yet. -[private] -_publish-test: - #!/usr/bin/env bash - set -euo pipefail - - metadata=$(cargo metadata --locked --format-version 1 --no-deps) - stale=$(jq -r ' - .packages as $packages - | $packages[] as $package - | select($package.publish != []) - | $package.dependencies[] - | select(.source == null and .path != null and .kind != "dev") - | . as $dependency - | ($packages[] | select(.name == $dependency.name)) as $current - | select($dependency.req != ("^" + $current.version)) - | "\($package.name): \($dependency.name) requires \($dependency.req), current is ^\($current.version)" - ' <<< "$metadata") - - if [[ -n "$stale" ]]; then - echo "rs: published workspace dependency lower bounds are stale:" >&2 - echo "$stale" >&2 - exit 1 - fi - - versioned=$(jq -r ' - .packages[] - | select(.publish != []) - | .name as $package - | .dependencies[] - | select(.source == null and .path != null and .kind == "dev" and .req != "*") - | "\($package): \(.name) \(.req)" - ' <<< "$metadata") - - if [[ -n "$versioned" ]]; then - echo "rs: published crates have versioned workspace dev-dependencies:" >&2 - echo "$versioned" >&2 - echo "use { path = \"../\" } instead of { workspace = true }." >&2 - exit 1 - fi - - echo "rs: published workspace dependencies current" - -# Takes a newline-separated list of changed files; skips when no crate is -# affected. The `just rs ...` calls below go through the root justfile's -# `mod rs` because these recipes run from the repo root, where a bare `check` -# would resolve to the root recipe of that name. - -# Like `check`, but only the crates a diff touches plus their dependents. -# TEST=true runs `check-test` in place of `check`. -check-changed $FILES $TEST="false": - #!/usr/bin/env bash - set -euo pipefail - recipe=check - [[ "$TEST" == true ]] && recipe=check-test - packages=$(just rs _select "$FILES") - case "$packages" in - "") echo "rs: no crates affected; skipping." ;; - ALL) just rs "$recipe" --workspace {{ no_fuzz }} ;; - *) echo "rs: checking $(just rs _names "$packages")"; just rs "$recipe" $packages ;; - esac - - if [[ "$packages" == "ALL" ]] || just rs _wants-tokio-features "$packages"; then - just rs tokio-features - fi - - # moq-wasm's crate root is entirely `#![cfg(target_arch = "wasm32")]`, so - # however it got selected the host-target pass above compiled it to nothing - # and saw no errors in it. This wasm32 pass is what actually checks it. - # - # The alternation must list every package `wasm` compiles, not just moq-wasm: - # nothing in the workspace depends on moq-ffi, so an moq-ffi-only diff selects - # only itself and would never reach the gate that exists to cover it. moq-mux - # is selected by its dependents, none of which are moq-wasm. - if [[ "$packages" == "ALL" ]] || just rs _wants-wasm "$packages"; then - just rs wasm - fi - - # moq-video's and moq-audio's device code is behind `capture`, which is off - # by default, so the pass above never compiles it. Only nightly's - # `--all-features` did, and a break there lands on main. With TEST=true the - # lint rides the test build, as in `check-test`. - if [[ "$packages" == "ALL" ]] || just rs _wants-capture "$packages"; then - if [[ "$TEST" == true ]]; then - CARGO_BUILD_WARNINGS=deny just rs capture-test - else - just rs capture - fi - fi - - # moq-cli's media tasks are behind `play`, off by default, so the test pass - # above never runs them. Only `check` tests, so the lint-only CI job skips it. - if [[ "$TEST" == true ]] && { [[ "$packages" == "ALL" ]] || just rs _wants-play "$packages"; }; then - just rs play - fi - - # moq-relay's io_uring listener is behind `io-uring`, off by default, so the - # pass above never compiles it. `check --all` reaches it through `bench - # check`, but only for a diff under `bench/`; a relay change would otherwise - # land on the nightly. - if [[ "$packages" == "ALL" ]] || just rs _wants-uring "$packages"; then - just rs uring-check - fi - - # Each media feature shape compiles on its own (see rs/scripts/media-features.sh), - # about two minutes, so only a diff that reaches the media crates, or the - # contract script itself, pays for it. - if [[ "$packages" == "ALL" ]] || just rs _wants-media-features "$packages" "$FILES"; then - just rs media-features - fi +# Like `test`, but only the crates in a changed-file list plus their dependents. +test-changed $LIST: + sh/rs/select.sh test "$LIST" # Compile moq-tokio by itself at the feature extremes and with each crypto provider. -# Its default-feature build is already part of the ordinary clippy pass above. tokio-features: - #!/usr/bin/env bash - set -euo pipefail - - cargo clippy --locked -p moq-tokio --no-default-features -- -D warnings - cargo clippy --locked -p moq-tokio --all-features -- -D warnings - cargo clippy --locked -p moq-tokio --no-default-features --features noq,aws-lc-rs -- -D warnings - cargo clippy --locked -p moq-tokio --no-default-features --features noq,ring -- -D warnings - - for feature in aws-lc-rs ring tcp uds websocket; do - cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings - done - - want='a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' - if output=$(cargo check --locked -p moq-tokio --no-default-features --features noq 2>&1); then - echo "rs: moq-tokio with only noq should reject its missing crypto provider" >&2 - exit 1 - fi - if ! grep -qF "$want" <<< "$output"; then - echo "rs: moq-tokio with only noq should fail on: $want" >&2 - echo "$output" | grep -E '^error' >&2 || true - exit 1 - fi - -# Same as `fix`, but scoped the same way as `check-changed`. -fix-changed $FILES: - #!/usr/bin/env bash - set -euo pipefail - packages=$(just rs _select "$FILES") - case "$packages" in - "") echo "rs: no crates affected; skipping." ;; - ALL) just rs fix --workspace {{ no_fuzz }} ;; - *) echo "rs: fixing $(just rs _names "$packages")"; just rs fix $packages ;; - esac + sh/rs/tokio-features.sh - # Mirrors `check-changed`'s gate, alternation included: without this a wasm32 - # lint is never auto-fixed, and `check`'s wasm32 clippy pass fails on it later. - if [[ "$packages" == "ALL" ]] || just rs _wants-wasm "$packages"; then - just rs wasm-fix - fi - -# Runs the `#[cfg(windows)]` code past the compiler, which the Linux gate skips -# entirely: moq-video's Media Foundation capture/encode/decode and its D3D11 -# frames, moq-nvenc's Windows loader, and the Winsock paths in moq-tokio. A -# native host rather than a cross-compile, because openh264-sys2's vendored C++ -# needs an MSVC toolchain. platform.yml runs this on every pull request and -# every push to main and dev. -# -# Default features rather than `--all-features`: jemalloc doesn't build on -# MSVC, and the Linux-only features (vaapi, pipewire, io-uring) have nothing to -# compile here. +# Runs the `#[cfg(windows)]` code past the compiler, which the Linux gate +# skips; platform.yml runs it on a Windows runner, scoped like `check-changed` +# so a one-crate PR compiles one crate on these slow runners too. Default +# features, since jemalloc doesn't build on MSVC. # Compile what the branch changed since BASE on a Windows host, or everything with `--all`. windows $BASE="": - just rs _platform "$BASE" + sh/rs/platform.sh "$BASE" # Compile and test the io_uring relay path, which is off the default feature set. # Linux-only, and the tests below the 6.12 kernel floor skip themselves. -# -# `qlog` is a second axis, and the traces are what a debugging run depends on, -# so it gets a separate pass. +[positional-arguments] uring *args: - just rs uring-check {{ args }} - cargo nextest run --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets {{ args }} - cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring,moq-relay/qlog --all-targets {{ args }} -- -D warnings - cargo nextest run --locked -p moq-uring --features qlog --all-targets {{ args }} + just rs uring-check "$@" + cargo nextest run --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets "$@" + cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring,moq-relay/qlog --all-targets "$@" -- -D warnings + cargo nextest run --locked -p moq-uring --features qlog --all-targets "$@" -# Compile the io_uring relay path, off the default feature set. `check-changed` -# runs this whenever moq-relay is selected, so a change that breaks the ring -# listener fails its own PR rather than landing on the nightly. +# Compile the io_uring relay path, off the default feature set. +[positional-arguments] uring-check *args: - cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets {{ args }} -- -D warnings + cargo clippy --locked -p moq-uring -p moq-relay --features moq-relay/io-uring --all-targets "$@" -- -D warnings -# Runs the Apple-gated code past the compiler: moq-video's VideoToolbox -# encode/decode and its ScreenCaptureKit / AVFoundation capture, moq-audio's -# ScreenCaptureKit system audio and TCC permission pre-check, and moq-sock's -# BSD socket-buffer sysctl. platform.yml runs this alongside `windows`. +# The Apple-gated code in moq-video, moq-audio, and moq-sock, which +# platform.yml compiles on a Mac runner alongside `windows`. # Compile what the branch changed since BASE on a macOS host, or everything with `--all`. macos $BASE="": - just rs _platform "$BASE" - -# The shared body of `windows` and `macos`, scoped like `check-changed` so a -# one-crate PR compiles one crate on these slow runners too. moq-gst is dropped -# because it links GStreamer via pkg-config, which the runners don't have. -# -# `moq-cli/play` and `moq-cli/capture` are named explicitly because they are -# off by default, and they are what turns on the device and render code in -# moq-video, moq-audio, and the cli itself. Only when moq-cli is selected, -# since cargo rejects a feature of a package outside the selection; any change -# to the crates those features reach selects moq-cli as a dependent anyway. - -# Compile what the branch changed since BASE plus its dependents, or everything with `--all`. -[private] -_platform $BASE: - #!/usr/bin/env bash - set -euo pipefail - packages=ALL - if [[ "$BASE" != --all ]]; then - files=$(just _changed "$BASE") - # `_select` reads an empty list as everything, so an empty diff stops here. - if [[ -z "$files" ]]; then - echo "rs: nothing changed; skipping." - exit 0 - fi - [[ "$files" == ALL ]] || packages=$(just rs _select "$files") - fi - - features="--features moq-cli/play,moq-cli/capture" - if [[ "$packages" == ALL ]]; then - cargo check --locked --workspace --exclude moq-gst {{ no_fuzz }} --all-targets $features - exit 0 - fi - - packages=$(sed -E 's|--package [^ ]*/moq-gst#[^ ]* ?||' <<< "$packages") - if [[ -z "${packages// }" ]]; then - echo "rs: no crates affected; skipping." - exit 0 - fi - just rs _wants-play "$packages" || features="" - echo "rs: checking $(just rs _names "$packages")" - cargo check --locked $packages --all-targets $features - -# The Linux device code: moq-video's V4L2 and X11 capture and moq-audio's cpal -# capture, all behind the off-by-default `capture` feature. `check-changed` runs -# this whenever either crate is selected. The per-crate `--features` spelling -# keeps it to those two instead of widening what their dependencies compile. -# `--all-features` would also build the bindgen codecs, which nightly covers. + sh/rs/platform.sh "$BASE" # Compile and lint the capture feature, which the default check skips. +[positional-arguments] capture *args: - cargo clippy --locked -p moq-video -p moq-audio --all-targets --features moq-video/capture,moq-audio/capture {{ args }} -- -D warnings + cargo clippy --locked -p moq-video -p moq-audio --all-targets --features moq-video/capture,moq-audio/capture "$@" -- -D warnings -# The capture publishers' tests (the clock fixtures among them) drive synthetic -# devices, so they need no hardware and run with every PR that reaches either crate. +# The capture tests drive synthetic devices, so they need no hardware. # Test the capture feature, which the default test pass skips. +[positional-arguments] capture-test *args: - just rs test -p moq-video -p moq-audio --features moq-video/capture,moq-audio/capture {{ args }} + just rs test -p moq-video -p moq-audio --features moq-video/capture,moq-audio/capture "$@" -# Same idea as `windows`/`macos`, for Android: moq-video's MediaCodec encoder and -# decoder, its `Surface::HardwareBuffer` variant, and the `frame::android` module -# are all `#[cfg(target_os = "android")]`, so a host-target check compiles none of -# them and every error in them stays invisible. -# -# Unlike `windows` and `macos` this needs no special host: the Android targets -# cross-compile from Linux in about half a minute. It needs an NDK, which -# `cargo-ndk` finds through `ANDROID_NDK_HOME` or the standard SDK layout. The -# dev shell provides neither, so android.yml runs this outside Nix on a GitHub -# runner, whose image ships an NDK, for every PR that touches moq-video or -# anything it is built from. -# -# arm64-v8a alone: it is the only ABI anything here ships, and the code is not -# ABI-specific, so the other two would compile the same source twice more. -# -# `check` rather than a `-D warnings` clippy, matching `windows` and `macos`: -# the capture module's plumbing is dead on a target with no capture backend, so -# a denying lint would fail on code this recipe is not here to police. +# moq-video's MediaCodec code is `#[cfg(target_os = "android")]`. Needs an NDK, +# which the dev shell lacks, so android.yml runs it outside Nix. `check` rather +# than a denying clippy: the capture plumbing is dead on a target without a +# capture backend. # Compile the Android backends, which the host-target check skips. +[positional-arguments] android *args: - cargo ndk -t arm64-v8a --platform 26 check --locked -p moq-video --all-targets {{ args }} + cargo ndk -t arm64-v8a --platform 26 check --locked -p moq-video --all-targets "$@" -# Same idea as `windows`/`macos`, for the browser: moq-wasm's whole crate root is -# `#![cfg(target_arch = "wasm32")]`, so a host-target `cargo check --workspace` -# compiles an empty crate and every error in it stays invisible. Only a wasm32 -# build sees the code. -# -# Unlike those two this needs no special host, so `ci` runs it on every Rust PR. -# The wasm32 target and the `getrandom`/`web-sys` cfg flags come from the Nix dev -# shell and `.cargo/config.toml`. Not to be confused with the root `just wasm`, -# which builds the shippable `@moq/wasm` package (wasm-bindgen, release profile); -# this is the compile gate. - -# `moq-mux` and `moq-ffi` ride along in the same invocation on purpose: `moq-mux` -# needs `getrandom`'s `wasm_js` backend that only its siblings declare, and feature -# unification only happens within one cargo invocation. Splitting them fails on -# getrandom. `--lib` rather than `--all-targets` because moq-mux's tests use tokio's -# native timers and moq-ffi's drive a native relay; moq-wasm has no test targets, so -# it loses no coverage. +# moq-wasm's crate root is `#![cfg(target_arch = "wasm32")]`, so only a wasm32 +# build sees the code. Not to be confused with `just js wasm`, which builds the +# shippable `@moq/wasm` package. `moq-mux` and `moq-ffi` share the invocation +# because moq-mux needs the getrandom backend only its siblings declare, and +# features unify only within one cargo invocation. # Compile and lint the browser/WASM bindings, which the host-target check skips. +[positional-arguments] wasm *args: - cargo clippy --locked -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib {{ args }} -- -D warnings - -# The `fix` counterpart to `wasm`. `cargo fmt` is scoped rather than `--all` -# because this runs alongside `fix`, which already formats everything else. + cargo clippy --locked -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib "$@" -- -D warnings # Auto-fix the browser/WASM bindings. +[positional-arguments] wasm-fix *args: - cargo clippy --locked --fix --allow-staged --allow-dirty -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib {{ args }} + cargo clippy --locked --fix --allow-staged --allow-dirty -p moq-wasm -p moq-mux -p moq-ffi --target wasm32-unknown-unknown --lib "$@" cargo fmt -p moq-wasm -p moq-mux -p moq-ffi -# Dependency advisories plus the license/ban policy. Time-based rather than -# diff-based: an advisory lands without this repo changing, so a per-PR run both -# fires on PRs that cannot affect it and stays silent on the days nobody pushes. -# Runs nightly instead (.github/workflows/nightly.yml), which also keeps a fresh -# advisory from blocking an unrelated PR mid-review. - -# Audit dependencies against deny.toml. +# Audit dependencies against deny.toml. Nightly: advisories land without a diff. audit: cargo deny check --show-stats @@ -750,109 +157,16 @@ audit: relay-minimal: cargo nextest run --locked -p moq-relay --no-default-features --lib -E 'test(workers_require_a_quic_backend)' -# libmoq's C fixtures (`rs/libmoq/c-tests/*.c`) link `libmoq.a` the way an -# embedder does: an external `cc` against the generated `moq.h`, plus the native -# libraries from `rs/libmoq/native-libs` that cargo can't inject into a link it -# doesn't drive. Each fixture pins part of the C ABI at compile time and -# exercises it at runtime, so it has to run, not just compile. -# -# Outside nextest on purpose. The staticlib link alone takes ~2 minutes, past the -# hang budget in `.config/nextest.toml` once the rest of the workspace builds -# beside it, and `rs/AGENTS.md` says to make a slow test fast, not to raise the -# ceiling. obs.yml runs this on every PR that touches rs/libmoq, right after it -# has built the same debug libmoq.a for the plugin, and nightly.yml covers the -# paths that filter misses. -# -# Unix only: linking a Rust staticlib from MSVC spells the flags and the -# archive (`moq.lib`) differently, and this recipe doesn't model that. +# Outside nextest because the staticlib link alone outlasts its hang budget. # Compile and run libmoq's C fixtures, and the C doc samples, against libmoq.a. Must run on Unix. c-tests: - #!/usr/bin/env bash - set -euo pipefail - cc="${CC:-cc}" - if ! command -v "$cc" >/dev/null; then - echo "no C compiler at '$cc'" >&2 - exit 1 - fi - - cargo build --locked -p libmoq - - # Ask cargo where it put the staticlib and the header rather than assuming - # target/debug: CARGO_TARGET_DIR and a configured build target - # (`target//debug`) both move them, and guessing wrong would link a - # stale copy or nothing. Replays the build above from cache, so it costs a - # process, not a compile. - log=$(cargo build --locked -p libmoq --message-format=json) - lib=$(jq -r 'select(.reason == "compiler-artifact") | .filenames[] | select(endswith("/libmoq.a"))' <<<"$log" | tail -1) - if [ -z "$lib" ]; then - echo "cargo reported no libmoq.a for libmoq" >&2 - exit 1 - fi - profile=$(dirname "$lib") - # rs/libmoq/build.rs writes the header into its OUT_DIR. - out_dir=$(jq -r 'select(.reason == "build-script-executed") | select(.package_id | test("libmoq")) | .out_dir' <<<"$log" | tail -1) - include="$out_dir/include" - if [ -z "$out_dir" ] || [ ! -f "$include/moq.h" ]; then - echo "$include/moq.h missing after 'cargo build --locked -p libmoq'" >&2 - exit 1 - fi + sh/rs/c-tests.sh - # Same list nix/overlay.nix (moq.pc), CMakeLists.txt, and test/interop/interop.sh read: - # `framework:Foo` is a linker framework flag, anything else a plain library. - case "$(uname -s)" in - Darwin) native_libs=rs/libmoq/native-libs/apple.txt ;; - *) native_libs=rs/libmoq/native-libs/linux.txt ;; - esac - libs=() - while read -r entry; do - case "$entry" in - '' | '#'*) continue ;; - framework:*) libs+=(-framework "${entry#framework:}") ;; - *) libs+=("-l$entry") ;; - esac - done <"$native_libs" - - out=$(mktemp -d) - trap 'rm -rf "$out"' EXIT - - # The C docs compile against this header too: each sample beside the inputs - # doc-samples.h declares, and every name the prose cites. A `moq_decode_*` - # wildcard ends in `_` and is skipped. - bash doc/lib/samples.sh c doc/lib/c/index.md >"$out/doc-samples.c" - "$cc" -fsyntax-only -Werror=implicit-function-declaration -include rs/libmoq/c-tests/doc-samples.h -I"$include" "$out/doc-samples.c" - for name in $(grep -oE '\b(moq|MOQ)_[A-Za-z0-9_]*' doc/lib/c/index.md | grep -v '_$' | sort -u); do - if ! grep -qw "$name" "$include/moq.h"; then - echo "doc/lib/c/index.md cites $name, which moq.h does not declare" >&2 - exit 1 - fi - done - - for source in rs/libmoq/c-tests/*.c; do - bin="$out/$(basename "$source" .c)" - "$cc" "$source" -I"$include" -L"$profile" -lmoq "${libs[@]}" -o "$bin" - echo "running $source" - "$bin" - done - -# The feature permutations `check` leaves out. Each is a full workspace compile -# at a feature set that shares almost no artifacts with the default one (~6min -# for the all-features clippy alone), which is why this is nightly rather than -# part of the per-PR gate. -# -# `--all-features` is the only thing that compiles moq-cli's play/capture, the -# moq-video and moq-audio device features, the Linux bindgen codecs (vaapi, -# v4l2), PipeWire, and jemalloc. `--no-default-features` is -# the only thing that compiles the `#[cfg(not(feature = ...))]` arms. -# -# Cargo unifies features across a workspace selection. Check each package alone -# to catch missing dependencies in minimal builds. The nightly sweep adds each -# feature to aws-lc-rs; tokio-features checks provider-free transports, standalone -# providers, and the intentional refusal of rustls QUIC without a provider. -# -# Both carry `--all-targets`. A `#[cfg(test)]` module leans on whatever the -# default features happened to import, so a test that only compiles by accident -# is invisible to a lib-only check. +# The feature permutations `check` leaves out, nightly. `--all-features` is the +# only thing that compiles moq-cli's play/capture, the device features, the +# Linux bindgen codecs, PipeWire, and jemalloc; `--no-default-features` the only +# thing that compiles the `#[cfg(not(feature = ...))]` arms. # Compile the workspace at the feature extremes. features: @@ -871,258 +185,120 @@ features: # Compile the media crates at supported feature boundaries and prove expensive # native/render dependencies are absent when their features are excluded. media-features: - rs/scripts/media-features.sh + sh/rs/media-features.sh -# nextest, not `cargo test`, so a wedged test is killed instead of pinning a core -# until someone notices (`.config/nextest.toml` sets the timeout). The trade is -# doctests, which nextest does not execute; `just rs doctest` covers those. -# -# Takes cargo arguments, so `just rs test -p moq-net` works. `test-changed` is -# the diff-scoped variant that `just ci test` dispatches to. -# -# clippy-driver as the compiler wrapper is rustc plus the lint passes, so this -# one compilation both builds the test binaries and reports clippy findings. -# `cargo clippy` can't be reused here: it asks cargo for metadata only, and a -# metadata build and a codegen build have different fingerprints, so running -# both compiles the dependency tree twice. Linting here is free, which is what -# lets `check-test` (and so the root `just check`) compile Rust once. -# -# RUSTC_WORKSPACE_WRAPPER, not RUSTC_WRAPPER, for two reasons. Cargo folds it -# into the fingerprint, so a crate already built by a plain `cargo test` is -# rebuilt through clippy rather than silently reported as fresh with no -# findings. And it applies to workspace crates only, so dependencies keep the -# artifacts they share with plain cargo instead of duplicating the whole tree. -# It is also what `cargo clippy` sets internally. -# -# Findings stay warnings rather than errors. `check-test` is the gate (it sets -# `build.warnings=deny`), and a hard failure here would block running the tests -# over an unused import while you are mid-edit. -# -# No RUSTFLAGS: it is part of cargo's fingerprint too, so setting it here would -# split the artifact cache against every other cargo invocation, including CI's -# warm cache. Skips the wrapper entirely when clippy-driver is absent. +# nextest kills a wedged test (`.config/nextest.toml` sets the timeout), but +# skips doctests; `just rs doctest` covers those. clippy-driver as the workspace +# wrapper reports lint findings from the same compile, as warnings: `check-test` +# is the gate. It is part of cargo's fingerprint, and RUSTFLAGS would split the +# artifact cache, which is why neither is set any other way. # Run the test suite, reporting clippy findings from the same compile. +[positional-arguments] test *args: - RUSTC_WORKSPACE_WRAPPER="$(command -v clippy-driver || true)" cargo nextest run --locked --all-targets {{ args }} + RUSTC_WORKSPACE_WRAPPER="$(command -v clippy-driver || true)" cargo nextest run --locked --all-targets "$@" -# Exercise the Vulkan producer -> CUDA image contract, the GPU color conversion -# and resize, and NVENC encoding of the result on Linux/NVIDIA hardware. The -# driver directory is added explicitly because the Nix shell's dynamic loader -# path does not include Ubuntu's host driver directory. +# Exercise Vulkan -> CUDA -> NVENC on Linux/NVIDIA hardware. vulkan-cuda: - #!/usr/bin/env bash - set -euo pipefail - driver=$(/usr/sbin/ldconfig -p | awk '/libcuda\.so\.1/{print $NF; exit}') - if [[ -z "$driver" ]]; then - echo "libcuda.so.1 is not installed" >&2 - exit 1 - fi - driver_dir=$(dirname "$driver") - export LD_LIBRARY_PATH="$driver_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" - cargo nextest run --locked -p moq-video --run-ignored only -E 'test(/^frame::(vulkan|cuda)::tests::vulkan_cuda_/)' - -# Same selection as `check-changed`: compiling a crate's test binaries is the -# expensive half of a test run, so scoping matters more here than anywhere else. -# -# `--no-tests=pass` only on the scoped branch. A selection can legitimately hold -# nothing testable (moq-wasm has no host-target tests, since its crate root is -# `#![cfg(target_arch = "wasm32")]`), and nextest exits 4 on that by default. -# The unscoped branch keeps the default, where finding no tests really is wrong. - -# Like `test`, but only the crates a diff touches plus their dependents. -test-changed $FILES: - #!/usr/bin/env bash - set -euo pipefail - packages=$(just rs _select "$FILES") - case "$packages" in - "") echo "rs: no crates affected; skipping tests." ;; - ALL) just rs test --workspace {{ no_fuzz }} ;; - *) echo "rs: testing $(just rs _names "$packages")"; just rs test --no-tests=pass $packages ;; - esac - - # Mirrors `check-changed`: the capture tests are off the default feature set. - if [[ "$packages" == "ALL" ]] || just rs _wants-capture "$packages"; then - just rs capture-test - fi - - if [[ "$packages" == "ALL" ]] || just rs _wants-play "$packages"; then - just rs play - fi + sh/rs/vulkan-cuda.sh # The media tasks run against a recorder rather than a device (see -# `rs/moq-cli/src/play/fake.rs`), so this needs no speaker or display, only the -# device crates' build dependencies, which the dev shell carries. +# `rs/moq-cli/src/play/fake.rs`), so this needs no speaker or display. # Test moq-cli's `play` feature, which default features leave out. +[positional-arguments] play *args: - just rs test -p moq-cli --features play {{ args }} + just rs test -p moq-cli --features play "$@" # Compile and run the `/// ```` examples, which nextest skips. +[positional-arguments] doctest *args: - cargo test --locked --doc {{ args }} + cargo test --locked --doc "$@" + +# `--cfg loom` swaps kio's primitives for loom's, so it rebuilds the world and +# stays out of `check`. `--release` because a model check runs the body once +# per interleaving (411s -> 51s); the assertions are plain `assert!`. kio's +# models search every interleaving; moq-net's cap preemptions (see +# `tests/loom.rs`, `LOOM_MAX_PREEMPTIONS`). kio's own unit tests are +# `cfg(not(loom))`. # Permutation-test the concurrent handoffs with loom. -# -# `--cfg loom` swaps kio's Mutex/atomics for loom's instrumented ones, so it -# rebuilds the world and can't share artifacts with a normal `cargo test`, so -# it stays separate from `check`/`ci`. -# -# `--release` because a model check runs the body once per interleaving, so the -# optimizer pays for itself many times over: 411s -> 51s across the two suites. -# Our assertions are plain `assert!`, not `debug_assert!`, so nothing is -# compiled out (verified by re-running the mutation check in release). -# -# kio's models search every interleaving. moq-net's compose far more operations -# per path, so they cap preemptions (see `tests/loom.rs`) to stop the search -# growing exponentially with each lock the model layer adds. Set -# `LOOM_MAX_PREEMPTIONS` to search deeper. -# -# kio's own unit tests are `cfg(not(loom))`: they'd construct loom primitives -# outside `loom::model`, which panics. +[positional-arguments] loom *args: - RUSTFLAGS="--cfg loom" cargo test --locked --release -p kio --lib loom:: {{ args }} - RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test loom {{ args }} - -# Coverage-guided fuzzing of the moq-net wire codecs (libFuzzer, via cargo-fuzz). -# -# The one recipe that steps off the pinned toolchain: libFuzzer needs `-Zsanitizer`, -# which is nightly-only, so this stays out of `check` and `test` the same way `loom` -# does. What CI gets instead is the replay: the target bodies live in moq-net's hidden -# `fuzz` module, and its own tests run the generated corpus plus everything committed -# under `fuzz/regressions/` on stable. A crash found here becomes a regression test -# nobody has to install cargo-fuzz to run. -# -# Seeds are regenerated first so the corpus follows the dispatch rather than a stale -# run, and libFuzzer writes what it discovers to the FIRST corpus directory, which is -# why the generated seeds are passed second and stay read-only. -# -# TARGET is lite, announce, ietf, varint, path, or pattern. Arguments pass through, so -# `just rs fuzz lite -- -max_total_time=300` bounds a run. See rs/moq-net/fuzz/README.md. -fuzz target *args: - #!/usr/bin/env bash - set -euo pipefail - - if ! command -v cargo-fuzz > /dev/null 2>&1; then - echo "rs: cargo-fuzz is missing: cargo install --locked cargo-fuzz" >&2 - exit 1 - fi - - if ! rustup run nightly rustc --version > /dev/null 2>&1; then - echo "rs: the nightly toolchain is missing: rustup toolchain install nightly" >&2 - exit 1 - fi - - cargo run --locked -q -p moq-net --features fuzz --example fuzz-seeds -- rs/moq-net/fuzz/seeds - mkdir -p rs/moq-net/fuzz/corpus/{{ target }} + RUSTFLAGS="--cfg loom" cargo test --locked --release -p kio --lib loom:: "$@" + RUSTFLAGS="--cfg loom" cargo test --locked --release -p moq-net --test loom "$@" - # Nightly by PATH rather than by `cargo +nightly`: the dev shell puts a - # Nix-provided cargo ahead of the rustup shim, so the `+` form is not - # understood, and cargo-fuzz shells out to a bare `cargo` and `rustc` anyway. - # Both have to be the nightly ones or the sanitizer flags are rejected. - export PATH="$(rustup run nightly rustc --print sysroot)/bin:$PATH" +# Needs nightly and cargo-fuzz; CI replays the corpus on stable through +# moq-net's own tests instead. See rs/moq-net/fuzz/README.md. - cargo fuzz run --fuzz-dir rs/moq-net/fuzz {{ target }} \ - rs/moq-net/fuzz/corpus/{{ target }} rs/moq-net/fuzz/seeds/{{ target }} {{ args }} +# Fuzz the moq-net wire codecs: `just rs fuzz lite -- -max_total_time=300`. +[positional-arguments] +fuzz target *args: + sh/rs/fuzz.sh "$@" +# Build the default workspace members. build: cargo build --locked -# Run a moq-bench preset against a relay, e.g. `just rs bench chat https://relay.example.com`. -# Extra args pass through, so `--connections 500 --output stats.jsonl` work as-is. -bench preset url *args: - cargo run --release -p moq-bench -- --file 'rs/moq-bench/config/{{ preset }}.toml' --connect '{{ url }}' {{ args }} - -# Measure the Linux Tokio/epoll UDP syscall path. Extra args pass to Criterion, -# e.g. `just rs bench-udp-tokio --quick`. -bench-udp-tokio *args: - cargo bench -p moq-uring --bench udp_tokio -- {{ args }} - -# Measure the Linux io_uring UDP completion path with multishot recvmsg. -bench-udp-uring *args: - cargo bench -p moq-uring --bench udp_uring -- {{ args }} - # Run the comparable epoll and io_uring UDP matrices. +[positional-arguments] bench-udp *args: - cargo bench -p moq-uring --bench udp_tokio -- {{ args }} - cargo bench -p moq-uring --bench udp_uring -- {{ args }} + cargo bench -p moq-uring --bench udp_tokio -- "$@" + cargo bench -p moq-uring --bench udp_uring -- "$@" # Measure a noq echo through the io_uring worker, ablating receive # batching, GRO, and GSO one at a time. Linux 6.12+ (skips loudly otherwise). +[positional-arguments] bench-echo *args: - cargo bench -p moq-uring --bench echo_noq -- {{ args }} + cargo bench -p moq-uring --bench echo_noq -- "$@" # Measure a full moq-lite session (publisher to subscriber) through the # io_uring worker, same ablations and per-iteration unit as bench-echo. # Linux 6.12+ (skips loudly otherwise). +[positional-arguments] bench-session *args: - cargo bench -p moq-uring --bench session_lite -- {{ args }} - -# Sample a process's CPU/memory/context switches on this host (run it where the -# relay runs; it only reads /proc). E.g. `just rs bench-host --name moq-relay`. -bench-host *args: - cargo run --release -p moq-bench --bin moq-bench-host -- {{ args }} - -# Remove the Rust target directory. -clean: - cargo clean - -# Check semver compatibility against crates.io (default-members only). -semver: - cargo semver-checks check-release - -# Update versions and changelogs via release-plz. -bump: - release-plz update + cargo bench -p moq-uring --bench session_lite -- "$@" # Create release PRs and publish crates via release-plz. release: release-plz release-pr --git-token "$GITHUB_TOKEN" release-plz release --git-token "$GITHUB_TOKEN" -update: - cargo update - cargo upgrade --incompatible - -# Build a .deb or .rpm for one of the Rust binaries locally. nfpm comes -# from the flake's dev shell (`nix develop`). For .rpm produced this way, -# the linkage matches the host's glibc; CI uses an AlmaLinux 9 container -# to produce broadly compatible artifacts instead. -# -# Examples: -# just rs package moq-relay deb - -# just rs package moq-cli rpm +# Build a .deb or .rpm locally: `just rs package moq-relay deb`. package crate packager: - #!/usr/bin/env bash - set -euo pipefail - case "{{ crate }}" in - moq-relay) bin=moq-relay ;; - moq-cli) bin=moq ;; - *) echo "Unknown crate: {{ crate }} (use moq-relay or moq-cli)" >&2; exit 1 ;; - esac - case "{{ packager }}" in - deb) - if command -v dpkg >/dev/null 2>&1; then - arch=$(dpkg --print-architecture) - else - case "$(uname -m)" in - x86_64) arch=amd64 ;; - aarch64|arm64) arch=arm64 ;; - *) echo "Cannot infer deb arch from host $(uname -m)" >&2; exit 1 ;; - esac - fi - ;; - rpm) arch=$(uname -m) ;; - *) echo "Unknown packager: {{ packager }} (use deb or rpm)" >&2; exit 1 ;; - esac - version=$(grep -m1 '^version' rs/{{ crate }}/Cargo.toml | sed 's/.*"\(.*\)".*/\1/') - cargo build --locked --release -p {{ crate }} - mkdir -p dist - VERSION="$version" ARCH="$arch" BINARY_PATH="target/release/$bin" \ - rs/scripts/package-nfpm.sh packaging/{{ crate }}/nfpm.yaml {{ packager }} dist/ - if [[ "{{ packager }}" == deb && -f "packaging/{{ crate }}/transition.yaml" ]]; then - VERSION="$version" ARCH="$arch" \ - rs/scripts/package-nfpm.sh packaging/{{ crate }}/transition.yaml deb dist/ - fi - ls -1 dist/ + sh/rs/package.sh {{ crate }} {{ packager }} + +# Build and package a workspace binary as a release tarball. +[positional-arguments] +package-binary *args: + sh/rs/package-binary.sh "$@" + +# Build and package a workspace binary as a Windows .zip. +[positional-arguments] +package-windows *args: + sh/rs/package-windows.sh "$@" + +# Render an nfpm config from the environment and build a .deb or .rpm: CONFIG PACKAGER OUTPUT. +[positional-arguments] +package-nfpm *args: + sh/rs/package-nfpm.sh "$@" + +# Build and package the moq-ffi native libraries, or only its bindings. +[positional-arguments] +package-ffi *args: + rs/moq-ffi/build.sh "$@" + +# Build and package libmoq. +[positional-arguments] +package-libmoq *args: + rs/libmoq/build.sh "$@" + +# Build and package the moq-gst GStreamer plugin. +[positional-arguments] +package-gst *args: + rs/moq-gst/build.sh "$@" + +# Load a packaged moq-gst plugin into the host's GStreamer and find its elements. +[positional-arguments] +gst-smoke *args: + rs/moq-gst/smoke.sh "$@" diff --git a/rs/moq-gst/package.sh b/rs/moq-gst/package.sh index b48f392eb20..85ff0731b67 100755 --- a/rs/moq-gst/package.sh +++ b/rs/moq-gst/package.sh @@ -145,7 +145,7 @@ mkdir -p "$OUTPUT_DIR" echo ">> Running nfpm ($PACKAGER)..." export VERSION ARCH="$PKG_ARCH" PKG_NAME PLUGIN_PATH="$BUILT_SO" PLUGIN_DIR -"$WORKSPACE_DIR/rs/scripts/package-nfpm.sh" \ +"$WORKSPACE_DIR/sh/rs/package-nfpm.sh" \ "$WORKSPACE_DIR/packaging/moq-gst/nfpm.yaml" "$PACKAGER" "$OUTPUT_DIR/" echo ">> Done. Artifacts in: $OUTPUT_DIR" diff --git a/rs/moq-gst/scrub.sh b/rs/moq-gst/scrub.sh index 5d4699deee1..09ec56ad34a 100755 --- a/rs/moq-gst/scrub.sh +++ b/rs/moq-gst/scrub.sh @@ -40,7 +40,7 @@ scrub_macos() { # Framework path covers the official .pkg installer, and /usr/lib lets # dyld resolve system libs (libiconv, libc++) via the dyld_shared_cache at # @rpath substitution time. - "$SCRIPT_DIR/../scripts/scrub-macho.sh" "$dylib" \ + "$SCRIPT_DIR/../../sh/rs/scrub-macho.sh" "$dylib" \ /opt/homebrew/lib \ /Library/Frameworks/GStreamer.framework/Libraries \ /usr/lib diff --git a/rs/moq-wasm/README.md b/rs/moq-wasm/README.md index 7a0609c710b..34a90b2ccc5 100644 --- a/rs/moq-wasm/README.md +++ b/rs/moq-wasm/README.md @@ -7,7 +7,7 @@ wire implementation in `@moq/net` (~10k LOC) with the canonical Rust one, so the protocol lives in exactly one place. This crate is the Rust half; the generated JS package is -[`@moq/wasm`](../../js/wasm) (`just wasm` builds it). It is **not** the same as +[`@moq/wasm`](../../js/wasm) (`just js wasm` builds it). It is **not** the same as `moq-ffi`: that crate uses UniFFI, which targets the C ABI (Kotlin/Swift/Python/ Go). Browsers need `wasm-bindgen`, so this is a separate sibling crate. (For *React Native* JS, `uniffi-bindgen-react-native` can reuse `moq-ffi` directly; @@ -66,7 +66,7 @@ dependency is commented out in `Cargo.toml` until then. ## Building -`just wasm` (from the repo root) does everything: builds for wasm and runs +`just js wasm` (from the repo root) does everything: builds for wasm and runs `wasm-bindgen` (web target) into `js/wasm/dist`. The wasm target, the cfg flags (`getrandom` wasm backend + web-sys unstable WebTransport APIs), and the `wasm-bindgen-cli` tool come from `.cargo/config.toml` and the Nix dev shell. diff --git a/rs/scripts/doc-names.jq b/rs/scripts/doc-names.jq deleted file mode 100644 index 5dfad17c57c..00000000000 --- a/rs/scripts/doc-names.jq +++ /dev/null @@ -1,31 +0,0 @@ -# Find workspace targets that would render to the same `target/doc/`. -# -# `cargo doc` writes each documented target to a directory named after the target with -# dashes folded to underscores, so two targets that differ only there collide: both -# rustdoc processes race to clean it, and the warning escalates to a hard error -# depending on which wins. -# -# Reads `cargo metadata --no-deps` and prints one line per colliding group. -[ .packages[] - | . as $pkg - # Cargo skips a binary that would land on the same directory as a library in its own - # package, documenting only the library, so such a pair never actually collides. - # Compared after folding, matching cargo: a `foo_bar` lib beside a `foo-bar` bin is - # suppressed too (verified against cargo 1.95). Only cross-package pairs collide. - | [ $pkg.targets[] - | select([.kind[]] | any(. == "lib" or . == "rlib" or . == "dylib" - or . == "cdylib" or . == "staticlib" or . == "proc-macro")) - | (.name | gsub("-"; "_")) - ] as $libs - | $pkg.targets[] - # `doc = false` in the manifest, which is how a reported pair gets resolved. - | select(.doc) - | select([.kind[]] | any(. == "lib" or . == "rlib" or . == "dylib" - or . == "cdylib" or . == "staticlib" or . == "proc-macro" or . == "bin")) - | select((([.kind[]] | any(. == "bin")) and ((.name | gsub("-"; "_")) | IN($libs[]))) | not) - | { dir: (.name | gsub("-"; "_")), package: $pkg.name, target: .name, kind: .kind[0] } -] -| group_by(.dir) -| map(select(length > 1)) -| .[] -| " target/doc/\(.[0].dir) <- " + (map("\(.package) [\(.kind)] \(.target)") | join(" and ")) diff --git a/rs/scripts/package-nfpm.test.sh b/rs/scripts/package-nfpm.test.sh deleted file mode 100755 index 5582669426e..00000000000 --- a/rs/scripts/package-nfpm.test.sh +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env bash -set -euo pipefail - -if ! command -v envsubst >/dev/null 2>&1; then - exit 0 -fi - -scratch=$(mktemp -d) -trap 'rm -rf "$scratch"' EXIT -mkdir -p "$scratch/bin" "$scratch/capture" - -cat >"$scratch/bin/nfpm" <<'SH' -#!/usr/bin/env bash -set -euo pipefail - -while (($#)); do - case "$1" in - --config) - config=$2 - shift 2 - ;; - --packager) - printf '%s\n' "$2" >"$NFPM_CAPTURE/packager" - shift 2 - ;; - --target) - printf '%s\n' "$2" >"$NFPM_CAPTURE/target" - shift 2 - ;; - *) shift ;; - esac -done - -cp "$config" "$NFPM_CAPTURE/config.yaml" -SH -chmod +x "$scratch/bin/nfpm" - -cat >"$scratch/nfpm.yaml" <<'YAML' -name: test -version: ${VERSION} -arch: ${ARCH} -contents: - - src: ${BINARY_PATH} - dst: /usr/bin/test -YAML - -PATH="$scratch/bin:$PATH" \ - NFPM_CAPTURE="$scratch/capture" \ - VERSION=1.2.3 \ - ARCH=amd64 \ - BINARY_PATH=target/release/test \ - rs/scripts/package-nfpm.sh "$scratch/nfpm.yaml" deb "$scratch/dist" - -grep -qx 'version: 1.2.3' "$scratch/capture/config.yaml" -grep -qx 'arch: amd64' "$scratch/capture/config.yaml" -grep -qx ' - src: target/release/test' "$scratch/capture/config.yaml" -grep -qx 'deb' "$scratch/capture/packager" -grep -qx "$scratch/dist" "$scratch/capture/target" - -if PATH="$scratch/bin:$PATH" \ - NFPM_CAPTURE="$scratch/capture" \ - VERSION=1.2.3 \ - ARCH=amd64 \ - rs/scripts/package-nfpm.sh "$scratch/nfpm.yaml" deb "$scratch/dist" 2>"$scratch/missing.log"; then - echo "package-nfpm accepted a missing BINARY_PATH" >&2 - exit 1 -fi -grep -qx "Missing environment variable in $scratch/nfpm.yaml: BINARY_PATH" "$scratch/missing.log" diff --git a/sh/changed.sh b/sh/changed.sh new file mode 100755 index 00000000000..287b950ac30 --- /dev/null +++ b/sh/changed.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Print the paths the branch changed since BASE, one per line. +# +# Usage: sh/changed.sh [BASE] +# +# BASE: the argument, then $GITHUB_BASE_REF (a PR checkout has no upstream), +# then the upstream, then origin/main. `git push -u` points the upstream at the +# branch's own remote copy, which would diff HEAD against itself, so that case +# falls through to origin/main. +set -euo pipefail + +base=${1:-} + +cd "$(git rev-parse --show-toplevel)" + +if [[ -z "$base" && -n "${GITHUB_BASE_REF:-}" ]]; then + base="origin/$GITHUB_BASE_REF" +fi +if [[ -z "$base" ]]; then + base=$(git rev-parse --abbrev-ref '@{upstream}' 2>/dev/null || true) + if [[ -z "$base" || "$base" == */"$(git branch --show-current)" ]]; then + base=origin/main + fi +fi +merge_base=$(git merge-base "$base" HEAD) || { + echo "error: cannot resolve merge-base against $base (is full history fetched?)" >&2 + exit 1 +} +echo "changed: base $base" >&2 + +# Untracked files count too: a brand new crate or module is the whole change. +{ + git diff --name-only "$merge_base" + git ls-files --others --exclude-standard +} | sort -u diff --git a/sh/clean.sh b/sh/clean.sh new file mode 100755 index 00000000000..24f495cafe7 --- /dev/null +++ b/sh/clean.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Delete build artifacts and caches: sh/clean.sh [here|all] +# +# Only this checkout by default. Agent worktrees each carry their own +# artifacts, and another agent is usually building in one: `cargo clean` under +# a running build fails it, and there is no telling a finished worktree from a +# busy one from out here. `all` is the explicit opt-in for an idle machine. +# +# Source is never touched, dirty or untracked: this deletes build output, not +# work. Nothing here reaches a machine-wide store (Nix, cargo/bun/uv home +# caches), because those are shared with every other checkout. +set -euo pipefail + +scope=${1:-here} +case "$scope" in + here | all) ;; + *) + echo "usage: sh/clean.sh [here|all]" >&2 + exit 2 + ;; +esac + +cd "$(git rev-parse --show-toplevel)" + +# Rust. +cargo clean + +# JS: the bun workspace spans the repo, so sweep from the root. +find . -name .claude -prune -o \ + -type d \( -name node_modules -o -name dist -o -name out -o -name pkg \) \ + -prune -exec rm -rf {} + +find . -name .claude -prune -o -type f -name '*.tsbuildinfo' -exec rm -f {} + + +# Python: the venv, release dist, bytecode, and the bindings maturin drops in. +rm -rf .venv py/.venv py/dist py/moq-ffi/moq_ffi/_uniffi +find py -name .claude -prune -o -type d -name __pycache__ -prune -exec rm -rf {} + +find py -name .claude -prune -o -type f -name '*.pyc' -exec rm -f {} + + +# Kotlin: gradle output plus the bindings and native libs sh/kt generates. +find kt -type d \( -name build -o -name .gradle -o -name .kotlin \) -prune -exec rm -rf {} + +rm -rf kt/local.properties \ + kt/moq-ffi/src/jvmAndAndroidMain/kotlin/uniffi \ + kt/moq-ffi/src/jvmMain/resources \ + kt/moq-ffi/src/androidMain/jniLibs + +# Swift: SwiftPM output plus the XCFramework and bindings sh/swift lays down. +rm -rf swift/.build swift/.swiftpm swift/Package.resolved \ + swift/Sources/MoqFFI/Generated.swift swift/MoqFFI.xcframework + +# Go: the gitignored bindings, native libs, and vendoring in go/. +rm -rf go/ffi/moq/moq.go go/ffi/moq/moq.h go/ffi/moq/lib go/ffi/go.sum \ + go/ffi/vendor go/wrapper/go.sum go/wrapper/vendor + +# Dart: tool state and build output. +rm -rf dart/moq/.dart_tool dart/moq_ffi/.dart_tool dart/moq/build dart/moq_ffi/build + +# Rendered drafts and their reference cache. +rm -rf drafts/draft-*.xml drafts/draft-*.txt drafts/draft-*.html drafts/.refcache + +# Caches no one language owns: nix build result, direnv, wrangler. +rm -rf result .direnv +find . -name .claude -prune -o -type d -name .wrangler -prune -exec rm -rf {} + + +# Worktrees don't nest, so this recurses exactly one level. Tolerate stale +# worktrees on branches that predate this script. +if [[ "$scope" == all ]]; then + for wt in .claude/worktrees/*/; do + [[ -f "${wt}justfile" ]] || continue + echo "==> cleaning ${wt}" + (cd "$wt" && just clean) || echo " (skipped: just clean failed in ${wt})" + done +fi diff --git a/dart/scripts/check.sh b/sh/dart/check.sh similarity index 88% rename from dart/scripts/check.sh rename to sh/dart/check.sh index 5e658e9b187..eaa4f579bef 100755 --- a/dart/scripts/check.sh +++ b/sh/dart/check.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) ACTION=check if [[ "${1:-}" == --fix ]]; then @@ -9,13 +9,6 @@ if [[ "${1:-}" == --fix ]]; then shift fi -# `just check` passes the whole changed-file list, so match the scope anywhere in -# it rather than only at the start. -files="${1:-}" -if [[ -n "$files" ]] && ! grep -qE '(^|[[:space:]])(dart/|rs/moq-ffi/)' <<<"$files"; then - exit 0 -fi - for tool in cargo dart uniffi_bindgen_dart; do if ! command -v "$tool" >/dev/null 2>&1; then echo "dart check: skipping, $tool is not installed" >&2 @@ -58,7 +51,7 @@ done generated=$(mktemp -d) trap 'rm -rf "$generated"' EXIT -"$DART_DIR/scripts/generate.sh" "$generated" +"$DART_DIR/../sh/dart/generate.sh" "$generated" for file in moq.dart uniffi_runtime.dart; do if ! diff -u -B "$DART_DIR/moq_ffi/lib/src/$file" "$generated/$file"; then diff --git a/dart/scripts/generate.sh b/sh/dart/generate.sh similarity index 96% rename from dart/scripts/generate.sh rename to sh/dart/generate.sh index bbbe65ad976..7a2c9c24fc3 100755 --- a/dart/scripts/generate.sh +++ b/sh/dart/generate.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) WORKSPACE_DIR=$(cd "$DART_DIR/.." && pwd) OUTPUT_DIR="${1:-$DART_DIR/moq_ffi/lib/src}" diff --git a/dart/scripts/package.sh b/sh/dart/package.sh similarity index 97% rename from dart/scripts/package.sh rename to sh/dart/package.sh index 21c4d7b135c..cb984776957 100755 --- a/dart/scripts/package.sh +++ b/sh/dart/package.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) PACKAGE="" VERSION="" OUTPUT_DIR="dist" diff --git a/dart/scripts/publish.sh b/sh/dart/publish.sh similarity index 84% rename from dart/scripts/publish.sh rename to sh/dart/publish.sh index 6a6caaa50e8..1348e0ec64b 100755 --- a/dart/scripts/publish.sh +++ b/sh/dart/publish.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd) +DART_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")/../../dart" && pwd) PACKAGE="${1:?usage: publish.sh }" VERSION="${2:?usage: publish.sh }" OUTPUT=$(mktemp -d) trap 'rm -rf "$OUTPUT"' EXIT package_dir=$( - "$DART_DIR/scripts/package.sh" \ + "$DART_DIR/../sh/dart/package.sh" \ --package "$PACKAGE" \ --version "$VERSION" \ --output "$OUTPUT" diff --git a/sh/demo/ca.sh b/sh/demo/ca.sh new file mode 100755 index 00000000000..01cfbd0c057 --- /dev/null +++ b/sh/demo/ca.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Generate the relay cluster's self-signed CA (ca.pem, ca.key, ca.cnf) in the +# current directory, unless it already exists. +# +# Usage: sh/demo/ca.sh, from demo/relay. +set -euo pipefail +umask 077 + +[ -f ca.pem ] && [ -f ca.key ] && exit 0 +rm -f ca.pem ca.key +printf '[req]\ndistinguished_name = req_dn\n[req_dn]\n' >ca.cnf +export OPENSSL_CONF="ca.cnf" +openssl req -x509 -sha256 -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ + -days 365 -subj "/CN=moq cluster CA" \ + -keyout ca.key -out ca.pem diff --git a/sh/demo/cert.sh b/sh/demo/cert.sh new file mode 100755 index 00000000000..916ece90f61 --- /dev/null +++ b/sh/demo/cert.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Issue NAME.crt and NAME.key for localhost from the CA that sh/demo/ca.sh +# generates in the current directory. Chrome requires a <=14 day validity for +# self-signed certs. +# +# Usage: sh/demo/cert.sh NAME, from demo/relay. +set -euo pipefail +umask 077 + +name=${1:?usage: sh/demo/cert.sh NAME} + +export OPENSSL_CONF="ca.cnf" +openssl req -newkey ec -pkeyopt ec_paramgen_curve:prime256v1 -nodes \ + -subj "/CN=$name" \ + -keyout "$name.key" -out "$name.csr" +openssl x509 -req -sha256 -in "$name.csr" \ + -CA ca.pem -CAkey ca.key -CAcreateserial \ + -days 14 -extfile <(printf "subjectAltName=DNS:localhost\n") \ + -out "$name.crt" +rm "$name.csr" diff --git a/sh/demo/download.sh b/sh/demo/download.sh new file mode 100755 index 00000000000..3b1a4f57a51 --- /dev/null +++ b/sh/demo/download.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Download URL to FILE unless FILE already exists. The download lands in a +# temporary file first, so an interrupted one is retried instead of cached. +# +# Usage: sh/demo/download.sh URL FILE +set -euo pipefail + +usage="usage: sh/demo/download.sh URL FILE" +url=${1:?$usage} +file=${2:?$usage} + +[ -f "$file" ] && exit 0 +mkdir -p "$(dirname "$file")" +curl -fsSL -o "$file.tmp" "$url" +mv "$file.tmp" "$file" diff --git a/sh/demo/hls.sh b/sh/demo/hls.sh new file mode 100755 index 00000000000..f2c15aa1f6d --- /dev/null +++ b/sh/demo/hls.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Encode media/NAME.mp4 into a looping HLS stream (1280x720 + 256x144 + audio) +# under media/NAME/, and import it into RELAY as NAME.hang. +# +# Usage: sh/demo/hls.sh NAME RELAY, from demo/pub. +set -euo pipefail + +usage="usage: sh/demo/hls.sh NAME RELAY" +name=${1:?$usage} +relay=${2:?$usage} + +input="media/$name.mp4" +out="media/$name" + +rm -rf "$out" +mkdir -p "$out" + +echo ">>> Generating HLS stream to disk (1280x720 + 256x144)..." + +ffmpeg -hide_banner -loglevel warning -re -stream_loop -1 -i "$input" \ + -filter_complex "[0:v]split=2[v0][v1]; [v0]scale=-2:720[v720]; [v1]scale=-2:144[v144]" \ + -map "[v720]" -map "[v144]" -map 0:a:0 \ + -r 25 -preset veryfast -g 50 -keyint_min 50 -sc_threshold 0 \ + -c:v:0 libx264 -profile:v:0 high -level:v:0 4.1 -pix_fmt:v:0 yuv420p -tag:v:0 avc1 \ + -b:v:0 4M -maxrate:v:0 4.4M -bufsize:v:0 8M \ + -c:v:1 libx264 -profile:v:1 high -level:v:1 4.1 -pix_fmt:v:1 yuv420p -tag:v:1 avc1 \ + -b:v:1 300k -maxrate:v:1 330k -bufsize:v:1 600k \ + -c:a aac -b:a 128k \ + -f hls -hls_time 2 -hls_list_size 6 \ + -hls_flags independent_segments+delete_segments \ + -hls_segment_type fmp4 \ + -master_pl_name master.m3u8 \ + -var_stream_map "v:0,agroup:audio,name:720 v:1,agroup:audio,name:144 a:0,agroup:audio,name:audio" \ + -hls_segment_filename "$out/v%v/segment_%09d.m4s" \ + "$out/v%v/stream.m3u8" & +ffmpeg=$! + +cleanup() { + echo "Shutting down..." + kill "$ffmpeg" 2>/dev/null || true + sleep 0.5 + kill -9 "$ffmpeg" 2>/dev/null || true +} +trap cleanup EXIT +# A signal exits, so the EXIT trap cleans up instead of the script resuming its waits. +trap 'exit 130' INT TERM + +echo ">>> Waiting for HLS playlist generation..." +for _ in {1..30}; do + if [ -f "$out/master.m3u8" ]; then break; fi + sleep 0.5 +done + +if [ ! -f "$out/master.m3u8" ]; then + echo "Error: master.m3u8 not generated in time" >&2 + exit 1 +fi + +echo ">>> Waiting for variant playlists..." +sleep 2 +for _ in {1..20}; do + if [ -f "$out/v720/stream.m3u8" ] || [ -f "$out/v144/stream.m3u8" ] || [ -f "$out/vaudio/stream.m3u8" ]; then + break + fi + sleep 0.5 +done + +echo ">>> Importing HLS via moq-cli" +cargo run --bin moq -- --connect "$relay" --broadcast "$name.hang" import hls "$out/master.m3u8" diff --git a/sh/demo/key.sh b/sh/demo/key.sh new file mode 100755 index 00000000000..1c199dafcbd --- /dev/null +++ b/sh/demo/key.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# Generate the relay's signing key (root.jwk) in the current directory, unless +# it already exists. A new key invalidates every token, so they are removed. +# +# Usage: sh/demo/key.sh, from demo/relay. +set -euo pipefail +umask 077 + +[ -f root.jwk ] && exit 0 +rm -f ./*.jwt +cargo run --bin moq -- auth generate --out root.jwk diff --git a/sh/demo/port.sh b/sh/demo/port.sh new file mode 100755 index 00000000000..814314a7038 --- /dev/null +++ b/sh/demo/port.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash +# Print the first port at/after START that is free on both TCP and UDP. +# +# `lsof` isn't available everywhere (e.g. Git Bash on Windows); without it we +# can't probe, so print START and let the relay surface a bind error if it's +# actually taken. +# +# Usage: sh/demo/port.sh START +set -euo pipefail + +port=${1:?usage: sh/demo/port.sh START} +if command -v lsof >/dev/null 2>&1; then + while lsof -nP -iTCP:"$port" -sTCP:LISTEN -t >/dev/null 2>&1 || + lsof -nP -iUDP:"$port" -t >/dev/null 2>&1; do + port=$((port + 1)) + done +fi +echo "$port" diff --git a/sh/demo/run.sh b/sh/demo/run.sh new file mode 100755 index 00000000000..af9d85555ba --- /dev/null +++ b/sh/demo/run.sh @@ -0,0 +1,24 @@ +#!/usr/bin/env bash +# Run the web demo: a localhost relay, Big Buck Bunny, and the web player. +# +# Picks the first free port at/after 4443 (both UDP for QUIC and TCP for HTTP) +# so multiple worktrees can run `just dev` without colliding. The relay reads +# the port from MOQ_LISTEN/MOQ_WEB_HTTP_LISTEN, overriding localhost.toml. +# +# Usage: sh/demo/run.sh, from demo/ so `just` resolves the demo recipes. +set -euo pipefail + +port=$("$(dirname "$0")/port.sh" 4443) +if [ "$port" != "4443" ]; then + echo ">>> Port 4443 is busy, using $port instead" >&2 +fi + +base="http://localhost:$port" + +# Scope the bind env vars to the relay only. The unified `moq` binary also +# reads MOQ_LISTEN, so exporting it globally would make the `pub` client +# try to start a server (and fail without a TLS cert). +bun run concurrently --kill-others --names rly,bbb,web --prefix-colors auto \ + "MOQ_LISTEN='[::]:$port' MOQ_WEB_HTTP_LISTEN='[::]:$port' just relay" \ + "just wait $base/certificate.sha256 && just pub bbb $base" \ + "just wait $base/certificate.sha256 && just web serve $base" diff --git a/sh/demo/serve-hls.sh b/sh/demo/serve-hls.sh new file mode 100755 index 00000000000..bf5257bc11a --- /dev/null +++ b/sh/demo/serve-hls.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Encode media/NAME.mp4 into a looping HLS stream under media/NAME/ and serve +# it over HTTP on PORT, for testing HLS import. +# +# Usage: sh/demo/serve-hls.sh NAME PORT, from demo/pub. +set -euo pipefail + +usage="usage: sh/demo/serve-hls.sh NAME PORT" +name=${1:?$usage} +port=${2:?$usage} + +input="media/$name.mp4" +out="media/$name" + +rm -rf "$out" +mkdir -p "$out" + +echo ">>> Starting HLS stream generation..." +echo ">>> Master playlist: http://localhost:$port/master.m3u8" + +ffmpeg -loglevel warning -re -stream_loop -1 -i "$input" \ + -map 0:v:0 -map 0:v:0 -map 0:a:0 \ + -r 25 -preset veryfast -g 50 -keyint_min 50 -sc_threshold 0 \ + -c:v:0 libx264 -profile:v:0 high -level:v:0 4.1 -pix_fmt:v:0 yuv420p -tag:v:0 avc1 -bsf:v:0 dump_extra -b:v:0 4M -vf:0 "scale=1920:-2" \ + -c:v:1 libx264 -profile:v:1 high -level:v:1 4.1 -pix_fmt:v:1 yuv420p -tag:v:1 avc1 -bsf:v:1 dump_extra -b:v:1 300k -vf:1 "scale=256:-2" \ + -c:a aac -b:a 128k \ + -f hls \ + -hls_time 2 -hls_list_size 12 \ + -hls_flags independent_segments+delete_segments \ + -hls_segment_type fmp4 \ + -master_pl_name master.m3u8 \ + -var_stream_map "v:0,agroup:audio v:1,agroup:audio a:0,agroup:audio" \ + -hls_segment_filename "$out/v%v/segment_%09d.m4s" \ + "$out/v%v/stream.m3u8" & +ffmpeg=$! + +trap 'echo "Shutting down..."; exit 0' INT TERM +trap 'kill "$ffmpeg" 2>/dev/null || true' EXIT + +sleep 2 +echo ">>> HTTP server: http://localhost:$port/" +cd "$out" +python3 -m http.server "$port" diff --git a/sh/demo/sync.sh b/sh/demo/sync.sh new file mode 100755 index 00000000000..af5f2fb8860 --- /dev/null +++ b/sh/demo/sync.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Upload every file in DIR to the R2 bucket BUCKET, keyed by its file name. +# +# Usage: sh/demo/sync.sh BUCKET DIR +set -euo pipefail + +usage="usage: sh/demo/sync.sh BUCKET DIR" +bucket=${1:?$usage} +dir=${2:?$usage} + +for file in "$dir"/*; do + [ -e "$file" ] || continue + key=$(basename "$file") + echo "Uploading $key..." + bun wrangler r2 object put "$bucket/$key" --file "$file" --remote +done diff --git a/sh/demo/token.sh b/sh/demo/token.sh new file mode 100755 index 00000000000..e85ebc7bace --- /dev/null +++ b/sh/demo/token.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Sign the local development tokens (demo-web.jwt, demo-cli.jwt) with root.jwk +# in the current directory, skipping any that already exist. +# +# Usage: sh/demo/token.sh, from demo/relay. +set -euo pipefail +umask 077 + +if [ ! -f demo-web.jwt ]; then + cargo run --quiet --bin moq -- auth sign --key root.jwk \ + --root demo --subscribe '**' --publish 'me/**' \ + >demo-web.jwt +fi + +if [ ! -f demo-cli.jwt ]; then + cargo run --quiet --bin moq -- auth sign --key root.jwk \ + --root demo --publish '**' \ + >demo-cli.jwt +fi diff --git a/sh/demo/wait.sh b/sh/demo/wait.sh new file mode 100755 index 00000000000..e556127c3a3 --- /dev/null +++ b/sh/demo/wait.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash +# Wait up to ten minutes for URL to respond, e.g. a relay's certificate +# fingerprint once it is listening. +# +# Usage: sh/demo/wait.sh URL +set -euo pipefail + +url=${1:?usage: sh/demo/wait.sh URL} +for _ in $(seq 1 600); do + curl -sf "$url" >/dev/null 2>&1 && exit 0 + sleep 1 +done +echo "timed out waiting for $url" >&2 +exit 1 diff --git a/sh/dispatch.sh b/sh/dispatch.sh new file mode 100755 index 00000000000..09ec5c13fd9 --- /dev/null +++ b/sh/dispatch.sh @@ -0,0 +1,205 @@ +#!/usr/bin/env bash +# Run `just check`, `just fix`, or `just ci check|test` over what the branch +# changed. +# +# Usage: sh/dispatch.sh check|fix|ci-check|ci-test [BASE|--all] +# +# The branch diff is resolved once and matched against the impact map below, +# the one place that says which paths put which module in scope. `check` is +# `ci-check` plus `ci-test`, which CI runs as parallel jobs with MOQ_STRICT=1, +# so there is no second definition of "checked". +set -euo pipefail + +usage="usage: sh/dispatch.sh check|fix|ci-check|ci-test [BASE|--all]" +action=${1:?$usage} +base=${2:-} +case "$action" in + check | fix | ci-check | ci-test) ;; + *) + echo "$usage" >&2 + exit 2 + ;; +esac + +# A host toolchain fails deep in a build instead: its gcc keeps the +# _FORTIFY_SOURCE the dev shell disables, which breaks jemalloc's -Werror +# configure probes. This flake's dev shell sets MOQ_DEV_SHELL. Windows has no +# Nix, so it always runs on the host. +if [[ -z "${MOQ_DEV_SHELL:-}" && -z "${MOQ_ALLOW_HOST:-}" && "$OSTYPE" != msys && "$OSTYPE" != cygwin ]]; then + echo "error: run inside the Nix dev shell ('nix develop' or direnv) so tools match CI" >&2 + echo " or set MOQ_ALLOW_HOST=1 to use the host toolchain anyway" >&2 + exit 1 +fi + +cd "$(git rev-parse --show-toplevel)" + +changed=$(mktemp) +trap 'rm -f "$changed"' EXIT + +all= +if [[ "$base" == --all ]]; then + all=1 +else + sh/changed.sh "$base" >"$changed" + + # These files hold the map and the recipes that call it, and match no + # module, so a change to them would otherwise validate none of it. + if grep -qE '^(justfile|sh/(dispatch|changed)\.sh)$' "$changed"; then + echo "$action: root orchestration changed; running everything." >&2 + all=1 + fi +fi + +# The impact map: a module is in scope when a changed path matches its +# pattern. An empty pattern is a repository-wide lint that runs on every diff. +declare -A scope=( + [js]='^(js/|doc/|drafts/|demo/(boy|web)/|test/interop/clients/js|test/wasm/|sh/js/|sh/rs/stats-docs\.py$|package\.json$|bun\.lock(b)?$|biome\.jsonc$)' + # Workers with lockfiles outside the Bun workspace. + [workers]='^(infra/apt/|infra/rpm/|demo/pub/|sh/js/workers\.sh$)' + # sh/rs/select.sh widens to the whole workspace for inputs every crate shares. + [rs]='^(rs/|sh/rs/|Cargo\.(toml|lock)$|rust-toolchain\.toml$|\.config/nextest\.toml$)' + [bench]='^bench/' + [drafts]='^(drafts/|sh/drafts/|doc/\.vitepress/drafts\.ts$)' + # Quest documents form one graph, so any change validates the whole tree, + # as does a bump of the flake-pinned validator. + [quest]='^(quest/|flake\.lock$)' + # The drill mutations patch Rust source, so a Rust change can move the code + # they target. Only nightly runs the drills; this catches a stale patch. + [drill]='^(rs/|test/drill/)' + # maturin bundles rs/moq-ffi into the moq-ffi wheel, and the other + # bindings generate from it. Each check also compiles its doc samples. + [py]='^(py/|sh/py/|pyproject\.toml$|uv\.lock$|rs/moq-ffi/|doc/lib/py/|doc/lib/samples\.sh$)' + [kt]='^(kt/|sh/kt/|rs/moq-ffi/|doc/lib/kt/|doc/lib/samples\.sh$)' + [swift]='^(swift/|sh/swift/|rs/moq-ffi/|doc/lib/swift/|doc/lib/samples\.sh$)' + [go]='^(go/|sh/go/|rs/moq-ffi/)' + [dart]='^(dart/|sh/dart/|rs/moq-ffi/)' + # The plugin calls libmoq through its generated header, and flake.nix owns + # the libobs headers it compiles against. + [obs_compile]='^(cpp/obs/|sh/obs/|rs/libmoq/|flake\.nix$)' + # `obs check` compares the OBS pinned in buildspec.json, flake.nix, and + # nixpkgs, and the last moves on a flake.lock bump alone. + [obs]='^(cpp/obs/|sh/obs/|flake\.(nix|lock)$)' + [flake]='(^rs/|^Cargo\.(toml|lock)$|^flake\.lock$|\.nix$)' + [markdown]='' + [shell]='' + [toml]='' + [nix]='' + [justfile]='' + [gh]='' +) + +# The tools each module needs. A module missing one is skipped locally, so an +# incomplete toolchain checks less; under MOQ_STRICT (CI) it is an error, up +# front, because there a skip is indistinguishable from a pass. swift needs +# none: it skips off macOS by design, and swift.yml is its real gate. +declare -A tools=( + [js]='bun python3' + [workers]='bun' + [rs]='cargo jq' + [bench]='cargo' + [drafts]='bun kramdown-rfc xml2rfc' + [quest]='quest' + [drill]='' + [py]='uv' + [kt]='cargo gradle java' + [swift]='' + [go]='cargo go uniffi-bindgen-go' + [dart]='cargo dart uniffi_bindgen_dart' + [obs_compile]='cargo jq pkg-config' + [obs]='clang-format cmake gersemi jq' + [flake]='nix' + [markdown]='bun' + [shell]='shellcheck shfmt' + [toml]='taplo' + [nix]='nixfmt' + [justfile]='' + [gh]='actionlint bun' +) + +case "$action" in + check | ci-check) modules=(js workers drafts rs bench quest drill py kt swift go dart obs_compile obs flake markdown shell toml nix justfile gh) ;; + fix) modules=(js rs py dart obs markdown shell toml nix justfile) ;; + ci-test) modules=(js rs py) ;; +esac + +selected=() +missing=() +for module in "${modules[@]}"; do + pattern=${scope[$module]} + if [[ -z "$all" && -n "$pattern" ]] && ! grep -qE "$pattern" "$changed"; then + continue + fi + absent=() + for tool in ${tools[$module]}; do + command -v "$tool" >/dev/null 2>&1 || absent+=("$tool") + done + if ((${#absent[@]} == 0)); then + selected+=("$module") + elif [[ -n "${MOQ_STRICT:-}" ]]; then + missing+=("${absent[@]}") + else + echo "$action: skipping $module; missing ${absent[*]}" >&2 + fi +done + +if ((${#missing[@]})); then + echo "error: MOQ_STRICT is set but these tools are missing: $(printf '%s\n' "${missing[@]}" | sort -u | tr '\n' ' ')" >&2 + echo " run inside 'nix develop', or unset MOQ_STRICT to skip what isn't installed" >&2 + exit 1 +fi + +# rs selects crates from the same list, or the whole workspace. +rs_list=$changed +[[ -z "$all" ]] || rs_list=--all + +# Tracked and new files, skipping whatever .gitignore does (node_modules, target). +files() { + git ls-files -z --cached --others --exclude-standard -- "$@" +} + +# `check` and `ci-check` share every lint; they differ in how rs compiles and +# whether the tests run. +verb=${action#ci-} + +for module in "${selected[@]}"; do + case "$verb:$module" in + check:js) just js check ;; + check:workers) just js workers ;; + # `check` lints Rust through its test build, so the crates compile once; + # `ci-check` uses clippy, which is quicker when the tests run elsewhere. + check:rs) if [[ "$action" == check ]]; then just rs check-test-changed "$rs_list"; else just rs check-changed "$rs_list"; fi ;; + check:bench) cargo check --locked --package moq-relay --package moq-bench --features moq-relay/io-uring ;; + check:quest) quest check ;; + check:drill) just test drill-sensitivity --apply-only ;; + check:obs_compile) just obs compile ;; + check:flake) nix flake check ;; + check:markdown) + bun install --frozen-lockfile + sh/markdown.sh check + ;; + check:toml) RUST_LOG=error taplo format --check ;; + check:nix) files '*.nix' | xargs -0 nixfmt --check ;; + check:justfile) files justfile '*/justfile' | xargs -0 -n1 just --fmt --check --justfile ;; + fix:rs) just rs fix-changed "$rs_list" ;; + fix:markdown) + bun install + sh/markdown.sh fix + ;; + fix:toml) RUST_LOG=error taplo format ;; + fix:nix) files '*.nix' | xargs -0 nixfmt ;; + fix:justfile) files justfile '*/justfile' | xargs -0 -n1 just --fmt --justfile ;; + test:rs) just rs test-changed "$rs_list" ;; + *:shell) sh/shell.sh "$verb" ;; + *) just "$module" "$verb" ;; + esac +done + +# The rest of `check`'s tests. rs ran its own above, and the kt, swift, go, and +# dart checks always run theirs. +if [[ "$action" == check ]]; then + for module in "${selected[@]}"; do + case "$module" in + js | py) just "$module" test ;; + esac + done +fi diff --git a/sh/drafts/build.sh b/sh/drafts/build.sh new file mode 100755 index 00000000000..e050b63a3c1 --- /dev/null +++ b/sh/drafts/build.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Render drafts to .txt and .html: sh/drafts/build.sh [NAME...] +# +# The local editor's copy (gitignored), with the -latest docname as-is; use +# `just drafts publish` for a version. No NAME renders every draft. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)/drafts" + +names=("$@") +if ((${#names[@]} == 0)); then + for f in draft-*.md; do names+=("${f%.md}"); done +fi + +for name in "${names[@]}"; do + kramdown-rfc --v3 <"$name.md" >"$name.xml" + xml2rfc -q --allow-local-file-access --text "$name.xml" -o "$name.txt" + xml2rfc -q --allow-local-file-access --html "$name.xml" -o "$name.html" + echo "built $name.txt and $name.html" +done diff --git a/sh/drafts/check.sh b/sh/drafts/check.sh new file mode 100755 index 00000000000..3b8f884cab5 --- /dev/null +++ b/sh/drafts/check.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Validate that every draft parses and that its generated XML passes validation. +# +# Both halves are needed. kramdown-rfc catches markdown and frontmatter errors, +# but happily emits XML for a cross-reference that points at no such section; +# xml2rfc is what rejects the dangling IDREF. Checking only the first half lets +# a broken `[text](#anchor)` pass here and then fail at `publish` time. +# +# --preptool stops after validation instead of rendering txt/html, and -N keeps +# it off the network (kramdown-rfc already inlined the references), which is +# what makes this a couple of seconds per draft rather than a minute. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" +bun install --frozen-lockfile +cd drafts + +if grep -n "$(printf '\t')" draft-*.md; then + echo "tabs render unpredictably in xml2rfc; indent with spaces" >&2 + exit 1 +fi + +for f in draft-*.md; do + echo "checking $f" + kramdown-rfc --v3 <"$f" >"${f%.md}.xml" + xml2rfc -q --allow-local-file-access --preptool -N "${f%.md}.xml" -o "${f%.md}.prepped.xml" || { + rm -f "${f%.md}.prepped.xml" + exit 1 + } + rm -f "${f%.md}.prepped.xml" +done + +for f in moq-e2ee-*.ts; do + bun tsc --noEmit --skipLibCheck --target esnext --module esnext --moduleResolution bundler --types bun "$f" + bun "$f" +done diff --git a/sh/drafts/publish.sh b/sh/drafts/publish.sh new file mode 100755 index 00000000000..9ef5e8ad5c0 --- /dev/null +++ b/sh/drafts/publish.sh @@ -0,0 +1,44 @@ +#!/usr/bin/env bash +# Submit a new draft version to the IETF datatracker: sh/drafts/publish.sh NAME VERSION EMAIL +# +# The datatracker emails the submitter a confirmation link; the submission is +# not final until that link is clicked. For a brand-new draft (-00) set +# "Replaces" on the confirmation page. +set -euo pipefail + +usage="usage: sh/drafts/publish.sh NAME VERSION EMAIL" +name=${1:?$usage} +version=${2:?$usage} +email=${3:?$usage} + +cd "$(git rev-parse --show-toplevel)/drafts" +case "$version" in + [0-9][0-9]) ;; + *) + echo "version must be two digits, e.g. 05" >&2 + exit 1 + ;; +esac +doc="$name-$version" +if [ ! -f "$name.md" ]; then + echo "no such draft: $name.md" >&2 + exit 1 +fi +echo "Building $doc.xml" +sed "s/$name-latest/$doc/g" "$name.md" | kramdown-rfc --v3 >"$doc.xml" +echo "Submitting $doc.xml to the datatracker as $email" +resp="$(mktemp)" +code="$(curl -sS -o "$resp" -w '%{http_code}' \ + -F "user=$email" -F "xml=@$doc.xml" \ + https://datatracker.ietf.org/api/submission)" +echo "HTTP $code" +cat "$resp" +echo +rm -f "$resp" +case "$code" in + 200 | 201) echo "Submitted. Check $email for the confirmation link." ;; + *) + echo "Submission failed." >&2 + exit 1 + ;; +esac diff --git a/.github/scripts/alert.sh b/sh/gh/alert.sh similarity index 93% rename from .github/scripts/alert.sh rename to sh/gh/alert.sh index 6a795dc47ad..ee95b2d7e39 100755 --- a/.github/scripts/alert.sh +++ b/sh/gh/alert.sh @@ -18,7 +18,7 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -WORKFLOWS_DIR="$(cd "$SCRIPT_DIR/../workflows" && pwd)" +WORKFLOWS_DIR="$(cd "$SCRIPT_DIR/../../.github/workflows" && pwd)" # Post a red embed naming the workflow, what it was reacting to, and a link. # jq builds the body so a commit subject with quotes or newlines can't produce @@ -88,7 +88,7 @@ check_coverage() { return 1 } -# Names of workflows carrying at least one non-pull_request trigger. +# Names of workflows with a trigger that starts a run outside a pull request. # # The YAML is parsed rather than pattern-matched. GitHub accepts `on:` as a # block map, block sequence, flow sequence or bare scalar, any of which may wrap @@ -113,9 +113,10 @@ non_pr_workflow_names() { printf "%s\n" "${files[@]}" | bun -e ' const files = (await Bun.stdin.text()).split("\n").filter(Boolean); -// Both report their failure as a check on the PR itself, so alert.yml skips -// them at runtime and a workflow triggered only by these needs no entry. -const PR_EVENTS = new Set(["pull_request", "pull_request_target"]); +// The PR events report their failure as a check on the PR itself, and +// workflow_call runs inside the run that called it, so a workflow triggered +// only by these needs no entry. +const UNWATCHED = new Set(["pull_request", "pull_request_target", "workflow_call"]); const names = []; for (const file of files) { const doc = Bun.YAML.parse(await Bun.file(file).text()); @@ -132,7 +133,7 @@ for (const file of files) { console.error("alert.sh: cannot read the on: value of " + file); process.exit(2); } - if (!triggers.some((t) => !PR_EVENTS.has(t))) continue; + if (!triggers.some((t) => !UNWATCHED.has(t))) continue; const name = doc.name; if (typeof name !== "string" || name.trim() === "") { diff --git a/sh/gh/check-runs.ts b/sh/gh/check-runs.ts new file mode 100644 index 00000000000..c979f47b73b --- /dev/null +++ b/sh/gh/check-runs.ts @@ -0,0 +1,26 @@ +// Fail when a workflow `run:` step names a script file. Workflows call `just` +// recipes, so a script can move or change its arguments without touching them, +// and the same command works locally. +import { Glob } from "bun"; + +type Step = { run?: string }; +type Job = { steps?: Step[] }; +type Workflow = { jobs?: Record }; + +const violations: string[] = []; +for await (const path of new Glob(".github/workflows/*.{yml,yaml}").scan({ dot: true })) { + const workflow = Bun.YAML.parse(await Bun.file(path).text()) as Workflow; + for (const [name, job] of Object.entries(workflow.jobs ?? {})) { + for (const step of job.steps ?? []) { + for (const line of step.run?.split("\n") ?? []) { + if (/\.(sh|py|ts|js|mjs)\b/.test(line)) violations.push(`${path} (${name}): ${line.trim()}`); + } + } + } +} + +if (violations.length > 0) { + console.error("workflow steps must run a `just` recipe, not a script:"); + for (const violation of violations.sort()) console.error(` ${violation}`); + process.exit(1); +} diff --git a/.github/scripts/install-formula.py b/sh/gh/install-formula.py similarity index 100% rename from .github/scripts/install-formula.py rename to sh/gh/install-formula.py diff --git a/.github/scripts/install-formula.test.py b/sh/gh/install-formula.test.py similarity index 100% rename from .github/scripts/install-formula.test.py rename to sh/gh/install-formula.test.py diff --git a/.github/scripts/release.sh b/sh/gh/release.sh similarity index 100% rename from .github/scripts/release.sh rename to sh/gh/release.sh diff --git a/.github/scripts/release.test.sh b/sh/gh/release.test.sh similarity index 100% rename from .github/scripts/release.test.sh rename to sh/gh/release.test.sh diff --git a/.github/scripts/render-formula.sh b/sh/gh/render-formula.sh similarity index 100% rename from .github/scripts/render-formula.sh rename to sh/gh/render-formula.sh diff --git a/.github/scripts/trigger-repo-publish.sh b/sh/gh/trigger-repo-publish.sh similarity index 100% rename from .github/scripts/trigger-repo-publish.sh rename to sh/gh/trigger-repo-publish.sh diff --git a/go/scripts/check-errors.sh b/sh/go/check-errors.sh similarity index 100% rename from go/scripts/check-errors.sh rename to sh/go/check-errors.sh diff --git a/go/scripts/check.sh b/sh/go/check.sh similarity index 84% rename from go/scripts/check.sh rename to sh/go/check.sh index 46f39f23afc..4f0d7de59bd 100755 --- a/go/scripts/check.sh +++ b/sh/go/check.sh @@ -3,7 +3,7 @@ set -euo pipefail # Local smoke check for the Go module. # -# Stages the ffi + wrapper modules from this checkout (scripts/stage.sh) and +# Stages the ffi + wrapper modules from this checkout (sh/go/stage.sh) and # runs `go build`/`go vet`/`go test` against them. Intended for `just go check`. # # The main repo stays binary-free: no `.a` or generated `.go` files land @@ -13,17 +13,13 @@ set -euo pipefail # Skipped cleanly on hosts without `go`, `cargo`, or `uniffi-bindgen-go`. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_DIR="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_DIR/.." && pwd)" if ! command -v go >/dev/null 2>&1; then echo "go check: no go on PATH, skipping" >&2 exit 0 fi -# The publish state machine needs neither cargo nor generated bindings, so it -# runs ahead of the gates that skip a partial toolchain. -bash "$SCRIPT_DIR/publish-wrapper.test.sh" - if ! command -v cargo >/dev/null 2>&1; then echo "go check: no cargo on PATH, skipping" >&2 exit 0 diff --git a/go/scripts/package-ffi.sh b/sh/go/package-ffi.sh similarity index 98% rename from go/scripts/package-ffi.sh rename to sh/go/package-ffi.sh index 04450fd28f1..b2d89cd51d2 100755 --- a/go/scripts/package-ffi.sh +++ b/sh/go/package-ffi.sh @@ -10,7 +10,7 @@ set -euo pipefail # output at $BINDINGS_DIR/moq/moq.go. # # Usage: -# go/scripts/package-ffi.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist +# sh/go/package-ffi.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist # # Optional: # --source-dir DIR in-tree ffi module skeleton (default: go/ffi) @@ -24,7 +24,7 @@ set -euo pipefail # $LIB_DIR/x86_64-pc-windows-msvc/moq_ffi.lib SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_ROOT="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_ROOT/.." && pwd)" VERSION="" diff --git a/go/scripts/package-wrapper.sh b/sh/go/package-wrapper.sh similarity index 97% rename from go/scripts/package-wrapper.sh rename to sh/go/package-wrapper.sh index 210d0db854c..52159e7932e 100755 --- a/go/scripts/package-wrapper.sh +++ b/sh/go/package-wrapper.sh @@ -13,7 +13,7 @@ set -euo pipefail # produces an identical tree and no release. # # Usage: -# go/scripts/package-wrapper.sh --line 0.3 --ffi-version 0.2.18 --output dist +# sh/go/package-wrapper.sh --line 0.3 --ffi-version 0.2.18 --output dist # # Optional: # --source-dir DIR in-tree wrapper skeleton (default: go/wrapper) @@ -22,7 +22,7 @@ set -euo pipefail # --no-archive SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_ROOT="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_ROOT/.." && pwd)" LINE="" diff --git a/go/scripts/publish-ffi.sh b/sh/go/publish-ffi.sh similarity index 100% rename from go/scripts/publish-ffi.sh rename to sh/go/publish-ffi.sh diff --git a/go/scripts/publish-wrapper.sh b/sh/go/publish-wrapper.sh similarity index 100% rename from go/scripts/publish-wrapper.sh rename to sh/go/publish-wrapper.sh diff --git a/go/scripts/publish-wrapper.test.sh b/sh/go/publish-wrapper.test.sh similarity index 97% rename from go/scripts/publish-wrapper.test.sh rename to sh/go/publish-wrapper.test.sh index 423778218be..39e57fe7943 100755 --- a/go/scripts/publish-wrapper.test.sh +++ b/sh/go/publish-wrapper.test.sh @@ -12,7 +12,7 @@ set -euo pipefail # missing. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_DIR="$(cd "$SCRIPT_DIR/../../go" && pwd)" if ! command -v go >/dev/null 2>&1; then echo "publish-wrapper test: no go on PATH, skipping" >&2 @@ -75,7 +75,7 @@ LINE=$(tr -d '[:space:]' <"$GO_DIR/wrapper/VERSION") # Stage a real wrapper tarball. --skip-tidy keeps it offline (no go.sum), which # is fine: publish-wrapper.sh only reads VERSION out of the staged tree. -bash "$GO_DIR/scripts/package-wrapper.sh" \ +bash "$SCRIPT_DIR/package-wrapper.sh" \ --line "$LINE" \ --ffi-version "0.0.0-test" \ --source-dir "$GO_DIR/wrapper" \ @@ -93,7 +93,7 @@ publish() { ( cd "$WORK/run" GO_MIRROR_URL="$MIRROR" GO_MIRROR_TOKEN=unused \ - bash "$GO_DIR/scripts/publish-wrapper.sh" + bash "$SCRIPT_DIR/publish-wrapper.sh" ) } diff --git a/go/scripts/stage.sh b/sh/go/stage.sh similarity index 98% rename from go/scripts/stage.sh rename to sh/go/stage.sh index dcb09e22897..86db6e827ca 100755 --- a/go/scripts/stage.sh +++ b/sh/go/stage.sh @@ -10,7 +10,7 @@ set -euo pipefail # its own module against them and build Go code against these bindings. # # Usage: -# go/scripts/stage.sh [--output DIR] +# sh/go/stage.sh [--output DIR] # # Optional: # --output DIR stage parent (default: /dist, gitignored) @@ -19,7 +19,7 @@ set -euo pipefail # for an optimized cdylib. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -GO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +GO_DIR="$(cd "$SCRIPT_DIR/../../go" && pwd)" WORKSPACE_DIR="$(cd "$GO_DIR/.." && pwd)" STAGE_PARENT="$WORKSPACE_DIR/dist" diff --git a/sh/js/workers.sh b/sh/js/workers.sh new file mode 100755 index 00000000000..4fb2178c2ac --- /dev/null +++ b/sh/js/workers.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Validate each Worker's deploy bundle without publishing or credentials. +# +# These Workers have lockfiles outside the Bun workspace, so `just js check` +# never installs or builds them. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +for project in infra/apt infra/rpm demo/pub; do + ( + cd "$project" + bun install --frozen-lockfile + bun run deploy --dry-run + ) +done diff --git a/kt/scripts/check.sh b/sh/kt/check.sh similarity index 92% rename from kt/scripts/check.sh rename to sh/kt/check.sh index 4a00e10eb73..57c3be14a05 100644 --- a/kt/scripts/check.sh +++ b/sh/kt/check.sh @@ -2,7 +2,7 @@ set -euo pipefail # Full check for the Kotlin packages: regenerate the bindings + native lib -# (scripts/generate.sh), then run the raw-bindings and wrapper JVM tests. +# (sh/kt/generate.sh), then run the raw-bindings and wrapper JVM tests. # # Unlike generation, this needs a JDK and Gradle. Both ship in the `nix # develop` dev shell (see flake.nix ktDeps), so a missing one is an error @@ -11,7 +11,7 @@ set -euo pipefail # should run `just kt generate` instead. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +KT_DIR="$(cd "$SCRIPT_DIR/../../kt" && pwd)" bash "$SCRIPT_DIR/generate.sh" diff --git a/kt/scripts/generate.sh b/sh/kt/generate.sh similarity index 96% rename from kt/scripts/generate.sh rename to sh/kt/generate.sh index b1716373fa6..4ddc0d5947e 100755 --- a/kt/scripts/generate.sh +++ b/sh/kt/generate.sh @@ -6,11 +6,11 @@ set -euo pipefail # and regenerate the uniffi bindings. No Gradle, no JDK required. # # Intended for environments that intentionally lack Gradle (e.g. regenerating -# checked-in bindings after a moq-ffi change). `scripts/check.sh` calls this +# checked-in bindings after a moq-ffi change). `sh/kt/check.sh` calls this # first and then compiles + tests the result. Requires cargo. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +KT_DIR="$(cd "$SCRIPT_DIR/../../kt" && pwd)" WORKSPACE_DIR="$(cd "$KT_DIR/.." && pwd)" if ! command -v cargo >/dev/null 2>&1; then diff --git a/kt/scripts/package.sh b/sh/kt/package.sh similarity index 97% rename from kt/scripts/package.sh rename to sh/kt/package.sh index 73e9011d708..03c9cc13382 100755 --- a/kt/scripts/package.sh +++ b/sh/kt/package.sh @@ -10,7 +10,7 @@ set -euo pipefail # uniffi-bindgen kotlin output at $BINDINGS_DIR/uniffi/moq/moq.kt. # # Usage: -# kt/scripts/package.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist +# sh/kt/package.sh --version 0.0.0-dev --lib-dir libs --bindings-dir bindings --output dist # # Expected $LIB_DIR layout (per target, populated by the build matrix): # $LIB_DIR/aarch64-linux-android/libmoq_ffi.so @@ -20,7 +20,7 @@ set -euo pipefail # ... etc. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -KT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +KT_DIR="$(cd "$SCRIPT_DIR/../../kt" && pwd)" VERSION="" LIB_DIR="" diff --git a/sh/markdown.sh b/sh/markdown.sh new file mode 100755 index 00000000000..cd811fa9119 --- /dev/null +++ b/sh/markdown.sh @@ -0,0 +1,69 @@ +#!/usr/bin/env bash +# Lint or format every Markdown file: sh/markdown.sh check|fix +# +# remark-cli has no check mode: `--frail` raises the exit code on lint +# messages, and only `--output` formats, so a file that is merely misformatted +# passes both ways. `check` formats a scratch mirror and diffs it, so it stays +# read-only while `fix` writes in place. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +case "${1:-}" in + check) ;; + fix) + bun remark . --quiet --output + exit 0 + ;; + *) + echo "usage: sh/markdown.sh check|fix" >&2 + exit 2 + ;; +esac + +mirror=$(mktemp -d) +trap 'rm -rf "$mirror"' EXIT + +extension_list=$(bun -e \ + 'import extensions from "markdown-extensions"; console.log(extensions.join("\n"))') +patterns=() +while IFS= read -r extension; do + [[ -n "$extension" ]] && patterns+=("*.$extension") +done <<<"$extension_list" +((${#patterns[@]})) || { + echo "error: remark reported no Markdown extensions" >&2 + exit 1 +} + +# Untracked files are in scope so a new doc is linted before it is staged; +# --exclude-standard keeps build output out. +files=() +while IFS= read -r -d '' file; do + [[ -f "$file" ]] || continue + files+=("$file") + mkdir -p "$mirror/$(dirname "$file")" + cp "$file" "$mirror/$file" +done < <(git ls-files -z --cached --others --exclude-standard -- "${patterns[@]}") +((${#files[@]})) || exit 0 + +# The config rides along so every .remarkignore pattern resolves against the +# mirror root exactly as it does here, and node_modules is where the plugins +# named by .remarkrc.mjs come from. +cp .remarkrc.mjs .remarkignore "$mirror/" +ln -s "$PWD/node_modules" "$mirror/node_modules" + +status=0 +(cd "$mirror" && bun remark . --quiet --frail --output) || status=$? + +stale=() +for file in "${files[@]}"; do + cmp -s "$file" "$mirror/$file" || stale+=("$file") +done + +if ((${#stale[@]})); then + echo "error: these files are not formatted, run 'just fix':" >&2 + printf ' %s\n' "${stale[@]}" >&2 + status=1 +fi + +exit "$status" diff --git a/sh/obs/check.sh b/sh/obs/check.sh new file mode 100755 index 00000000000..b8b651b41b5 --- /dev/null +++ b/sh/obs/check.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Lint formatting. Skips a tool silently if it isn't on $PATH (matches repo convention). +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" + +if grep -Fq '"MOQ_VERSION"' CMakePresets.json; then + echo "CMakePresets.json must not pin MOQ_VERSION" >&2 + exit 1 +fi +# The headers `just obs compile` type-checks against are a second copy of +# the OBS version this spec downloads, so a bump here has to reach flake.nix +# or the two gates check different libobs. +spec_obs=$(jq -r '.dependencies["obs-studio"].version // ""' buildspec.json) +flake_obs=$(sed -n '/pname = "libobs-headers"/,/}/p' ../../flake.nix | sed -n 's/.*version = "\([^"]*\)".*/\1/p') +# Both are pattern matches against files this recipe doesn't own the shape +# of. An empty parse compares equal to an empty parse, so the guard would +# pass by reporting nothing rather than by finding the versions in sync. +if [ -z "$spec_obs" ] || [ -z "$flake_obs" ]; then + echo "couldn't read the obs-studio version: '$spec_obs' from buildspec.json, '$flake_obs' from flake.nix" >&2 + exit 1 +fi +if [ "$spec_obs" != "$flake_obs" ]; then + echo "obs-studio is $spec_obs in buildspec.json but $flake_obs in flake.nix (libobs-headers)" >&2 + exit 1 +fi +# And a third: `just obs ci` links against nixpkgs' obs-studio, the one OBS +# this repo doesn't pick. Without this the released macOS/Windows binaries +# could be a whole libobs API apart from what CI ever compiles. Patch +# releases carry no API change, so only the major.minor has to agree; that +# also keeps the guard quiet until flake.lock actually moves OBS, which is +# the change that opens the gap and so the one that should fail. +# OBS_LINKED_VERSION comes from the dev shell (see flake.nix), so this leg +# is skipped outside it, where there is no nixpkgs obs-studio to compare. +# MOQ_STRICT turns that skip into an error for the same reason it turns a +# missing tool into one: in CI a leg that checks nothing still reports green. +if [ -z "${OBS_LINKED_VERSION:-}" ]; then + if [ -n "${MOQ_STRICT:-}" ]; then + echo "MOQ_STRICT is set but OBS_LINKED_VERSION is unset; run inside 'nix develop', which exports it" >&2 + exit 1 + fi +else + spec_api=$(echo "$spec_obs" | cut -d. -f1,2) + linked_api=$(echo "$OBS_LINKED_VERSION" | cut -d. -f1,2) + if [ "$spec_api" != "$linked_api" ]; then + echo "obs-studio is $spec_obs in buildspec.json but $OBS_LINKED_VERSION in nixpkgs, which is what 'just obs ci' links" >&2 + echo "bump buildspec.json and flake.nix's libobs-headers to $OBS_LINKED_VERSION, or pin nixpkgs back" >&2 + exit 1 + fi +fi +if command -v cmake >/dev/null; then + build_dir=$(mktemp -d) + trap 'rm -rf "$build_dir"' EXIT + if cmake -S . -B "$build_dir" -DMOQ_LOCAL= >"$build_dir/configure.log" 2>&1; then + echo "CMake accepted a release build without MOQ_VERSION" >&2 + exit 1 + fi + if ! grep -Fq "MOQ_VERSION is required when MOQ_LOCAL does not contain rs/libmoq" "$build_dir/configure.log"; then + cat "$build_dir/configure.log" >&2 + exit 1 + fi +fi +if command -v clang-format >/dev/null; then + git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format --dry-run --Werror +fi +if command -v gersemi >/dev/null; then + gersemi --check CMakeLists.txt cmake +fi diff --git a/sh/obs/compile.sh b/sh/obs/compile.sh new file mode 100755 index 00000000000..6253cc15860 --- /dev/null +++ b/sh/obs/compile.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Type-check every plugin source and unit test, without linking or an obs-deps +# download. +# +# This is the gate to run in a worktree. `just obs build` needs the +# multi-hundred-MB obs-deps bundle (macOS/Windows) and a full cargo build, per +# worktree; this needs neither, because the dev shell carries all three header +# sets: libobs pinned to the same OBS release buildspec.json downloads (see +# `obs-headers` in flake.nix), Qt6, and ffmpeg. CI links the real thing on Linux +# (.github/workflows/obs.yml). +# +# It compiles the Qt sources too, which the CMake build only does when +# ENABLE_QT and ENABLE_FRONTEND_API are on, so the definitions they gate on are +# set here to match. +# +# test/ is in scope because each test file defines the libmoq entry points the +# plugin calls, so a signature that drifts from the generated moq.h is a +# conflicting C declaration. Catching that needs only headers, which is why it +# belongs here: `just obs ci` finds the same drift, but only where obs.yml's +# path filter reaches, and `just obs test` is manual. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$here/../../cpp/obs" + +# Never skips, for the reason `test` doesn't: a compile gate that reports +# success without compiling is worse than no gate. Everything it needs is in +# the dev shell, so the fix is always to enter it. +cxx="${CXX:-c++}" +if ! command -v "$cxx" >/dev/null; then + echo "no C++ compiler at '$cxx'" >&2 + exit 1 +fi +for pkgs in "Qt6Widgets Qt6Gui Qt6Core" "libavcodec libavutil libswscale libswresample"; do + # shellcheck disable=SC2086 + if ! pkg-config --exists $pkgs; then + echo "missing headers for: $pkgs" >&2 + echo "run inside 'nix develop', which supplies Qt6 and ffmpeg on every platform" >&2 + exit 1 + fi +done + +# Assign first so a failure in `_includes` still aborts under `set -e`, +# then split on newlines only: an include path may contain spaces. +includes_raw=$("$here/includes.sh") +includes=() +while IFS= read -r flag; do includes+=("$flag"); done <<<"$includes_raw" + +qt=$(pkg-config --cflags Qt6Widgets Qt6Gui Qt6Core) +ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale libswresample) + +# MOQ_VERSION_STRING only reaches a label in the dock, so any value +# type-checks the same; CMake stamps the real libmoq version. +status=0 +for source in src/*.cpp test/*.cpp; do + # shellcheck disable=SC2086 + # -Wno-unused-command-line-argument: the nix compiler wrapper injects + # link flags, which a syntax-only run reports as unused, once per file. + # $qt and $ffmpeg stay unquoted on purpose: pkg-config hands back one + # space-separated string, and splitting it is the intended reading. + if ! "$cxx" -std=c++17 -fsyntax-only -Wno-unused-command-line-argument \ + "${includes[@]}" $qt $ffmpeg \ + -DMOQ_FRONTEND_ENABLED -DMOQ_VERSION_STRING='"0.0.0"' "$source"; then + status=1 + fi +done +exit $status diff --git a/sh/obs/fix.sh b/sh/obs/fix.sh new file mode 100755 index 00000000000..46cf8d27f86 --- /dev/null +++ b/sh/obs/fix.sh @@ -0,0 +1,12 @@ +#!/usr/bin/env bash +# Auto-fix formatting. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" + +if command -v clang-format >/dev/null; then + git ls-files 'src/*.cpp' 'src/*.h' 'test/*.cpp' | xargs clang-format -i +fi +if command -v gersemi >/dev/null; then + gersemi --in-place CMakeLists.txt cmake +fi diff --git a/sh/obs/includes.sh b/sh/obs/includes.sh new file mode 100755 index 00000000000..45a8f83f316 --- /dev/null +++ b/sh/obs/includes.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Print the -I flags for libobs and libmoq, shared by compile.sh and unit.sh. +# +# moq.h is generated by cbindgen from rs/libmoq/src, so regenerate rather than +# just locating it: a stale header is how a plugin call to a since-changed +# libmoq function type-checks anyway. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" + +# The dev shell's pinned headers first, so every platform checks against the +# same libobs. Then the obs-deps framework (macOS/Windows), the OBS sources +# CMake unpacks beside it, and a system install (Linux without nix). +obs_include="" +for candidate in \ + "${OBS_INCLUDE_DIR:-}" \ + .deps/Frameworks/libobs.framework/Versions/A/Headers \ + .deps/obs-studio-*/libobs; do + if [ -n "$candidate" ] && [ -f "$candidate/obs.h" ]; then + obs_include="$candidate" + break + fi +done +if [ -z "$obs_include" ] && command -v pkg-config >/dev/null && pkg-config --exists libobs; then + obs_include=$(pkg-config --variable=includedir libobs)/obs +fi +if [ -z "$obs_include" ]; then + echo "libobs headers not found; run inside 'nix develop' (or set OBS_INCLUDE_DIR)" >&2 + exit 1 +fi + +# obs.h reaches as of OBS 32, which used to be vendored +# under libobs/util/ and so came along with the headers above. The dev shell +# carries it (see flake.nix); the obs-deps bundle ships its own copy beside +# libobs, which is how the macOS and Windows builds get it. +simde_include="" +if command -v pkg-config >/dev/null && pkg-config --exists simde; then + simde_include=$(pkg-config --variable=includedir simde) +else + for candidate in .deps/obs-deps-*/include; do + if [ -f "$candidate/simde/x86/sse2.h" ]; then + simde_include="$candidate" + break + fi + done +fi +if [ -z "$simde_include" ]; then + echo "simde headers not found; run inside 'nix develop', which supplies them" >&2 + exit 1 +fi + +(cd ../.. && cargo check --locked --quiet -p libmoq) + +# rs/libmoq/build.rs writes the header into its OUT_DIR, so ask cargo where +# that is. Replays the check above from cache, so it costs a process rather +# than a compile. +out_dir=$(cd ../.. && cargo check --locked -p libmoq --message-format=json | + jq -r 'select(.reason == "build-script-executed") | select(.package_id | tostring | test("libmoq")) | .out_dir' | + tail -1) +if [ -z "$out_dir" ]; then + echo "cargo reported no build script output for libmoq" >&2 + exit 1 +fi +moq_include="$out_dir/include" +if [ ! -f "$moq_include/moq.h" ]; then + echo "$moq_include/moq.h missing after 'cargo check --locked -p libmoq'" >&2 + exit 1 +fi + +# One flag per line, path glued to the -I, so a caller can rebuild the +# arguments without splitting on the spaces in a path. +printf -- '-I%s\n' src "$obs_include" "$simde_include" "$moq_include" diff --git a/sh/obs/preset.sh b/sh/obs/preset.sh new file mode 100755 index 00000000000..c6bd950dc62 --- /dev/null +++ b/sh/obs/preset.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash +# Detect the CMake preset for the current platform (or use the override). +# Pass `ci` as the second argument for the warnings-as-errors variant, whose +# name upstream spells differently on each platform. +set -euo pipefail + +cd "$(dirname "${BASH_SOURCE[0]}")/../../cpp/obs" +override=${1:-} +variant=${2:-} + +if [[ -n "$override" ]]; then + echo "$override" +elif [[ "$OSTYPE" == "darwin"* ]]; then + [[ "$variant" == "ci" ]] && echo "macos-ci" || echo "macos" +elif [[ "$OSTYPE" == "linux-gnu"* ]]; then + [[ "$variant" == "ci" ]] && echo "ubuntu-ci-x86_64" || echo "ubuntu-x86_64" +elif [[ "$OSTYPE" == "msys" || "$OSTYPE" == "cygwin" ]]; then + [[ "$variant" == "ci" ]] && echo "windows-ci-x64" || echo "windows-x64" +else + echo "Unknown platform: $OSTYPE" >&2 + exit 1 +fi diff --git a/sh/obs/test.sh b/sh/obs/test.sh new file mode 100755 index 00000000000..d8e596050c3 --- /dev/null +++ b/sh/obs/test.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Unit-test the plugin sources against stubbed libobs/libmoq/ffmpeg, under +# ThreadSanitizer. +# +# Manual because ThreadSanitizer needs its own build. `just obs ci` runs the +# same tests without it, so a regression these assertions catch still turns PR +# CI red; the sanitizer is what adds the races on top. Run this whenever you touch src/, especially the session +# callback plumbing, whose orderings the build can't check. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$here/../../cpp/obs" + +# Unlike the lint recipes, this one never skips: a test gate that reports +# success without running anything is worse than no gate. +cxx="${CXX:-c++}" +unsupported="'$cxx' cannot build and run -fsanitize=thread binaries. These tests need a +Clang or GCC whose ThreadSanitizer runtime works on this host: set CXX, or on Windows run +them from WSL, since neither MSVC nor Clang on Windows implements ThreadSanitizer. +They also run without the sanitizer as part of 'just obs ci'." +if ! command -v "$cxx" >/dev/null; then + echo "no C++ compiler at '$cxx'." >&2 + echo "$unsupported" >&2 + exit 1 +fi + +probe=$(mktemp -d) +trap 'rm -rf "$probe"' EXIT + +# Run the probe, don't just link it: a mismatch between the compiler's TSan +# runtime and the host kernel (macOS is where this shows up) links fine and +# then segfaults before main, which is an unreadable way for the real test +# to fail. +# The subshell keeps bash's own "Segmentation fault" job message out of the +# output, so the explanation below is all the reader gets. +if ! printf 'int main(){}\n' | "$cxx" -x c++ -fsanitize=thread -o "$probe/probe" - >/dev/null 2>&1 || + ! ("$probe/probe" >/dev/null 2>&1) 2>/dev/null; then + echo "$unsupported" >&2 + exit 1 +fi + +"$here/unit.sh" tsan diff --git a/sh/obs/unit.sh b/sh/obs/unit.sh new file mode 100755 index 00000000000..54b775baace --- /dev/null +++ b/sh/obs/unit.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# Compile and run the unit tests: sh/obs/unit.sh [plain|tsan] +# +# Shared by `just obs test` (ThreadSanitizer) and `just obs ci` (plain), so +# both run the same assertions. +set -euo pipefail + +here=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +cd "$here/../../cpp/obs" +mode=${1:-plain} + +cxx="${CXX:-c++}" +if ! command -v "$cxx" >/dev/null; then + echo "no C++ compiler at '$cxx'" >&2 + exit 1 +fi +# moq-source.cpp decodes with ffmpeg, so its test needs the headers even +# though it stubs every function out of them. +if ! pkg-config --exists libavcodec libavutil libswscale; then + echo "missing ffmpeg headers" >&2 + echo "run inside 'nix develop', which supplies ffmpeg on every platform" >&2 + exit 1 +fi + +includes_raw=$("$here/includes.sh") +includes=() +while IFS= read -r flag; do includes+=("$flag"); done <<<"$includes_raw" + +ffmpeg=$(pkg-config --cflags libavcodec libavutil libswscale) + +flags=(-std=c++17 -g -O0 -pthread) +if [ "$mode" = "tsan" ]; then + flags+=(-fsanitize=thread) +fi + +out=$(mktemp -d) +trap 'rm -rf "$out"' EXIT + +# One binary per source under test: each test file defines its own libobs and +# libmoq stubs, so two of them can't share a link. Header-only helpers +# (quality defaults) build as a third binary with no plugin source. +for name in moq-output moq-source; do + # shellcheck disable=SC2086 + # $ffmpeg stays unquoted on purpose: pkg-config hands back one + # space-separated string, and splitting it is the intended reading. + "$cxx" "${flags[@]}" "${includes[@]}" $ffmpeg \ + -o "$out/$name-test" "test/$name-test.cpp" "src/$name.cpp" + TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" +done +for name in moq-quality-defaults moq-dock-stop moq-error moq-encoder-latency moq-dial moq-spark; do + "$cxx" "${flags[@]}" "${includes[@]}" \ + -o "$out/$name-test" "test/$name-test.cpp" + TSAN_OPTIONS="halt_on_error=1" "$out/$name-test" +done diff --git a/sh/py/samples.sh b/sh/py/samples.sh new file mode 100755 index 00000000000..dadc35e5d62 --- /dev/null +++ b/sh/py/samples.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Type-check every Python sample in the docs against the installed wrapper. +# +# Samples leave their inputs (`payload`, `pts`) to the reader, so they follow +# the declarations in moq-rs/tests/doc_prelude.py. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)/py" + +out=$(mktemp -d) +trap 'rm -rf "$out"' EXIT +{ + cat moq-rs/tests/doc_prelude.py + bash ../doc/lib/samples.sh python ../doc/lib/py/index.md moq-rs/README.md moq-rs/docs/index.md +} >"$out/samples.py" +echo '{"pythonVersion": "3.10"}' >"$out/pyrightconfig.json" +uv run --no-sync pyright --project "$out" "$out/samples.py" diff --git a/sh/rs/c-tests.sh b/sh/rs/c-tests.sh new file mode 100755 index 00000000000..225780bdcd3 --- /dev/null +++ b/sh/rs/c-tests.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Compile and run libmoq's C fixtures (`rs/libmoq/c-tests/*.c`), and the C doc +# samples, against libmoq.a, linked the way an embedder does: an external `cc` +# against the generated `moq.h`, plus the native libraries from +# `rs/libmoq/native-libs` that cargo can't inject into a link it doesn't drive. +# Unix only. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +cc="${CC:-cc}" +if ! command -v "$cc" >/dev/null; then + echo "no C compiler at '$cc'" >&2 + exit 1 +fi + +cargo build --locked -p libmoq + +# Ask cargo where it put the staticlib and the header rather than assuming +# target/debug: CARGO_TARGET_DIR and a configured build target +# (`target//debug`) both move them, and guessing wrong would link a +# stale copy or nothing. Replays the build above from cache, so it costs a +# process, not a compile. +log=$(cargo build --locked -p libmoq --message-format=json) +lib=$(jq -r 'select(.reason == "compiler-artifact") | .filenames[] | select(endswith("/libmoq.a"))' <<<"$log" | tail -1) +if [ -z "$lib" ]; then + echo "cargo reported no libmoq.a for libmoq" >&2 + exit 1 +fi +profile=$(dirname "$lib") +# rs/libmoq/build.rs writes the header into its OUT_DIR. +out_dir=$(jq -r 'select(.reason == "build-script-executed") | select(.package_id | test("libmoq")) | .out_dir' <<<"$log" | tail -1) +include="$out_dir/include" +if [ -z "$out_dir" ] || [ ! -f "$include/moq.h" ]; then + echo "$include/moq.h missing after 'cargo build --locked -p libmoq'" >&2 + exit 1 +fi + +# Same list nix/overlay.nix (moq.pc), CMakeLists.txt, and test/interop/interop.sh read: +# `framework:Foo` is a linker framework flag, anything else a plain library. +case "$(uname -s)" in + Darwin) native_libs=rs/libmoq/native-libs/apple.txt ;; + *) native_libs=rs/libmoq/native-libs/linux.txt ;; +esac +libs=() +while read -r entry; do + case "$entry" in + '' | '#'*) continue ;; + framework:*) libs+=(-framework "${entry#framework:}") ;; + *) libs+=("-l$entry") ;; + esac +done <"$native_libs" + +out=$(mktemp -d) +trap 'rm -rf "$out"' EXIT + +# The C docs compile against this header too: each sample beside the inputs +# doc-samples.h declares, and every name the prose cites. A `moq_decode_*` +# wildcard ends in `_` and is skipped. +bash doc/lib/samples.sh c doc/lib/c/index.md >"$out/doc-samples.c" +"$cc" -fsyntax-only -Werror=implicit-function-declaration -include rs/libmoq/c-tests/doc-samples.h -I"$include" "$out/doc-samples.c" +grep -oE '\b(moq|MOQ)_[A-Za-z0-9_]*' doc/lib/c/index.md | grep -v '_$' | sort -u | while read -r name; do + if ! grep -qw "$name" "$include/moq.h"; then + echo "doc/lib/c/index.md cites $name, which moq.h does not declare" >&2 + exit 1 + fi +done + +for source in rs/libmoq/c-tests/*.c; do + bin="$out/$(basename "$source" .c)" + "$cc" "$source" -I"$include" -L"$profile" -lmoq "${libs[@]}" -o "$bin" + echo "running $source" + "$bin" +done diff --git a/sh/rs/fuzz.sh b/sh/rs/fuzz.sh new file mode 100755 index 00000000000..c23912f1e93 --- /dev/null +++ b/sh/rs/fuzz.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Coverage-guided fuzzing of the moq-net wire codecs: sh/rs/fuzz.sh TARGET [ARGS...] +# +# TARGET is lite, announce, ietf, varint, path, or pattern; ARGS pass to +# `cargo fuzz run`, so `just rs fuzz lite -- -max_total_time=300` bounds a run. +# See rs/moq-net/fuzz/README.md. +# +# Seeds are regenerated first so the corpus follows the dispatch rather than a +# stale run, and libFuzzer writes what it discovers to the FIRST corpus +# directory, which is why the generated seeds are passed second and stay +# read-only. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +target=${1:?usage: sh/rs/fuzz.sh TARGET [ARGS...]} +shift + +if ! command -v cargo-fuzz >/dev/null 2>&1; then + echo "rs: cargo-fuzz is missing: cargo install --locked cargo-fuzz" >&2 + exit 1 +fi + +if ! rustup run nightly rustc --version >/dev/null 2>&1; then + echo "rs: the nightly toolchain is missing: rustup toolchain install nightly" >&2 + exit 1 +fi + +cargo run --locked -q -p moq-net --features fuzz --example fuzz-seeds -- rs/moq-net/fuzz/seeds +mkdir -p rs/moq-net/fuzz/corpus/"$target" + +# Nightly by PATH rather than by `cargo +nightly`: the dev shell puts a +# Nix-provided cargo ahead of the rustup shim, so the `+` form is not +# understood, and cargo-fuzz shells out to a bare `cargo` and `rustc` anyway. +# Both have to be the nightly ones or the sanitizer flags are rejected. +sysroot=$(rustup run nightly rustc --print sysroot) +export PATH="$sysroot/bin:$PATH" + +cargo fuzz run --fuzz-dir rs/moq-net/fuzz "$target" \ + rs/moq-net/fuzz/corpus/"$target" rs/moq-net/fuzz/seeds/"$target" "$@" diff --git a/rs/scripts/media-features.sh b/sh/rs/media-features.sh similarity index 100% rename from rs/scripts/media-features.sh rename to sh/rs/media-features.sh diff --git a/rs/scripts/package-binary.sh b/sh/rs/package-binary.sh similarity index 99% rename from rs/scripts/package-binary.sh rename to sh/rs/package-binary.sh index 0984857a204..cfda98d1ad7 100755 --- a/rs/scripts/package-binary.sh +++ b/sh/rs/package-binary.sh @@ -16,7 +16,7 @@ set -euo pipefail # produces an extensionless executable beside the archive. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -RS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +RS_DIR="$(cd "$SCRIPT_DIR/../../rs" && pwd)" WORKSPACE_DIR="$(cd "$RS_DIR/.." && pwd)" CRATE="" diff --git a/rs/scripts/package-nfpm.sh b/sh/rs/package-nfpm.sh similarity index 100% rename from rs/scripts/package-nfpm.sh rename to sh/rs/package-nfpm.sh diff --git a/rs/scripts/package-rename.test.py b/sh/rs/package-rename.test.py similarity index 98% rename from rs/scripts/package-rename.test.py rename to sh/rs/package-rename.test.py index d4222ce00fd..484dda146aa 100644 --- a/rs/scripts/package-rename.test.py +++ b/sh/rs/package-rename.test.py @@ -27,7 +27,7 @@ def test_packages(self): for config, packager in (("nfpm", "deb"), ("transition", "deb"), ("nfpm", "rpm")): output = scratch / f"{name}-{config}.{packager}" subprocess.run([ - "bash", "rs/scripts/package-nfpm.sh", + "bash", "sh/rs/package-nfpm.sh", f"packaging/{old}/{config}.yaml", packager, str(output), ], env=env, cwd=ROOT, check=True) outputs[config, packager] = output diff --git a/rs/scripts/package-windows.sh b/sh/rs/package-windows.sh similarity index 98% rename from rs/scripts/package-windows.sh rename to sh/rs/package-windows.sh index 33658a71d51..19aee9b6a9b 100755 --- a/rs/scripts/package-windows.sh +++ b/sh/rs/package-windows.sh @@ -15,7 +15,7 @@ set -euo pipefail # releases, so wingetcreate can carry it forward unchanged. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -RS_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +RS_DIR="$(cd "$SCRIPT_DIR/../../rs" && pwd)" WORKSPACE_DIR="$(cd "$RS_DIR/.." && pwd)" CRATE="" diff --git a/sh/rs/package.sh b/sh/rs/package.sh new file mode 100755 index 00000000000..970d7c8ef7f --- /dev/null +++ b/sh/rs/package.sh @@ -0,0 +1,51 @@ +#!/usr/bin/env bash +# Build a .deb or .rpm for moq-relay or moq-cli locally: sh/rs/package.sh CRATE deb|rpm +# +# nfpm comes from the dev shell. An .rpm built this way links against the +# host's glibc; CI builds in an AlmaLinux 9 container for broad compatibility. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +crate=${1:?usage: sh/rs/package.sh CRATE deb|rpm} +packager=${2:?usage: sh/rs/package.sh CRATE deb|rpm} + +case "$crate" in + moq-relay) bin=moq-relay ;; + moq-cli) bin=moq ;; + *) + echo "Unknown crate: $crate (use moq-relay or moq-cli)" >&2 + exit 1 + ;; +esac +case "$packager" in + deb) + if command -v dpkg >/dev/null 2>&1; then + arch=$(dpkg --print-architecture) + else + case "$(uname -m)" in + x86_64) arch=amd64 ;; + aarch64 | arm64) arch=arm64 ;; + *) + echo "Cannot infer deb arch from host $(uname -m)" >&2 + exit 1 + ;; + esac + fi + ;; + rpm) arch=$(uname -m) ;; + *) + echo "Unknown packager: $packager (use deb or rpm)" >&2 + exit 1 + ;; +esac +version=$(grep -m1 '^version' "rs/$crate/Cargo.toml" | sed 's/.*"\(.*\)".*/\1/') +cargo build --locked --release -p "$crate" +mkdir -p dist +VERSION="$version" ARCH="$arch" BINARY_PATH="target/release/$bin" \ + sh/rs/package-nfpm.sh "packaging/$crate/nfpm.yaml" "$packager" dist/ +if [[ "$packager" == deb && -f "packaging/$crate/transition.yaml" ]]; then + VERSION="$version" ARCH="$arch" \ + sh/rs/package-nfpm.sh "packaging/$crate/transition.yaml" deb dist/ +fi +ls -1 dist/ diff --git a/sh/rs/platform.sh b/sh/rs/platform.sh new file mode 100755 index 00000000000..ae807fca7f9 --- /dev/null +++ b/sh/rs/platform.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Compile what the branch changed since BASE, plus its dependents, on a Windows +# or macOS host. `--all` compiles the whole workspace. +# +# Usage: sh/rs/platform.sh [BASE|--all] +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +if [[ "${1:-}" == --all ]]; then + exec sh/rs/select.sh platform --all +fi + +list=$(mktemp) +trap 'rm -f "$list"' EXIT +sh/changed.sh "${1:-}" >"$list" +sh/rs/select.sh platform "$list" diff --git a/rs/scripts/scrub-macho.sh b/sh/rs/scrub-macho.sh similarity index 100% rename from rs/scripts/scrub-macho.sh rename to sh/rs/scrub-macho.sh diff --git a/sh/rs/select.sh b/sh/rs/select.sh new file mode 100755 index 00000000000..28bf44afa05 --- /dev/null +++ b/sh/rs/select.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# Run `just rs check|check-test|fix|test` on the crates a diff touches, plus +# their dependents. `platform` is the Windows and macOS compile. +# +# Usage: sh/rs/select.sh check|check-test|fix|test|platform LISTFILE|--all +# +# LISTFILE holds the changed paths, one per line. `--all` is the whole +# workspace, as is a change to anything every crate is built or tested by. +set -euo pipefail + +usage="usage: sh/rs/select.sh check|check-test|fix|test|platform LISTFILE|--all" +action=${1:?$usage} +list=${2:?$usage} + +cd "$(git rev-parse --show-toplevel)" + +# Print `ALL`, nothing (no crate affected), or cargo package ids. +select_packages() { + if [[ "$list" == --all ]] || grep -qE '^(Cargo\.(toml|lock)|rust-toolchain\.toml|rs/justfile|sh/rs/(select|platform)\.sh|\.config/nextest\.toml)$' "$list"; then + echo ALL + return + fi + + grep -q '^rs/' "$list" || return 0 + + # `--no-deps` keeps this to workspace members and off the network. + local metadata + metadata=$(cargo metadata --format-version 1 --no-deps) + + # A seed is the first path segment under `rs/`, so only a crate whose + # manifest sits directly at `rs//Cargo.toml` can be one. That drops + # `moq-net-fuzz` (at `rs/moq-net/fuzz`), which a moq-net diff would + # otherwise select as a dependent and compile libFuzzer for. The separator + # is normalized first so Windows paths split too. + metadata=$(jq '.packages |= map((.manifest_path |= gsub("\\\\"; "/")) | select(.manifest_path | split("/")[-3] == "rs"))' <<<"$metadata") + + # A changed path is a seed only because every crate directory is named + # after its crate. Under-selecting checks nothing, so fall back to all. + if jq -e '.packages[] | select((.manifest_path | split("/")[-2]) != .name)' <<<"$metadata" >/dev/null; then + echo "rs: a crate directory no longer matches its crate name; selecting everything." >&2 + echo ALL + return + fi + + local seeds edges selected + seeds=$(sed -n 's|^rs/\([^/]*\)/.*|\1|p' "$list" | sort -u) + edges=$(jq -r '.packages[] | .name as $name | .dependencies[] | "\($name) \(.name)"' <<<"$metadata") + + # A crate is rebuilt when anything it depends on changed, so walk the edges + # backwards until the selection stops growing. Test the value, not the key: + # awk creates a key on every `want[dep[i]]` read. The id lookup below drops + # names that are not workspace crates, such as a seed from `rs/AGENTS.md`. + # The seeds go through the environment because BSD awk (macOS) rejects a + # newline in a `-v` value. + selected=$(SEEDS="$seeds" awk ' + BEGIN { split(ENVIRON["SEEDS"], s, "\n"); for (i in s) want[s[i]] = 1 } + { pkg[NR] = $1; dep[NR] = $2 } + END { + do { + grew = 0 + for (i = 1; i <= NR; i++) + if (want[dep[i]] && !want[pkg[i]]) { want[pkg[i]] = 1; grew = 1 } + } while (grew) + for (name in want) if (want[name]) print name + } + ' <<<"$edges" | sort -u) + + [[ -n "$selected" ]] || return 0 + + # Cargo's own package ids, not bare names: a workspace crate that is also + # pulled from crates.io (kio) makes `--package kio` ambiguous, and cargo + # refuses before compiling anything. + jq -r --arg selected "$selected" ' + ($selected | split("\n")) as $want + | .packages[] + | select(.name as $name | $want | index($name) != null) + | "--package \(.id)" + ' <<<"$metadata" | sort -u | tr '\n' ' ' +} + +packages=$(select_packages) + +# Ids are `path+file://#` for a directory named after its crate. +names=$(tr ' ' '\n' <<<"$packages" | sed -n 's|.*/\([^/#]*\)#.*|\1|p' | sort -u | tr '\n' ' ') + +# True when the selection includes a crate matching the pattern. +wants() { + [[ "$packages" == ALL ]] || grep -qwE "$1" <<<"$names" +} + +# The media feature contract script is no crate, so its own diff selects +# nothing and still has to reach the gate below. +media_script= +if [[ -f "$list" ]] && grep -qx 'sh/rs/media-features.sh' "$list"; then media_script=1; fi + +case "$packages" in + "") + if [[ "$action" == check* && -n "$media_script" ]]; then + just rs media-features + else + echo "rs: no crates affected; skipping." + fi + exit 0 + ;; + ALL) echo "rs: selecting the workspace" ;; + *) echo "rs: selecting $names" ;; +esac + +# `moq-net-fuzz` needs a nightly toolchain; only `just rs fuzz` compiles it. +if [[ "$packages" == ALL ]]; then + flags=(--workspace --exclude moq-net-fuzz) +else + # Space-separated `--package ` pairs; ids contain no spaces. + # shellcheck disable=SC2206 + flags=($packages) +fi + +case "$action" in + check | check-test) + just rs "$action" "${flags[@]}" + # Workspace feature unification hides a broken moq-tokio feature set + # whenever any dependent enables a transport. + if wants moq-tokio; then just rs tokio-features; fi + # moq-wasm's crate root is `#![cfg(target_arch = "wasm32")]`, so only a + # wasm32 pass checks it. moq-mux and moq-ffi ride along in that pass, and + # nothing depends on moq-ffi, so each has to reach it on its own. + if wants '(moq-wasm|moq-mux|moq-ffi)'; then just rs wasm; fi + # Device code behind the off-by-default `capture` feature. check-test + # lints it through the test build, as for the default features. + if wants '(moq-video|moq-audio)'; then + if [[ "$action" == check-test ]]; then CARGO_BUILD_WARNINGS=deny just rs capture-test; else just rs capture; fi + fi + # moq-cli's media tasks are behind `play`; only check-test runs tests. + if [[ "$action" == check-test ]] && wants moq-cli; then just rs play; fi + # The relay's io_uring listener, off the default feature set. + if wants moq-relay; then just rs uring-check; fi + # Each media feature shape compiles on its own, about two minutes. + if [[ -n "$media_script" ]] || wants '(moq-video|moq-audio|moq-transcode)'; then just rs media-features; fi + ;; + fix) + just rs fix "${flags[@]}" + if wants '(moq-wasm|moq-mux|moq-ffi)'; then just rs wasm-fix; fi + ;; + test) + # A selection can hold nothing testable (moq-wasm has no host tests), and + # nextest exits 4 on that; the whole workspace finding none really is wrong. + [[ "$packages" == ALL ]] || flags+=(--no-tests=pass) + just rs test "${flags[@]}" + # Tests behind the off-by-default `capture` and `play` features. + if wants '(moq-video|moq-audio)'; then just rs capture-test; fi + if wants moq-cli; then just rs play; fi + ;; + platform) + # moq-gst links GStreamer via pkg-config, which the runners don't have. + if [[ "$packages" == ALL ]]; then + flags+=(--exclude moq-gst) + else + flags=() + for id in $packages; do + [[ "$id" == --package || "$id" == */moq-gst#* ]] || flags+=(--package "$id") + done + if ((${#flags[@]} == 0)); then + echo "rs: no crates affected; skipping." + exit 0 + fi + fi + # `play` and `capture` are off by default and turn on the device and + # render code in moq-video, moq-audio, and the cli. Only when moq-cli is + # selected, since cargo rejects a feature of a package outside the + # selection; a change to anything they reach selects moq-cli anyway. + if wants moq-cli; then flags+=(--features "moq-cli/play moq-cli/capture"); fi + cargo check --locked "${flags[@]}" --all-targets + ;; + *) + echo "$usage" >&2 + exit 2 + ;; +esac diff --git a/rs/scripts/stats-docs.py b/sh/rs/stats-docs.py similarity index 100% rename from rs/scripts/stats-docs.py rename to sh/rs/stats-docs.py diff --git a/sh/rs/tokio-features.sh b/sh/rs/tokio-features.sh new file mode 100755 index 00000000000..b80de4f8e83 --- /dev/null +++ b/sh/rs/tokio-features.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Compile moq-tokio by itself at the feature extremes and with each crypto +# provider. Its default-feature build is already part of the ordinary clippy +# pass; this catches what workspace feature unification hides. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +cargo clippy --locked -p moq-tokio --no-default-features -- -D warnings +cargo clippy --locked -p moq-tokio --all-features -- -D warnings +cargo clippy --locked -p moq-tokio --no-default-features --features noq,aws-lc-rs -- -D warnings +cargo clippy --locked -p moq-tokio --no-default-features --features noq,ring -- -D warnings + +for feature in aws-lc-rs ring tcp uds websocket; do + cargo clippy --locked -p moq-tokio --no-default-features --features "$feature" -- -D warnings +done + +# shellcheck disable=SC2016 # the backticks are literal, as rustc prints them +want='a rustls QUIC backend requires a crypto provider: enable either the `aws-lc-rs` or `ring` feature' +if output=$(cargo check --locked -p moq-tokio --no-default-features --features noq 2>&1); then + echo "rs: moq-tokio with only noq should reject its missing crypto provider" >&2 + exit 1 +fi +if ! grep -qF "$want" <<<"$output"; then + echo "rs: moq-tokio with only noq should fail on: $want" >&2 + echo "$output" | grep -E '^error' >&2 || true + exit 1 +fi diff --git a/sh/rs/vulkan-cuda.sh b/sh/rs/vulkan-cuda.sh new file mode 100755 index 00000000000..bcdc12944c0 --- /dev/null +++ b/sh/rs/vulkan-cuda.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Exercise the Vulkan producer -> CUDA image contract, the GPU color conversion +# and resize, and NVENC encoding of the result on Linux/NVIDIA hardware. The +# driver directory is added explicitly because the Nix shell's dynamic loader +# path does not include Ubuntu's host driver directory. +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +driver=$(/usr/sbin/ldconfig -p | awk '/libcuda\.so\.1/{print $NF; exit}') +if [[ -z "$driver" ]]; then + echo "libcuda.so.1 is not installed" >&2 + exit 1 +fi +driver_dir=$(dirname "$driver") +export LD_LIBRARY_PATH="$driver_dir${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +cargo nextest run --locked -p moq-video --run-ignored only -E 'test(/^frame::(vulkan|cuda)::tests::vulkan_cuda_/)' diff --git a/sh/shell.sh b/sh/shell.sh new file mode 100755 index 00000000000..fb6ae022c8b --- /dev/null +++ b/sh/shell.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Lint or format every tracked shell script: sh/shell.sh check|fix +set -euo pipefail + +cd "$(git rev-parse --show-toplevel)" + +scripts=() +while IFS= read -r -d '' file; do + if git --literal-pathspecs ls-files --error-unmatch -- "$file" >/dev/null 2>&1; then + scripts+=("$file") + fi +done < <(shfmt -f=0 .) +((${#scripts[@]})) || exit 0 + +case "${1:-}" in + check) + shfmt --diff "${scripts[@]}" + shellcheck "${scripts[@]}" + ;; + fix) + shfmt --write "${scripts[@]}" + ;; + *) + echo "usage: sh/shell.sh check|fix" >&2 + exit 2 + ;; +esac diff --git a/swift/scripts/check.sh b/sh/swift/check.sh similarity index 98% rename from swift/scripts/check.sh rename to sh/swift/check.sh index a05b47cb250..a1330ad4bff 100755 --- a/swift/scripts/check.sh +++ b/sh/swift/check.sh @@ -8,7 +8,7 @@ set -euo pipefail # `cargo`. Intended for `just swift check`. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SWIFT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SWIFT_DIR="$(cd "$SCRIPT_DIR/../../swift" && pwd)" WORKSPACE_DIR="$(cd "$SWIFT_DIR/.." && pwd)" if ! command -v swift >/dev/null 2>&1; then diff --git a/swift/scripts/package-ffi.sh b/sh/swift/package-ffi.sh similarity index 98% rename from swift/scripts/package-ffi.sh rename to sh/swift/package-ffi.sh index 5958c213ae1..8166429c734 100755 --- a/swift/scripts/package-ffi.sh +++ b/sh/swift/package-ffi.sh @@ -12,7 +12,7 @@ set -euo pipefail # Only macOS hosts can run this (xcodebuild is required). # # Usage: -# swift/scripts/package-ffi.sh --version 0.0.0-dev --lib-dir dist --output dist +# sh/swift/package-ffi.sh --version 0.0.0-dev --lib-dir dist --output dist # # --version Version baked into Package.swift (the moq-ffi crate version). # --lib-dir Directory containing per-target moq-ffi outputs. @@ -24,7 +24,7 @@ set -euo pipefail # override when publishing from a fork. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SWIFT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SWIFT_DIR="$(cd "$SCRIPT_DIR/../../swift" && pwd)" WORKSPACE_DIR="$(cd "$SWIFT_DIR/.." && pwd)" VERSION="" diff --git a/swift/scripts/package.sh b/sh/swift/package.sh similarity index 98% rename from swift/scripts/package.sh rename to sh/swift/package.sh index e1d40527192..4d281925ee1 100755 --- a/swift/scripts/package.sh +++ b/sh/swift/package.sh @@ -12,7 +12,7 @@ set -euo pipefail # package depends on at .upToNextMinor(from: ). # # Usage: -# swift/scripts/package.sh --version 0.3.0 --output dist +# sh/swift/package.sh --version 0.3.0 --output dist # # --version Wrapper version (from swift/VERSION). Names the tarball only; # the SPM version comes from the mirror's git tag. @@ -21,7 +21,7 @@ set -euo pipefail # --output Destination directory for the .tar.gz. SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" -SWIFT_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" +SWIFT_DIR="$(cd "$SCRIPT_DIR/../../swift" && pwd)" WORKSPACE_DIR="$(cd "$SWIFT_DIR/.." && pwd)" VERSION="" diff --git a/swift/scripts/publish-ffi.sh b/sh/swift/publish-ffi.sh similarity index 100% rename from swift/scripts/publish-ffi.sh rename to sh/swift/publish-ffi.sh diff --git a/swift/scripts/publish.sh b/sh/swift/publish.sh similarity index 100% rename from swift/scripts/publish.sh rename to sh/swift/publish.sh diff --git a/swift/scripts/verify-ffi.sh b/sh/swift/verify-ffi.sh similarity index 96% rename from swift/scripts/verify-ffi.sh rename to sh/swift/verify-ffi.sh index 65c8f60d851..886fb75c6d9 100755 --- a/swift/scripts/verify-ffi.sh +++ b/sh/swift/verify-ffi.sh @@ -13,8 +13,8 @@ set -euo pipefail # never reaches consumers. # # Usage: -# swift/scripts/verify-ffi.sh --staged-dir -# swift/scripts/verify-ffi.sh --tarball +# sh/swift/verify-ffi.sh --staged-dir +# sh/swift/verify-ffi.sh --tarball # # Exactly one of --staged-dir / --tarball must be passed. diff --git a/swift/scripts/verify.sh b/sh/swift/verify.sh similarity index 96% rename from swift/scripts/verify.sh rename to sh/swift/verify.sh index 865495798d9..6df5a561b59 100755 --- a/swift/scripts/verify.sh +++ b/sh/swift/verify.sh @@ -13,8 +13,8 @@ set -euo pipefail # already published (true on a normal release; a correct failure otherwise). # # Usage: -# swift/scripts/verify.sh --staged-dir -# swift/scripts/verify.sh --tarball +# sh/swift/verify.sh --staged-dir +# sh/swift/verify.sh --tarball # # Exactly one of --staged-dir / --tarball must be passed. diff --git a/swift/.gitignore b/swift/.gitignore index b72f40122ba..40a03d7ac31 100644 --- a/swift/.gitignore +++ b/swift/.gitignore @@ -3,7 +3,7 @@ .swiftpm/ Package.resolved -# Generated by swift/scripts/{check,package}.sh — never check these in. +# Generated by sh/swift/{check,package}.sh — never check these in. Sources/MoqFFI/Generated.swift Tests/MoqTests/Docs/Samples.swift MoqFFI.xcframework/ diff --git a/swift/.spi.yml b/swift/.spi.yml index 0259deae9cc..9fb2ec6db2c 100644 --- a/swift/.spi.yml +++ b/swift/.spi.yml @@ -3,7 +3,7 @@ # served at https://swiftpackageindex.com/moq-dev/moq-swift/documentation. # # This file is copied into the published moq-dev/moq-swift mirror by -# swift/scripts/package.sh, since SPI reads it from the package root. +# sh/swift/package.sh, since SPI reads it from the package root. version: 1 builder: configs: diff --git a/swift/Package.swift b/swift/Package.swift index 97256a74c93..e06be8154a6 100644 --- a/swift/Package.swift +++ b/swift/Package.swift @@ -1,9 +1,9 @@ // swift-tools-version:5.9 -// Local-development manifest. swift/scripts/check.sh builds a single-slice +// Local-development manifest. sh/swift/check.sh builds a single-slice // MoqFFI.xcframework next to this file so the path-based binaryTarget below // resolves locally without a network fetch. // -// The released form (URL + checksum) is generated by swift/scripts/package.sh +// The released form (URL + checksum) is generated by sh/swift/package.sh // from Package.swift.template at release time. Edit the template, not this // file, when changing the released manifest. diff --git a/swift/Package.swift.template b/swift/Package.swift.template index 0cdfbf06060..f6339a07a8d 100644 --- a/swift/Package.swift.template +++ b/swift/Package.swift.template @@ -1,7 +1,7 @@ // swift-tools-version:5.9 // Released manifest for the ergonomic wrapper at moq-dev/moq-swift. The // source-of-truth template lives at swift/Package.swift.template in -// moq-dev/moq; swift/scripts/package.sh substitutes the moq-ffi version pin +// moq-dev/moq; sh/swift/package.sh substitutes the moq-ffi version pin // (REPLACE_FFI_VERSION) at release time. // // The wrapper versions independently of the bindings (see swift/VERSION). The diff --git a/swift/README.md b/swift/README.md index 766f1ff96ec..613831a5810 100644 --- a/swift/README.md +++ b/swift/README.md @@ -83,7 +83,7 @@ is an `AsyncSequence` of the decoded type. Pass matching `compression` on both s ## Local development -`swift/scripts/check.sh` builds `moq-ffi` for the host, regenerates the UniFFI Swift bindings, builds a single-slice `MoqFFI.xcframework`, and runs `swift test`, which also compiles every Swift sample in this README and `doc/lib/swift`, extracted by `doc/lib/samples.sh`. Requires macOS with `xcodebuild` and `swift` on `$PATH`. Run via `just swift check`; skips cleanly on non-macOS hosts. +`sh/swift/check.sh` builds `moq-ffi` for the host, regenerates the UniFFI Swift bindings, builds a single-slice `MoqFFI.xcframework`, and runs `swift test`, which also compiles every Swift sample in this README and `doc/lib/swift`, extracted by `doc/lib/samples.sh`. Requires macOS with `xcodebuild` and `swift` on `$PATH`. Run via `just swift check`; skips cleanly on non-macOS hosts. Local development uses one **monolithic** `Package.swift` containing both the `Moq` and `MoqFFI` targets plus the path-based XCFramework, so `swift test` and Xcode work against a single package. The split into two packages exists only in the released artifacts, assembled from the two templates below at release time. Because the FFI module is named `MoqFFI` in both layouts, the wrapper sources (`import MoqFFI`) compile identically either way. @@ -99,7 +99,6 @@ swift/ Moq/ Ergonomic wrapper (Client, Server, Origin, Broadcast, Track, Media, Audio, …) MoqFFI/ UniFFI-generated swift (populated by check.sh/package-ffi.sh, gitignored) Tests/MoqTests/ Smoke tests - scripts/ check.sh, package{,-ffi}.sh, verify{,-ffi}.sh, publish{,-ffi}.sh ``` Edit the templates when changing a released manifest; never copy the monolithic dev-mode form into the release path. @@ -118,8 +117,8 @@ To release a new wrapper version: bump `swift/VERSION` in a PR. On merge, `relea To dry-run a publish locally against a staged tarball: ```bash -BUILD_VERSION= ./swift/scripts/publish.sh --dry-run # wrapper -> moq-swift -BUILD_VERSION= ./swift/scripts/publish-ffi.sh --dry-run # bindings -> moq-swift-ffi +BUILD_VERSION= ./sh/swift/publish.sh --dry-run # wrapper -> moq-swift +BUILD_VERSION= ./sh/swift/publish-ffi.sh --dry-run # bindings -> moq-swift-ffi ``` No Apple Developer account or App Store Connect setup needed. diff --git a/swift/Tests/MoqTests/Docs/Prelude.swift b/swift/Tests/MoqTests/Docs/Prelude.swift index 7be98118e6e..f900542855a 100644 --- a/swift/Tests/MoqTests/Docs/Prelude.swift +++ b/swift/Tests/MoqTests/Docs/Prelude.swift @@ -1,4 +1,4 @@ -// The inputs the documentation samples leave to the reader. swift/scripts/check.sh +// The inputs the documentation samples leave to the reader. sh/swift/check.sh // compiles every Swift sample in the docs beside this file, so a sample that // names a missing symbol or the wrong labels fails the check. Never executed. import Foundation diff --git a/swift/ffi/Package.swift.template b/swift/ffi/Package.swift.template index d66fb3d01b6..55588ffc092 100644 --- a/swift/ffi/Package.swift.template +++ b/swift/ffi/Package.swift.template @@ -1,7 +1,7 @@ // swift-tools-version:5.9 // Released manifest for the raw UniFFI bindings package at moq-dev/moq-swift-ffi. // The source-of-truth template lives at swift/ffi/Package.swift.template in -// moq-dev/moq; swift/scripts/package-ffi.sh substitutes the xcframework URL and +// moq-dev/moq; sh/swift/package-ffi.sh substitutes the xcframework URL and // SHA-256 (REPLACE_URL / REPLACE_CHECKSUM) at release time. // // Lockstep with the moq-ffi Rust crate: each moq-ffi-v* tag publishes a matching diff --git a/swift/justfile b/swift/justfile index 8ab3c71bf91..075ae923142 100644 --- a/swift/justfile +++ b/swift/justfile @@ -9,29 +9,47 @@ default: just check # Skips cleanly on non-macOS hosts, which is why swift.yml runs it on a Mac. -# -# This runs the tests too, so there is no separate `just swift test`. Takes an -# optional newline-separated list of changed files and skips when none are -# Swift-scoped; `just swift check` (no FILES) always runs. +# This runs the tests too, so there is no separate `just swift test`. # Build moq-ffi for the host, regenerate uniffi bindings, run swift test. -check $FILES="": - #!/usr/bin/env bash - set -euo pipefail - if [[ -n "$FILES" ]] && ! grep -qE '^(swift/|rs/moq-ffi/|doc/lib/swift/|doc/lib/samples\.sh$)' <<< "$FILES"; then - echo "swift: no Swift changes; skipping." - exit 0 - fi - bash scripts/check.sh +check: + bash ../sh/swift/check.sh -# Assemble the released wrapper package from Package.swift.template. +# Release recipes run in the caller's directory, so a relative `--output` or +# `--tarball` means what it would to the script itself. -# Used by .github/workflows/release-swift-lib.yml. +# Assemble the released wrapper package from Package.swift.template. +[no-cd] +[positional-arguments] package *args: - bash scripts/package.sh {{ args }} - -# Remove SwiftPM build output plus the XCFramework and generated bindings - -# that scripts/check.sh lays down (see swift/.gitignore). -clean: - rm -rf .build .swiftpm Package.resolved Sources/MoqFFI/Generated.swift MoqFFI.xcframework + {{ quote(justfile_directory() / "sh/swift/package.sh") }} "$@" + +# Assemble the MoqFFI package from per-target static libs and the generated bindings. +[no-cd] +[positional-arguments] +package-ffi *args: + {{ quote(justfile_directory() / "sh/swift/package-ffi.sh") }} "$@" + +# Build a throwaway consumer of a staged wrapper package. +[no-cd] +[positional-arguments] +verify *args: + {{ quote(justfile_directory() / "sh/swift/verify.sh") }} "$@" + +# Build a throwaway consumer of a staged MoqFFI package. +[no-cd] +[positional-arguments] +verify-ffi *args: + {{ quote(justfile_directory() / "sh/swift/verify-ffi.sh") }} "$@" + +# Push the staged wrapper package to the moq-dev/moq-swift mirror. +[no-cd] +[positional-arguments] +publish *args: + {{ quote(justfile_directory() / "sh/swift/publish.sh") }} "$@" + +# Push the staged MoqFFI package to the moq-dev/moq-swift-ffi mirror. +[no-cd] +[positional-arguments] +publish-ffi *args: + {{ quote(justfile_directory() / "sh/swift/publish-ffi.sh") }} "$@" diff --git a/test/README.md b/test/README.md index ae54517fa27..968b286777b 100644 --- a/test/README.md +++ b/test/README.md @@ -88,25 +88,3 @@ with the `rm -rf` the run prints; nothing expires it for you. In CI the harness writes under `MOQ_TEST_RUNS`, and `interop.yml` and `wasm.yml` upload that directory as a short-lived artifact when the job fails. - -## Worktrees - -`just worktree` reports what a checkout can actually do before anything is built: -its base and how stale that base is, and whether the Git metadata is reachable for -fetch, branch creation, and rebase. A linked worktree keeps its shared metadata -under `--git-common-dir`, inside the main repository, so write access to the -source tree does not imply any of the three. - -```bash -just worktree # report, change nothing -just worktree setup # fetch, set the branch upstream, record the base SHA -``` - -`setup` never resets, rebases, or cleans, so it is safe to run against a checkout -with work in progress. - -The upstream is the only place the base survives, so `setup BASE` fails when it -cannot be written -- a detached HEAD has no branch to hang it on, and the shared -config may be read-only. Left as a warning, `just check` would go on scoping -against `origin/main` while setup reported success. Without a `BASE`, that -fallback is what would have been written anyway, so it warns instead. diff --git a/test/interop/README.md b/test/interop/README.md index 3d961b65ad7..8d862ba42e5 100644 --- a/test/interop/README.md +++ b/test/interop/README.md @@ -29,7 +29,7 @@ player survive the publication lifecycle. See [Media QA](#media-qa). |---|---|---|---| | Rust | `rs/moq-relay` + `rs/moq-cli` | `cargo build` | publish (video) + subscribe | | Python | `py/moq-rs` (+ `rs/moq-ffi`, import `moq`) | `uv build` wheels (maturin + hatchling), installed into a venv in the run directory | publish (video + audio) + subscribe | -| Go | `go/wrapper` (+ `rs/moq-ffi`, import `moq-go/moq`) | `go/scripts/stage.sh` (uniffi-bindgen-go) + `go build` | publish (video + audio) + subscribe | +| Go | `go/wrapper` (+ `rs/moq-ffi`, import `moq-go/moq`) | `sh/go/stage.sh` (uniffi-bindgen-go) + `go build` | publish (video + audio) + subscribe | | Browser | `js/watch` + `js/publish` | `vite build` + headless Chromium (Playwright) | publish (video + audio) + rendered playback | | Native JS | `js/net` + `js/hang` + the npm `@moq/web-transport` polyfill | `node` (tsx) and `bun` | subscribe | | C | `rs/libmoq` | `cargo build -p libmoq` + `cc` | subscribe | @@ -40,7 +40,7 @@ The browser, native JS, C, and GStreamer clients subscribe only by choice intentionally minimal, and `moqsink` publishing needs request-pad muxing this client doesn't drive). Rust, Python, Go, and the browser publish. -The Go client builds against the modules `go/scripts/stage.sh` assembles from +The Go client builds against the modules `sh/go/stage.sh` assembles from this checkout: `moq-ffi` compiled for the host, bindings regenerated with `uniffi-bindgen-go`, and the `go/wrapper` module wired to them by a `replace`. That is the same staging `just go check` uses, so this cell covers the Go diff --git a/test/interop/clients/go/go.mod b/test/interop/clients/go/go.mod index a57ce62b534..bd1ae2e2045 100644 --- a/test/interop/clients/go/go.mod +++ b/test/interop/clients/go/go.mod @@ -1,6 +1,6 @@ // Not a published module. test/interop/interop.sh builds this out of a scratch copy // and rewrites the require below into a `replace` pointing at the wrapper that -// go/scripts/stage.sh assembles from this checkout, so nothing resolves from the +// sh/go/stage.sh assembles from this checkout, so nothing resolves from the // module proxy. The placeholder version keeps the committed file honest: there // is no such tag, and there is no go.sum because every dependency is local. module moq.dev/interop diff --git a/test/interop/interop.sh b/test/interop/interop.sh index 0b7a45a1e6a..166257d7ae8 100755 --- a/test/interop/interop.sh +++ b/test/interop/interop.sh @@ -80,6 +80,14 @@ while [[ $# -gt 0 ]]; do MEDIA=1 shift ;; + # The full matrix. --timeout 30 gives headless Chromium cold-start + # headroom; flags after it still override. + --all) + PUBLISHERS="rust,python,go,js" + SUBSCRIBERS="rust,python,go,js,js-native-node,js-native-bun,c,gst" + TIMEOUT=30 + shift + ;; *) echo "unknown arg: $1" >&2 exit 2 @@ -250,7 +258,7 @@ prepare_js() { fi } -# Stage the Go modules from this checkout (go/scripts/stage.sh builds moq-ffi for +# Stage the Go modules from this checkout (sh/go/stage.sh builds moq-ffi for # the host, regenerates the bindings, and wires the wrapper to them by replace), # then build the interop client against that exact tree. The client is copied to a # scratch dir first so the committed go.mod keeps its placeholder require; every @@ -266,8 +274,8 @@ prepare_go() { } echo "building go client (workspace moq-go via uniffi-bindgen-go)..." local staged ffi_pkg wrapper_pkg src="$HARNESS_RUN/go-client" - if ! staged=$(bash "$WORKSPACE/go/scripts/stage.sh" --output "$HARNESS_RUN/go-stage" 2>"$HARNESS_RUN/go-stage.log"); then - mark_broken go "go/scripts/stage.sh failed" + if ! staged=$(bash "$WORKSPACE/sh/go/stage.sh" --output "$HARNESS_RUN/go-stage" 2>"$HARNESS_RUN/go-stage.log"); then + mark_broken go "sh/go/stage.sh failed" sed 's/^/ /' "$HARNESS_RUN/go-stage.log" >&2 || true return fi diff --git a/test/justfile b/test/justfile index d98acad97df..4b466640516 100644 --- a/test/justfile +++ b/test/justfile @@ -3,9 +3,6 @@ # Tests that span more than one language or need a live relay. Unit tests run # in the root `just check`. # -# Shebang recipes read their arguments as $1..$n. -set positional-arguments - # Two cross-language interop tests exist: # - https://github.com/moq-dev/smoke installs each client from its PUBLIC # package registry to catch packaging breakage in a release. @@ -15,8 +12,6 @@ set positional-arguments set fallback set working-directory := '.' -# `interop` reads its args as `$1`/`$@`; the `positional-arguments` setting above -# is what lets a shebang recipe see them. # List the cross-language suites. default: @@ -35,27 +30,21 @@ harness: # relay and runs the publisher x subscriber matrix. Default: rust only (fast # sanity check). `interop --all` runs the full matrix: rust + python + go + # browser publish; everyone subscribes (incl. native node/bun JS, the C libmoq -# client, and the GStreamer moqsrc plugin). --timeout 30 gives headless Chromium -# cold-start headroom. Other flags pass through, e.g. +# client, and the GStreamer moqsrc plugin). Other flags pass through, e.g. # `just test interop --publishers rust,python --subscribers rust,c`. # Both runs first check js/net's varints against moq-net's encoder at every size boundary, # and on lite-06/07 messages (the filter also matches `lite_varint_interop`). +[positional-arguments] interop *args: - #!/usr/bin/env bash - set -euo pipefail just rs test -p moq-net --lib varint_interop --run-ignored all - if [[ "${1:-}" == --all ]]; then - shift - ./interop/interop.sh --publishers rust,python,go,js --subscribers rust,python,go,js,js-native-node,js-native-bun,c,gst --timeout 30 "$@" - else - ./interop/interop.sh "$@" - fi + ./interop/interop.sh "$@" # Negative control: no publisher, every subscriber must time out (proves the # harness can actually report failure). +[positional-arguments] interop-negative *args: - ./interop/interop.sh --negative {{ args }} + ./interop/interop.sh --negative "$@" # Browser-to-browser media QA against a deterministic fixture: presented frame progress, audible # tone, audio/video skew, and the publication lifecycle (cold start, pause, rejoin, detach, @@ -63,8 +52,9 @@ interop-negative *args: # shifted tone, and a leaked session; each must fail the assertion it names. --timeout 30 gives # headless Chromium cold-start headroom. +[positional-arguments] media *args: - ./interop/interop.sh --media --timeout 30 {{ args }} + ./interop/interop.sh --media --timeout 30 "$@" # Disrupts a real relay over real QUIC three ways: cancel a stalled reader, kill # the relay mid-group, and republish an interrupted publisher's name. Ordinary @@ -73,16 +63,18 @@ media *args: # no-publisher negative control. See drill/README.md. # Transport and lifecycle failure drills. +[positional-arguments] drill *args: - just rs test -p moq-relay --test drills {{ args }} + just rs test -p moq-relay --test drills "$@" # Removes one recovery behavior at a time in a disposable copy of the tree and # requires the drill covering it to fail, for the reason the mutation names. The # checkout is never modified. `--list` shows the mutations. # Prove the drills fail without the fixes they cover. +[positional-arguments] drill-sensitivity *args: - ./drill/sensitivity.sh {{ args }} + ./drill/sensitivity.sh "$@" # Round-trips a PCR-paced TS through a relay and checks the subscriber's `export # ts` output with TSDuck plus a custom analyzer (PCR jitter/repetition, @@ -94,21 +86,24 @@ drill-sensitivity *args: # ts/README.md. # Check the subscriber's MPEG-TS output for IRD compliance. +[positional-arguments] ts *args: - ./ts/run.sh {{ args }} + ./ts/run.sh "$@" # Round-trip the EIT fixtures: the sparse schedule must survive and # pending-version sections must be dropped at import. +[positional-arguments] ts-eit *args: - ./ts/eit-roundtrip.sh {{ args }} + ./ts/eit-roundtrip.sh "$@" # Runs the @moq/wasm bindings in headless Chromium against a real relay, one per # protocol flavour. The crate is `#![cfg(target_arch = "wasm32")]`, so nothing in # `just check` or `just rs wasm` gets past compiling it. See wasm/README.md. # Browser coverage for the @moq/wasm bindings. +[positional-arguments] wasm *args: - ./wasm/run.sh {{ args }} + ./wasm/run.sh "$@" # Cross-language subscription termination over in-memory transports. bare-fin: diff --git a/test/wasm/README.md b/test/wasm/README.md index a65cb85dd8a..7e20248ccf6 100644 --- a/test/wasm/README.md +++ b/test/wasm/README.md @@ -21,7 +21,7 @@ just test wasm just test wasm --timeout 60 ``` -Everything is built from this checkout: `just wasm` for the bindings, `cargo +Everything is built from this checkout: `just js wasm` for the bindings, `cargo build -p moq-relay` for the relay. Each relay reserves its own port (see [the harness contract](../README.md)); `WASM_PORT` pins the first one instead. `WASM_PROFILE` picks the relay's cargo profile, and `RELAY_BIN` points at a diff --git a/test/wasm/run.sh b/test/wasm/run.sh index e2e649743e5..71ed51ba538 100755 --- a/test/wasm/run.sh +++ b/test/wasm/run.sh @@ -92,7 +92,7 @@ command -v wasm-bindgen >/dev/null 2>&1 || { # ── build ─────────────────────────────────────────────────────────────────── echo "building @moq/wasm..." -(cd "$WORKSPACE" && just wasm) +(cd "$WORKSPACE" && just js wasm) echo "building moq-relay ($PROFILE)..." flag=()