From a3f33ac9d6382ae3822dd8c62cb0f62104ec42e0 Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 10:01:52 -0800 Subject: [PATCH 1/3] chore(quest): claim quest/m0/ci/libav-sources.md Quest-Claim: 75f77211c989a4e5 From 9a6d8902a377cd43ebfac861a47b8cb9970335fe Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 10:04:08 -0800 Subject: [PATCH 2/3] ci: publish @moq/libav and its sources to nix.moq.dev Check's `bun install` builds `moq-libav` outside the dev shell, so the Cache workflow never pushed it and every hosted Check compiled it from source, fetching ffmpeg, opus and emfiberthreads from their upstream mirrors. Push the built package and a link farm of everything its build downloads alongside the dev shell. Co-Authored-By: Claude Opus 5.5 --- .github/justfile | 2 +- .github/workflows/cache.yml | 2 +- infra/README.md | 4 +++- js/libav/package.nix | 14 +++++++++++++- quest/m0/ci/README.md | 1 - quest/m0/ci/libav-sources.md | 26 -------------------------- sh/gh/cache.sh | 17 ++++++++++++----- 7 files changed, 30 insertions(+), 36 deletions(-) delete mode 100644 quest/m0/ci/libav-sources.md diff --git a/.github/justfile b/.github/justfile index c7c07caac1c..e1b2b59d4a9 100644 --- a/.github/justfile +++ b/.github/justfile @@ -1,6 +1,6 @@ set working-directory := '..' -# Build and publish the development environment to nix.moq.dev. +# Build and publish the development environment and @moq/libav to nix.moq.dev. cache: bash sh/gh/cache.sh diff --git a/.github/workflows/cache.yml b/.github/workflows/cache.yml index 6ffce7cb10d..8a63655fb9b 100644 --- a/.github/workflows/cache.yml +++ b/.github/workflows/cache.yml @@ -27,7 +27,7 @@ jobs: persist-credentials: false - uses: ./.github/actions/nix - uses: ./.github/actions/just - - name: Cache the dev shell + - name: Cache the dev shell and @moq/libav env: NIX_SIGNING_KEY: ${{ secrets.NIX_SIGNING_KEY }} AWS_ACCESS_KEY_ID: ${{ secrets.NIX_R2_ACCESS_KEY_ID }} diff --git a/infra/README.md b/infra/README.md index 62d7ade9aaf..9d8b7387343 100644 --- a/infra/README.md +++ b/infra/README.md @@ -268,7 +268,9 @@ gh workflow run rpm-repo.yml --ref release -f tag=moq-relay-v1.2.3 released packages, pushed by `release-nix.yml` on a release tag, and never expires. `nix-ci-moq-dev` holds the dev shells `cache.yml` pushes after every merge to `main`, for x86 Linux, ARM Linux, and ARM macOS. It publishes the full -build closure from `nix develop --profile`, uploading only missing paths. +build closure from `nix develop --profile`, uploading only missing paths, plus +`moq-libav` and its downloaded sources, which `bun install` builds outside the +dev shell. Its lifecycle rule (`infra/nix/lifecycle.json`) deletes each object 30 days after upload; the next warm re-uploads what it still needs. `flake.nix` sets `fallback = true` because that expiry can remove a path before one that diff --git a/js/libav/package.nix b/js/libav/package.nix index 2296397580d..f62af9400f7 100644 --- a/js/libav/package.nix +++ b/js/libav/package.nix @@ -18,6 +18,7 @@ fetchurl, fetchFromGitHub, importNpmLock, + linkFarm, emscripten, nodejs, python3, @@ -143,7 +144,18 @@ stdenvNoCC.mkDerivation { runHook postInstall ''; - passthru = { inherit wasm; }; + passthru = { + inherit wasm; + # Everything the wasm build downloads, which sh/gh/cache.sh publishes so a + # rebuild never waits on ffmpeg.org, xiph.org, GitHub or npm. + sources = linkFarm "moq-libav-sources" ( + sources + // { + "libav.js" = wasm.src; + "npm-deps" = wasm.npmDeps; + } + ); + }; meta = { description = "libav.js decoding Opus and AAC for the @moq/hang WebCodecs polyfill"; diff --git a/quest/m0/ci/README.md b/quest/m0/ci/README.md index 72e2fcaf392..d109c85b2d7 100644 --- a/quest/m0/ci/README.md +++ b/quest/m0/ci/README.md @@ -175,4 +175,3 @@ The historical samples locate candidates; they are not controlled benchmarks. - [A release merge opens no stale release PR](/quest/m0/ci/release-pr-after-release.md) - `just rs release` publishes before it proposes, so a merged release isn't re-proposed - [Renovate keeps the workspace-hack generated](/quest/m0/ci/dependabot-workspace-hack.md) - a protected scheduled Renovate runs `cargo hakari generate` in cargo update PRs, replacing Dependabot's cargo updates - [Build two queue entries at once if it helps](/quest/m0/ci/queue-concurrency.md) - a measured trial of `maximumEntriesToBuild: 2` against the serial lane's burst latency -- [CI never downloads libav's sources](/quest/m0/ci/libav-sources.md) - `moq-libav` substitutes from `nix.moq.dev`, so an upstream mirror can't fail Check diff --git a/quest/m0/ci/libav-sources.md b/quest/m0/ci/libav-sources.md deleted file mode 100644 index 961fd9536a1..00000000000 --- a/quest/m0/ci/libav-sources.md +++ /dev/null @@ -1,26 +0,0 @@ -# [XS] CI never downloads libav's sources from upstream mirrors - -## Goal - -A `just ci` run never fetches `@moq/libav`'s source tarballs from their -upstream sites: `moq-libav` (or at least its fixed-output sources) substitutes -from `nix.moq.dev`, so a flaky upstream mirror can't fail a check. Done when a -clean checkout's Check builds `moq-libav` with no request to `ffmpeg.org`, -`downloads.xiph.org` or GitHub archives. - -## Plan - -Planned 2026-11-04 with the maintainer. Check failed twice on 2026-11-03/04 -for this alone: [moq-dev/moq#7685](https://github.com/moq-dev/moq/pull/7685) -and [moq-dev/moq#7760](https://github.com/moq-dev/moq/pull/7760) (run -41777407147: "cannot download ffmpeg-9.0.tar.xz from any mirror", then -`moq-libav` and the `moq` postinstall failed). `js/libav/package.nix` -`fetchurl`s `ffmpeg-9.0.tar.xz`, `opus-1.6.1.tar.gz` and -`emfiberthreads-1.3.tar.gz` whenever `moq-libav` isn't substituted. - -Find why `moq-libav` was built at all: whether `sh/gh/cache.sh` pushes it (and -its sources) with the dev shell's build closure, or whether its derivation -changed without being pushed. Prefer pushing the built `moq-libav` and its -source fixed-output derivations to `nix-ci-moq-dev` over mirroring tarballs -by hand. Related: [A cache error is a miss](/quest/m0/ci/nix-cache-5xx.md) -covers `nix.moq.dev` errors, not upstream ones. diff --git a/sh/gh/cache.sh b/sh/gh/cache.sh index 1beaed69496..bfe2417f943 100644 --- a/sh/gh/cache.sh +++ b/sh/gh/cache.sh @@ -1,14 +1,21 @@ #!/usr/bin/env bash -# Publish the dev shell's build closure, including its build-time inputs. +# Publish what CI builds to run checks: the dev shell's build closure, +# including its build-time inputs, and @moq/libav with its sources. set -euo pipefail : "${NIX_SIGNING_KEY:?not set; the job needs the cache-write Environment}" : "${AWS_ACCESS_KEY_ID:?not set}" : "${AWS_SECRET_ACCESS_KEY:?not set}" -profile=$(mktemp -d) -trap 'rm -rf "$profile"' EXIT +roots=$(mktemp -d) +trap 'rm -rf "$roots"' EXIT # Building the devShell derivation alone omits inputs that nix develop needs. -nix develop --profile "$profile/devshell" --command true -just infra nix push nix-ci-moq-dev "$profile/devshell" +nix develop --profile "$roots/devshell" --command true +just infra nix push nix-ci-moq-dev "$roots/devshell" + +# `bun install` builds @moq/libav outside the dev shell (js/libav/build.ts). +# With its sources here too, a PR that changes its build still never +# downloads from the upstream mirrors. +nix build .#moq-libav .#moq-libav.sources --out-link "$roots/libav" +just infra nix push nix-ci-moq-dev "$roots/libav" "$roots/libav-1" From 9cc461c8978095bedce76fe6620f17dbac16f87d Mon Sep 17 00:00:00 2001 From: Luke Curley Date: Wed, 4 Nov 2026 10:35:11 -0800 Subject: [PATCH 3/3] ci: also publish the @moq/libav wasm build to nix.moq.dev moq-libav copies the wasm build's files instead of referencing them, so pushing its closure left the wasm derivation behind. A change to the package alone (version bump, README, index.d.ts) would then recompile ffmpeg in Check. Build and push .#moq-libav.wasm alongside it. Co-Authored-By: Claude Opus 5.5 --- infra/README.md | 4 ++-- sh/gh/cache.sh | 11 +++++++---- 2 files changed, 9 insertions(+), 6 deletions(-) diff --git a/infra/README.md b/infra/README.md index 9d8b7387343..ff4d6a36191 100644 --- a/infra/README.md +++ b/infra/README.md @@ -269,8 +269,8 @@ released packages, pushed by `release-nix.yml` on a release tag, and never expires. `nix-ci-moq-dev` holds the dev shells `cache.yml` pushes after every merge to `main`, for x86 Linux, ARM Linux, and ARM macOS. It publishes the full build closure from `nix develop --profile`, uploading only missing paths, plus -`moq-libav` and its downloaded sources, which `bun install` builds outside the -dev shell. +`moq-libav`, its wasm build, and its downloaded sources, which `bun install` +builds outside the dev shell. Its lifecycle rule (`infra/nix/lifecycle.json`) deletes each object 30 days after upload; the next warm re-uploads what it still needs. `flake.nix` sets `fallback = true` because that expiry can remove a path before one that diff --git a/sh/gh/cache.sh b/sh/gh/cache.sh index bfe2417f943..42467b20152 100644 --- a/sh/gh/cache.sh +++ b/sh/gh/cache.sh @@ -15,7 +15,10 @@ nix develop --profile "$roots/devshell" --command true just infra nix push nix-ci-moq-dev "$roots/devshell" # `bun install` builds @moq/libav outside the dev shell (js/libav/build.ts). -# With its sources here too, a PR that changes its build still never -# downloads from the upstream mirrors. -nix build .#moq-libav .#moq-libav.sources --out-link "$roots/libav" -just infra nix push nix-ci-moq-dev "$roots/libav" "$roots/libav-1" +# The package copies the wasm build's files rather than referencing them, so +# the wasm build goes up separately: a change to the package alone (a version +# bump, README or index.d.ts) still substitutes the ffmpeg compile. With the +# sources here too, a PR that changes the wasm build never downloads from the +# upstream mirrors. +nix build .#moq-libav .#moq-libav.wasm .#moq-libav.sources --out-link "$roots/libav" +just infra nix push nix-ci-moq-dev "$roots"/libav*