From 6bf21055cc24bfcacc75700567f7de845c180cc2 Mon Sep 17 00:00:00 2001 From: vjan-nie Date: Wed, 7 Oct 2026 22:11:42 +0200 Subject: [PATCH] ci(studio-pack): publish the studio pack as a pinned OCI image workflow_dispatch with a full commit sha (reachable from develop or main) builds the pack from git archive with a digest-pinned Dockerfile that mirrors scripts/studio-pack.sh, and pushes ghcr.io/univers42/graph-studio-pack:sha- (FROM scratch, /pack/). Nothing is pushed unless a --no-cache rebuild and the commit's own studio-pack.sh produce byte-identical packs. Additive: no existing file changes. --- .github/workflows/studio-pack.yml | 170 ++++++++++++++++++++++++++++++ packaging/studio-pack-manifest.sh | 79 ++++++++++++++ packaging/studio-pack.Dockerfile | 104 ++++++++++++++++++ 3 files changed, 353 insertions(+) create mode 100644 .github/workflows/studio-pack.yml create mode 100755 packaging/studio-pack-manifest.sh create mode 100644 packaging/studio-pack.Dockerfile diff --git a/.github/workflows/studio-pack.yml b/.github/workflows/studio-pack.yml new file mode 100644 index 00000000..b3446e1d --- /dev/null +++ b/.github/workflows/studio-pack.yml @@ -0,0 +1,170 @@ +# Publishes the studio pack of one commit as ghcr.io/univers42/graph-studio-pack:sha-, +# a FROM-scratch image holding the six pack files under /pack/ (docs/contract/packaging.md). +# +# Manual only. The commit is built from `git archive`, with packaging/ from the ref this +# workflow runs on, so a commit older than the workflow can be packed. The commit must be a +# full sha reachable from develop or main. A tag is written once and never moved; there is +# no :latest. +# +# Three builds before anything is pushed: the Dockerfile twice (the second with no cache, so a +# cold cargo and build-std run must agree with the first), then the commit's own +# scripts/studio-pack.sh as the oracle — the published pack must be byte-identical to it. After +# the push the image is pulled back by digest and compared with the first build. +# +# Ponytail: all builds run on one runner on one day, so they cannot catch drift in what apt or +# the registries serve between days — only what varies inside one build. The tag-existence check +# reads GHCR's error text; an answer it does not recognise as "absent" fails the run. +# +# Every action is pinned to a full commit sha with its version in a comment. + +name: studio-pack + +on: + workflow_dispatch: + inputs: + commit: + description: Full 40-hex graph_render commit to pack (reachable from develop or main) + required: true + type: string + +permissions: + contents: read + packages: write + +concurrency: + group: studio-pack-${{ inputs.commit }} + cancel-in-progress: false + +env: + COMMIT: ${{ inputs.commit }} + IMAGE: ghcr.io/univers42/graph-studio-pack + +jobs: + pack: + runs-on: ubuntu-24.04 + timeout-minutes: 180 + steps: + - name: Validate the commit input + run: | + [[ $COMMIT =~ ^[0-9a-f]{40}$ ]] || { + echo "::error::commit must be a full lowercase 40-hex sha, got '$COMMIT'"; exit 1; } + + - name: Checkout + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + fetch-depth: 0 + persist-credentials: false + + - name: Reachable from develop or main + run: | + git cat-file -e "$COMMIT^{commit}" || { echo "::error::$COMMIT is not in this repository's history"; exit 1; } + git merge-base --is-ancestor "$COMMIT" origin/develop || + git merge-base --is-ancestor "$COMMIT" origin/main || + { echo "::error::$COMMIT is reachable from neither origin/develop nor origin/main"; exit 1; } + + # Tracked files only: no .git, no submodule, nothing the commit did not commit. + - name: Archive the commit + run: | + mkdir "$RUNNER_TEMP/tree" + git archive "$COMMIT" | tar -x -C "$RUNNER_TEMP/tree" + + # packaging/studio-pack.Dockerfile mirrors these files as they are at 77b68d9b. A commit + # whose copy differs would be packed by a stale mirror, so it is refused here, before an + # hour of building; update the Dockerfile and these sums together. + - name: Recipe matches the mirror + working-directory: ${{ runner.temp }}/tree + run: | + sha256sum -c --strict - <<'SUMS' + 82ed220894f046f52875635c5cc18631f79dcbf82a1c93ee256cfa571f8126a4 scripts/studio-pack.sh + 7c02695dedb34186f43aae2e0571a3146981d9243f88110e5425a921ba045e51 scripts/orch/wasm-threads.sh + 522a1f7129c7086d776b74a9173227c66a1630f4cb9f5ea04f67dff48e07e668 docker/rust.Dockerfile + 44504494f468fd7e3de00e328deeb2fbe325f2f38ab27c12691360a170b06314 docker/wasm-threads.Dockerfile + 534d81f7ff402a094c1c929e0ea574ffea31d4d90175ca09f8c5c4416414521d app/vite.pack.config.ts + SUMS + build_wasm=" \"\$gr\" cargo build -p graph-wasm --target wasm32-unknown-unknown --release" + grep -qxF "$build_wasm" scripts/studio.sh || + { echo "::error::scripts/studio.sh build_wasm changed; the Dockerfile's serial cargo build mirrors it"; exit 1; } + # image.sh moves for unrelated reasons, so only its node pin is compared. + ours=$(sed -n 's/^ARG NODE_IMAGE=//p' "$GITHUB_WORKSPACE/packaging/studio-pack.Dockerfile") + theirs=$(sed -n "s/^export GM_NODE_IMAGE=\${NODE_IMAGE:-\(.*\)}\$/\1/p" scripts/orch/image.sh) + [[ -n $ours && $ours == "$theirs" ]] || + { echo "::error::NODE_IMAGE: the Dockerfile has '$ours', the commit's scripts/orch/image.sh has '$theirs'"; exit 1; } + + - name: Log in to ghcr.io + env: + TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: printf '%s' "$TOKEN" | docker login ghcr.io -u "$GITHUB_ACTOR" --password-stdin + + - name: Tag is not published yet + run: | + if out=$(docker buildx imagetools inspect "$IMAGE:sha-$COMMIT" 2>&1); then + echo "::error::$IMAGE:sha-$COMMIT already exists; a published tag is never rewritten"; exit 1 + fi + grep -qiE 'not found|manifest unknown|name unknown' <<<"$out" || + { echo "::error::cannot tell whether $IMAGE:sha-$COMMIT exists: $out"; exit 1; } + echo "absent: $out" + + - name: Build twice and compare + run: | + build() { + docker buildx build "$@" --target pack-files \ + --build-context ctl=packaging --build-arg GRAPH_RENDER_SHA="$COMMIT" \ + -f packaging/studio-pack.Dockerfile "$RUNNER_TEMP/tree" + } + build --output "type=local,dest=$RUNNER_TEMP/pack-a" + build --no-cache --output "type=local,dest=$RUNNER_TEMP/pack-b" + diff -r "$RUNNER_TEMP/pack-a" "$RUNNER_TEMP/pack-b" + + - name: Byte-identical to the commit's own studio-pack.sh + run: | + git worktree add --detach "$RUNNER_TEMP/upstream" "$COMMIT" + "$RUNNER_TEMP/upstream/scripts/studio-pack.sh" target/pack + packs=("$RUNNER_TEMP"/upstream/target/pack/graph-studio-*/) + ((${#packs[@]} == 1)) || { echo "::error::expected one pack directory, got ${packs[*]}"; exit 1; } + diff -r "${packs[0]}" "$RUNNER_TEMP/pack-a" + + - name: Push + run: | + docker buildx build --push --provenance=false --sbom=false --target image \ + --build-context ctl=packaging \ + --build-arg GRAPH_RENDER_SHA="$COMMIT" \ + --build-arg RECIPE_REV="$GITHUB_SHA" \ + --build-arg SOURCE_URL="$GITHUB_SERVER_URL/$GITHUB_REPOSITORY" \ + --build-arg CREATED="$(date -u +%Y-%m-%dT%H:%M:%SZ)" \ + --metadata-file "$RUNNER_TEMP/meta.json" \ + -t "$IMAGE:sha-$COMMIT" \ + -f packaging/studio-pack.Dockerfile "$RUNNER_TEMP/tree" + + - name: Pushed bytes equal the verified build + run: | + digest=$(jq -r '."containerimage.digest"' "$RUNNER_TEMP/meta.json") + [[ $digest =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "::error::no digest in the build metadata"; exit 1; } + echo "DIGEST=$digest" >>"$GITHUB_ENV" + docker pull "$IMAGE@$digest" + id=$(docker create "$IMAGE@$digest" none) + docker cp "$id:/pack" "$RUNNER_TEMP/pushed" + docker rm "$id" + diff -r "$RUNNER_TEMP/pack-a" "$RUNNER_TEMP/pushed" + + - name: Job summary + if: always() + run: | + { + echo "### graph-studio-pack" + echo + echo "- commit (revision): \`$COMMIT\`" + echo "- recipe_rev (this workflow): \`$GITHUB_SHA\`" + if [[ -n ${DIGEST-} ]]; then + echo "- pushed: \`$IMAGE@$DIGEST\` as \`:sha-$COMMIT\`" + else + echo "- nothing pushed" + fi + echo + if [[ -d $RUNNER_TEMP/pack-a ]]; then + echo '| file | bytes | sha256 |' + echo '|---|---:|---|' + for f in "$RUNNER_TEMP"/pack-a/*; do + echo "| \`${f##*/}\` | $(wc -c <"$f") | \`$(sha256sum "$f" | cut -d' ' -f1)\` |" + done + fi + } >>"$GITHUB_STEP_SUMMARY" diff --git a/packaging/studio-pack-manifest.sh b/packaging/studio-pack-manifest.sh new file mode 100755 index 00000000..83d4a206 --- /dev/null +++ b/packaging/studio-pack-manifest.sh @@ -0,0 +1,79 @@ +#!/usr/bin/env bash +# studio-pack-manifest.sh PACK_DIR SOURCE_REV TREE — writes PACK_DIR/pack.json and verifies the +# pack, for packaging/studio-pack.Dockerfile. A port of scripts/studio-pack.sh's write_manifest, +# verify_files and verify_references (same pack.json line format, same checks, its worker-form.sh +# taken from TREE), plus one check of its own: the pack is exactly the six files of +# docs/contract/packaging.md. SOURCE_REV is the commit the workflow archived, so never `-dirty`. +# +# Exit: 0 written and verified · 1 the pack is bad · 2 usage. +set -uo pipefail + +(($# == 3)) || { echo "usage: studio-pack-manifest.sh PACK_DIR SOURCE_REV TREE" >&2; exit 2; } +pack=$1 rev=$2 tree=$3 +[[ $rev =~ ^[0-9a-f]{40}$ ]] || { echo "studio-pack: SOURCE_REV is not a full sha: '$rev'" >&2; exit 2; } +readonly EXPECTED_FILES="graph-studio.js graph_wasm.wasm graph_wasm_threads.wasm helper.js pack.json worker.js" +name=graph-studio +version=$(sed -n 's/^ "version": "\([^"]*\)",$/\1/p' "$tree/packages/graph-studio/package.json" | head -1) +abi=$(sed -n 's/^pub const ABI_VERSION: u32 = \([0-9][0-9]*\);$/\1/p' "$tree/crates/graph-wasm/src/lib.rs" | head -1) +problems=0 + +bad() { problems=$((problems + 1)); printf 'studio-pack: %s\n' "$*" >&2; } + +write_manifest() { + local file bytes digest first=1 + printf '{\n "name": "%s",\n "version": "%s",\n "abi_version": %s,\n "source_rev": "%s",\n "files": {\n' \ + "$name" "$version" "$abi" "$rev" >"$pack/pack.json" + while IFS= read -r file; do + bytes=$(wc -c <"$pack/$file") + digest=$(sha256sum "$pack/$file" | cut -d' ' -f1) + [[ $first == 1 ]] || printf ',\n' >>"$pack/pack.json" + first=0 + printf ' "%s": {"bytes": %s, "sha256": "%s"}' "$file" "$bytes" "$digest" >>"$pack/pack.json" + done < <(cd "$pack" && find . -type f ! -name pack.json | sed 's|^\./||' | LC_ALL=C sort) + printf '\n }\n}\n' >>"$pack/pack.json" +} + +manifest() { + sed -n 's/^ "\([^"]*\)": {"bytes": \([0-9][0-9]*\), "sha256": "\([0-9a-f]\{64\}\)"}\(,\)\?$/\1 \2 \3/p' \ + "$pack/pack.json" +} + +verify_file_set() { + local found + found=$(cd "$pack" && find . -type f | sed 's|^\./||' | LC_ALL=C sort | tr '\n' ' ') + [[ ${found% } == "$EXPECTED_FILES" ]] || bad "pack files are '${found% }', packaging.md lists '$EXPECTED_FILES'" +} + +verify_files() { + local listed=0 file bytes digest got + while read -r file bytes digest; do + listed=$((listed + 1)) + if [[ ! -f $pack/$file ]]; then bad "missing $file (pack.json lists it)"; continue; fi + got=$(sha256sum "$pack/$file" | cut -d' ' -f1) + [[ $got == "$digest" ]] || bad "$file: sha256 $got, pack.json says $digest" + [[ $(wc -c <"$pack/$file") == "$bytes" ]] || bad "$file: not $bytes bytes, as pack.json says" + done < <(manifest) + ((listed > 0)) || bad "pack.json lists no file in the shape the verify reads" +} + +verify_references() { + local line offenders verdict + grep -qsF 'graph_wasm_threads.wasm' "$pack"/*.js || + bad "no pack .js names graph_wasm_threads.wasm: the worker would fall back to the serial module" + # Comments dropped: rollup's //#region banner names each module's own path, which is provenance. + offenders=$(awk '!/^[[:space:]]*(\/\/|\/\*|\*)/ && ($0 ~ /\/src\// || $0 ~ /node_modules/ || $0 ~ /\.\.\//) { + print FILENAME ":" FNR ": " $0 }' "$pack"/*.js | head -5) + while IFS= read -r line; do + [[ -n $line ]] && bad "the pack reaches outside itself: $line" + done <<<"$offenders" + verdict=$(bash "$tree/scripts/worker-form.sh" "$pack" 2>&1 >/dev/null) || + bad "worker-form.sh refused the pack: $verdict" +} + +[[ -n $version && -n $abi ]] || { echo "studio-pack: no version/abi_version in $tree" >&2; exit 1; } +write_manifest +verify_file_set +verify_files +verify_references +((problems == 0)) || { echo "studio-pack: $problems problem(s): the pack is not shippable" >&2; exit 1; } +echo "studio-pack: ok, $name-$version abi $abi source_rev $rev" diff --git a/packaging/studio-pack.Dockerfile b/packaging/studio-pack.Dockerfile new file mode 100644 index 00000000..3b611a20 --- /dev/null +++ b/packaging/studio-pack.Dockerfile @@ -0,0 +1,104 @@ +# The studio pack (docs/contract/packaging.md) as an OCI image: FROM scratch, the six pack files +# under /pack/ and nothing else. Built by .github/workflows/studio-pack.yml, whose context is +# `git archive ` — so it can pack a commit older than this file. +# +# It mirrors scripts/studio-pack.sh step for step instead of running it: that script drives docker +# itself and builds ge-rust FROM an unpinned debian:trixie-slim, and a published image is built +# from bases pinned by digest. The cost is drift: a change to studio-pack.sh, studio.sh's wasm +# build, scripts/orch/wasm-threads.sh or the toolchain pins must be repeated here. The workflow +# refuses a commit whose copies of those files differ from the ones this mirrors (77b68d9b), and +# publishes nothing unless this pack is byte-identical to the commit's own studio-pack.sh output. +# +# Ponytail: apt packages are not pinned (whatever trixie ships on the build day). gcc only links +# host build scripts and the wasm links with rust-lld, so the pack is expected not to move with +# them — expected, not proven; two builds on one day cannot see it. The mirror is of +# studio-pack.sh at 77b68d9b: when the workflow's drift guard fails, update this file first. + +ARG DEBIAN_IMAGE=debian:trixie-slim@sha256:a99cfc517144bc59b1978475ec53b46ecabec7e43635402ee5b77cc54cd1b20a +ARG NODE_IMAGE=node:22.23.3-slim@sha256:43ac6c60b8f89723f746e8a92ce91abd5017e627ce1ddfe4238355d3a30b772c + +# hadolint ignore=DL3006 +FROM ${DEBIAN_IMAGE} AS toolchain +ARG RUST_VERSION=1.98.1 +ARG RUSTUP_VERSION=1.29.1 +ARG RUSTUP_SHA256=dda7234360b7f578ca8b0ddcb80145646fa61a67c1720a5abc7051b35c9fcb71 +# Same homes as docker/rust.Dockerfile: the paths rustc embeds in the wasm stay those of a local build. +ENV CARGO_HOME=/opt/cargo RUSTUP_HOME=/opt/rustup PATH=/opt/cargo/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin +SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# hadolint ignore=DL3008 +RUN set -eu; \ + [ "$(dpkg --print-architecture)" = amd64 ] || { echo "studio-pack: amd64 only (RUSTUP_SHA256 is the amd64 one)" >&2; exit 1; }; \ + apt-get update; \ + apt-get install -y --no-install-recommends ca-certificates curl gcc libc6-dev; \ + rm -rf /var/lib/apt/lists/*; \ + curl -fsSLo /tmp/rustup-init "https://static.rust-lang.org/rustup/archive/${RUSTUP_VERSION}/x86_64-unknown-linux-gnu/rustup-init"; \ + echo "${RUSTUP_SHA256} /tmp/rustup-init" | sha256sum -c -; \ + chmod +x /tmp/rustup-init; \ + /tmp/rustup-init -y --no-modify-path --profile minimal --default-toolchain "${RUST_VERSION}" \ + --target wasm32-unknown-unknown; \ + rm /tmp/rustup-init + +# Two stages, as upstream has two images (ge-rust, ge-wasm-threads): with rust-src installed, +# rustc embeds std's real source paths in panic locations instead of /rustc//, and the +# serial wasm came out 377 bytes off upstream's (measured at 77b68d9b). So rust-src exists only +# for the threads build. Each runs `cargo fetch --locked` first; the builds are the exact +# commands of scripts/studio.sh build_wasm and scripts/orch/wasm-threads.sh, without --locked. +FROM toolchain AS wasm-serial +WORKDIR /w +COPY . . +RUN cargo fetch --locked && cargo build -p graph-wasm --target wasm32-unknown-unknown --release + +FROM toolchain AS wasm-threads +RUN rustup component add rust-src +WORKDIR /w +COPY . . +RUN cargo fetch --locked && \ + RUSTC_BOOTSTRAP=1 \ + RUSTFLAGS="-C target-feature=+atomics,+bulk-memory,+mutable-globals -C link-arg=--shared-memory -C link-arg=--import-memory -C link-arg=--initial-memory=67108864 -C link-arg=--max-memory=4294967296 -C link-arg=--export=__wasm_init_tls -C link-arg=--export=__tls_size -C link-arg=--export=__tls_align -C link-arg=--export=__tls_base -C link-arg=--export=__stack_pointer" \ + cargo build --release -p graph-wasm --target wasm32-unknown-unknown \ + -Z build-std=std,panic_abort --features threads --target-dir target/wasm-threads + +# hadolint ignore=DL3006 +FROM ${NODE_IMAGE} AS pack +ARG GRAPH_RENDER_SHA +WORKDIR /w/app +COPY app/package.json app/package-lock.json ./ +RUN NPM_CONFIG_UPDATE_NOTIFIER=false npm ci --ignore-scripts --no-audit --no-fund +WORKDIR /w +COPY . . +WORKDIR /w/app +RUN PACK_MODE=pack PACK_OUT_DIR=/out node_modules/.bin/vite build --config vite.pack.config.ts +COPY --from=wasm-serial /w/target/wasm32-unknown-unknown/release/graph_wasm.wasm /out/graph_wasm.wasm +COPY --from=wasm-threads /w/target/wasm-threads/wasm32-unknown-unknown/release/graph_wasm.wasm /out/graph_wasm_threads.wasm +# scripts/orch/wasm-threads.sh's import/export check, verbatim (single quotes: it is JavaScript) +# hadolint ignore=SC2016 +RUN node -e ' \ +const m = new WebAssembly.Module(require("fs").readFileSync(process.argv[1])); \ +const im = WebAssembly.Module.imports(m).map((i) => `${i.module}.${i.name}:${i.kind}`); \ +const ex = new Set(WebAssembly.Module.exports(m).map((e) => e.name)); \ +const need = ["__wasm_init_tls", "__tls_size", "__stack_pointer", "gm_thread_serve", "gm_run_threaded", "gm_force_session_tick_threaded"]; \ +const missing = need.filter((n) => !ex.has(n)); \ +const ok = im.length === 1 && im[0] === "env.memory:memory" && missing.length === 0; \ +console.log(`imports ${im.join(",")} · missing exports ${missing.join(",") || "none"}`); \ +process.exit(ok ? 0 : 1); \ +' /out/graph_wasm_threads.wasm +# ctl is a named build context (--build-context ctl=packaging), not a stage. +# hadolint ignore=DL3022 +COPY --from=ctl studio-pack-manifest.sh /usr/local/bin/ +RUN studio-pack-manifest.sh /out "${GRAPH_RENDER_SHA}" /w + +# The bare pack, for `--output type=local` (the workflow's two-build comparison). +FROM scratch AS pack-files +COPY --from=pack /out/ / + +FROM scratch AS image +ARG GRAPH_RENDER_SHA +ARG SOURCE_URL +ARG CREATED +ARG RECIPE_REV +# revision is the packed commit; recipe_rev the commit this Dockerfile and its workflow came from. +LABEL org.opencontainers.image.revision="${GRAPH_RENDER_SHA}" \ + org.univers42.graph-studio-pack.recipe-rev="${RECIPE_REV}" \ + org.opencontainers.image.source="${SOURCE_URL}" \ + org.opencontainers.image.created="${CREATED}" +COPY --from=pack-files / /pack/