ci: canonicalize and merge-gate pixi lockfiles

.github/workflows/ci-pixi-lockfile-freshness-check.yml

@@ -19,23 +19,31 @@ concurrency:
 
 on:
   pull_request:
+    # A required workflow must run on every PR: a workflow skipped by a path
+    # filter leaves its required check pending. Label changes rerun the matrix
+    # so the aggregate gate immediately reflects the override state.
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - labeled
+      - unlabeled
+  push:
+    # `pull_request` already covers PRs, including those from forks: this check
+    # needs no secrets or GPU runner. Watching copy-pr-bot's `pull-request/N`
+    # mirror too would run the whole matrix a second time per PR.
+    branches:
+      - "main"
     # `pyproject.toml` is a lockfile input too: the manifests consume sibling
     # packages (including cuda_python_test_helpers) as path dependencies, so
     # their metadata can stale a lock without any pixi.toml edit. Matched by
     # glob rather than by name so the filter cannot drift as packages move.
-    paths: &lockfile_inputs
+    paths:
       - "**/pixi.toml"
       - "**/pixi.lock"
       - "**/pyproject.toml"
       - "ci/tools/list_pixi_workspaces.py"
       - ".github/workflows/ci-pixi-lockfile-freshness-check.yml"
-  push:
-    # `pull_request` already covers PRs, including those from forks: this check
-    # needs no secrets or GPU runner. Watching copy-pr-bot's `pull-request/N`
-    # mirror too would run the whole matrix a second time per PR.
-    branches:
-      - "main"
-    paths: *lockfile_inputs
   workflow_dispatch: {}
 
 defaults:
@@ -46,6 +54,8 @@ env:
   # Keep in sync with ci-pixi-source-test.yml. Must be >=0.71.0 so the
   # committed lockfile format (v7) matches what this job checks against.
   PIXI_VERSION: "v0.73.0"
+  REFRESH_WORKFLOW_URL: "https://github.com/NVIDIA/cuda-python/actions/workflows/ci-pixi-lockfile-refresh.yml"
+  OVERRIDE_LABEL: "ci-pixi-lockfile-override"
 
 permissions: {}
 
@@ -100,9 +110,11 @@ jobs:
           run-install: false
 
       - name: Check lockfile is current with its manifest
+        env:
+          MANIFEST: ${{ matrix.manifest }}
         run: |
-          if ! pixi lock --check --manifest-path "${{ matrix.manifest }}"; then
-            echo "::error::Lockfile is stale for '${{ matrix.manifest }}'. Regenerate with: pixi lock --manifest-path ${{ matrix.manifest }}"
+          if ! pixi lock --check --manifest-path "${MANIFEST}"; then
+            echo "::error::Lockfile is stale for '${MANIFEST}'. Run and commit: pixi lock --manifest-path ${MANIFEST}. For default-branch dependency drift, maintainers can run ${REFRESH_WORKFLOW_URL}"
             exit 1
           fi
 
@@ -112,8 +124,75 @@ jobs:
       # is invisible: the check prints "Updated lock file" and still passes,
       # while every later pixi run keeps rewriting the committed file (#2298).
       - name: Check lockfile is byte-for-byte canonical
+        env:
+          LOCKFILE: ${{ matrix.lockfile }}
+          MANIFEST: ${{ matrix.manifest }}
         run: |
-          if ! git diff --exit-code -- "${{ matrix.lockfile }}"; then
-            echo "::error::pixi ${PIXI_VERSION} rewrote ${{ matrix.lockfile }} during the check, so the committed bytes are not what it generates. Regenerate with pixi ${PIXI_VERSION}: pixi lock --manifest-path ${{ matrix.manifest }}"
+          if ! git diff --exit-code -- "${LOCKFILE}"; then
+            echo "::error::pixi ${PIXI_VERSION} rewrote ${LOCKFILE} during the check, so the committed bytes are not canonical. Run and commit with pixi ${PIXI_VERSION}: pixi lock --manifest-path ${MANIFEST}. For default-branch dependency drift, maintainers can run ${REFRESH_WORKFLOW_URL}"
             exit 1
           fi
+
+  gate:
+    name: Pixi lockfile freshness gate
+    if: ${{ always() && github.repository_owner == 'nvidia' }}
+    needs:
+      - plan
+      - lockfile-fresh
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions: {}
+    steps:
+      - name: Evaluate freshness results
+        env:
+          EVENT_ACTION: ${{ github.event.action }}
+          EVENT_NAME: ${{ github.event_name }}
+          LABELS_JSON: ${{ toJSON(github.event.pull_request.labels.*.name) }}
+          LOCKFILE_RESULT: ${{ needs.lockfile-fresh.result }}
+          PLAN_RESULT: ${{ needs.plan.result }}
+        run: |
+          has_override=false
+          if jq -e --arg label "${OVERRIDE_LABEL}" \
+            '(. // []) | index($label) != null' <<<"${LABELS_JSON}" >/dev/null; then
+            has_override=true
+          fi
+
+          {
+            echo "### Pixi lockfile freshness gate"
+            echo
+            echo "- Workspace discovery: \`${PLAN_RESULT}\`"
+            echo "- Lockfile checks: \`${LOCKFILE_RESULT}\`"
+            echo "- Override label present: \`${has_override}\`"
+          } >> "${GITHUB_STEP_SUMMARY}"
+
+          if [ "${PLAN_RESULT}" != "success" ]; then
+            echo "::error::Pixi workspace discovery did not succeed (${PLAN_RESULT})."
+            exit 1
+          fi
+
+          if [ "${LOCKFILE_RESULT}" = "success" ]; then
+            exit 0
+          fi
+
+          # Override approval applies only to a known diagnostic failure on the
+          # current PR head. opened/synchronize/reopened events cannot carry
+          # prior approval; the base-controlled reset workflow removes the
+          # label when one is present.
+          if [ "${EVENT_NAME}" = "pull_request" ] && \
+             [ "${EVENT_ACTION}" != "opened" ] && \
+             [ "${EVENT_ACTION}" != "synchronize" ] && \
+             [ "${EVENT_ACTION}" != "reopened" ] && \
+             [ "${LOCKFILE_RESULT}" = "failure" ] && \
+             [ "${has_override}" = "true" ]; then
+            echo "::warning::${OVERRIDE_LABEL} overrides the failing aggregate gate; the failed workspace diagnostics remain visible."
+            {
+              echo
+              echo "> [!WARNING]"
+              echo "> A maintainer override allows this gate to pass. Review the failed"
+              echo "> workspace diagnostics before merging."
+            } >> "${GITHUB_STEP_SUMMARY}"
+            exit 0
+          fi
+
+          echo "::error::Pixi lockfile checks did not succeed (${LOCKFILE_RESULT})."
+          exit 1

.github/workflows/ci-pixi-lockfile-override-reset.yml

@@ -0,0 +1,41 @@
+# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+#
+# SPDX-License-Identifier: Apache-2.0
+
+name: "CI: invalidate pixi lockfile override"
+
+on:
+  # pull_request_target loads this workflow from the trusted base branch. It
+  # must never check out or execute code from the pull request.
+  pull_request_target:
+    types:
+      - opened
+      - synchronize
+      - reopened
+
+permissions: {}
+
+jobs:
+  invalidate:
+    name: Invalidate Pixi lockfile override
+    if: >-
+      github.repository_owner == 'nvidia' &&
+      contains(github.event.pull_request.labels.*.name, 'ci-pixi-lockfile-override')
+    runs-on: ubuntu-latest
+    timeout-minutes: 5
+    permissions:
+      issues: write
+    steps:
+      - name: Remove override from changed PR
+        env:
+          GH_TOKEN: ${{ github.token }}
+          OVERRIDE_LABEL: ci-pixi-lockfile-override
+          PR_NUMBER: ${{ github.event.pull_request.number }}
+        run: |
+          endpoint="repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels"
+          labels="$(gh api "${endpoint}")"
+          if jq -e --arg label "${OVERRIDE_LABEL}" \
+            'map(.name) | index($label) != null' <<<"${labels}" >/dev/null; then
+            gh api --method DELETE "${endpoint}/${OVERRIDE_LABEL}"
+            echo "::notice::Removed ${OVERRIDE_LABEL}; a new PR head requires fresh maintainer approval."
+          fi

.github/workflows/ci-pixi-lockfile-refresh.yml

@@ -83,11 +83,10 @@ jobs:
           fetch-depth: 0
           ref: ${{ github.event.repository.default_branch }}
 
-      # GitHub suppresses workflow-triggering events for pushes and PRs made
-      # with GITHUB_TOKEN, so a refresh PR opened with the default token cannot
-      # reach a green required-check state on its own. An App token restores
-      # normal event delivery; without one the PR needs a manual nudge, which
-      # the PR body and a run warning spell out.
+      # Pull-request runs created by GITHUB_TOKEN wait for maintainer approval,
+      # and other workflow-triggering events remain suppressed. An App token
+      # avoids that approval step and restores normal event delivery. The PR
+      # body and run warning explain the fallback behavior.
       - name: Mint App token for the refresh PR
         id: app-token
         if: ${{ vars.PIXI_LOCK_REFRESH_APP_ID != '' }}
@@ -104,8 +103,25 @@ jobs:
           pixi-version: ${{ env.PIXI_VERSION }}
           run-install: false
 
-      - name: Update lockfile without installing
-        run: pixi update --no-install --manifest-path "${{ matrix.manifest }}"
+      # Pixi 0.73.0's update and lock-check paths can serialize the same
+      # solution differently (#2804). The first check canonicalizes the update
+      # result; the second proves that the committed bytes reached a fixed
+      # point before create-pull-request sees them.
+      - name: Update and canonicalize lockfile without installing
+        env:
+          LOCKFILE: ${{ matrix.lockfile }}
+          MANIFEST: ${{ matrix.manifest }}
+        run: |
+          pixi update --no-install --manifest-path "${MANIFEST}"
+          pixi lock --check --manifest-path "${MANIFEST}"
+
+          canonical_lock="${RUNNER_TEMP}/pixi-lock-after-canonicalization.lock"
+          cp -- "${LOCKFILE}" "${canonical_lock}"
+          pixi lock --check --manifest-path "${MANIFEST}"
+          if ! cmp --silent "${canonical_lock}" "${LOCKFILE}"; then
+            echo "::error::pixi ${PIXI_VERSION} did not reach a byte-stable lockfile after canonicalization: ${LOCKFILE}"
+            exit 1
+          fi
 
       - name: Compose PR body
         id: pr-body
@@ -117,17 +133,18 @@ jobs:
           body_path="${RUNNER_TEMP}/pixi-lock-refresh-body.md"
           {
             echo "Automated lockfile refresh for \`${WORKSPACE_ID}\`" \
-              "(\`pixi update --no-install --manifest-path ${MANIFEST}\`)."
+              "(\`pixi update --no-install --manifest-path ${MANIFEST}\`, followed"
+            echo "by pinned-pixi canonicalization and a byte-stability check)."
             echo
             echo "This PR is workspace-scoped so other packages are not forced to review"
             echo "unrelated solver churn. See #2298."
             echo
             if [ "${HAS_APP_TOKEN}" != "true" ]; then
               echo "> [!IMPORTANT]"
               echo "> No refresh App token is configured, so this PR was opened with"
-              echo "> \`GITHUB_TOKEN\` and GitHub will not deliver its \`pull_request\` events."
-              echo "> Required checks stay pending until a maintainer nudges the branch, by"
-              echo "> closing and reopening the PR or pushing an empty commit to it."
+              echo "> \`GITHUB_TOKEN\`. GitHub creates its \`opened\`, \`synchronize\`, and"
+              echo "> \`reopened\` workflow runs in an approval-required state; a maintainer"
+              echo "> must approve those runs before checks start. Other events remain suppressed."
               echo
             fi
             echo "If the diff looks wrong, close this PR and leave the branch deleted; the"
@@ -136,7 +153,7 @@ jobs:
           echo "path=${body_path}" >> "${GITHUB_OUTPUT}"
 
           if [ "${HAS_APP_TOKEN}" != "true" ]; then
-            echo "::warning::Opened with GITHUB_TOKEN; the refresh PR's checks must be triggered manually."
+            echo "::warning::Opened with GITHUB_TOKEN; a maintainer must approve the refresh PR's workflow runs."
           fi
 
       - name: Open or update a lockfile-only PR
@@ -148,7 +165,8 @@ jobs:
           commit-message: |
             chore: refresh pixi.lock for ${{ matrix.id }}
 
-            Scheduled pixi update --no-install for this workspace only.
+            Scheduled pixi update --no-install followed by pinned-pixi
+            canonicalization for this workspace only.
           signoff: true
           sign-commits: true
           branch: ci/pixi-lock-refresh/${{ matrix.id }}
@@ -168,6 +186,7 @@ jobs:
             echo ""
             echo "- Manifest: \`${{ matrix.manifest }}\`"
             echo "- Lockfile: \`${{ matrix.lockfile }}\`"
+            echo "- Canonicalization: byte-stable fixed point verified"
             echo "- Operation: \`${{ steps.cpr.outputs.pull-request-operation }}\`"
             if [ -n "${{ steps.cpr.outputs.pull-request-url }}" ]; then
               echo "- PR: ${{ steps.cpr.outputs.pull-request-url }}"

CONTRIBUTING.md

@@ -218,21 +218,30 @@ The freshness check additionally fails when the check itself rewrote a lockfile.
 not canonical for the pinned pixi version, quietly normalizing the file instead,
 which leaves every later pixi run rewriting the committed lockfile.
 
-A scheduled workflow (`CI: pixi lockfile refresh`)
-runs `pixi update --no-install` per workspace and opens a dedicated PR when that
-lockfile changes, so broad dependency churn is reviewed as maintenance rather
-than landing inside unrelated feature work. The workflow can also be dispatched
-manually for one workspace or for all of them. Its dispatch input and every
-lockfile CI matrix resolve through `ci/tools/list_pixi_workspaces.py`, which
-derives the workspace list from the committed manifests, so a newly added
-workspace is picked up without editing any workflow.
-
-Those refresh PRs need an App token to pick up CI on their own: GitHub does not
-deliver workflow-triggering events for branches pushed with `GITHUB_TOKEN`. Set
-the `PIXI_LOCK_REFRESH_APP_ID` variable and `PIXI_LOCK_REFRESH_APP_PRIVATE_KEY`
-secret to enable that path. Until they are set, the workflow says so in the PR
-body and in a run warning, and required checks stay pending until a maintainer
-closes and reopens the PR or pushes to its branch.
+The scheduled [`CI: pixi lockfile refresh`](https://github.com/NVIDIA/cuda-python/actions/workflows/ci-pixi-lockfile-refresh.yml)
+runs `pixi update --no-install` per workspace, canonicalizes the result with the
+pinned pixi version, and opens a dedicated PR when that lockfile changes. This
+keeps broad dependency churn in reviewable maintenance changes rather than
+unrelated feature work. Maintainers can also dispatch the workflow manually for
+one workspace or for all of them. Its dispatch input and every lockfile CI
+matrix resolve through `ci/tools/list_pixi_workspaces.py`, which derives the
+workspace list from the committed manifests, so a newly added workspace is
+picked up without editing any workflow.
+
+An App token lets refresh PRs start CI without manual approval and permits the
+full intended event flow. Set the `PIXI_LOCK_REFRESH_APP_ID` variable and
+`PIXI_LOCK_REFRESH_APP_PRIVATE_KEY` secret to enable that path. Without it, the
+workflow uses `GITHUB_TOKEN`; GitHub creates `opened`, `synchronize`, and
+`reopened` workflow runs in an approval-required state, while other
+workflow-triggering events remain suppressed. The generated PR body and run
+warning tell maintainers when approval is required.
+
+The stable `Pixi lockfile freshness gate` is intended to be a required check
+for pull requests targeting `main`. Its per-workspace jobs remain visible as
+diagnostics. In an emergency, a maintainer may apply the
+`ci-pixi-lockfile-override` label to let only the aggregate gate pass despite a
+known lockfile failure. The label is removed when the pull request head changes
+or the pull request is reopened, so the new head requires explicit approval.
 
 ## Secret Scanning