diff --git a/.github/workflows/ci-pixi-lockfile-freshness-check.yml b/.github/workflows/ci-pixi-lockfile-freshness-check.yml index d7f9d265c9d..096bbef386e 100644 --- a/.github/workflows/ci-pixi-lockfile-freshness-check.yml +++ b/.github/workflows/ci-pixi-lockfile-freshness-check.yml @@ -19,23 +19,31 @@ concurrency: on: pull_request: + # A required workflow must run on every PR: a workflow skipped by a path + # filter leaves its required check pending. Label changes rerun the matrix + # so the aggregate gate immediately reflects the override state. + types: + - opened + - reopened + - synchronize + - labeled + - unlabeled + push: + # `pull_request` already covers PRs, including those from forks: this check + # needs no secrets or GPU runner. Watching copy-pr-bot's `pull-request/N` + # mirror too would run the whole matrix a second time per PR. + branches: + - "main" # `pyproject.toml` is a lockfile input too: the manifests consume sibling # packages (including cuda_python_test_helpers) as path dependencies, so # their metadata can stale a lock without any pixi.toml edit. Matched by # glob rather than by name so the filter cannot drift as packages move. - paths: &lockfile_inputs + paths: - "**/pixi.toml" - "**/pixi.lock" - "**/pyproject.toml" - "ci/tools/list_pixi_workspaces.py" - ".github/workflows/ci-pixi-lockfile-freshness-check.yml" - push: - # `pull_request` already covers PRs, including those from forks: this check - # needs no secrets or GPU runner. Watching copy-pr-bot's `pull-request/N` - # mirror too would run the whole matrix a second time per PR. - branches: - - "main" - paths: *lockfile_inputs workflow_dispatch: {} defaults: @@ -46,6 +54,8 @@ env: # Keep in sync with ci-pixi-source-test.yml. Must be >=0.71.0 so the # committed lockfile format (v7) matches what this job checks against. PIXI_VERSION: "v0.73.0" + REFRESH_WORKFLOW_URL: "https://github.com/NVIDIA/cuda-python/actions/workflows/ci-pixi-lockfile-refresh.yml" + OVERRIDE_LABEL: "ci-pixi-lockfile-override" permissions: {} @@ -100,9 +110,11 @@ jobs: run-install: false - name: Check lockfile is current with its manifest + env: + MANIFEST: ${{ matrix.manifest }} run: | - if ! pixi lock --check --manifest-path "${{ matrix.manifest }}"; then - echo "::error::Lockfile is stale for '${{ matrix.manifest }}'. Regenerate with: pixi lock --manifest-path ${{ matrix.manifest }}" + if ! pixi lock --check --manifest-path "${MANIFEST}"; then + echo "::error::Lockfile is stale for '${MANIFEST}'. Run and commit: pixi lock --manifest-path ${MANIFEST}. For default-branch dependency drift, maintainers can run ${REFRESH_WORKFLOW_URL}" exit 1 fi @@ -112,8 +124,75 @@ jobs: # is invisible: the check prints "Updated lock file" and still passes, # while every later pixi run keeps rewriting the committed file (#2298). - name: Check lockfile is byte-for-byte canonical + env: + LOCKFILE: ${{ matrix.lockfile }} + MANIFEST: ${{ matrix.manifest }} run: | - if ! git diff --exit-code -- "${{ matrix.lockfile }}"; then - echo "::error::pixi ${PIXI_VERSION} rewrote ${{ matrix.lockfile }} during the check, so the committed bytes are not what it generates. Regenerate with pixi ${PIXI_VERSION}: pixi lock --manifest-path ${{ matrix.manifest }}" + if ! git diff --exit-code -- "${LOCKFILE}"; then + echo "::error::pixi ${PIXI_VERSION} rewrote ${LOCKFILE} during the check, so the committed bytes are not canonical. Run and commit with pixi ${PIXI_VERSION}: pixi lock --manifest-path ${MANIFEST}. For default-branch dependency drift, maintainers can run ${REFRESH_WORKFLOW_URL}" exit 1 fi + + gate: + name: Pixi lockfile freshness gate + if: ${{ always() && github.repository_owner == 'nvidia' }} + needs: + - plan + - lockfile-fresh + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: {} + steps: + - name: Evaluate freshness results + env: + EVENT_ACTION: ${{ github.event.action }} + EVENT_NAME: ${{ github.event_name }} + LABELS_JSON: ${{ toJSON(github.event.pull_request.labels.*.name) }} + LOCKFILE_RESULT: ${{ needs.lockfile-fresh.result }} + PLAN_RESULT: ${{ needs.plan.result }} + run: | + has_override=false + if jq -e --arg label "${OVERRIDE_LABEL}" \ + '(. // []) | index($label) != null' <<<"${LABELS_JSON}" >/dev/null; then + has_override=true + fi + + { + echo "### Pixi lockfile freshness gate" + echo + echo "- Workspace discovery: \`${PLAN_RESULT}\`" + echo "- Lockfile checks: \`${LOCKFILE_RESULT}\`" + echo "- Override label present: \`${has_override}\`" + } >> "${GITHUB_STEP_SUMMARY}" + + if [ "${PLAN_RESULT}" != "success" ]; then + echo "::error::Pixi workspace discovery did not succeed (${PLAN_RESULT})." + exit 1 + fi + + if [ "${LOCKFILE_RESULT}" = "success" ]; then + exit 0 + fi + + # Override approval applies only to a known diagnostic failure on the + # current PR head. opened/synchronize/reopened events cannot carry + # prior approval; the base-controlled reset workflow removes the + # label when one is present. + if [ "${EVENT_NAME}" = "pull_request" ] && \ + [ "${EVENT_ACTION}" != "opened" ] && \ + [ "${EVENT_ACTION}" != "synchronize" ] && \ + [ "${EVENT_ACTION}" != "reopened" ] && \ + [ "${LOCKFILE_RESULT}" = "failure" ] && \ + [ "${has_override}" = "true" ]; then + echo "::warning::${OVERRIDE_LABEL} overrides the failing aggregate gate; the failed workspace diagnostics remain visible." + { + echo + echo "> [!WARNING]" + echo "> A maintainer override allows this gate to pass. Review the failed" + echo "> workspace diagnostics before merging." + } >> "${GITHUB_STEP_SUMMARY}" + exit 0 + fi + + echo "::error::Pixi lockfile checks did not succeed (${LOCKFILE_RESULT})." + exit 1 diff --git a/.github/workflows/ci-pixi-lockfile-override-reset.yml b/.github/workflows/ci-pixi-lockfile-override-reset.yml new file mode 100644 index 00000000000..0b20443d3f8 --- /dev/null +++ b/.github/workflows/ci-pixi-lockfile-override-reset.yml @@ -0,0 +1,41 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# +# SPDX-License-Identifier: Apache-2.0 + +name: "CI: invalidate pixi lockfile override" + +on: + # pull_request_target loads this workflow from the trusted base branch. It + # must never check out or execute code from the pull request. + pull_request_target: + types: + - opened + - synchronize + - reopened + +permissions: {} + +jobs: + invalidate: + name: Invalidate Pixi lockfile override + if: >- + github.repository_owner == 'nvidia' && + contains(github.event.pull_request.labels.*.name, 'ci-pixi-lockfile-override') + runs-on: ubuntu-latest + timeout-minutes: 5 + permissions: + issues: write + steps: + - name: Remove override from changed PR + env: + GH_TOKEN: ${{ github.token }} + OVERRIDE_LABEL: ci-pixi-lockfile-override + PR_NUMBER: ${{ github.event.pull_request.number }} + run: | + endpoint="repos/${GITHUB_REPOSITORY}/issues/${PR_NUMBER}/labels" + labels="$(gh api "${endpoint}")" + if jq -e --arg label "${OVERRIDE_LABEL}" \ + 'map(.name) | index($label) != null' <<<"${labels}" >/dev/null; then + gh api --method DELETE "${endpoint}/${OVERRIDE_LABEL}" + echo "::notice::Removed ${OVERRIDE_LABEL}; a new PR head requires fresh maintainer approval." + fi diff --git a/.github/workflows/ci-pixi-lockfile-refresh.yml b/.github/workflows/ci-pixi-lockfile-refresh.yml index 2ee774cbf1b..5d997a177ef 100644 --- a/.github/workflows/ci-pixi-lockfile-refresh.yml +++ b/.github/workflows/ci-pixi-lockfile-refresh.yml @@ -83,11 +83,10 @@ jobs: fetch-depth: 0 ref: ${{ github.event.repository.default_branch }} - # GitHub suppresses workflow-triggering events for pushes and PRs made - # with GITHUB_TOKEN, so a refresh PR opened with the default token cannot - # reach a green required-check state on its own. An App token restores - # normal event delivery; without one the PR needs a manual nudge, which - # the PR body and a run warning spell out. + # Pull-request runs created by GITHUB_TOKEN wait for maintainer approval, + # and other workflow-triggering events remain suppressed. An App token + # avoids that approval step and restores normal event delivery. The PR + # body and run warning explain the fallback behavior. - name: Mint App token for the refresh PR id: app-token if: ${{ vars.PIXI_LOCK_REFRESH_APP_ID != '' }} @@ -104,8 +103,25 @@ jobs: pixi-version: ${{ env.PIXI_VERSION }} run-install: false - - name: Update lockfile without installing - run: pixi update --no-install --manifest-path "${{ matrix.manifest }}" + # Pixi 0.73.0's update and lock-check paths can serialize the same + # solution differently (#2804). The first check canonicalizes the update + # result; the second proves that the committed bytes reached a fixed + # point before create-pull-request sees them. + - name: Update and canonicalize lockfile without installing + env: + LOCKFILE: ${{ matrix.lockfile }} + MANIFEST: ${{ matrix.manifest }} + run: | + pixi update --no-install --manifest-path "${MANIFEST}" + pixi lock --check --manifest-path "${MANIFEST}" + + canonical_lock="${RUNNER_TEMP}/pixi-lock-after-canonicalization.lock" + cp -- "${LOCKFILE}" "${canonical_lock}" + pixi lock --check --manifest-path "${MANIFEST}" + if ! cmp --silent "${canonical_lock}" "${LOCKFILE}"; then + echo "::error::pixi ${PIXI_VERSION} did not reach a byte-stable lockfile after canonicalization: ${LOCKFILE}" + exit 1 + fi - name: Compose PR body id: pr-body @@ -117,7 +133,8 @@ jobs: body_path="${RUNNER_TEMP}/pixi-lock-refresh-body.md" { echo "Automated lockfile refresh for \`${WORKSPACE_ID}\`" \ - "(\`pixi update --no-install --manifest-path ${MANIFEST}\`)." + "(\`pixi update --no-install --manifest-path ${MANIFEST}\`, followed" + echo "by pinned-pixi canonicalization and a byte-stability check)." echo echo "This PR is workspace-scoped so other packages are not forced to review" echo "unrelated solver churn. See #2298." @@ -125,9 +142,9 @@ jobs: if [ "${HAS_APP_TOKEN}" != "true" ]; then echo "> [!IMPORTANT]" echo "> No refresh App token is configured, so this PR was opened with" - echo "> \`GITHUB_TOKEN\` and GitHub will not deliver its \`pull_request\` events." - echo "> Required checks stay pending until a maintainer nudges the branch, by" - echo "> closing and reopening the PR or pushing an empty commit to it." + echo "> \`GITHUB_TOKEN\`. GitHub creates its \`opened\`, \`synchronize\`, and" + echo "> \`reopened\` workflow runs in an approval-required state; a maintainer" + echo "> must approve those runs before checks start. Other events remain suppressed." echo fi echo "If the diff looks wrong, close this PR and leave the branch deleted; the" @@ -136,7 +153,7 @@ jobs: echo "path=${body_path}" >> "${GITHUB_OUTPUT}" if [ "${HAS_APP_TOKEN}" != "true" ]; then - echo "::warning::Opened with GITHUB_TOKEN; the refresh PR's checks must be triggered manually." + echo "::warning::Opened with GITHUB_TOKEN; a maintainer must approve the refresh PR's workflow runs." fi - name: Open or update a lockfile-only PR @@ -148,7 +165,8 @@ jobs: commit-message: | chore: refresh pixi.lock for ${{ matrix.id }} - Scheduled pixi update --no-install for this workspace only. + Scheduled pixi update --no-install followed by pinned-pixi + canonicalization for this workspace only. signoff: true sign-commits: true branch: ci/pixi-lock-refresh/${{ matrix.id }} @@ -168,6 +186,7 @@ jobs: echo "" echo "- Manifest: \`${{ matrix.manifest }}\`" echo "- Lockfile: \`${{ matrix.lockfile }}\`" + echo "- Canonicalization: byte-stable fixed point verified" echo "- Operation: \`${{ steps.cpr.outputs.pull-request-operation }}\`" if [ -n "${{ steps.cpr.outputs.pull-request-url }}" ]; then echo "- PR: ${{ steps.cpr.outputs.pull-request-url }}" diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 6f78500b6cd..512ad37382b 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -218,21 +218,30 @@ The freshness check additionally fails when the check itself rewrote a lockfile. not canonical for the pinned pixi version, quietly normalizing the file instead, which leaves every later pixi run rewriting the committed lockfile. -A scheduled workflow (`CI: pixi lockfile refresh`) -runs `pixi update --no-install` per workspace and opens a dedicated PR when that -lockfile changes, so broad dependency churn is reviewed as maintenance rather -than landing inside unrelated feature work. The workflow can also be dispatched -manually for one workspace or for all of them. Its dispatch input and every -lockfile CI matrix resolve through `ci/tools/list_pixi_workspaces.py`, which -derives the workspace list from the committed manifests, so a newly added -workspace is picked up without editing any workflow. - -Those refresh PRs need an App token to pick up CI on their own: GitHub does not -deliver workflow-triggering events for branches pushed with `GITHUB_TOKEN`. Set -the `PIXI_LOCK_REFRESH_APP_ID` variable and `PIXI_LOCK_REFRESH_APP_PRIVATE_KEY` -secret to enable that path. Until they are set, the workflow says so in the PR -body and in a run warning, and required checks stay pending until a maintainer -closes and reopens the PR or pushes to its branch. +The scheduled [`CI: pixi lockfile refresh`](https://github.com/NVIDIA/cuda-python/actions/workflows/ci-pixi-lockfile-refresh.yml) +runs `pixi update --no-install` per workspace, canonicalizes the result with the +pinned pixi version, and opens a dedicated PR when that lockfile changes. This +keeps broad dependency churn in reviewable maintenance changes rather than +unrelated feature work. Maintainers can also dispatch the workflow manually for +one workspace or for all of them. Its dispatch input and every lockfile CI +matrix resolve through `ci/tools/list_pixi_workspaces.py`, which derives the +workspace list from the committed manifests, so a newly added workspace is +picked up without editing any workflow. + +An App token lets refresh PRs start CI without manual approval and permits the +full intended event flow. Set the `PIXI_LOCK_REFRESH_APP_ID` variable and +`PIXI_LOCK_REFRESH_APP_PRIVATE_KEY` secret to enable that path. Without it, the +workflow uses `GITHUB_TOKEN`; GitHub creates `opened`, `synchronize`, and +`reopened` workflow runs in an approval-required state, while other +workflow-triggering events remain suppressed. The generated PR body and run +warning tell maintainers when approval is required. + +The stable `Pixi lockfile freshness gate` is intended to be a required check +for pull requests targeting `main`. Its per-workspace jobs remain visible as +diagnostics. In an emergency, a maintainer may apply the +`ci-pixi-lockfile-override` label to let only the aggregate gate pass despite a +known lockfile failure. The label is removed when the pull request head changes +or the pull request is reopened, so the new head requires explicit approval. ## Secret Scanning diff --git a/benchmarks/cuda_core/pixi.lock b/benchmarks/cuda_core/pixi.lock index e4d72270d4c..6aea14145f3 100644 --- a/benchmarks/cuda_core/pixi.lock +++ b/benchmarks/cuda_core/pixi.lock @@ -93,7 +93,7 @@ environments: - conda: https://conda.anaconda.org/conda-forge/noarch/typing_extensions-4.16.0-pyhcf101f3_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/tzdata-2026c-h151e31d_0.conda - conda: https://conda.anaconda.org/conda-forge/noarch/virtualenv-21.7.9-pyh5ded981_0.conda - - conda_source: cuda-core[23592f13] @ ../../cuda_core + - conda_source: cuda-core[8ae17e81] @ ../../cuda_core wheel: channels: - url: https://conda.anaconda.org/conda-forge/ @@ -1437,7 +1437,7 @@ packages: run_exports: {} size: 3895514 timestamp: 1788948703859 -- conda_source: cuda-core[23592f13] @ ../../cuda_core +- conda_source: cuda-core[8ae17e81] @ ../../cuda_core variants: c_stdlib: sysroot c_stdlib_version: '2.28' @@ -1455,8 +1455,8 @@ packages: - libstdcxx >=16 - __glibc >=2.28,<3.0.a0 - python_abi 3.14.* *_cp314 - - cuda-cudart >=13.4.49,<14.0a0 - cuda-nvrtc >=13.4.59,<14.0a0 + - cuda-cudart >=13.4.49,<14.0a0 license: Apache-2.0 build_packages: - conda: https://conda.anaconda.org/conda-forge/linux-64/_openmp_mutex-4.5-20_gnu.conda