diff --git a/.github/RELEASE-bindings.md b/.github/RELEASE-bindings.md index 3764f265764..84b90cd901e 100644 --- a/.github/RELEASE-bindings.md +++ b/.github/RELEASE-bindings.md @@ -39,14 +39,25 @@ intended toolkit inputs. Follow the Tag-triggered CI selects that line's bindings and metapackage, validates them with the published Pathfinder dependency, and builds release artifacts. The tag does not have to be on `main`. Wait for this exact tag's CI to pass. -4. Run **CI: Release** with `release-action=dry-run`, the tag, and - `component=cuda-bindings`; repeat for `cuda-python`. Leave `run-id` blank - to exercise automatic lookup, and leave `dry-run-docs-branch` blank to avoid +4. Release `cuda-bindings` before `cuda-python`. For each component, run + **CI: Release** with `release-action=dry-run`, the tag, and the chosen + `component`. Leave `run-id` blank to exercise automatic lookup, and leave + `dry-run-docs-branch` blank to avoid deploying docs. The workflow verifies a successful tag-triggered CI run with the exact source SHA, validates artifacts, and builds docs. -5. Review the dry runs, then separately approve and run `full-release` for - each component under the team's release procedure. A successful candidate - rehearsal or ordinary branch CI is not a substitute for exact-tag CI. + The `cuda-python` dry run also installs the candidate metapackage with + dependencies from PyPI in clean environments across the tag's supported + Python versions, including free-threaded builds, on Linux x86-64, Linux + ARM64, and Windows x86-64. It tests both the default install and `[all]` + using only binary wheels and runs `pip check`. Publish the matching + bindings and any required Core or Pathfinder versions to PyPI first; + unpublished CI dependency artifacts cannot satisfy this check. +5. Review that component's dry run, then separately approve and run + `full-release` for the same component under the team's release procedure. + A successful candidate rehearsal or ordinary branch CI is not a substitute + for exact-tag CI. + The metapackage dependency check runs again for `full-release` and must + pass before GitHub asset uploads or TestPyPI/PyPI publication. 6. After both component releases succeed, review and publish the shared `vX.Y.Z` GitHub Release draft. Check its release notes, versioned documentation links, source archive, checksum, and wheels. These component diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index e7ddb425881..9e7a6d2e2a4 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -334,6 +334,7 @@ jobs: - resolve-bindings-package outputs: artifact-id: ${{ steps.upload.outputs.artifact-id }} + python-versions: ${{ steps.dependency-test-matrix.outputs.python-versions }} env: ARCHIVE_NAME: ${{ github.event.repository.name }}-${{ inputs.git-tag }} BINDINGS_PACKAGE: ${{ needs.resolve-bindings-package.outputs.bindings-package }} @@ -388,6 +389,14 @@ jobs: sha256sum "release/$ARCHIVE_NAME.tar.gz" \ | awk '{print $1}' > "release/$ARCHIVE_NAME.tar.gz.sha256sum" + - name: Read metapackage dependency test Python versions + if: ${{ inputs.component == 'cuda-python' }} + id: dependency-test-matrix + run: | + config=$(python -m ci.tools.bindings_config --config release-source/ci/versions.yml) + python_versions=$(jq -c '.python.supported' <<< "$config") + echo "python-versions=$python_versions" >> "$GITHUB_OUTPUT" + # Consumers use this immutable artifact ID, never re-download build outputs. - name: Upload prepared release artifacts id: upload @@ -398,11 +407,50 @@ jobs: if-no-files-found: error retention-days: 3 + test-cuda-python-dependencies: + name: Test cuda-python${{ matrix.extra }} dependencies (${{ matrix.os }}, Python ${{ matrix.python-version }}) + if: ${{ inputs.component == 'cuda-python' }} + needs: prepare-artifacts + runs-on: ${{ matrix.os }} + permissions: {} + strategy: + fail-fast: false + matrix: + os: [ubuntu-latest, ubuntu-24.04-arm, windows-latest] + python-version: ${{ fromJSON(needs.prepare-artifacts.outputs.python-versions || '[]') }} + extra: ["", "[all]"] + steps: + - name: Download prepared release artifacts + uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + artifact-ids: ${{ needs.prepare-artifacts.outputs.artifact-id }} + path: release/ + merge-multiple: true + + - name: Create clean dependency test environment + # No checkout or CI-built dependencies: test the wheel that will be published. + uses: $/.github/actions/setup-hardened-python + with: + python-version: ${{ matrix.python-version }} + + - name: Install metapackage with published dependencies + env: + EXTRA: ${{ matrix.extra }} + run: | + wheels=(release/wheels/cuda_python-*.whl) + python -m pip --isolated install \ + --index-url https://pypi.org/simple/ \ + --only-binary=:all: "${wheels[0]}${EXTRA}" + python -m pip check + upload-archive: name: Upload GitHub release artifacts + if: >- + ${{ !cancelled() && needs.prepare-artifacts.result == 'success' && needs.doc.result == 'success' && + (inputs.component != 'cuda-python' || needs.test-cuda-python-dependencies.result == 'success') }} permissions: contents: write - needs: [prepare-artifacts, doc] + needs: [prepare-artifacts, doc, test-cuda-python-dependencies] uses: ./.github/workflows/release-upload.yml with: git-tag: ${{ inputs.git-tag }} @@ -415,9 +463,10 @@ jobs: if: >- ${{ !cancelled() && inputs.release-action == 'full-release' && needs.prepare-artifacts.result == 'success' && needs.doc.result == 'success' && + (inputs.component != 'cuda-python' || needs.test-cuda-python-dependencies.result == 'success') && (inputs.component != 'cuda-pathfinder' || needs.ensure-pathfinder-draft.result == 'success') }} runs-on: ubuntu-latest - needs: [prepare-artifacts, doc, ensure-pathfinder-draft] + needs: [prepare-artifacts, doc, ensure-pathfinder-draft, test-cuda-python-dependencies] environment: name: testpypi url: https://test.pypi.org/p/${{ inputs.component }}/ @@ -439,7 +488,11 @@ jobs: verify-testpypi: name: Verify Pathfinder on TestPyPI - if: ${{ inputs.component == 'cuda-pathfinder' && inputs.release-action == 'full-release' }} + # An intentionally skipped metapackage test is also an ancestor of these jobs. + # Override implicit success(), then require each mandatory prerequisite explicitly. + if: >- + ${{ !cancelled() && inputs.component == 'cuda-pathfinder' && inputs.release-action == 'full-release' && + needs.check-release-notes.result == 'success' && needs.publish-testpypi.result == 'success' }} needs: [check-release-notes, publish-testpypi] runs-on: ubuntu-latest permissions: {} @@ -515,7 +568,9 @@ jobs: verify-pypi: name: Verify Pathfinder on PyPI - if: ${{ inputs.component == 'cuda-pathfinder' && inputs.release-action == 'full-release' }} + if: >- + ${{ !cancelled() && inputs.component == 'cuda-pathfinder' && inputs.release-action == 'full-release' && + needs.check-release-notes.result == 'success' && needs.publish-pypi.result == 'success' }} needs: [check-release-notes, publish-pypi] runs-on: ubuntu-latest permissions: {} @@ -528,7 +583,11 @@ jobs: finalize-pathfinder: name: Publish Pathfinder GitHub release - if: ${{ inputs.component == 'cuda-pathfinder' && inputs.release-action == 'full-release' }} + if: >- + ${{ !cancelled() && inputs.component == 'cuda-pathfinder' && inputs.release-action == 'full-release' && + needs.doc.result == 'success' && needs.check-release-notes.result == 'success' && + needs.ensure-pathfinder-draft.result == 'success' && needs.publish-pypi.result == 'success' && + needs.verify-pypi.result == 'success' && needs.upload-archive.result == 'success' }} needs: [doc, check-release-notes, ensure-pathfinder-draft, publish-pypi, verify-pypi, upload-archive] runs-on: ubuntu-latest permissions: diff --git a/ci/tools/tests/test_release_job_gating.py b/ci/tools/tests/test_release_job_gating.py new file mode 100644 index 00000000000..3bebaa0f6a9 --- /dev/null +++ b/ci/tools/tests/test_release_job_gating.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +"""Exercise release job eligibility, without executing steps or reusable workflows.""" + +import re +from pathlib import Path + +import pytest +import yaml + +WORKFLOW = Path(__file__).resolve().parents[3] / ".github/workflows/release.yml" +PUBLISHERS = ("upload-archive", "publish-testpypi", "publish-pypi", "finalize-pathfinder") +PATHFINDER_JOBS = ("ensure-pathfinder-draft", "verify-testpypi", "verify-pypi", "finalize-pathfinder") + + +@pytest.fixture +def jobs(): + return yaml.safe_load(WORKFLOW.read_text(encoding="utf-8"))["jobs"] + + +def _needs(job): + needs = job.get("needs", []) + return [needs] if isinstance(needs, str) else needs + + +def _eligible(jobs, name, results, inputs, *, cancelled=False): + def ancestors(name): + direct = _needs(jobs[name]) + return set(direct).union(*(ancestors(parent) for parent in direct)) + + # Actions implicitly adds success() unless an explicit status function is + # present. Its dependency status includes transitive skipped ancestors. + # https://docs.github.com/en/actions/reference/workflows-and-actions/expressions#status-check-functions + successful = not cancelled and all(results[parent] == "success" for parent in ancestors(name)) + expression = jobs[name].get("if", "${{ true }}").removeprefix("${{").removesuffix("}}").strip() + if not re.search(r"\b(?:always|cancelled|failure|success)\s*\(", expression) and not successful: + return False + expression = re.sub(r"inputs\.([\w-]+)", lambda match: repr(inputs[match[1]]), expression) + expression = expression.replace("github.repository_owner", repr("nvidia")) + direct = _needs(jobs[name]) + expression = re.sub( + r"needs\.([\w-]+)\.result", lambda match: repr(results[match[1]] if match[1] in direct else ""), expression + ) + expression = expression.replace("cancelled()", repr(cancelled)).replace("success()", repr(successful)) + expression = expression.replace("always()", "True").replace("true", "True").replace("false", "False") + expression = re.sub(r"!(?!=)", " not ", expression).replace("&&", " and ").replace("||", " or ") + return eval(f"({expression})", {"__builtins__": {}}) # noqa: S307 - trusted checked-in workflow conditions. + + +def _run(jobs, component, *, action="full-release", tag="cuda-pathfinder-v1.9.1", outcomes=None): + results = {} + inputs = {"component": component, "release-action": action, "git-tag": tag} + + def finish(name): + if name in results: + return + for parent in _needs(jobs[name]): + finish(parent) + results[name] = (outcomes or {}).get(name, "success") if _eligible(jobs, name, results, inputs) else "skipped" + + for name in jobs: + finish(name) + return results + + +@pytest.mark.agent_authored(model="GPT-6") +def test_pathfinder_release_survives_intentionally_skipped_metapackage_test(jobs): + results = _run(jobs, "cuda-pathfinder") + + assert results["test-cuda-python-dependencies"] == "skipped" + for name in (*PUBLISHERS, *PATHFINDER_JOBS): + assert results[name] == "success", (name, results) + + +@pytest.mark.parametrize("component", ["cuda-bindings", "cuda-python"]) +@pytest.mark.agent_authored(model="GPT-6") +def test_other_releases_publish_without_pathfinder_jobs(jobs, component): + results = _run(jobs, component, tag="v12.9.11") + + for name in PATHFINDER_JOBS: + assert results[name] == "skipped", (name, results) + for name in PUBLISHERS[:-1]: + assert results[name] == "success", (name, results) + assert results["test-cuda-python-dependencies"] == ("success" if component == "cuda-python" else "skipped") + + +@pytest.mark.parametrize("component", ["cuda-pathfinder", "cuda-bindings", "cuda-python"]) +@pytest.mark.agent_authored(model="GPT-6") +def test_dry_run_prepares_artifacts_without_package_publication(jobs, component): + results = _run(jobs, component, action="dry-run") + + assert results["doc"] == results["prepare-artifacts"] == results["upload-archive"] == "success" + for name in (*PATHFINDER_JOBS, "publish-testpypi", "publish-pypi"): + assert results[name] == "skipped", (name, results) + + +@pytest.mark.agent_authored(model="GPT-6") +def test_failed_docs_prevent_publication(jobs): + results = _run(jobs, "cuda-pathfinder", outcomes={"doc": "failure"}) + + assert results["prepare-artifacts"] == "success" + for name in PUBLISHERS: + assert results[name] == "skipped", (name, results) + + +@pytest.mark.parametrize("test_result", ["skipped", "failure"]) +@pytest.mark.agent_authored(model="GPT-6") +def test_required_metapackage_tests_gate_publication(jobs, test_result): + results = _run(jobs, "cuda-python", outcomes={"test-cuda-python-dependencies": test_result}) + + assert results["prepare-artifacts"] == results["doc"] == "success" + for name in PUBLISHERS: + assert results[name] == "skipped", (name, results) + + +@pytest.mark.agent_authored(model="GPT-6") +def test_cancellation_prevents_remaining_release_writes_and_verification(jobs): + # Cancellation arrives after prerequisites completed, so only the job's + # own cancellation gate can stop it from starting. + results = dict.fromkeys(jobs, "success") + results["test-cuda-python-dependencies"] = "skipped" + inputs = {"component": "cuda-pathfinder", "release-action": "full-release"} + for name in (*PUBLISHERS, *PATHFINDER_JOBS): + assert not _eligible(jobs, name, results, inputs, cancelled=True), name + + +@pytest.mark.parametrize( + ("name", "required"), + [ + ("publish-testpypi", "prepare-artifacts"), + ("publish-testpypi", "doc"), + ("publish-testpypi", "ensure-pathfinder-draft"), + ("verify-testpypi", "check-release-notes"), + ("verify-testpypi", "publish-testpypi"), + ("publish-pypi", "prepare-artifacts"), + ("publish-pypi", "publish-testpypi"), + ("publish-pypi", "verify-testpypi"), + ("verify-pypi", "check-release-notes"), + ("verify-pypi", "publish-pypi"), + ("finalize-pathfinder", "doc"), + ("finalize-pathfinder", "check-release-notes"), + ("finalize-pathfinder", "ensure-pathfinder-draft"), + ("finalize-pathfinder", "publish-pypi"), + ("finalize-pathfinder", "verify-pypi"), + ("finalize-pathfinder", "upload-archive"), + ], +) +@pytest.mark.parametrize("required_result", ["skipped", "failure"]) +@pytest.mark.agent_authored(model="GPT-6") +def test_pathfinder_verification_and_publication_require_successful_prerequisites( + jobs, name, required, required_result +): + results = dict.fromkeys(jobs, "success") + results["test-cuda-python-dependencies"] = "skipped" + results[required] = required_result + inputs = {"component": "cuda-pathfinder", "release-action": "full-release"} + + assert not _eligible(jobs, name, results, inputs), (name, required, required_result)