python-package.yml

name: Python package

on:
  workflow_call:
    # This workflow can be called by other workflows (like intelligent-testing.yml)
    inputs:
      concurrency_key:
        required: false
        type: string
      matrix_json:
        required: true
        type: string
      pytest_paths_json:
        required: false
        type: string
        default: "[]"
      functional_scripts_json:
        required: false
        type: string
        default: "[]"
      full_suite:
        required: false
        type: boolean
        default: false
      pip_overrides:
        required: false
        type: string
        default: ""
      continue_on_error:
        required: false
        type: boolean
        default: false
      strict_cells:
        required: false
        type: boolean
        default: true

  workflow_dispatch:
    inputs:
      concurrency_key:
        description: "Optional stable key to dedupe manual runs (for example: pr-123)"
        required: false
        type: string
      matrix_json:
        description: "JSON matrix definition"
        required: true
        type: string
        default: '{"include":[{"os":"ubuntu-latest","python-version":"3.12","extras": ""}]}'
      pytest_paths_json:
        description: "JSON array of pytest paths"
        required: false
        type: string
        default: "[]"
      functional_scripts_json:
        description: "JSON array of functional scripts"
        required: false
        type: string
        default: "[]"
      full_suite:
        description: "Run full pytest and default functional suite"
        required: false
        type: boolean
        default: false
      pip_overrides:
        description: "Space-separated pip specs to force-install after normal deps (e.g. 'numpy>=2,<3' or 'numpy>=2,<3 pandas>=3')"
        required: false
        type: string
        default: ""
      continue_on_error:
        description: "Allow test steps to fail without marking the job as failed"
        required: false
        type: boolean
        default: false
      strict_cells:
        description: "Make canary lane checks strict (set false for non-blocking behavior)"
        required: false
        type: boolean
        default: true

jobs:
  build:
    runs-on: ${{ matrix.os }}
    # Matrix cell wins when set (canary lane); otherwise fall back to the workflow
    # input so callers / workflow_dispatch can apply one override to every cell.
    env:
      PIP_OVERRIDES: ${{ matrix.pip_overrides || inputs.pip_overrides || '' }}
    # Cancel outdated runs on the same OS and Python version when new commits are pushed
    # Only cancels on PRs.
    # Use a stable concurrency key only when one is explicitly provided
    # (e.g. PR/workflow_call/manual dedupe). Otherwise fall back to github.run_id
    # so pushes to main never cancel each other.
    # Use matrix.id (not full pip_overrides) so canary cells that share os/python
    # do not cancel each other.
    concurrency:
      group: >-
        tests-${{ github.workflow }}-
        ${{ github.event_name == 'workflow_call' && inputs.concurrency_key
            || github.event_name == 'workflow_dispatch' && inputs.concurrency_key
            || github.run_id }}-
        ${{ matrix.os }}-${{ matrix.python-version }}
        ${{ matrix.id && format('-{0}', matrix.id) || '' }}
      cancel-in-progress: true

    strategy:
      fail-fast: false
      matrix: ${{ fromJson(inputs.matrix_json) }}

    steps:
      - name: Checkout code
        uses: actions/checkout@v6

      - name: Free up disk space
        if: runner.os == 'Linux'
        run: |
          echo "Disk space before cleanup:"
          df -h
          # Remove unnecessary software to free up disk space
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf /usr/local/lib/android
          sudo rm -rf /opt/ghc
          sudo rm -rf /opt/hostedtoolcache/CodeQL
          sudo docker image prune --all --force
          echo "Disk space after cleanup:"
          df -h

      - name: Set up Python
        uses: conda-incubator/setup-miniconda@v3
        with:
          channels: conda-forge
          channel-priority: strict
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        shell: bash -el {0} # Important to enable conda env
        run: |
          python -m ensurepip --upgrade
          python -m pip install --upgrade pip setuptools wheel
          python -m pip install dependency-groups
          # knowledge-index: tests/tools/knowledge_indexing exercises the real
          # extractors, which import griffe, docutils and markdown-it.
          python -m pip install --no-cache-dir -e ".${{ matrix.extras }}" --group dev --group knowledge-index

      - name: Force-install upgrade packages
        if: ${{ env.PIP_OVERRIDES != '' }}
        shell: bash -el {0}
        run: |
          # Split on whitespace (not commas): pip version specs legitimately
          # contain commas to combine constraints, e.g. "numpy>=2,<3".
          read -ra pkgs <<< "$PIP_OVERRIDES"
          for pkg in "${pkgs[@]}"; do
            echo "Force-installing (--no-deps): $pkg"
            python -m pip install --no-cache-dir --upgrade --no-deps "$pkg"
          done

      - name: Install ffmpeg (Linux/macOS)
        if: runner.os != 'Windows'
        shell: bash
        run: |
          if [ "$RUNNER_OS" == "Linux" ]; then
            sudo apt-get update
            sudo apt-get install -y ffmpeg
          elif [ "$RUNNER_OS" == "macOS" ]; then
            brew install ffmpeg || true
          fi
      - name: Install ffmpeg (Windows, pinned monthly BtbN build)
      # NOTE: The pinned version should be retained for ~2 years. This WILL fail if the BtbN release is removed,
      # so if you are two years in the future and this step fails, please check the builds. Thanks.
        if: runner.os == 'Windows'
        shell: pwsh
        env:
          FFMPEG_TAG: autobuild-2026-03-31-13-11
          FFMPEG_ASSET: ffmpeg-N-123777-g53537f6cf5-win64-gpl-shared.zip
        run: |
          $ErrorActionPreference = "Stop"

          $tag = $env:FFMPEG_TAG
          $asset = $env:FFMPEG_ASSET

          $baseUrl = "https://github.com/BtbN/FFmpeg-Builds/releases/download/$tag"
          $url = "$baseUrl/$asset"
          $checksumsUrl = "$baseUrl/checksums.sha256"

          $tmpRoot = Join-Path $env:RUNNER_TEMP "ffmpeg-install"
          $zip = Join-Path $tmpRoot $asset
          $checksums = Join-Path $tmpRoot "checksums.sha256"
          $dest = Join-Path $tmpRoot "ffmpeg"

          if (Test-Path -LiteralPath $tmpRoot) {
            Remove-Item -LiteralPath $tmpRoot -Recurse -Force
          }
          New-Item -ItemType Directory -Path $tmpRoot | Out-Null

          Invoke-WebRequest -Uri $url -OutFile $zip
          Invoke-WebRequest -Uri $checksumsUrl -OutFile $checksums

          $expected = Get-Content -LiteralPath $checksums |
            ForEach-Object {
              if ($_ -match '^(?<sha>[0-9A-Fa-f]{64})\s+\*?(?<name>.+)$' -and $matches.name.Trim() -eq $asset) {
                $matches.sha.ToLowerInvariant()
              }
            } |
            Select-Object -First 1

          if (-not $expected) {
            throw "Could not find checksum for $asset in $checksums"
          }

          $actual = (Get-FileHash -LiteralPath $zip -Algorithm SHA256).Hash.ToLowerInvariant()
          if ($actual -ne $expected) {
            throw "FFmpeg checksum mismatch. Expected $expected but got $actual"
          }

          Expand-Archive -LiteralPath $zip -DestinationPath $dest -Force
          $ffdir = Get-ChildItem -LiteralPath $dest -Directory | Select-Object -First 1
          if (-not $ffdir) {
            throw "Could not find extracted FFmpeg directory."
          }

          $binDir = Join-Path $ffdir.FullName "bin"
          $binDir | Out-File -FilePath $env:GITHUB_PATH -Encoding utf8 -Append

      - name: Verify ffmpeg/ffprobe available
        shell: bash -el {0}
        run: |
          set -e
          ffmpeg -version
          ffprobe -version

      - name: Run pytest
        id: run_pytest
        continue-on-error: ${{ inputs.continue_on_error && !(inputs.strict_cells && matrix.strict == 'true') }}
        shell: bash -el {0}
        env:
          FULL_SUITE: ${{ inputs.full_suite }}
          PYTEST_PATHS_JSON: ${{ inputs.pytest_paths_json }}
        run: |
          python - << 'PY'
          import json
          import os
          import subprocess
          import sys

          full_suite = os.environ["FULL_SUITE"].lower() == "true"

          if full_suite:
              cmd = [sys.executable, "-m", "pytest"]
              print("Running full pytest suite")
          else:
              paths = json.loads(os.environ.get("PYTEST_PATHS_JSON", "[]"))
              if not paths:
                  print("No pytest paths selected; skipping pytest.")
                  raise SystemExit(0)
              cmd = [sys.executable, "-m", "pytest", *paths]
              print("Running targeted pytest:", " ".join(paths))

          raise SystemExit(subprocess.call(cmd))
          PY

      - name: Run functional scripts
        id: run_functional_scripts
        continue-on-error: ${{ inputs.continue_on_error && !(inputs.strict_cells && matrix.strict == 'true') }}
        shell: bash -el {0}
        env:
          FULL_SUITE: ${{ inputs.full_suite }}
          FUNCTIONAL_SCRIPTS_JSON: ${{ inputs.functional_scripts_json }}
        run: |
          python - << 'PY'
          import json
          import os
          import subprocess
          import sys

          def is_windows_python_3_11_or_greater() -> bool:
              """Aligned with pyproject: .[tf*] omits TensorFlow on Windows for Python 3.11+."""
              return sys.platform == "win32" and sys.version_info >= (3, 11)

          full_suite = os.environ["FULL_SUITE"].lower() == "true"
          if full_suite:
              scripts = [
                  "examples/testscript_tensorflow_single_animal.py",
                  "examples/testscript_tensorflow_multi_animal.py",
                  "examples/testscript_pytorch_single_animal.py",
                  "examples/testscript_pytorch_multi_animal.py",
              ]
          else:
              scripts = json.loads(os.environ.get("FUNCTIONAL_SCRIPTS_JSON", "[]"))
              if not scripts:
                  print("No functional scripts selected; skipping functional tests.")
                  raise SystemExit(0)

          for script in scripts:
              if "tensorflow" in script and is_windows_python_3_11_or_greater():
                  ver = f"{sys.version_info.major}.{sys.version_info.minor}"
                  print(
                      f"Skipping TensorFlow example on Windows {ver} (no TF in .[tf*] for 3.11+): {script}"
                  )
                  continue
              print("Running:", script)
              rc = subprocess.call([sys.executable, script])
              if rc != 0:
                  raise SystemExit(rc)
          PY

      - name: Report continue-on-error failures
        # continue-on-error hides failures behind a small icon on the step itself and
        # reports the job as passing overall, so make failures hard to miss: emit a
        # workflow annotation and a job summary entry whenever this happens. Strict
        # cells fail the job outright, so they need no annotation.
        if: >-
          ${{ inputs.continue_on_error && !(inputs.strict_cells && matrix.strict == 'true') &&
              (steps.run_pytest.outcome == 'failure' || steps.run_functional_scripts.outcome == 'failure') }}

        shell: bash
        run: |
          MATRIX_DESC="${{ matrix.os }}, py${{ matrix.python-version }}, pip_overrides='$PIP_OVERRIDES'"
          echo "::warning title=Non-blocking test failure::[$MATRIX_DESC] pytest=${{ steps.run_pytest.outcome }}, functional_scripts=${{ steps.run_functional_scripts.outcome }}. This job uses continue-on-error and will still report success overall, but the failure should be investigated."
          {
            echo "### :warning: Non-blocking test failure"
            echo ""
            echo "This job runs with \`continue_on_error: true\`, so it will still report as **passing** overall -- but at least one test step actually failed and should be investigated."
            echo ""
            echo "| Matrix | pytest | functional scripts |"
            echo "| --- | --- | --- |"
            echo "| $MATRIX_DESC | ${{ steps.run_pytest.outcome }} | ${{ steps.run_functional_scripts.outcome }} |"
          } >> "$GITHUB_STEP_SUMMARY"