diff --git a/.github/workflows/conda-package-cf.yml b/.github/workflows/conda-package-cf.yml index e75623a3fa..b1c8e76d58 100644 --- a/.github/workflows/conda-package-cf.yml +++ b/.github/workflows/conda-package-cf.yml @@ -13,7 +13,7 @@ env: PACKAGE_NAME: dpctl MODULE_NAME: dpctl TEST_ENV_NAME: test_dpctl - RECIPE_DIR: conda-recipe-cf + RECIPE_DIR: conda-recipe CHANNELS: "-c conda-forge --override-channels" VER_SCRIPT1: "import json; f = open('ver.json', 'r'); j = json.load(f); f.close(); d = j['dpctl'][0];" VER_SCRIPT2: "print('='.join((d[s] for s in ('version', 'build'))))" @@ -60,7 +60,7 @@ jobs: with: path: ~/.conda/pkgs key: - ${{ runner.os }}-conda-cf-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('conda-recipe-cf/meta.yaml') }} + ${{ runner.os }}-conda-cf-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('conda-recipe/meta.yaml') }} restore-keys: | ${{ runner.os }}-conda-cf-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- ${{ runner.os }}-conda-cf-${{ env.CACHE_NUMBER }}- @@ -405,7 +405,7 @@ jobs: EXAMPLES_ENV_NAME: examples BUILD_ENV_NAME: build_env DPCPP_CMPLR: "dpcpp_linux-64>=2026.0" - # keep in sync with conda-recipe-cf/conda_build_config.yaml + # keep in sync with conda-recipe/conda_build_config.yaml GCC_CMPLR_VERSION: '15' steps: diff --git a/.github/workflows/conda-package.yml b/.github/workflows/conda-package.yml deleted file mode 100644 index bd630c9db7..0000000000 --- a/.github/workflows/conda-package.yml +++ /dev/null @@ -1,995 +0,0 @@ -name: Conda package - -on: - push: - branches: - - master - pull_request: - -permissions: - contents: read - -env: - PACKAGE_NAME: dpctl - MODULE_NAME: dpctl - TEST_ENV_NAME: test_dpctl - VER_SCRIPT1: "import json; f = open('ver.json', 'r'); j = json.load(f); f.close(); d = j['dpctl'][0];" - VER_SCRIPT2: "print('='.join((d[s] for s in ('version', 'build'))))" - VER_SCRIPT3: "print(' '.join(map(lambda s: chr(34) + s + chr(34), [comp for comp in d['depends'] if 'dpcpp' in comp][1:])))" - INTEL_CHANNEL: "https://software.repos.intel.com/python/conda/" - CONDA_BUILD_VERSION: 26.3.0 - CONDA_INDEX_VERSION: 0.11.0 - -jobs: - build_linux: - # Pinned: ubuntu-latest is moving to Ubuntu 26.04, whose image has no - # preinstalled Miniconda ($CONDA is empty), and Linux jobs below rely on it - runs-on: ubuntu-24.04 - timeout-minutes: 90 - - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - - python: '3.14' - python_spec: '3.14.* *_cp314t' - - python: '3.15' - python_spec: '3.15.* *_cp315' - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - name: Set pkgs_dirs - run: | - echo "pkgs_dirs: [~/.conda/pkgs]" >> ~/.condarc - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - name: Add conda to system path - run: echo "$CONDA/bin" >> "$GITHUB_PATH" - - name: Update conda - run: | - conda update -n base --all - - name: Install conda-build - run: | - conda install -n base conda-build=${{ env.CONDA_BUILD_VERSION }} -c conda-forge --override-channels - - name: Show Conda info - run: | - conda info --all - - name: List base environment packages - run: | - conda list -n base - - name: Store conda paths as envs - shell: bash -l {0} - run: | - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE${{ runner.os == 'Linux' && '/' || '\\' }}" >> "$GITHUB_ENV" - - name: Build conda package - run: | - # use bootstrap channel to pull NumPy linked with OpenBLAS - if [ -n "${{ matrix.python_spec }}" ]; then - conda build --no-test --python "${{ matrix.python_spec }}" --numpy 2.0 -c conda-forge --override-channels conda-recipe - else - conda build --no-test --python ${{ matrix.python }} --numpy 2.0 -c conda-forge --override-channels conda-recipe - fi - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - path: /usr/share/miniconda/conda-bld/linux-64/${{ env.PACKAGE_NAME }}-*.conda - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - build_windows: - runs-on: windows-latest - timeout-minutes: 150 - defaults: - run: - shell: cmd /C CALL {0} - - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - - python: '3.14' - python_spec: '3.14.* *_cp314t' - - python: '3.15' - python_spec: '3.15.* *_cp315' - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - steps: - - name: Cancel Previous Runs - uses: styfle/cancel-workflow-action@d07a454dad7609a92316b57b23c9ccfd4f59af66 # 0.13.1 - with: - access_token: ${{ github.token }} - - - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - activate-environment: build - channels: conda-forge - python-version: ${{ matrix.python }} - conda-build-version: ${{ env.CONDA_BUILD_VERSION }} - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: /home/runner/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('**/meta.yaml') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - name: Store conda paths as envs - shell: bash -l {0} - run: | - echo "CONDA_BLD=$CONDA/conda-bld/win-64/" | tr "\\\\" '/' >> "$GITHUB_ENV" - echo "WHEELS_OUTPUT_FOLDER=$GITHUB_WORKSPACE${{ runner.os == 'Linux' && '/' || '\\' }}" >> "$GITHUB_ENV" - - name: Show Conda info - run: | - conda info --all - - name: List base environment packages - run: | - conda list -n base - - name: Build conda package - env: - OVERRIDE_INTEL_IPO: 1 # IPO requires more resources that GH actions VM provides - run: | - set "PYTHON_VER=${{ matrix.python_spec }}" - if "%PYTHON_VER%" NEQ "" ( - conda build --no-test --python "%PYTHON_VER%" --numpy 2.0 -c conda-forge --override-channels conda-recipe - ) else ( - conda build --no-test --python ${{ matrix.python }} --numpy 2.0 -c conda-forge --override-channels conda-recipe - ) - - - name: Upload artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - path: ${{ env.CONDA_BLD }}${{ env.PACKAGE_NAME }}-*.conda - - - name: Upload wheels artifact - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - path: ${{ env.WHEELS_OUTPUT_FOLDER }}${{ env.PACKAGE_NAME }}-*.whl - - test_linux: - needs: build_linux - runs-on: ${{ matrix.runner }} - timeout-minutes: 30 - - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - experimental: false - runner: ubuntu-24.04 - - python: '3.14' - python_spec: '3.14.* *_cp314t' - experimental: false - runner: ubuntu-24.04 - - python: '3.15' - python_spec: '3.15.* *_cp315' - experimental: false - runner: ubuntu-24.04 - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - experimental: [false] - runner: [ubuntu-24.04] - continue-on-error: ${{ matrix.experimental }} - - steps: - - name: Construct channels line - run: | - echo "CHANNELS=-c ${{ env.INTEL_CHANNEL }} -c conda-forge --override-channels" >> "$GITHUB_ENV" - - name: Display channels line - run: | - echo "${{ env.CHANNELS }}" - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - name: Add conda to system path - run: echo "$CONDA/bin" >> "$GITHUB_PATH" - - name: Update conda - run: | - conda update -n base --all - - name: Install conda-index - run: | - conda install -n base conda-index=${{ env.CONDA_INDEX_VERSION }} -c conda-forge --override-channels - - name: Show Conda info - run: | - conda info --all - - name: List base environment packages - run: | - conda list -n base - - name: Create conda channel - run: | - mkdir -p "$GITHUB_WORKSPACE/channel/linux-64" - conda index "$GITHUB_WORKSPACE/channel" || exit 1 - mv "${PACKAGE_NAME}"-*.conda "$GITHUB_WORKSPACE/channel/linux-64" || exit 1 - conda index "$GITHUB_WORKSPACE/channel" || exit 1 - # Test channel - conda search "$PACKAGE_NAME" -c "$GITHUB_WORKSPACE/channel" --override-channels --info --json > "$GITHUB_WORKSPACE/ver.json" - cat ver.json - - name: Collect dependencies - run: | - CHANNELS="-c $GITHUB_WORKSPACE/channel ${{ env.CHANNELS }}" - export CHANNELS - PACKAGE_VERSION=$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}") - export PACKAGE_VERSION - PYTHON_VER="${{ matrix.python_spec }}" - if [ -z "${PYTHON_VER}" ]; then - PYTHON_VER="${{ matrix.python }}" - fi - # shellcheck disable=SC2086 - conda create -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=${PACKAGE_VERSION} python="${PYTHON_VER}" $CHANNELS --only-deps --dry-run > lockfile - cat lockfile - - name: Set pkgs_dirs - run: | - echo "pkgs_dirs: [~/.conda/pkgs]" >> ~/.condarc - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - name: Install dpctl - run: | - CHANNELS="-c $GITHUB_WORKSPACE/channel ${{ env.CHANNELS }}" - export CHANNELS - TEST_DEPENDENCIES="pytest cython setuptools" - export TEST_DEPENDENCIES - PACKAGE_VERSION=$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}") - PYTHON_VER="${{ matrix.python_spec }}" - if [ -z "${PYTHON_VER}" ]; then - PYTHON_VER="${{ matrix.python }}" - fi - export PACKAGE_VERSION - # shellcheck disable=SC2086 - conda create -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=${PACKAGE_VERSION} $TEST_DEPENDENCIES python="${PYTHON_VER}" $CHANNELS - # Test installed packages - conda list -n "${{ env.TEST_ENV_NAME }}" - - name: Smoke test - run: | - . "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.TEST_ENV_NAME }}" - python -c "import dpctl; dpctl.lsplatform(verbosity=2)" - - name: Create test temp dir - # create temporary empty folder to runs tests from - # https://github.com/pytest-dev/pytest/issues/11904 - run: mkdir -p "${GITHUB_WORKSPACE}/test_tmp" - - name: Run tests - working-directory: ${{ github.workspace }}/test_tmp - env: - SYCL_CACHE_PERSISTENT: 1 - run: | - . "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.TEST_ENV_NAME }}" - python -m pytest -v --pyargs "${{ env.MODULE_NAME }}" - - test_windows: - needs: build_windows - runs-on: ${{ matrix.runner }} - timeout-minutes: 60 - defaults: - run: - shell: cmd /C CALL {0} - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - experimental: false - runner: windows-latest - - python: '3.14' - python_spec: '3.14.* *_cp314t' - experimental: false - runner: windows-latest - - python: '3.15' - python_spec: '3.15.* *_cp315' - experimental: false - runner: windows-latest - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - experimental: [false] - runner: [windows-latest] - continue-on-error: ${{ matrix.experimental }} - env: - workdir: '${{ github.workspace }}' - - steps: - - name: Construct channels line - shell: pwsh - run: | - echo "CHANNELS=-c ${{ env.INTEL_CHANNEL }} -c conda-forge --override-channels" >> $env:GITHUB_ENV - - - name: Display channels line - run: | - echo "${{ env.CHANNELS }}" - - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - channels: conda-forge - activate-environment: ${{ env.TEST_ENV_NAME }} - python-version: ${{ matrix.python }} - - - name: Install conda-index - run: | - conda install -n base conda-index - - - name: Show Conda info - run: | - conda info --all - - - name: List base environment packages - run: | - conda list -n base - - - name: Create conda channel with the artifact bit - shell: cmd /C CALL {0} - run: | - @echo on - echo ${{ env.workdir }} - mkdir ${{ env.workdir }}\channel - mkdir ${{ env.workdir }}\channel\win-64 - move ${{ env.PACKAGE_NAME }}-*.conda ${{ env.workdir }}\channel\win-64 - dir ${{ env.workdir }}\channel\win-64\ - - - name: Index the channel - shell: cmd /C CALL {0} - run: | - @echo on - conda index ${{ env.workdir }}\channel - - - name: List content of the channels - shell: cmd /C CALL {0} - run: | - dir ${{ env.workdir }}\channel - dir ${{ env.workdir }}\channel\win-64 - - - name: Dump dpctl version info from created channel into ver.json - shell: cmd /C CALL {0} - run: | - @echo on - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.workdir }}/channel --override-channels --info --json > ${{ env.workdir }}\ver.json - dir ${{ env.workdir }} - - - name: Output content of produced ver.json - shell: pwsh - run: Get-Content -Path ${{ env.workdir }}\ver.json - - - name: Collect dependencies - shell: cmd /C CALL {0} - run: | - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - SET "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - SET "PYTHON_VER=${{ matrix.python_spec }}" - IF "%PYTHON_VER%"=="" SET "PYTHON_VER=${{ matrix.python }}" - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% python="%PYTHON_VER%" -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} --only-deps --dry-run > lockfile - - - name: Display lockfile content - shell: pwsh - run: Get-Content -Path .\lockfile - - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: /home/runner/conda_pkgs_dir - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - - name: Install opencl_rt - shell: cmd /C CALL {0} - run: conda install -n ${{ env.TEST_ENV_NAME }} opencl_rt -c ${{ env.INTEL_CHANNEL }} --override-channels - - - name: Install dpctl - shell: cmd /C CALL {0} - run: | - @ECHO ON - IF NOT EXIST ver.json ( - copy /Y ${{ env.workdir }}\ver.json . - ) - set "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - SET "PYTHON_VER=${{ matrix.python_spec }}" - IF "%PYTHON_VER%"=="" SET "PYTHON_VER=${{ matrix.python }}" - SET TEST_DEPENDENCIES=pytest"<8" cython setuptools - conda install -n ${{ env.TEST_ENV_NAME }} ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% %TEST_DEPENDENCIES% python="%PYTHON_VER%" -c ${{ env.workdir }}/channel ${{ env.CHANNELS }} - - - name: Report content of test environment - shell: cmd /C CALL {0} - run: | - echo "Value of CONDA environment variable was: " %CONDA% - echo "Value of CONDA_PREFIX environment variable was: " %CONDA_PREFIX% - conda info && conda list -n ${{ env.TEST_ENV_NAME }} - - - name: Configure Intel OpenCL CPU RT - shell: pwsh - run: | - $script_path="$env:CONDA_PREFIX\Scripts\set-intel-ocl-icd-registry.ps1" - if (Test-Path $script_path) { - &$script_path - } else { - Write-Warning "File $script_path was NOT found!" - } - # Check the variable assisting OpenCL CPU driver to find TBB DLLs which are not located where it expects them by default - $cl_cfg="$env:CONDA_PREFIX\Library\bin\cl.cfg" - Get-Content -Tail 5 -Path $cl_cfg - - - name: Smoke test, step 1 - shell: cmd /C CALL {0} - run: >- - conda activate ${{ env.TEST_ENV_NAME }} && python -c "import sys; print(sys.executable)" - - - name: Smoke test, step 2 - shell: cmd /C CALL {0} - run: >- - conda activate ${{ env.TEST_ENV_NAME }} && python -m dpctl -f - - - name: Create empty temporary directory to run tests from - shell: cmd /C CALL {0} - # create temporary empty folder to runs tests from - # https://github.com/pytest-dev/pytest/issues/11904 - run: >- - mkdir "${{ env.workdir }}\test_tmp" - - - name: List content of workdir folder - shell: cmd /C CALL {0} - run: dir "${{ env.workdir }}" - - - name: Run tests - shell: cmd /C CALL {0} - env: - SYCL_CACHE_PERSISTENT: 1 - working-directory: ${{ env.workdir }}\test_tmp - run: >- - conda activate ${{ env.TEST_ENV_NAME }} && python -m pytest -v -s --pyargs ${{ env.MODULE_NAME }} - - upload_linux: - needs: test_linux - if: | - (github.repository == 'IntelPython/dpctl') && - (github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true) || github.event_name == 'push' && contains(github.ref, 'refs/tags/')) - runs-on: ubuntu-24.04 - timeout-minutes: 20 - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - - python: '3.14' - python_spec: '3.14.* *_cp314t' - - python: '3.15' - python_spec: '3.15.* *_cp315' - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - steps: - - name: Download conda artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - - name: Download wheel artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - - name: Install anaconda-client - run: conda install anaconda-client -c conda-forge --override-channels - - name: Add conda to system path - run: echo "$CONDA/bin" >> "$GITHUB_PATH" - - name: Package version - run: echo "PACKAGE_VERSION=$(basename ${{ env.PACKAGE_NAME }}-*.conda | sed 's/^${{ env.PACKAGE_NAME }}-\([^-]*\).*/\1/')" >> "$GITHUB_ENV" - - - name: Upload - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - if: ${{ env.ANACONDA_TOKEN != '' }} - run: | - anaconda --token "${{ env.ANACONDA_TOKEN }}" upload --user dppy --label dev "${{ env.PACKAGE_NAME }}"-*.conda - - - name: Upload Wheels - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - run: anaconda --token "${{ env.ANACONDA_TOKEN }}" upload --user dppy --label dev "${{ env.PACKAGE_NAME }}"-*.whl --version "${{ env.PACKAGE_VERSION }}" - - upload_windows: - needs: test_windows - if: | - (github.repository == 'IntelPython/dpctl') && - (github.ref == 'refs/heads/master' || (startsWith(github.ref, 'refs/heads/release') == true) || github.event_name == 'push' && contains(github.ref, 'refs/tags/')) - runs-on: windows-latest - timeout-minutes: 20 - defaults: - run: - shell: cmd /C CALL {0} - strategy: - matrix: - python: ['3.10', '3.11', '3.12', '3.13'] - python_spec: [''] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314' - - python: '3.14' - python_spec: '3.14.* *_cp314t' - - python: '3.15' - python_spec: '3.15.* *_cp315' - # TODO: add 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - steps: - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - - name: Download wheel artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Wheels Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - miniforge-version: latest - channels: conda-forge - activate-environment: upload - - - name: Install anaconda-client - run: conda install anaconda-client -c conda-forge --override-channels - - - name: Package version - shell: bash -el {0} - run: echo "PACKAGE_VERSION=$(basename ${{ env.PACKAGE_NAME }}-*.conda | sed 's/^${{ env.PACKAGE_NAME }}-\([^-]*\).*/\1/')" >> "$GITHUB_ENV" - - - name: Upload - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - run: | - anaconda --token "${{ env.ANACONDA_TOKEN }}" upload --user dppy --label dev "${{ env.PACKAGE_NAME }}"-*.conda - - - name: Upload Wheels - env: - ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} - run: anaconda --token "${{ env.ANACONDA_TOKEN }}" upload --user dppy --label dev "${{ env.PACKAGE_NAME }}"-*.whl --version "${{ env.PACKAGE_VERSION }}" - - test_examples_linux: - needs: build_linux - runs-on: ${{ matrix.runner }} - strategy: - matrix: - python: ['3.15'] - python_spec: ['3.15.* *_cp315'] - experimental: [false] - runner: [ubuntu-24.04] - include: - # TODO: switch to 3.15t (cp315t) when NumPy cp315t builds are available on conda-forge - - python: '3.14' - python_spec: '3.14.* *_cp314t' - experimental: false - runner: ubuntu-24.04 - continue-on-error: ${{ matrix.experimental }} - timeout-minutes: 60 - env: - EXAMPLES_ENV_NAME: examples - BUILD_ENV_NAME: build_env - - steps: - - name: Construct channels line - run: | - echo "CHANNELS=-c ${{ env.INTEL_CHANNEL }} -c conda-forge --override-channels" >> "$GITHUB_ENV" - - name: Display channels line - run: | - echo ${{ env.CHANNELS }} - - name: Install conda-index - # Needed to be able to run conda index - run: | - conda update -n base --all - conda install conda-index=${{ env.CONDA_INDEX_VERSION }} -c conda-forge --override-channels - - name: Checkout dpctl repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - name: Add conda to system path - run: echo "$CONDA/bin" >> "$GITHUB_PATH" - - name: Create conda channel - run: | - mkdir -p "$GITHUB_WORKSPACE/channel/linux-64" - conda index "$GITHUB_WORKSPACE/channel" || exit 1 - mv "${{ env.PACKAGE_NAME }}"-*.conda "$GITHUB_WORKSPACE/channel/linux-64" || exit 1 - conda index "$GITHUB_WORKSPACE/channel" || exit 1 - # Test channel - conda search "$PACKAGE_NAME" -c "$GITHUB_WORKSPACE/channel" --override-channels --info --json > "$GITHUB_WORKSPACE/ver.json" - cat ver.json - - name: Collect dependencies - run: | - CHANNELS="-c $GITHUB_WORKSPACE/channel ${{ env.CHANNELS }}" - export CHANNELS - PACKAGE_VERSION=$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}") - export PACKAGE_VERSION - PYTHON_VER="${{ matrix.python_spec }}" - if [ -z "${PYTHON_VER}" ]; then - PYTHON_VER="${{ matrix.python }}" - fi - export PYTHON_VER - # shellcheck disable=SC2086 - conda create -n ${{ env.EXAMPLES_ENV_NAME }} ${{ env.PACKAGE_NAME }}=${PACKAGE_VERSION} python="${PYTHON_VER}" $CHANNELS --only-deps --dry-run > lockfile - cat lockfile - - name: Set pkgs_dirs - run: | - echo "pkgs_dirs: [~/.conda/pkgs]" >> ~/.condarc - - name: Cache conda packages - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - env: - CACHE_NUMBER: 3 # Increase to reset cache - with: - path: ~/.conda/pkgs - key: - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}-${{hashFiles('lockfile') }} - restore-keys: | - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-python-${{ matrix.python }}-spec-${{ matrix.python == '3.14' && contains(matrix.python_spec, 'cp314t') && 't' || '' }}- - ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}- - - name: Install example requirements - shell: bash -ex -l {0} - env: - DPCPP_CMPLR: "dpcpp_linux-64>=2026.0" - run: | - export CHANNELS="${{ env.CHANNELS }}" - . "$CONDA/etc/profile.d/conda.sh" - DPCTL_DEPENDS="$(python -c "${VER_SCRIPT1} ${VER_SCRIPT3}")" - export DPCTL_DEPENDS - echo "Dpctl dependencies: ${DPCTL_DEPENDS}" - PYTHON_VER="${{ matrix.python_spec }}" - if [ -z "${PYTHON_VER}" ]; then - PYTHON_VER="${{ matrix.python }}" - fi - # shellcheck disable=SC2086 - conda create -n ${{ env.EXAMPLES_ENV_NAME }} -y pytest python="${PYTHON_VER}" "setuptools<72.2.0" $CHANNELS - echo "Environment created" - # shellcheck disable=SC2086 - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y cmake ninja $CHANNELS || exit 1 - echo "Cmake and Ninja installed" - # shellcheck disable=SC2086 - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y pybind11 cython scikit-build $CHANNELS || exit 1 - echo "scikit-build installed" - # shellcheck disable=SC2086 - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y mkl-dpcpp \ - mkl-devel-dpcpp dpcpp_cpp_rt "${DPCTL_DEPENDS}" \ - $CHANNELS || exit 1 - echo "IPL installed" - # shellcheck disable=SC2086 - conda create -y -n ${{ env.BUILD_ENV_NAME }} $CHANNELS gcc_linux-64 gxx_linux-64 \ - "${{ env.DPCPP_CMPLR }}" "${DPCTL_DEPENDS}" \ - "sysroot_linux-64>=2.28" - echo "Compiler installed" - conda list -n "${{ env.BUILD_ENV_NAME }}" - - name: Install dpctl - shell: bash -l {0} - run: | - source "$CONDA/etc/profile.d/conda.sh" - CHANNELS="-c $GITHUB_WORKSPACE/channel -c dppy/label/dev -c ${{ env.INTEL_CHANNEL }} -c conda-forge --override-channels" - export CHANNELS - PACKAGE_VERSION=$(python -c "${VER_SCRIPT1} ${VER_SCRIPT2}") - export PACKAGE_VERSION - # shellcheck disable=SC2086 - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y $CHANNELS ${{ env.PACKAGE_NAME }}=${PACKAGE_VERSION} || exit 1 - - name: Build and run examples of pybind11 extensions - shell: bash -l {0} - run: | - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.EXAMPLES_ENV_NAME }}" - conda list - cd examples/pybind11 - while IFS= read -r d; do - pushd "$d" > /dev/null - conda activate --stack build_env - CC=icx CXX=icpx python setup.py build_ext --inplace -G Ninja || exit 1 - conda deactivate - if [ -e tests ]; then - LD_LIBRARY_PATH="${CONDA_PREFIX}/lib" python -m pytest tests || exit 1 - else - LD_LIBRARY_PATH="${CONDA_PREFIX}/lib" python example.py || exit 1 - fi - popd > /dev/null - done < <(find . -maxdepth 1 -type d -not -path ".") - - name: Build and run examples of Cython extensions - shell: bash -l {0} - run: | - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.EXAMPLES_ENV_NAME }}" - conda list - cd examples/cython - while IFS= read -r d; do - pushd "$d" > /dev/null - conda activate --stack "${{ env.BUILD_ENV_NAME }}" - CC=icx CXX=icpx python setup.py build_ext --inplace -G Ninja || exit 1 - conda deactivate - python -m pytest tests || exit 1 - popd > /dev/null - done < <(find . -maxdepth 1 -type d -not -path ".") - - name: Build and run examples of C-extensions - shell: bash -l {0} - run: | - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.EXAMPLES_ENV_NAME }}" - conda list - cd examples/c - while IFS= read -r d; do - pushd "$d" > /dev/null - conda activate --stack "${{ env.BUILD_ENV_NAME }}" - python setup.py build_ext --inplace || exit 1 - conda deactivate - python -m pytest tests || exit 1 - popd > /dev/null - done < <(find . -maxdepth 1 -type d -not -path ".") - - name: Run Python examples - shell: bash -l {0} - run: | - cd examples/python - source "$CONDA/etc/profile.d/conda.sh" - conda activate "${{ env.EXAMPLES_ENV_NAME }}" - while IFS= read -r script; do - echo "Executing ${script}" - python "${script}" --run all || exit 1 - done < <(find . \( -not -name "_*" -and -name "*.py" \)) - - test_examples_windows: - needs: build_windows - runs-on: ${{ matrix.runner }} - defaults: - run: - shell: cmd /C CALL {0} - strategy: - matrix: - python: ['3.14'] - python_spec: ['3.14.* *_cp314'] - experimental: [false] - runner: [windows-latest] - include: - - python: '3.14' - python_spec: '3.14.* *_cp314t' - experimental: false - runner: windows-latest - continue-on-error: ${{ matrix.experimental }} - timeout-minutes: 120 - env: - EXAMPLES_ENV_NAME: examples - BUILD_ENV_NAME: build_env - DPCPP_CMPLR: "dpcpp_win-64>=2026.0" - workdir: '${{ github.workspace }}' - - steps: - - name: Construct channels line - shell: pwsh - run: | - echo "CHANNELS=-c ${{ env.INTEL_CHANNEL }} -c conda-forge --override-channels" >> $env:GITHUB_ENV - - name: Display channels line - run: | - echo "${{ env.CHANNELS }}" - - name: Checkout dpctl repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - fetch-depth: 0 - - name: Download artifact - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - name: ${{ env.PACKAGE_NAME }} ${{ runner.os }} Python ${{ matrix.python == '3.14' && (contains(matrix.python_spec, 'cp314t') && '3.14t' || '3.14') || matrix.python }} - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - auto-update-conda: true - miniforge-version: latest - channels: conda-forge - activate-environment: ${{ env.EXAMPLES_ENV_NAME }} - python-version: ${{ matrix.python }} - - name: Install conda-index - run: | - conda install -n base conda-index=${{ env.CONDA_INDEX_VERSION }} - - name: Create conda channel with the artifact bit - run: | - @echo on - mkdir ${{ env.workdir }}\channel\win-64 - move ${{ env.PACKAGE_NAME }}-*.conda ${{ env.workdir }}\channel\win-64 - conda index ${{ env.workdir }}\channel - dir ${{ env.workdir }}\channel\win-64\ - - name: Dump dpctl version info from created channel into ver.json - run: | - @echo on - conda search ${{ env.PACKAGE_NAME }} -c ${{ env.workdir }}/channel --override-channels --info --json > ${{ env.workdir }}\ver.json - - name: Output content of produced ver.json - shell: pwsh - run: Get-Content -Path ${{ env.workdir }}\ver.json - - name: Install example requirements - run: | - @echo on - SET "PYTHON_VER=${{ matrix.python_spec }}" - IF "%PYTHON_VER%"=="" SET "PYTHON_VER=${{ matrix.python }}" - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y python="%PYTHON_VER%" pytest ^ - "setuptools<72.2.0" cmake ninja pybind11 cython scikit-build ^ - mkl-dpcpp mkl-devel-dpcpp ${{ env.CHANNELS }} || exit /b 1 - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y opencl_rt -c ${{ env.INTEL_CHANNEL }} --override-channels || exit /b 1 - REM the compiler lives in its own environment, stacked on top of the - REM examples environment while extensions are being built - conda create -n ${{ env.BUILD_ENV_NAME }} -y "${{ env.DPCPP_CMPLR }}" vs2022_win-64 ${{ env.CHANNELS }} || exit /b 1 - conda list -n ${{ env.BUILD_ENV_NAME }} - - name: Install dpctl - run: | - @echo on - set "SCRIPT=%VER_SCRIPT1% %VER_SCRIPT2%" - FOR /F "tokens=* USEBACKQ" %%F IN (`python -c "%SCRIPT%"`) DO ( - SET PACKAGE_VERSION=%%F - ) - conda install -n ${{ env.EXAMPLES_ENV_NAME }} -y ${{ env.PACKAGE_NAME }}=%PACKAGE_VERSION% ^ - -c ${{ env.workdir }}/channel -c dppy/label/dev ${{ env.CHANNELS }} || exit /b 1 - conda list -n ${{ env.EXAMPLES_ENV_NAME }} - - name: Configure Intel OpenCL CPU RT - shell: pwsh - run: | - $script_path="$env:CONDA_PREFIX\Scripts\set-intel-ocl-icd-registry.ps1" - if (Test-Path $script_path) { - &$script_path - } else { - Write-Warning "File $script_path was NOT found!" - } - # Check the variable assisting OpenCL CPU driver to find TBB DLLs which are not located where it expects them by default - $cl_cfg="$env:CONDA_PREFIX\Library\bin\cl.cfg" - Get-Content -Tail 5 -Path $cl_cfg - - name: Smoke test - # dpctl source directory shadows the package, so don't run from the - # checkout root - working-directory: ${{ env.workdir }}\examples - run: >- - conda activate ${{ env.EXAMPLES_ENV_NAME }} && python -m dpctl -f - - name: Build and run examples of pybind11 extensions - working-directory: ${{ env.workdir }}\examples\pybind11 - env: - CC: icx - CXX: icx - run: | - @echo on - call conda activate ${{ env.EXAMPLES_ENV_NAME }} || exit /b 1 - conda list - for /d %%d in (*) do ( - pushd %%d || exit /b 1 - call conda activate --stack ${{ env.BUILD_ENV_NAME }} || exit /b 1 - python setup.py build_ext --inplace -G Ninja || exit /b 1 - call conda deactivate || exit /b 1 - if exist tests ( - python -m pytest -v tests || exit /b 1 - ) else ( - python example.py || exit /b 1 - ) - popd - ) - - name: Build and run examples of Cython extensions - working-directory: ${{ env.workdir }}\examples\cython - env: - CC: icx - CXX: icx - run: | - @echo on - call conda activate ${{ env.EXAMPLES_ENV_NAME }} || exit /b 1 - conda list - for /d %%d in (*) do ( - pushd %%d || exit /b 1 - call conda activate --stack ${{ env.BUILD_ENV_NAME }} || exit /b 1 - python setup.py build_ext --inplace -G Ninja || exit /b 1 - call conda deactivate || exit /b 1 - python -m pytest -v tests || exit /b 1 - popd - ) - - name: Build and run examples of C-extensions - working-directory: ${{ env.workdir }}\examples\c - run: | - @echo on - call conda activate ${{ env.EXAMPLES_ENV_NAME }} || exit /b 1 - conda list - for /d %%d in (*) do ( - pushd %%d || exit /b 1 - python setup.py build_ext --inplace || exit /b 1 - python -m pytest -v tests || exit /b 1 - popd - ) - - name: Run Python examples - working-directory: ${{ env.workdir }}\examples\python - run: | - @echo on - call conda activate ${{ env.EXAMPLES_ENV_NAME }} || exit /b 1 - for /f "delims=" %%f in ('dir /b *.py ^| findstr /v "^_"') do ( - echo Executing %%f - python %%f --run all || exit /b 1 - ) - - cleanup_packages: - name: Clean up anaconda packages - needs: [upload_linux, upload_windows] - runs-on: 'ubuntu-latest' - timeout-minutes: 30 - defaults: - run: - shell: bash -el {0} - steps: - - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 - with: - run-post: false - channel-priority: "disabled" - channels: conda-forge - python-version: '3.11' - - - name: Install anaconda-client - run: conda install anaconda-client -c conda-forge --override-channels - - - name: Checkout repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - repository: IntelPython/devops-tools - fetch-depth: 0 - - - name: Cleanup old packages - run: | - python scripts/cleanup-old-packages.py \ - --verbose --force --token ${{ secrets.ANACONDA_TOKEN }} \ - --package dppy/${{ env.PACKAGE_NAME }} --label dev diff --git a/.github/workflows/intel-ci.yml b/.github/workflows/intel-ci.yml new file mode 100644 index 0000000000..416d2e75ee --- /dev/null +++ b/.github/workflows/intel-ci.yml @@ -0,0 +1,261 @@ +# Intel Python CI (gh-ci): builds and tests the conda packages and wheels for Intel's Jenkins, see ci/README.md +name: Intel Python CI + +on: + push: + branches: [master] + tags: ['*'] + pull_request: + +permissions: + contents: read + +concurrency: + group: intel-ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + matrix: + name: Platforms and Python versions + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 10 + outputs: + matrix: ${{ steps.declared.outputs.matrix }} + wheel: ${{ steps.declared.outputs.wheel }} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Read the declared platforms and Python versions + id: declared + run: | + python ci/matrix.py --event "$GITHUB_EVENT_NAME" --github-output matrix + python ci/matrix.py --system wheel --event "$GITHUB_EVENT_NAME" --github-output wheel + + # The recipe and pyproject.toml declare no macOS, as SYCL has none; Jenkins ignores it anyway. + build: + name: ${{ matrix.name }} Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # The recipe takes its version from git describe. + fetch-depth: 0 + + # Miniforge 26.9.4-0 fails setup-miniconda's conda init on Windows runners; 26.7.2-0 works. + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install conda-build + run: conda install -y -q -n base --override-channels -c conda-forge conda-build + + - name: Build and run the recipe tests + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dpctl ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.conda + if-no-files-found: error + # Jenkins downloads them within minutes of the run; tags keep them 30 days. + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test: + name: Test ${{ matrix.name }} Python ${{ matrix.python }} + needs: [matrix, build] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.matrix) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dpctl ${{ matrix.name }} Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the package and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" + + build-wheel: + name: ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: matrix + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 150 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + # versioneer takes the version from git describe. + fetch-depth: 0 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + # CMake builds with Ninja and icx, which need MSVC's environment. + - uses: ilammy/msvc-dev-cmd@0b201ec74fa43914dc39ae48a89fd1d8cb592756 # v1.13.0 + if: matrix.platform == 'windows' + + - name: Install cibuildwheel + run: python -m pip install --no-input cibuildwheel==4.3.5 + + - name: Build the wheel + env: + PYTHON_VERSION: ${{ matrix.python }} + run: python ci/build.py --system wheel --python "$PYTHON_VERSION" --output "$RUNNER_TEMP/output" + + - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 + with: + name: dpctl ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/output/*.whl + if-no-files-found: error + retention-days: ${{ startsWith(github.ref, 'refs/tags/') && 30 || 3 }} + + test-wheel: + name: Test ${{ matrix.name }} Wheels Python ${{ matrix.python }} + needs: [matrix, build-wheel] + runs-on: ${{ matrix.platform == 'windows' && (vars.GH_CI_WINDOWS_RUNNER || 'windows-2022') || matrix.platform == 'macos' && (vars.GH_CI_MACOS_RUNNER || 'macos-26-intel') || (vars.GH_CI_RUNNER || 'ubuntu-latest') }} + timeout-minutes: 60 + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.matrix.outputs.wheel) }} + defaults: + run: + shell: bash -el {0} + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + name: dpctl ${{ matrix.name }} Wheels Python ${{ matrix.python }} + path: ${{ runner.temp }}/artifacts + + - uses: conda-incubator/setup-miniconda@be893c923ea9cf1cf7cd510fbdde27c7e18cbdcb # v4.1.0 + with: + miniforge-version: 26.7.2-0 + activate-environment: "" + conda-remove-defaults: "true" + + - name: Install the wheel and run ci/test.py + env: + PYTHON_VERSION: ${{ matrix.python }} + EVENT: ${{ startsWith(github.ref, 'refs/tags/') && 'tag' || github.event_name == 'pull_request' && 'pr' || 'push' }} + COMMIT: ${{ github.event.pull_request.head.sha || github.sha }} + run: >- + python ci/hooks.py --kind wheel --python "$PYTHON_VERSION" --artifacts "$RUNNER_TEMP/artifacts" + --work "$RUNNER_TEMP/hooks" --event "$EVENT" --commit "$COMMIT" + + # The dev channel on anaconda.org, which docs and the nightly run-tests-from-dppy-bits.yaml read. + upload: + name: Upload to anaconda.org + needs: [test, test-wheel] + if: | + github.repository == 'IntelPython/dpctl' && + github.event_name == 'push' && + (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 20 + defaults: + run: + shell: bash + steps: + - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 + with: + pattern: dpctl * + merge-multiple: true + path: ${{ runner.temp }}/packages + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.14" + + - name: Install anaconda-client + run: python -m pip install --no-input anaconda-client + + - name: Upload the conda packages and the wheels + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + PACKAGES: ${{ runner.temp }}/packages + run: | + cd "$PACKAGES" + conda_packages=(dpctl-*.conda) + version=${conda_packages[0]#dpctl-} + version=${version%%-*} + anaconda --token "$ANACONDA_TOKEN" upload --user dppy --label dev dpctl-*.conda + anaconda --token "$ANACONDA_TOKEN" upload --user dppy --label dev dpctl-*.whl --version "$version" + + cleanup: + name: Clean up anaconda.org + needs: upload + if: | + github.repository == 'IntelPython/dpctl' && + github.event_name == 'push' && + (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/')) + runs-on: ${{ vars.GH_CI_RUNNER || 'ubuntu-latest' }} + timeout-minutes: 30 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + repository: IntelPython/devops-tools + ref: 08ffddf88627957e8b094817aac4ce0bde7cff63 # main + fetch-depth: 0 + + - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 + with: + python-version: "3.11" + + - name: Install anaconda-client + run: python -m pip install --no-input anaconda-client + + - name: Remove the old dev packages + env: + ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }} + run: python scripts/cleanup-old-packages.py --verbose --force --token "$ANACONDA_TOKEN" --package dppy/dpctl --label dev diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cc5c4bf8a..10382e1c6c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ### Maintenance +* Added Intel Python CI (`.github/workflows/intel-ci.yml` and `ci/`), which builds and tests the conda packages and wheels; `conda-recipe/` is now the only, conda-forge, recipe and `conda-package.yml` is removed [gh-2409](https://github.com/IntelPython/dpctl/pull/2409) + ## [0.23.0] - Oct. 22, 2026 This release extends the SYCL interface exposed by `dpctl` in several directions: `dpctl.compiler.SyclKernelBundle` objects can now be built from SYCL source strings via the DPC++ `kernel_compiler` extension, SPIR-V specialization constants are supported, USM-device allocations can be shared between processes using the SYCL IPC memory extension, and `dpctl.SyclQueue` gained `copy`, `memset`, and `fill` methods along with their asynchronous counterparts. The package can now be used with free-threaded builds of CPython 3.14 and supports Python 3.15. The `dpctl.program` submodule is deprecated in favor of `dpctl.compiler`, which better describes its purpose. diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000000..742d8338df --- /dev/null +++ b/ci/README.md @@ -0,0 +1,50 @@ +# How Intel Python CI tests dpctl + +Intel Python CI (gh-ci) builds and tests dpctl's conda packages and wheels in GitHub Actions. Intel's Jenkins then tests each package again on Intel machines, including GPU ones, and scans it, and on a release tag publishes the packages it tested as the tag's GitHub Release. Everything it needs from this repository is in this directory, one workflow, the recipe, and `pyproject.toml`. + +| File | Role | +|---|---| +| `../conda-recipe/` | The conda recipe, built against conda-forge only. `conda_build_config.yaml` lists the Python versions, which has no 3.15t as conda-forge has no NumPy for it. The recipe skips macOS in `meta.yaml`, and its own tests only import dpctl and list the devices, so they need no device. | +| `../pyproject.toml` | What the wheels are built from. Its Python classifiers list the wheels' Python versions, and its Operating System classifiers their platforms. `[tool.cibuildwheel]` holds the wheel build settings, and `[tool.gh-ci]` holds the run-time requirements the wheels need on top of `[project]`'s. | +| `../.github/workflows/intel-ci.yml` | One job reads both matrices. One job per platform and Python version builds and uploads `dpctl Python `, and one builds `dpctl Wheels Python `. Then one job per package tests it. On a push to `master` or a tag, two more jobs upload the tested packages to the `dev` label of `dppy` on anaconda.org and remove its old ones. | +| `matrix.py` | Reads the platforms and Python versions from the recipe, or with `--system wheel` from `pyproject.toml`. A pull request gets the oldest and the newest regular versions. | +| `build.py` | Builds one Python version: with conda-build, which runs the recipe's own tests, or with `--system wheel` with cibuildwheel, in a manylinux container on Linux. | +| `hooks.py` | The workflow's test jobs: installs one package into a clean environment with the requirements in `config.yml`, and runs `test.py` the way Jenkins does. A conda package's environment comes from conda-forge; a wheel's has only Python from conda-forge and everything else from PyPI. | +| `install_icx.py` | Installs Intel's icx, which the conda recipe builds with too, for the Windows wheel builds: `[tool.cibuildwheel.windows]` in `../pyproject.toml` runs it first. The Linux wheel builds get icx from Intel's yum repository. Both get the Level Zero headers from a pinned tag of its GitHub repository. | +| `config.yml` | Tells gh-ci which workflow builds the packages, what its artifacts are called, and what the tests and benchmarks need. | +| `test.py` | Runs the installed package's test suite with pytest, in Actions and again in Jenkins. | +| `benchmark.py` | Runs the asv suite in `../benchmarks/` against the installed package, in Jenkins only, on a bare-metal benchmarking machine. Jenkins requires it because `../benchmarks/` exists. | + +The scripts need only the Python standard library. `matrix.py`, `hooks.py` and `benchmark.py` are the same in every repository gh-ci tests, `build.py` adds a fix for the CRLF `METADATA` that setuptools writes on Windows, and `install_icx.py` and `test.py` are specific to dpctl. Scanning is not the repository's to define: Jenkins runs its own scans. + +## Devices + +dpctl's tests need a SYCL device, and GitHub's runners have none but a CPU. `test.py` prints the devices with `python -m dpctl -f` and fails if there is none, so a run that skipped every device test cannot pass. + +- **In Actions** the device is the OpenCL CPU runtime of `intel-opencl-rt`, which `config.yml` installs from conda-forge for a conda package and from PyPI for a wheel. `test.py` points the OpenCL loader at it with `OCL_ICD_FILENAMES`, and on Windows also registers it under `HKLM\SOFTWARE\Khronos\OpenCL\Vendors`, as the loader in `System32` reads only that, which needs the administrator rights a GitHub runner gives. +- **In Jenkins** the GPU agents set `GH_CI_DEVICE=gpu`. Then `test.py` leaves the loader alone, apart from letting a Linux environment's loader see the system's ICDs too, and fails unless a GPU device is present. + +Two tests compile an extension when they run, `test_c_headers` and `test_cython_api`. `test.py` deselects them, and says so, when setuptools finds no C and C++ compiler. A conda package's environment gets `c-compiler` and `cxx-compiler` from `config.yml`; a wheel's environment uses the machine's. + +## Wheels + +The wheels link the SYCL runtime that `intel-sycl-rt` and `intel-cmplr-lib-rt` ship, and find it through their RPATH, or the DLL search path on Windows, so `auditwheel` and `delvewheel` leave those libraries out. `[tool.gh-ci] wheel-requires` adds the two as requirements to each wheel, as the wheels on PyPI have. icx is pinned to 2026.2 to match. + +## Uploads + +A push to `master` or a tag, in `IntelPython/dpctl` only and after every test passed, uploads the conda packages and the wheels the workflow built to `dppy` on anaconda.org with the label `dev`, with the repository secret `ANACONDA_TOKEN`, and then removes the old dev packages. No other event or branch gets the secret. + +**A pull request into `master`.** Actions builds the oldest and the newest regular Python versions on Linux and Windows, as a conda package and as a wheel, and tests each package. Jenkins notices the pull request within 3 minutes, waits for that whole run, and then tests and scans every package at once, and benchmarks the newest Linux conda package, sampling each benchmark once. Each cell posts its own status on the pull request, and `intel-ci/pr` sums them up. + +**A release tag.** Actions builds every declared version, and Jenkins tests and scans all of them, and measures the benchmarks properly on the newest Linux conda package, before it publishes the release. A benchmark that fails fails the run; a slower one does not, as nothing compares the timings with a baseline yet. + +To support another Python, add it to `conda-recipe/conda_build_config.yaml` for the conda packages and as a classifier in `pyproject.toml` for the wheels; nothing else lists versions. Try it locally from a conda base environment with conda-build, or with cibuildwheel for wheels: + +```bash +python ci/matrix.py # every platform and Python version +python ci/matrix.py --event pull_request # what a pull request builds +python ci/build.py --python VERSION --output /tmp/out # VERSION: one matrix.py lists +python ci/hooks.py --python VERSION --artifacts /tmp/out --work /tmp/hooks +python ci/build.py --system wheel --python VERSION --output /tmp/wheels +python ci/hooks.py --kind wheel --python VERSION --artifacts /tmp/wheels --work /tmp/wheel-hooks +``` diff --git a/ci/benchmark.py b/ci/benchmark.py new file mode 100644 index 0000000000..af04d60b7b --- /dev/null +++ b/ci/benchmark.py @@ -0,0 +1,90 @@ +""" +Intel Python CI (gh-ci) benchmark of the installed package. + +Intel's Jenkins runs this because the repository has a benchmarks/ directory: +once per pull request or tag, with the conda package of the newest regular +Linux Python installed in a clean conda-forge environment with the benchmark +requirements from ci/config.yml, on a bare-metal benchmarking machine, from an +empty working directory. + +It runs the asv suite in benchmarks/ against the installed package, without a +build of its own, and records the results under the commit being tested, in +$GH_CI_RESULTS, which Jenkins keeps with the build. A pull request samples each +benchmark once, enough to show that every one still runs; a tag measures them +properly. A benchmark that fails fails the cell; a slower one does not, as +nothing compares the timings with a baseline yet. +""" + +import json +import os +import pathlib +import subprocess +import sys + +# asv's own dependency, which reads the comments in its configuration. +import json5 + +ASV_CONFIG = pathlib.Path("benchmarks", "asv.conf.json") + + +def asv_config(source: pathlib.Path, results: pathlib.Path) -> pathlib.Path: + """Writes this run's asv configuration into the working directory. + + It is the repository's own, with absolute paths, since asv reads them from + where the configuration is, and with results under GH_CI_RESULTS. It lists + one branch, HEAD: the checkout is the commit under test, detached, and asv + cannot resolve the repository's branches there. + """ + path = source / ASV_CONFIG + config = json5.loads(path.read_text(encoding="utf-8")) + work = pathlib.Path.cwd() + config.update( + repo=str((path.parent / config.get("repo", "..")).resolve()), + benchmark_dir=str( + (path.parent / config.get("benchmark_dir", "benchmarks")).resolve() + ), + branches=["HEAD"], + results_dir=str(results / "asv"), + html_dir=str(work / "html"), + env_dir=str(work / "env"), + ) + written = work / "asv.conf.json" + written.write_text(json.dumps(config, indent=2), encoding="utf-8") + return written + + +def main() -> int: + """Runs the asv suite against the installed package.""" + config = asv_config( + pathlib.Path(os.environ["GH_CI_SOURCE"]), + pathlib.Path(os.environ["GH_CI_RESULTS"]), + ) + asv = [sys.executable, "-m", "asv"] + # asv keeps its machine description in ~/.asv-machine.json; this one stays + # with the run rather than with the machine's user. + home = pathlib.Path.cwd() / "home" + home.mkdir(exist_ok=True) + env = dict(os.environ, HOME=str(home)) + described = subprocess.run( + asv + ["machine", "--yes", "--config", str(config)], env=env + ) + if described.returncode: + return described.returncode + run = [ + "run", + "--config", + str(config), + f"--environment=existing:{sys.executable}", + f"--set-commit-hash={os.environ['GH_CI_COMMIT']}", + "--no-pull", + "--show-stderr", + "--launch-method=spawn", + ] + if os.environ.get("GH_CI_EVENT") != "tag": + run.append("--quick") + # asv exits 2 when a benchmark failed. + return subprocess.run(asv + run, env=env).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/build.py b/ci/build.py new file mode 100644 index 0000000000..cd462e5ed4 --- /dev/null +++ b/ci/build.py @@ -0,0 +1,344 @@ +""" +Builds this repository's packages for one Python version. + +For the gh-ci GitHub Actions workflow, and for Coverity in Jenkins. + +Copy this file and ci/matrix.py into any repository gh-ci tests. They work out +the build system from the files present, and the Python versions from that +build system's own declaration, so most repositories need no changes to them: + + rattler-build .conda recipe.yaml or recipe/recipe.yaml, versions in + variants.yaml + conda-build .conda conda-recipe/meta.yaml or meta.yaml, versions in + conda_build_config.yaml + cibuildwheel .whl pyproject.toml, versions from its classifiers + +Usage: + python ci/build.py --python 3.14 --output OUT [--system rattler-build] + [--channel conda-forge] + +--python must be one of the declared versions; `python ci/matrix.py` lists them. + +Packages land directly in OUT, ready for actions/upload-artifact. A conda +recipe's own tests run during the build; Jenkins later runs ci/test.py against +the uploaded artifacts, and builds with this script once more under Coverity for +intel-innersource tags. Needs only the Python standard library. rattler-build is +used from PATH, or downloaded at a pinned version for Linux or Windows and +checked against its published sha256; conda-build, or cibuildwheel for wheels, +must already be installed for this interpreter, so run it with the conda base +install's Python to build with conda-build. + +Wheels build with cibuildwheel, in a manylinux container on Linux, as +pyproject.toml's [tool.cibuildwheel] says. Run-time requirements that +pyproject.toml cannot declare, because the conda package gets them from its +recipe instead, such as MKL's libraries, go in [tool.gh-ci] wheel-requires: +build.py adds them to each wheel's metadata. +""" + +import argparse +import base64 +import hashlib +import os +import platform +import shlex +import shutil +import subprocess +import sys +import tempfile +import urllib.request +import zipfile +from pathlib import Path + +import matrix + +ROOT = matrix.ROOT +PACKAGE_SUFFIXES = (".conda", ".tar.bz2", ".whl") +# cibuildwheel's platform part of a build identifier, per system. +WHEEL_PLATFORMS = { + "Linux": "manylinux_x86_64", + "Windows": "win_amd64", + "Darwin": "macosx_x86_64", +} + +# The rattler-build release fetched when none is on PATH, per system, with the +# file name it goes under and the sha256 its release publishes. +RATTLER_BUILD_VERSION = "v0.77.0" +RATTLER_BUILD_RELEASES = { + "Linux": ( + "rattler-build-x86_64-unknown-linux-musl", + "rattler-build", + "5a5e8150c703b21e88a01f9130ea49a7a680198e9ac9414170c0a293fed7e5af", + ), + "Windows": ( + "rattler-build-x86_64-pc-windows-msvc.exe", + "rattler-build.exe", + "e144334f782cee9aa23e26eb8d095c143e1696fbbec7ba7534a156663d4abb2c", + ), +} + + +class BuildError(Exception): + """Raised when the build cannot start or produced nothing.""" + + +def ensure_rattler_build(tools: Path) -> None: + """ + Puts rattler-build on PATH, downloading the pinned release when none is + there. + """ + if shutil.which("rattler-build"): + return + release = RATTLER_BUILD_RELEASES.get(platform.system()) + if release is None or platform.machine().lower() not in ("x86_64", "amd64"): + raise BuildError( + "no rattler-build on PATH, and only the Linux and Windows x86_64 " + "releases are pinned here" + ) + asset, name, sha256 = release + print(f"build: fetching rattler-build {RATTLER_BUILD_VERSION}", flush=True) + url = ( + "https://github.com/prefix-dev/rattler-build/releases/download/" + f"{RATTLER_BUILD_VERSION}/{asset}" + ) + with urllib.request.urlopen(url, timeout=300) as reply: + binary = reply.read() + if hashlib.sha256(binary).hexdigest() != sha256: + raise BuildError( + f"rattler-build {RATTLER_BUILD_VERSION} does not match its pinned " + "sha256" + ) + tools.mkdir(parents=True, exist_ok=True) + target = tools / name + target.write_bytes(binary) + target.chmod(0o755) + os.environ["PATH"] = f'{tools}{os.pathsep}{os.environ.get("PATH", "")}' + + +def cibuildwheel_identifier(python: matrix.Python) -> str: + """The one build cibuildwheel runs, such as cp314t-manylinux_x86_64.""" + tag = WHEEL_PLATFORMS.get(platform.system()) + if tag is None: + raise BuildError( + f"wheels build on Linux, Windows or macOS, not {platform.system()}" + ) + minor, threaded = matrix.version_key(python.version) + return f'cp3{minor}{"t" if threaded else ""}-{tag}' + + +def wheel_requires() -> list[str]: + """[tool.gh-ci] wheel-requires from pyproject.toml.""" + if matrix.TOMLLIB is None: + raise BuildError("reading pyproject.toml needs Python 3.11 or newer") + text = (ROOT / "pyproject.toml").read_text(encoding="utf-8") + tool = matrix.TOMLLIB.loads(text).get("tool", {}).get("gh-ci", {}) + requires = tool.get("wheel-requires", []) + if not isinstance(requires, list) or not all( + isinstance(r, str) for r in requires + ): + raise BuildError("[tool.gh-ci] wheel-requires must be a list of specs") + return requires + + +def record_hash(data: bytes) -> str: + """A file's hash as a wheel's RECORD spells it.""" + digest = base64.urlsafe_b64encode(hashlib.sha256(data).digest()) + return "sha256=" + digest.rstrip(b"=").decode("ascii") + + +def add_requirements(wheel: Path, requirements: list[str]) -> None: + """Adds Requires-Dist lines to a built wheel, and updates its RECORD.""" + with zipfile.ZipFile(wheel) as source: + members = [(info, source.read(info)) for info in source.infolist()] + names = [info.filename for info, _ in members] + metadata = next(n for n in names if n.endswith(".dist-info/METADATA")) + record = metadata.removesuffix("METADATA") + "RECORD" + for index, (info, data) in enumerate(members): + if info.filename == metadata: + metadata_text = data.decode("utf-8").replace("\r\n", "\n") + head, blank, body = metadata_text.partition("\n\n") + head = head.rstrip("\n") + "".join( + f"\nRequires-Dist: {r}" for r in requirements + ) + text = head + ("\n\n" + body if blank else "\n") + new_metadata = text.encode("utf-8") + members[index] = (info, new_metadata) + for index, (info, data) in enumerate(members): + if info.filename == record: + lines = [ + ( + f"{metadata},{record_hash(new_metadata)}," + f"{len(new_metadata)}" + if line.split(",")[0] == metadata + else line + ) + for line in data.decode("utf-8").splitlines() + ] + members[index] = (info, ("\n".join(lines) + "\n").encode()) + rewritten = wheel.with_suffix(".tmp") + with zipfile.ZipFile(rewritten, "w", zipfile.ZIP_DEFLATED) as target: + for info, data in members: + target.writestr(info, data) + rewritten.replace(wheel) + + +def commands( + system: str, + selector: Path, + python: matrix.Python, + channels: list[str], + scratch: Path, +) -> list[list[str]]: + """ + The commands that build one Python version's packages into scratch. + + Each build system gets the version back exactly as its own file declares it, + and keeps every other variant setting from that file. + """ + channel_args = [arg for channel in channels for arg in ("-c", channel)] + if system == "rattler-build": + return [ + [ + "rattler-build", + "build", + "-r", + str(selector), + "--variant", + f"python={python.spec}", + "--output-dir", + str(scratch / "conda"), + *channel_args, + ] + ] + if system == "conda-build": + # conda runs as a module of this interpreter, so Windows needs no + # conda.bat on PATH. + return [ + [ + sys.executable, + "-m", + "conda", + "build", + str(selector.parent), + "--python", + python.spec, + "--no-anaconda-upload", + "--output-folder", + str(scratch / "conda"), + "--override-channels", + *channel_args, + ] + ] + return [ + [ + sys.executable, + "-m", + "cibuildwheel", + "--only", + cibuildwheel_identifier(python), + "--output-dir", + str(scratch / "wheel"), + str(ROOT), + ] + ] + + +def collect(scratch: Path, output: Path) -> list[Path]: + """Moves built packages into output, flat, and lists them.""" + output.mkdir(parents=True, exist_ok=True) + for package in sorted(scratch.rglob("*")): + if ( + package.is_file() + and package.name.endswith(PACKAGE_SUFFIXES) + and "broken" not in package.parts + ): + shutil.move(str(package), output / package.name) + built = sorted( + path + for path in output.iterdir() + if path.name.endswith(PACKAGE_SUFFIXES) + ) + if not built: + raise BuildError(f"the build left no package in {output}") + return built + + +def main(argv: list[str] | None = None) -> int: + """Builds one Python version.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--output", required=True, type=Path, help="where the packages go" + ) + parser.add_argument( + "--system", + default="", + choices=("", *matrix.SYSTEMS), + help="build system to use instead of the detected one", + ) + parser.add_argument( + "--channel", + action="append", + default=None, + help="conda channel to build against, repeatable; default conda-forge", + ) + default_tools = ( + Path(os.environ.get("RUNNER_TEMP", tempfile.gettempdir())) + / "gh-ci-tools" + ) + parser.add_argument( + "--tools", + type=Path, + default=default_tools, + help="where a downloaded rattler-build goes", + ) + args = parser.parse_args(argv) + + system, selector = matrix.detect(ROOT, args.system) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise BuildError( + f"Python {args.python} is not declared for {system}; " + f'declared: {", ".join(declared)}' + ) + python = declared[args.python] + print( + f"build: {system} from {selector.relative_to(ROOT)} " + f"for Python {python.version} ({python.spec})", + flush=True, + ) + if system == "rattler-build": + ensure_rattler_build(args.tools) + output = args.output.resolve() + with tempfile.TemporaryDirectory() as scratch: + for command in commands( + system, + selector, + python, + args.channel or ["conda-forge"], + Path(scratch), + ): + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True) + built = collect(Path(scratch), output) + if system == "wheel" and (requires := wheel_requires()): + for wheel in built: + add_requirements(wheel, requires) + print("build: added Requires-Dist " + ", ".join(requires)) + print("build: " + ", ".join(path.name for path in built)) + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + BuildError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"build: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/config.yml b/ci/config.yml new file mode 100644 index 0000000000..8211f4d755 --- /dev/null +++ b/ci/config.yml @@ -0,0 +1,15 @@ +# Intel Python CI (gh-ci) settings for dpctl, see ci/README.md +version: 1 +workflow: intel-ci.yml +artifacts: + conda: "dpctl {platform} Python {python}" + wheel: "dpctl {platform} Wheels Python {python}" +test: + # The compilers build the extensions two tests compile; intel-opencl-rt is the OpenCL CPU device. + requires: [pytest, cython, setuptools, intel-opencl-rt, c-compiler, cxx-compiler] + # A wheel's environment has the system's compilers only, and the SYCL runtime comes with the wheel. + wheel_requires: [pytest, cython, setuptools, intel-opencl-rt] +benchmark: + requires: [numpy, intel-opencl-rt] + # conda-forge lacks asv for some Python the recipe declares; the abi3 wheel on PyPI fits all. + pip_requires: [asv] diff --git a/ci/hooks.py b/ci/hooks.py new file mode 100644 index 0000000000..718b61b1a1 --- /dev/null +++ b/ci/hooks.py @@ -0,0 +1,440 @@ +""" +Runs ci/test.py against one built artifact, the way Intel Python CI does. + +For the GitHub Actions test job. + +Each Python version's artifact is installed into a fresh conda environment from +a local channel plus conda-forge, pinned to its exact build, with the test +requirements ci/config.yml lists, and the test runs with that environment's +Python from an empty working directory, on Linux or Windows. A wheel's +environment has only Python from conda-forge: the wheel and the test +requirements install with pip, as in the wheel's users' environments. The +interpreter spec and the requirements are the ones Jenkins uses. It gets the +same variables, and nothing else from the runner, as it does in Jenkins, so a +test that passes here passes there. Jenkins runs ci/test.py again on its own +machines, GPUs included; the scans are Jenkins' own and never run here. + +Usage, from any Python 3.10 or newer; it runs itself again with the conda base +install's Python, from $CONDA, which setup-miniconda sets, or the conda on PATH: + python ci/hooks.py --python 3.14 --artifacts DIR --work DIR [--kind wheel] + +Needs conda, conda-package-handling and ruamel.yaml, which every conda base has, +and installs nothing into it. +""" + +import argparse +import hashlib +import json +import os +import re +import shlex +import shutil +import subprocess +import sys +import tempfile +from pathlib import Path + +import matrix + +# Dependencies of conda itself; YAML is read with PyYAML, else ruamel.yaml +try: + from conda_package_handling import api as CPH_API +except ModuleNotFoundError: + CPH_API = None +try: + import yaml as PYYAML +except ModuleNotFoundError: + PYYAML = None +try: + from ruamel.yaml import YAML as RUAMEL_YAML +except ModuleNotFoundError: + RUAMEL_YAML = None + +ROOT = matrix.ROOT +IS_WINDOWS = os.name == "nt" +if IS_WINDOWS: + PLATFORM, HOST_SUBDIR = "windows", "win-64" +elif sys.platform == "darwin": + PLATFORM, HOST_SUBDIR = "macos", "osx-64" +else: + PLATFORM, HOST_SUBDIR = "linux", "linux-64" +CONDA_SUFFIXES = (".conda", ".tar.bz2") +# All the test sees of the runner's environment, so no tokens reach it +KEEP = ( + "PATH", + "HOME", + "USER", + "LANG", + "TERM", + "TMPDIR", + "TZ", + "http_proxy", + "https_proxy", + "no_proxy", + "HTTP_PROXY", + "HTTPS_PROXY", + "NO_PROXY", + "SSL_CERT_FILE", + "REQUESTS_CA_BUNDLE", + "CONDA_PKGS_DIRS", + "SYSTEMROOT", + "SYSTEMDRIVE", + "WINDIR", + "COMSPEC", + "PATHEXT", + "TEMP", + "TMP", + "USERPROFILE", + "APPDATA", + "LOCALAPPDATA", + "PROGRAMDATA", + "NUMBER_OF_PROCESSORS", + "PROCESSOR_ARCHITECTURE", +) + + +class HookError(Exception): + """Raised when the artifact cannot be installed or ci/test.py is missing.""" + + +def run(command: list[str], **options: object) -> None: + """Echoes and runs a command, never through a shell.""" + print(f"+ {shlex.join(command)}", flush=True) + subprocess.run(command, check=True, **options) + + +def conda(*args: str) -> None: + """Runs conda from the base environment running this script.""" + run([sys.executable, "-m", "conda", *args]) + + +def empty_repodata(subdir: str) -> dict: + """The repodata.json of a channel subdir with no packages.""" + return { + "info": {"subdir": subdir}, + "packages": {}, + "packages.conda": {}, + "repodata_version": 1, + } + + +def package_record(package: Path) -> tuple[str, dict]: + """A conda package's subdir and repodata record, from its index.json.""" + if CPH_API is None: + raise HookError( + "run this with a conda base Python, " + "which has conda-package-handling" + ) + with tempfile.TemporaryDirectory() as scratch: + CPH_API.extract(str(package), scratch, components="info") + record = json.loads( + (Path(scratch) / "info" / "index.json").read_text(encoding="utf-8") + ) + data = package.read_bytes() + record.update( + { + "md5": hashlib.md5(data, usedforsecurity=False).hexdigest(), + "sha256": hashlib.sha256(data).hexdigest(), + "size": len(data), + } + ) + subdir = str(record.get("subdir") or "noarch") + if not re.fullmatch(r"[a-z0-9]+(-[a-z0-9_]+)?", subdir): + raise HookError( + f"{package.name} declares an unusable subdir {subdir!r}" + ) + return subdir, record + + +def stage_channel(packages: list[Path], channel: Path) -> None: + """ + Lays the packages out as a local channel, by the subdir each one declares, + and writes its repodata.json the way Jenkins does, without conda-index. + """ + repodata = { + subdir: empty_repodata(subdir) for subdir in ("noarch", HOST_SUBDIR) + } + for package in packages: + subdir, record = package_record(package) + (channel / subdir).mkdir(parents=True, exist_ok=True) + shutil.copy2(package, channel / subdir / package.name) + section = ( + "packages.conda" if package.name.endswith(".conda") else "packages" + ) + repodata.setdefault(subdir, empty_repodata(subdir))[section][ + package.name + ] = record + for subdir, data in repodata.items(): + (channel / subdir).mkdir(parents=True, exist_ok=True) + (channel / subdir / "repodata.json").write_text( + json.dumps(data, indent=2) + "\n", encoding="utf-8" + ) + + +def test_python(version: str) -> str: + """The interpreter spec for the test environment, as Jenkins spells it.""" + minor, threaded = matrix.version_key(version) + # The .* matters: conda reads a bare 3.11 as exactly 3.11.0 + if minor < matrix.FREE_THREADED_MINOR: + return f"3.{minor}.*" + # conda-forge marks the ABI from 3.13, so 3.14 can't resolve to 3.14t + return f'3.{minor}.* *_cp3{minor}{"t" if threaded else ""}' + + +def test_requirements() -> tuple[list[str], list[str], list[str] | None]: + """ + The conda-forge and pip packages ci/config.yml installs next to the + artifact, and a wheel's PyPI requirements, None when it gives none. + """ + text = (ROOT / "ci" / "config.yml").read_text(encoding="utf-8") + if PYYAML is not None: + config = PYYAML.safe_load(text) + elif RUAMEL_YAML is not None: + config = RUAMEL_YAML(typ="safe", pure=True).load(text) + else: + raise HookError("reading ci/config.yml needs PyYAML or ruamel.yaml") + test = (config or {}).get("test") or {} + lists = [] + for key in ("requires", "pip_requires", "wheel_requires"): + values = test.get(key) + if values is not None and ( + not isinstance(values, list) + or not all(isinstance(v, str) for v in values) + ): + raise HookError( + f"ci/config.yml: test.{key} must be a list of package specs" + ) + lists.append(values) + return lists[0] or [], lists[1] or [], lists[2] + + +def exact_spec(package: Path) -> str: + """name=version=build from a conda package file name.""" + stem = package.name + for suffix in CONDA_SUFFIXES: + stem = stem.removesuffix(suffix) + name, version, build = stem.rsplit("-", 2) + return f"{name}={version}={build}" + + +def env_python(env_dir: Path) -> Path: + """The interpreter of a conda environment.""" + return env_dir / "python.exe" if IS_WINDOWS else env_dir / "bin" / "python" + + +def env_path(env_dir: Path) -> list[Path]: + """ + The directories conda's activation would put first on PATH for an + environment. + """ + if not IS_WINDOWS: + return [env_dir / "bin"] + library = env_dir / "Library" + return [ + env_dir, + library / "mingw-w64" / "bin", + library / "usr" / "bin", + library / "bin", + env_dir / "Scripts", + env_dir / "bin", + ] + + +def test_env( + env_dir: Path, artifacts: Path, results: Path, args: argparse.Namespace +) -> dict: + """The variables Jenkins gives ci/test.py, and no runner tokens.""" + env = {key: value for key, value in os.environ.items() if key in KEEP} + env["PATH"] = os.pathsep.join( + [*map(str, env_path(env_dir)), env.get("PATH", "")] + ) + env.update( + { + "CONDA_PREFIX": str(env_dir), + "GH_CI_ARTIFACTS": str(artifacts), + "GH_CI_SOURCE": str(ROOT), + "GH_CI_RESULTS": str(results), + "GH_CI_KIND": args.kind, + "GH_CI_PYTHON": args.python, + "GH_CI_PLATFORM": PLATFORM, + "GH_CI_EVENT": args.event, + "GH_CI_COMMIT": args.commit, + } + ) + return env + + +def summarize(lines: list[str]) -> None: + """Adds the results to the job summary page, in GitHub Actions.""" + summary = os.environ.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as page: + page.write("\n".join(lines) + "\n") + + +def install(args: argparse.Namespace) -> Path: + """ + Creates the test environment with the artifact, pinned to its exact build; + returns it. + """ + system, selector = matrix.detect( + ROOT, "wheel" if args.kind == "wheel" else "" + ) + declared = { + python.version: python for python in matrix.declared(system, selector) + } + if args.python not in declared: + raise HookError( + f"Python {args.python} is not declared; " + f'declared: {", ".join(declared)}' + ) + suffixes = (".whl",) if args.kind == "wheel" else CONDA_SUFFIXES + packages = sorted( + path + for path in args.artifacts.rglob("*") + if path.name.endswith(suffixes) + ) + if not packages: + raise HookError(f"{args.artifacts} holds no {args.kind} package") + requires, pip_requires, wheel_requires = test_requirements() + env_dir = args.work / "env" + pip = [str(env_python(env_dir)), "-m", "pip", "install", "--no-input"] + python = f"python {test_python(args.python)}" + if args.kind == "wheel": + conda_create(env_dir, python, "pip") + wheel_specs = requires if wheel_requires is None else wheel_requires + run([*pip, *map(str, packages), *wheel_specs]) + else: + channel = args.work / "channel" + stage_channel(packages, channel) + conda_create( + env_dir, + python, + *requires, + *map(exact_spec, packages), + channels=(channel.as_uri(),), + ) + if pip_requires: + run([*pip, *pip_requires]) + return env_dir + + +def conda_create( + env_dir: Path, *specs: str, channels: tuple[str, ...] = () +) -> None: + """Creates a conda environment from conda-forge and channels.""" + conda( + "create", + "-y", + "-q", + "-p", + str(env_dir), + "--override-channels", + *(arg for channel in channels for arg in ("-c", channel)), + "-c", + "conda-forge", + *specs, + ) + + +def run_test(env_dir: Path, args: argparse.Namespace) -> int: + """Runs ci/test.py with the environment's Python; returns its status.""" + script = ROOT / "ci" / "test.py" + if not script.is_file(): + raise HookError("ci/test.py is required and missing") + results, scratch = ( + args.work / "results" / "test", + args.work / "scratch" / "test", + ) + for directory in (results, scratch): + directory.mkdir(parents=True, exist_ok=True) + command = [str(env_python(env_dir)), str(script)] + print(f"+ {shlex.join(command)}", flush=True) + status = subprocess.run( + command, + cwd=scratch, + check=False, + env=test_env(env_dir, args.artifacts, results, args), + ).returncode + result = "passed" if status == 0 else f"failed, exit {status}" + summarize( + [ + f"### {matrix.PLATFORMS[PLATFORM]} Python {args.python}" + f" {args.kind}", + "", + "| Test | Result |", + "|---|---|", + f"| ci/test.py | {result} |", + ] + ) + return status + + +def conda_base_python() -> Path: + """The conda base install's Python: from $CONDA, else the conda on PATH.""" + root = os.environ.get("CONDA", "") + if not root: + found = shutil.which("conda") + if not found: + raise HookError( + "no conda base install: set CONDA or put conda on PATH" + ) + # conda lives in bin, Scripts or condabin, one level below the install. + root = str(Path(found).resolve().parent.parent) + python = env_python(Path(root)) + if not python.is_file(): + raise HookError( + f"{root} is not a conda base install; it has no {python.name}" + ) + return python + + +def main(argv: list[str] | None = None) -> int: + """Installs the artifact, then runs ci/test.py against it.""" + if CPH_API is None: + # Re-run with base's Python, which has conda-package-handling + command = [ + str(conda_base_python()), + str(Path(__file__).resolve()), + *(sys.argv[1:] if argv is None else argv), + ] + return subprocess.run(command, check=False).returncode + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--python", required=True, help="a declared version, such as 3.14t" + ) + parser.add_argument( + "--artifacts", required=True, type=Path, help="the downloaded artifact" + ) + parser.add_argument( + "--work", required=True, type=Path, help="an empty scratch directory" + ) + parser.add_argument( + "--event", default="push", choices=("pr", "tag", "push") + ) + parser.add_argument("--commit", default="") + parser.add_argument( + "--kind", + default="conda", + choices=("conda", "wheel"), + help="what the artifact holds", + ) + args = parser.parse_args(argv) + args.artifacts, args.work = args.artifacts.resolve(), args.work.resolve() + if args.work.exists() and ( + not args.work.is_dir() or any(args.work.iterdir()) + ): + raise HookError(f"--work {args.work} must be an empty directory") + return run_test(install(args), args) + + +if __name__ == "__main__": + try: + sys.exit(main()) + except ( + HookError, + matrix.DeclarationError, + subprocess.CalledProcessError, + ) as error: + print(f"hooks: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/install_icx.py b/ci/install_icx.py new file mode 100644 index 0000000000..cbdfaa3cc6 --- /dev/null +++ b/ci/install_icx.py @@ -0,0 +1,78 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Installs Intel's icx for dpctl's Windows wheel builds. + +cibuildwheel runs this before the builds, as pyproject.toml says. It fetches +micromamba, checked against its published sha256, and creates PREFIX with +conda-forge's dpcpp_win-64, the compiler the conda recipe builds with. + +Usage: + python ci/install_icx.py PREFIX +""" + +import hashlib +import subprocess +import sys +import tempfile +import urllib.request +from pathlib import Path + +MICROMAMBA = ( + "https://github.com/mamba-org/micromamba-releases/releases/download/" + "2.10.0-0/micromamba-win-64" +) +MICROMAMBA_SHA256 = ( + "f8b4284bd4eb1134122eaf93b3ea6c7be567e596de5829e1a6f478e34e41974a" +) +COMPILER = "dpcpp_win-64=2026.2.0" + + +def main() -> int: + """Creates PREFIX with the compiler.""" + prefix = Path(sys.argv[1]) + with tempfile.TemporaryDirectory() as scratch: + micromamba = Path(scratch) / "micromamba.exe" + with urllib.request.urlopen(MICROMAMBA, timeout=300) as response: + micromamba.write_bytes(response.read()) + digest = hashlib.sha256(micromamba.read_bytes()).hexdigest() + if digest != MICROMAMBA_SHA256: + print( + f"install_icx: micromamba has sha256 {digest}, " + f"not {MICROMAMBA_SHA256}", + file=sys.stderr, + ) + return 1 + command = [ + str(micromamba), + "create", + "-y", + "-r", + str(Path(scratch) / "root"), + "-p", + str(prefix), + "--override-channels", + "-c", + "conda-forge", + COMPILER, + ] + print("+ " + " ".join(command), flush=True) + return subprocess.run(command, check=False).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/ci/matrix.py b/ci/matrix.py new file mode 100644 index 0000000000..e6a4b49abd --- /dev/null +++ b/ci/matrix.py @@ -0,0 +1,653 @@ +""" +The platforms and Python versions this repository builds. + +Both are read from the repository's own build files. + +Every build system already has a standard place for both, so each is written +once, where the build reads it, and the GitHub Actions matrix and Intel Python +CI (gh-ci) both follow it: + + rattler-build, selected by recipe.yaml + Python versions python: in variants.yaml next to the recipe + platforms build: skip: in the recipe + conda-build, selected by meta.yaml + Python versions python: in conda_build_config.yaml next to the recipe + platforms skip: true # [selector] lines in the recipe + wheel, selected by pyproject.toml + Python versions the Programming Language :: Python :: 3.N classifiers; + any Free Threading classifier adds 3.Nt for each from 3.14 + platforms the Operating System :: classifiers + +A recipe builds on every platform gh-ci builds, Linux, Windows and macOS on +Intel, unless a skip that depends on the platform alone rules one out, such as +`skip: [win]` or `skip: true # [osx]`; a skip that also depends on something +else, such as `win and py < 311` or `osx and arm64`, skips some builds, not the +platform. A pyproject.toml without Operating System classifiers, or with OS +Independent, builds on all three. Of the declared platforms, the matrix holds +those GitHub Actions has runners for, BUILT_PLATFORMS. macOS builds and tests in +GitHub Actions only: Jenkins tests, scans and releases Linux and Windows. + +meson-python, scikit-build-core and every other PEP 517 backend declare these in +pyproject.toml too, so the wheel row covers them. Another convention, such as +pixi.toml, is one more reader in READERS and PLATFORM_READERS and, if it is a +new build system, one more entry in SYSTEMS. + +Usage: + python ci/matrix.py [--system SYSTEM] [--event EVENT] [--github-output NAME] + +Prints the build matrix as JSON for strategy.matrix: {"include": [{"platform": +"linux", "name": "Linux", "python": "3.14"}, ...]}, every platform with every +Python version. For a pull_request event only the oldest and the newest regular +versions are listed, which is all gh-ci tests on a pull request. --github-output +also writes NAME= to $GITHUB_OUTPUT, for later jobs. Standard library +only; reading pyproject.toml needs Python 3.11 or newer for tomllib. +""" + +import argparse +import ast +import json +import os +import re +import sys +from collections.abc import Callable +from dataclasses import dataclass +from pathlib import Path + +# Only the wheel reader needs it, and only it needs Python 3.11 or newer. +try: + import tomllib as TOMLLIB +except ModuleNotFoundError: + TOMLLIB = None + +ROOT = Path(__file__).resolve().parent.parent + +# Build system -> files that select it, in the order they are looked for. Where +# a repository still has conda-recipe-cf/ next to conda-recipe/, the conda-forge +# one comes first: gh-ci tests from conda-forge alone. +SYSTEMS = { + "rattler-build": ("recipe.yaml", "recipe/recipe.yaml"), + "conda-build": ( + "conda-recipe-cf/meta.yaml", + "meta.yaml", + "conda-recipe/meta.yaml", + "recipe/meta.yaml", + ), + "wheel": ("pyproject.toml",), +} + +# The platforms gh-ci builds and tests, and how artifact names spell them. +PLATFORMS = {"linux": "Linux", "windows": "Windows", "macos": "macOS"} +# The platforms the workflow can build on: GitHub-hosted runners have all +# three. A declared platform missing here is reported and left out. +BUILT_PLATFORMS = ("linux", "windows", "macos") +# What a platform-only selector means on each platform, for both conda-build and +# rattler-build. +SELECTORS = { + "linux": { + "linux": True, + "linux64": True, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + "windows": { + "linux": False, + "linux64": False, + "unix": False, + "win": True, + "win64": True, + "win32": False, + "osx": False, + "osx64": False, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, + # Intel Macs only, as MKL has no Apple silicon build. + "macos": { + "linux": False, + "linux64": False, + "unix": True, + "win": False, + "win64": False, + "win32": False, + "osx": True, + "osx64": True, + "x86_64": True, + "x86": False, + "aarch64": False, + "arm64": False, + "ppc64le": False, + "s390x": False, + }, +} +CONDA_SKIP_RE = re.compile(r"\s*skip\s*:\s*(?:true|True|yes)\s*#\s*\[(.+)\]\s*") +OS_CLASSIFIERS = { + "Operating System :: OS Independent": tuple(PLATFORMS), + "Operating System :: POSIX": ("linux",), + "Operating System :: Unix": ("linux",), + "Operating System :: POSIX :: Linux": ("linux",), + "Operating System :: Microsoft :: Windows": ("windows",), + "Operating System :: MacOS": ("macos",), +} + +PULL_REQUEST_EVENTS = ("pull_request",) +OLDEST_MINOR = 10 +FREE_THREADED_MINOR = 13 +# A Free Threading classifier adds 3.Nt from 3.14, the first supported one, +# which is also the oldest cibuildwheel builds. +CLASSIFIED_FREE_THREADED_MINOR = 14 + +# A conda python variant: 3.12, 3.12.*, 3.12.* *_cpython, 3.13.* *_cp313 or +# 3.14.* *_cp314t. +CONDA_SPEC_RE = re.compile( + r"3\.(\d+)(?:\.\*)?(?:\s+\*_(?:cpython|cp3(\d+)(t?)))?" +) +CLASSIFIER_RE = re.compile(r"Programming Language :: Python :: 3\.(\d+)") +FREE_THREADING_CLASSIFIER = "Programming Language :: Python :: Free Threading" +LOWER_BOUND_RE = re.compile(r">=\s*3\.(\d+)") + + +class DeclarationError(Exception): + """ + Raised when the build files declare no Python versions, or ones this cannot + read. + """ + + +@dataclass(frozen=True) +class Python: + """One declared Python version.""" + + version: str # as artifact names and gh-ci spell it: 3.14 or 3.14t + spec: str # as the build system spells it, passed back to it unchanged + + +def version_key(version: str) -> tuple[int, bool]: + """(minor, free-threaded); also the sort order.""" + return int(version.rstrip("t").split(".")[1]), version.endswith("t") + + +def checked(minor: int, threaded: bool, where: str) -> str: + """The version string, once it is one gh-ci can test.""" + if minor < OLDEST_MINOR or (threaded and minor < FREE_THREADED_MINOR): + raise DeclarationError( + f'{where}: 3.{minor}{"t" if threaded else ""} is not supported; ' + f"the oldest is 3.{OLDEST_MINOR}, free-threaded from " + f"3.{FREE_THREADED_MINOR}" + ) + return f'3.{minor}{"t" if threaded else ""}' + + +def _strip_comment(line: str, where: str) -> str: + """ + The line without a trailing comment; conda-build selectors are refused, not + guessed at. + """ + in_quote = "" + for index, char in enumerate(line): + if char in "\"'" and in_quote in ("", char): + in_quote = "" if in_quote else char + elif ( + char == "#" + and not in_quote + and (index == 0 or line[index - 1].isspace()) + ): + if line[index:].lstrip("# ").startswith("["): + raise DeclarationError( + f"{where}: selectors in the python list are not supported" + ) + return line[:index].rstrip() + return line.rstrip() + + +def _unquote(value: str, where: str) -> str: + """A plain or quoted YAML scalar.""" + value = value.strip() + if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'": + return value[1:-1] + if not value or value[0] in "{[&*!|>" or ":" in value: + raise DeclarationError( + f"{where}: write each python entry as a plain value, not {value!r}" + ) + return value + + +def yaml_python_list(text: str, where: str) -> list[str]: + """ + The top-level python: list of a variants file, without a YAML library. + + Accepts a block list or a one-line [a, b] list of plain or quoted values. + Anything richer, such as if/then selectors or templating, is refused so the + matrix is never guessed. + """ + lines = text.splitlines() + for number, line in enumerate(lines): + if not re.match(r"python\s*:", line): + continue + rest = _strip_comment(line.split(":", 1)[1], where).strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + raise DeclarationError(f"{where}: python must be a list") + values = [] + for item in lines[number + 1 :]: + # Checked before the comment, as the next key may have a selector + if item[:1] not in ("", " ", "\t", "-", "#"): + break # the next top-level key + entry = _strip_comment(item, where).lstrip() + if not entry: + continue + if not entry.startswith("- "): + raise DeclarationError( + f"{where}: python must be a plain list of versions" + ) + values.append(_unquote(entry[2:], where)) + return values + raise DeclarationError(f"{where} has no top-level python: list") + + +def from_conda_variants(path: Path) -> list[Python]: + """The python: list of a variants.yaml or a conda_build_config.yaml.""" + where = ( + path.relative_to(ROOT).as_posix() + if path.is_relative_to(ROOT) + else str(path) + ) + try: + text = path.read_text(encoding="utf-8") + except FileNotFoundError as ex: + raise DeclarationError( + f"{where} is missing; list the Python versions to build there" + ) from ex + pythons = [] + for spec in yaml_python_list(text, where): + match = CONDA_SPEC_RE.fullmatch(spec) + if not match or (match[2] and match[2] != match[1]): + raise DeclarationError( + f"{where}: {spec!r} is not a CPython variant such as " + '"3.13.* *_cp313" or "3.14.* *_cp314t"' + ) + pythons.append( + Python(checked(int(match[1]), bool(match[3]), where), spec) + ) + return pythons + + +def from_pyproject(path: Path) -> list[Python]: + """ + The 3.N classifiers in pyproject.toml, plus 3.Nt from 3.14 if free-threading + is declared. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + project = TOMLLIB.loads(path.read_text(encoding="utf-8")).get("project", {}) + classifiers = project.get("classifiers", []) + minors = sorted( + { + int(match[1]) + for classifier in classifiers + if (match := CLASSIFIER_RE.fullmatch(classifier)) + } + ) + if not minors: + raise DeclarationError( + "pyproject.toml lists no Programming Language :: Python :: 3.N " + "classifiers" + ) + bound = LOWER_BOUND_RE.search(project.get("requires-python", "")) + if bound and minors[0] < int(bound[1]): + raise DeclarationError( + f"pyproject.toml classifies 3.{minors[0]}, below requires-python " + f'{project["requires-python"]}' + ) + threaded = any( + classifier.startswith(FREE_THREADING_CLASSIFIER) + for classifier in classifiers + ) + versions = [checked(minor, False, "pyproject.toml") for minor in minors] + versions += [ + f"3.{minor}t" + for minor in minors + if threaded and minor >= CLASSIFIED_FREE_THREADED_MINOR + ] + return [Python(version, version) for version in versions] + + +def skips_platform(expression: str, platform: str) -> bool | None: + """ + Whether a skip expression rules out a whole platform: True or False when it + depends on the platform alone, None when it also depends on anything else, + such as the Python version. + """ + try: + tree = ast.parse( + expression.strip().removeprefix("${{").removesuffix("}}").strip(), + mode="eval", + ).body + except SyntaxError: + return None + + def value(node: ast.AST) -> bool: + if isinstance(node, ast.BoolOp): + values = [value(item) for item in node.values] + return all(values) if isinstance(node.op, ast.And) else any(values) + if isinstance(node, ast.UnaryOp) and isinstance(node.op, ast.Not): + return not value(node.operand) + if isinstance(node, ast.Name) and node.id in SELECTORS[platform]: + return SELECTORS[platform][node.id] + if isinstance(node, ast.Constant) and isinstance(node.value, bool): + return node.value + raise ValueError(ast.dump(node)) + + try: + return value(tree) + except ValueError: + return None + + +def platforms_without(skips: list[str], where: str) -> list[str]: + """Every platform gh-ci builds, but those a platform-only skip rules out.""" + kept = [ + platform + for platform in PLATFORMS + if not any(skips_platform(skip, platform) for skip in skips) + ] + if not kept: + raise DeclarationError( + f"{where} skips every platform gh-ci builds: " + + ", ".join(PLATFORMS.values()) + ) + return kept + + +def yaml_build_skips(text: str, where: str) -> list[str]: + """ + The skip: entries directly under a rattler-build recipe's top-level build:, + as a list of plain or quoted values, a one-line [a, b] list, or one value; + none when there is no skip. + """ + lines = text.splitlines() + starts = [ + number + for number, line in enumerate(lines) + if re.match(r"build\s*:\s*(#.*)?$", line) + ] + if not starts: + return [] + child = "" + for number in range(starts[0] + 1, len(lines)): + line = ( + _strip_comment(lines[number], where) + if "#" in lines[number] + else lines[number] + ) + if not line.strip(): + continue + if not line[0].isspace(): + return [] # the next top-level key + indent = line[: len(line) - len(line.lstrip())] + child = child or indent + match = ( + re.match(r"skip\s*:(.*)", line.lstrip()) + if indent == child + else None + ) + if not match: + continue + rest = match[1].strip() + if rest.startswith("[") and rest.endswith("]"): + return [ + _unquote(item, where) + for item in rest[1:-1].split(",") + if item.strip() + ] + if rest: + return [_unquote(rest, where)] + values = [] + for item in lines[number + 1 :]: + entry = item.strip() + if not entry or entry.startswith("#"): + continue + if not entry.startswith("- "): + break + values.append(_unquote(entry[2:].split(" #")[0], where)) + return values + return [] + + +def from_rattler_recipe(recipe: Path) -> list[str]: + """The platforms a rattler-build recipe builds on, from its build: skip:.""" + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + return platforms_without( + yaml_build_skips(recipe.read_text(encoding="utf-8"), where), where + ) + + +def from_conda_recipe(recipe: Path) -> list[str]: + """ + The platforms a conda-build meta.yaml builds on, from its skip: true # + [selector] lines. + """ + where = ( + recipe.relative_to(ROOT).as_posix() + if recipe.is_relative_to(ROOT) + else str(recipe) + ) + skips = [ + match[1] + for line in recipe.read_text(encoding="utf-8").splitlines() + if (match := CONDA_SKIP_RE.fullmatch(line)) + ] + return platforms_without(skips, where) + + +def from_os_classifiers(path: Path) -> list[str]: + """ + The platforms pyproject.toml's Operating System classifiers name; all of + them when it names none. + """ + if TOMLLIB is None: + raise DeclarationError( + "reading pyproject.toml needs Python 3.11 or newer" + ) + classifiers = ( + TOMLLIB.loads(path.read_text(encoding="utf-8")) + .get("project", {}) + .get("classifiers", []) + ) + named = [ + classifier + for classifier in classifiers + if classifier.startswith("Operating System ::") + ] + if not named: + return list(PLATFORMS) + found = { + platform + for classifier in named + for prefix, platforms in OS_CLASSIFIERS.items() + if classifier == prefix or classifier.startswith(prefix + " ::") + for platform in platforms + } + if not found: + raise DeclarationError( + "pyproject.toml names no Operating System gh-ci builds: " + + ", ".join(PLATFORMS.values()) + ) + return [platform for platform in PLATFORMS if platform in found] + + +# Build system -> where it declares Python versions, given the file that +# selected it. +READERS: dict[str, Callable[[Path], list[Python]]] = { + "rattler-build": lambda recipe: from_conda_variants( + recipe.parent / "variants.yaml" + ), + "conda-build": lambda recipe: from_conda_variants( + recipe.parent / "conda_build_config.yaml" + ), + "wheel": from_pyproject, +} + +# Build system -> where it declares the platforms it builds on, given the file +# that selected it. +PLATFORM_READERS: dict[str, Callable[[Path], list[str]]] = { + "rattler-build": from_rattler_recipe, + "conda-build": from_conda_recipe, + "wheel": from_os_classifiers, +} + + +def detect(root: Path, system: str = "") -> tuple[str, Path]: + """The build system and the file that selects it.""" + for name, candidates in SYSTEMS.items(): + if system and name != system: + continue + for candidate in candidates: + if (root / candidate).is_file(): + return name, root / candidate + wanted = system or " or ".join(SYSTEMS) + raise DeclarationError( + f"no {wanted} input found; looked for " + + ", ".join(c for name, cs in SYSTEMS.items() for c in cs) + ) + + +def declared(system: str, selector: Path) -> list[Python]: + """The build system's Python versions, sorted, each listed once.""" + pythons = READERS[system](selector) + versions = [python.version for python in pythons] + if not pythons: + raise DeclarationError(f"{system} declares no Python versions") + if len(set(versions)) != len(versions): + raise DeclarationError( + f"{system} declares a Python version twice: {versions}" + ) + return sorted(pythons, key=lambda python: version_key(python.version)) + + +def declared_platforms(system: str, selector: Path) -> list[str]: + """The platforms the build system builds on, in PLATFORMS order.""" + return PLATFORM_READERS[system](selector) + + +def for_event(pythons: list[Python], event: str) -> list[Python]: + """All of them, or for a pull request the oldest and newest regular ones.""" + if event not in PULL_REQUEST_EVENTS: + return pythons + regular = [python for python in pythons if not python.version.endswith("t")] + regular = regular or pythons + return list(dict.fromkeys(regular[:1] + regular[-1:])) + + +def build_matrix(system: str, selector: Path, event: str) -> dict: + """ + Every declared platform Actions can build on, with every Python version the + event builds. + """ + pythons = for_event(declared(system, selector), event) + platforms = declared_platforms(system, selector) + waiting = [ + PLATFORMS[platform] + for platform in platforms + if platform not in BUILT_PLATFORMS + ] + if waiting: + print( + f'matrix: {", ".join(waiting)} declared but not built: ' + "no runners for it yet", + file=sys.stderr, + ) + built = [platform for platform in platforms if platform in BUILT_PLATFORMS] + if not built: + raise DeclarationError( + "the build files declare only platforms there are no runners " + "for yet: " + ", ".join(waiting) + ) + return { + "include": [ + { + "platform": platform, + "name": PLATFORMS[platform], + "python": python.version, + } + for platform in built + for python in pythons + ] + } + + +def main(argv: list[str] | None = None) -> int: + """Prints the matrix to build; writes it for GitHub Actions if asked.""" + parser = argparse.ArgumentParser(description=__doc__.splitlines()[1]) + parser.add_argument( + "--system", + default="", + choices=("", *SYSTEMS), + help="build system to read instead of the detected one", + ) + parser.add_argument( + "--event", default="", help="GitHub event name, such as pull_request" + ) + parser.add_argument( + "--github-output", + default="", + metavar="NAME", + help="also write NAME= to $GITHUB_OUTPUT", + ) + args = parser.parse_args(argv) + if args.github_output: + if not re.fullmatch(r"[A-Za-z_][A-Za-z0-9_-]*", args.github_output): + raise DeclarationError( + f"{args.github_output!r} is not an output name" + ) + if not os.environ.get("GITHUB_OUTPUT"): + raise DeclarationError( + "--github-output needs $GITHUB_OUTPUT, which Actions sets" + ) + system, selector = detect(ROOT, args.system) + matrix = build_matrix(system, selector, args.event) + cells = ", ".join( + f'{cell["name"]} {cell["python"]}' for cell in matrix["include"] + ) + print( + f"matrix: {system} from {selector.relative_to(ROOT)}: {cells}", + file=sys.stderr, + ) + text = json.dumps(matrix) + print(text) + if args.github_output: + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write(f"{args.github_output}={text}\n") + return 0 + + +if __name__ == "__main__": + try: + sys.exit(main()) + except DeclarationError as error: + print(f"matrix: {error}", file=sys.stderr) + sys.exit(1) diff --git a/ci/test.py b/ci/test.py new file mode 100644 index 0000000000..46a5131d59 --- /dev/null +++ b/ci/test.py @@ -0,0 +1,240 @@ +# Data Parallel Control (dpctl) +# +# Copyright 2026 Intel Corporation +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Intel Python CI (gh-ci) test for dpctl. + +GitHub Actions, through ci/hooks.py, and Intel's Jenkins both run this with one +built package installed in a clean environment, from an empty working +directory, so the tests import the installed dpctl rather than the source tree. +The test requirements come from ci/config.yml. Writes a JUnit report for +Jenkins. + +The environment is scrubbed, so this sets what dpctl needs itself. It prints +the SYCL devices and fails without any, so a run that skipped every device test +cannot pass. Jenkins sets GH_CI_DEVICE=gpu on its GPU agents, and then a GPU +device is required too. Without it the OpenCL CPU runtime from the test +environment is the device, as on GitHub's runners. +""" + +import json +import os +import pathlib +import subprocess +import sys +import sysconfig +import tempfile + +PACKAGE = "dpctl" + +CPU_ICD = {"win32": "intelocl64.dll", "linux": "libintelocl.so"} + +DEVICES = """ +import json +import dpctl +print(json.dumps([ + [d.backend.name, d.device_type.name, d.name] for d in dpctl.get_devices() +])) +""" + +PROBE_SETUP = """ +import setuptools + +setuptools.setup( + name="probe", + ext_modules=[ + setuptools.Extension("probe_c", ["probe_c.c"], language="c"), + setuptools.Extension("probe_cpp", ["probe_cpp.cpp"], language="c++"), + ], +) +""" + +PROBE_SOURCE = """ +#include +static struct PyModuleDef module = {PyModuleDef_HEAD_INIT, "NAME", 0, -1, 0}; +PyMODINIT_FUNC PyInit_NAME(void) { return PyModule_Create(&module); } +""" + +# These two tests build extensions with the Python environment's compiler. +NEED_COMPILER = ["test_c_headers", "test_cython_api"] + + +def running_python() -> str: + """This interpreter's version as gh-ci spells it, such as 3.14t.""" + version = f"{sys.version_info.major}.{sys.version_info.minor}" + threaded = sysconfig.get_config_var("Py_GIL_DISABLED") + return version + ("t" if threaded else "") + + +def has_compilers(env: dict) -> bool: + """Whether setuptools can build a C and a C++ extension.""" + with tempfile.TemporaryDirectory() as scratch: + directory = pathlib.Path(scratch) + (directory / "setup.py").write_text(PROBE_SETUP, encoding="utf-8") + for language, suffix in (("c", "c"), ("c++", "cpp")): + source = PROBE_SOURCE.replace("NAME", f"probe_{suffix}") + (directory / f"probe_{suffix}.{suffix}").write_text( + source, encoding="utf-8" + ) + built = subprocess.run( + [sys.executable, "setup.py", "build_ext", "--inplace"], + cwd=directory, + env=env, + capture_output=True, + text=True, + check=False, + ) + if built.returncode: + print( + f"no usable compiler:\n{built.stdout[-1500:]}{built.stderr[-1500:]}" + ) + return not built.returncode + + +def skipped_tests(env: dict) -> list: + """The tests this environment cannot run, said out loud.""" + skipped = [] + if os.environ.get("GH_CI_KIND") == "wheel": + print("deselecting test_syclinterface: a wheel keeps no symlinks") + skipped.append("test_syclinterface") + if not has_compilers(env): + print(f"deselecting {', '.join(NEED_COMPILER)}: they need a compiler") + skipped += NEED_COMPILER + return skipped + + +def cpu_runtime_icd() -> str | None: + """The OpenCL CPU runtime of the test environment, if it has one.""" + name = CPU_ICD.get(sys.platform, CPU_ICD["linux"]) + found = next(pathlib.Path(sys.prefix).rglob(name), None) + return str(found) if found else None + + +def merge_icd_vendors(env: dict, merged: pathlib.Path) -> None: + """Lets the environment's OpenCL loader see the system's GPU drivers too.""" + system = pathlib.Path("/etc/OpenCL/vendors") + own = pathlib.Path(sys.prefix, "etc", "OpenCL", "vendors") + if not (system.is_dir() and own.is_dir()): + return + for icd in [*own.glob("*.icd"), *system.glob("*.icd")]: + if not (merged / icd.name).exists(): + (merged / icd.name).symlink_to(icd) + env["OCL_ICD_VENDORS"] = str(merged) + + +def register_icd(icd: str) -> None: + """Registers the ICD in the registry, which Windows' loader reads.""" + import winreg + + try: + with winreg.CreateKeyEx( + winreg.HKEY_LOCAL_MACHINE, + r"SOFTWARE\Khronos\OpenCL\Vendors", + 0, + winreg.KEY_SET_VALUE, + ) as vendors: + winreg.SetValueEx(vendors, icd, 0, winreg.REG_DWORD, 0) + except OSError as error: + print(f"could not register {icd}: {error}") + + +def sycl_env(gpu: bool, merged: pathlib.Path) -> dict: + """The environment dpctl runs in.""" + env = dict(os.environ, SYCL_CACHE_PERSISTENT="1") + if sys.platform == "win32": + # setuptools finds Visual C++ from these, and gh-ci's scrubbed + # environment lacks them. + drive = env.get("SYSTEMDRIVE", "C:") + env.setdefault("PROGRAMFILES", drive + "\\Program Files") + env.setdefault("PROGRAMFILES(X86)", drive + "\\Program Files (x86)") + if gpu: + if sys.platform.startswith("linux"): + merge_icd_vendors(env, merged) + return env + # Linux and newer Windows loaders take the ICD from here. + icd = cpu_runtime_icd() + if icd: + env["OCL_ICD_FILENAMES"] = icd + if sys.platform == "win32": + register_icd(icd) + return env + + +def devices(env: dict) -> list: + """The [backend, type, name] of every SYCL device, in a fresh process.""" + listed = subprocess.run( + [sys.executable, "-c", DEVICES], + env=env, + capture_output=True, + text=True, + check=False, + ) + if listed.returncode: + print(listed.stderr) + return [] + return json.loads(listed.stdout) + + +def devices_ok(found: list, gpu: bool) -> bool: + """Prints the devices, and whether the cell has the ones it needs.""" + for backend, kind, name in found: + print(f"SYCL device: {backend}:{kind} {name}") + if not found: + print("no SYCL device, so the tests would skip everything") + return False + if gpu and not any(kind == "gpu" for _, kind, _ in found): + print("GH_CI_DEVICE=gpu, but there is no GPU device") + return False + return True + + +def main() -> int: + """Runs the installed package's test suite.""" + sys.stdout.reconfigure(line_buffering=True) + expected = os.environ["GH_CI_PYTHON"] + if running_python() != expected: + print(f"running Python {running_python()}, the cell is {expected}") + return 1 + gpu = os.environ.get("GH_CI_DEVICE") == "gpu" + with tempfile.TemporaryDirectory() as merged: + env = sycl_env(gpu, pathlib.Path(merged)) + subprocess.run([sys.executable, "-m", "dpctl", "-f"], env=env) + if not devices_ok(devices(env), gpu): + return 1 + # Every cell of a Jenkins build reports into the same build, so each + # names its own suite. + cell = f"{os.environ.get('GH_CI_PLATFORM', 'linux')}.py{expected}" + report = pathlib.Path(os.environ["GH_CI_RESULTS"]) / f"{PACKAGE}.xml" + pytest = [ + sys.executable, + "-m", + "pytest", + "-v", + "-p", + "no:cacheprovider", + f"--junitxml={report}", + "-o", + f"junit_suite_name={PACKAGE} {cell}", + "--pyargs", + PACKAGE, + ] + if skipped := skipped_tests(env): + pytest += ["-k", "not " + " and not ".join(skipped)] + return subprocess.run(pytest, env=env, check=False).returncode + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/conda-recipe-cf/bld.bat b/conda-recipe-cf/bld.bat deleted file mode 100644 index 1c9e08701d..0000000000 --- a/conda-recipe-cf/bld.bat +++ /dev/null @@ -1,46 +0,0 @@ -REM A workaround for activate-dpcpp.bat issue to be addressed in 2021.4 -set "LIB=%BUILD_PREFIX%\Library\lib;%BUILD_PREFIX%\compiler\lib;%LIB%" -set "INCLUDE=%BUILD_PREFIX%\include;%INCLUDE%" - -"%PYTHON%" setup.py clean --all - -REM useful for building in resources constrained VMs (public CI) -set "CMAKE_ARGS=%CMAKE_ARGS% -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE" - -REM Try to detect SYCL resource/include dir from icpx -for /f "usebackq delims=" %%R in (`icpx --print-resource-dir 2^>NUL`) do ( - set "SYCL_RESOURCE_DIR=%%R" -) - -if defined SYCL_RESOURCE_DIR ( - if exist "%SYCL_RESOURCE_DIR%\include" ( - set "SYCL_INCLUDE_DIR_HINT=%SYCL_RESOURCE_DIR%" - ) -) - -set "CC=icx" -set "CXX=icx" - -set "CMAKE_GENERATOR=Ninja" -:: Make CMake verbose -set "VERBOSE=1" - -:: set CMAKE to use less threads to avoid OOM -set CMAKE_BUILD_PARALLEL_LEVEL=%CPU_COUNT% - -set "CMAKE_ARGS=%CMAKE_ARGS% -DDPCTL_LEVEL_ZERO_INCLUDE_DIR=%PREFIX:\=/%/Library/include/level_zero" - -%PYTHON% -m build -w -n -x -if %ERRORLEVEL% neq 0 exit 1 - -:: wheel file was renamed -for /f %%f in ('dir /b /S .\dist') do ( - %PYTHON% -m pip install %%f ^ - --no-build-isolation ^ - --no-deps ^ - --only-binary :all: ^ - --no-index ^ - --prefix %PREFIX% ^ - -vv - if errorlevel 1 exit 1 -) diff --git a/conda-recipe-cf/build.sh b/conda-recipe-cf/build.sh deleted file mode 100755 index 983ba5d76c..0000000000 --- a/conda-recipe-cf/build.sh +++ /dev/null @@ -1,38 +0,0 @@ -#!/bin/bash - -# This is necessary to help DPC++ find Intel libraries such as SVML, IRNG, etc in build prefix -export LIBRARY_PATH="$LIBRARY_PATH:${BUILD_PREFIX}/lib" - -# Intel LLVM must cooperate with compiler and sysroot from conda -echo "--gcc-toolchain=${BUILD_PREFIX} --sysroot=${BUILD_PREFIX}/${HOST}/sysroot -target ${HOST}" > icpx_for_conda.cfg -ICPXCFG="$(pwd)/icpx_for_conda.cfg" -export ICPXCFG -ICXCFG="$(pwd)/icpx_for_conda.cfg" -export ICXCFG - -if [ -e "_skbuild" ]; then - ${PYTHON} setup.py clean --all -fi - -export CC=icx -export CXX=icpx - -export CMAKE_GENERATOR=Ninja -# Make CMake verbose -export VERBOSE=1 - -# set CMAKE to use less threads to avoid OOM -export CMAKE_BUILD_PARALLEL_LEVEL=${CPU_COUNT} - -CMAKE_ARGS="${CMAKE_ARGS} -DDPCTL_LEVEL_ZERO_INCLUDE_DIR=${PREFIX}/include/level_zero -DDPCTL_WITH_REDIST=ON" - -# -wnx flags mean: --wheel --no-isolation --skip-dependency-check -${PYTHON} -m build -w -n -x - -${PYTHON} -m pip install dist/dpctl*.whl \ - --no-build-isolation \ - --no-deps \ - --only-binary :all: \ - --no-index \ - --prefix "${PREFIX}" \ - -vv diff --git a/conda-recipe-cf/conda_build_config.yaml b/conda-recipe-cf/conda_build_config.yaml deleted file mode 100644 index 408d05b438..0000000000 --- a/conda-recipe-cf/conda_build_config.yaml +++ /dev/null @@ -1,18 +0,0 @@ -numpy: - - '2' -c_compiler: # [linux] - - gcc # [linux] -cxx_compiler: # [linux] - - gxx # [linux] -cxx_compiler_version: # [linux] - - '15' # [linux] -c_stdlib: # [linux] - - sysroot # [linux] -c_stdlib_version: # [linux] - - '2.28' # [linux] -c_stdlib: # [win] - - vs # [win] -cxx_compiler: # [win] - - vs2022 # [win] -c_compiler: # [win] - - vs2022 # [win] diff --git a/conda-recipe-cf/meta.yaml b/conda-recipe-cf/meta.yaml deleted file mode 100644 index 649d053129..0000000000 --- a/conda-recipe-cf/meta.yaml +++ /dev/null @@ -1,74 +0,0 @@ -{% set required_compiler_version = "2026.0.0" %} -package: - name: dpctl - version: {{ GIT_DESCRIBE_TAG }} - -source: - path: .. - -build: - number: {{ GIT_DESCRIBE_NUMBER }} - ignore_run_exports: - - level-zero - missing_dso_whitelist: - - $RPATH/DPCTLSyclInterface.dll # [win] - skip: true # [osx or py<39] - -requirements: - # TODO: keep in sync with /pyproject.toml - build: - - {{ compiler('cxx') }} - - {{ stdlib('c') }} - - {{ compiler('dpcpp') }} >={{ required_compiler_version }} - - ninja >=1.11.1 # [linux] - - cmake >=3.29.0 - host: - - python - - pip >=24.0 - - level-zero-devel >=1.16 - - pybind11 >=3.0.2 - - intel-sycl-rt - - intel-cmplr-lib-rt - # Ensure we are using latest version of setuptools, since we don't need - # editable environments for release. - - setuptools >=69 - - wheel >=0.43 - - python-build >=1.1 - - scikit-build >=0.17.0 - - cython >=3.1.0 # [py>=313] - - cython >=3.0.10 # [py<313] - - numpy - # WARNING: check with doc how to upgrade - - versioneer ==0.29 - # versioneer dependency - - tomli # [py<311] - run: - - python - - {{ pin_compatible('intel-sycl-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }} - - numpy - -test: - requires: - - pip - - pytest - - cython - - setuptools - commands: - - pip check - - python -c "import dpctl; print(dpctl.__version__)" - - python -m dpctl -f - -about: - home: https://github.com/IntelPython/dpctl.git - license: Apache-2.0 - license_file: LICENSE - summary: 'A lightweight Python wrapper for a subset of SYCL API.' - description: | - Data Parallel Control or dpctl is a Python library that allows users to control the execution placement of a compute kernel on an XPU. - -extra: - recipe-maintainers: - - ndgrigorian - - antonwolfy - - vlad-perevezentsev diff --git a/conda-recipe-cf/run_test.bat b/conda-recipe-cf/run_test.bat deleted file mode 100644 index 42816416d3..0000000000 --- a/conda-recipe-cf/run_test.bat +++ /dev/null @@ -1,10 +0,0 @@ -@echo on - -"%PYTHON%" -c "import dpctl; print(dpctl.__version__)" -if %ERRORLEVEL% neq 0 exit 1 - -"%PYTHON%" -m dpctl -f -if %ERRORLEVEL% neq 0 exit 1 - -python -m pytest -q -ra --disable-warnings --pyargs dpctl -vv -if %ERRORLEVEL% neq 0 exit 1 diff --git a/conda-recipe-cf/run_test.sh b/conda-recipe-cf/run_test.sh deleted file mode 100755 index de137e0caa..0000000000 --- a/conda-recipe-cf/run_test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e - -${PYTHON} -c "import dpctl; print(dpctl.__version__)" -${PYTHON} -m dpctl -f -${PYTHON} -m pytest -q -ra --disable-warnings --pyargs dpctl -vv diff --git a/conda-recipe/bld.bat b/conda-recipe/bld.bat index 2ff9e6cdcc..1c9e08701d 100644 --- a/conda-recipe/bld.bat +++ b/conda-recipe/bld.bat @@ -4,15 +4,17 @@ set "INCLUDE=%BUILD_PREFIX%\include;%INCLUDE%" "%PYTHON%" setup.py clean --all -REM Overriding IPO is useful for building in resources constrained VMs (public CI) -if DEFINED OVERRIDE_INTEL_IPO ( - set "CMAKE_ARGS=%CMAKE_ARGS% -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE" +REM useful for building in resources constrained VMs (public CI) +set "CMAKE_ARGS=%CMAKE_ARGS% -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE" + +REM Try to detect SYCL resource/include dir from icpx +for /f "usebackq delims=" %%R in (`icpx --print-resource-dir 2^>NUL`) do ( + set "SYCL_RESOURCE_DIR=%%R" ) -FOR %%V IN (17.0.0 17 18.0.0 18 19.0.0 19 20.0.0 20 21.0.0 21) DO @( - REM set DIR_HINT if directory exists - IF EXIST "%BUILD_PREFIX%\Library\lib\clang\%%V\" ( - SET "SYCL_INCLUDE_DIR_HINT=%BUILD_PREFIX%\Library\lib\clang\%%V" +if defined SYCL_RESOURCE_DIR ( + if exist "%SYCL_RESOURCE_DIR%\include" ( + set "SYCL_INCLUDE_DIR_HINT=%SYCL_RESOURCE_DIR%" ) ) @@ -23,18 +25,14 @@ set "CMAKE_GENERATOR=Ninja" :: Make CMake verbose set "VERBOSE=1" +:: set CMAKE to use less threads to avoid OOM +set CMAKE_BUILD_PARALLEL_LEVEL=%CPU_COUNT% + set "CMAKE_ARGS=%CMAKE_ARGS% -DDPCTL_LEVEL_ZERO_INCLUDE_DIR=%PREFIX:\=/%/Library/include/level_zero" %PYTHON% -m build -w -n -x if %ERRORLEVEL% neq 0 exit 1 -:: `pip install dist\numpy*.whl` does not work on windows, -:: so use a loop; there's only one wheel in dist/ anyway -for /f %%f in ('dir /b /S .\dist') do ( - %PYTHON% -m wheel tags --remove --build %GIT_DESCRIBE_NUMBER% %%f - if %ERRORLEVEL% neq 0 exit 1 -) - :: wheel file was renamed for /f %%f in ('dir /b /S .\dist') do ( %PYTHON% -m pip install %%f ^ @@ -44,11 +42,5 @@ for /f %%f in ('dir /b /S .\dist') do ( --no-index ^ --prefix %PREFIX% ^ -vv - if %ERRORLEVEL% neq 0 exit 1 -) - -:: Copy wheel package -if NOT "%WHEELS_OUTPUT_FOLDER%"=="" ( - copy dist\dpctl*.whl %WHEELS_OUTPUT_FOLDER% - if %ERRORLEVEL% neq 0 exit 1 + if errorlevel 1 exit 1 ) diff --git a/conda-recipe/build.sh b/conda-recipe/build.sh index e892602ede..983ba5d76c 100755 --- a/conda-recipe/build.sh +++ b/conda-recipe/build.sh @@ -10,9 +10,6 @@ export ICPXCFG ICXCFG="$(pwd)/icpx_for_conda.cfg" export ICXCFG -read -r GLIBC_MAJOR GLIBC_MINOR <<<"$(conda list '^sysroot_linux-64$' \ - | tail -n 1 | awk '{print $2}' | grep -oP '\d+' | head -n 2 | tr '\n' ' ')" - if [ -e "_skbuild" ]; then ${PYTHON} setup.py clean --all fi @@ -24,15 +21,14 @@ export CMAKE_GENERATOR=Ninja # Make CMake verbose export VERBOSE=1 +# set CMAKE to use less threads to avoid OOM +export CMAKE_BUILD_PARALLEL_LEVEL=${CPU_COUNT} + CMAKE_ARGS="${CMAKE_ARGS} -DDPCTL_LEVEL_ZERO_INCLUDE_DIR=${PREFIX}/include/level_zero -DDPCTL_WITH_REDIST=ON" # -wnx flags mean: --wheel --no-isolation --skip-dependency-check ${PYTHON} -m build -w -n -x -${PYTHON} -m wheel tags --remove --build "$GIT_DESCRIBE_NUMBER" \ - --platform-tag "manylinux_${GLIBC_MAJOR}_${GLIBC_MINOR}_x86_64" \ - dist/dpctl*.whl - ${PYTHON} -m pip install dist/dpctl*.whl \ --no-build-isolation \ --no-deps \ @@ -40,8 +36,3 @@ ${PYTHON} -m pip install dist/dpctl*.whl \ --no-index \ --prefix "${PREFIX}" \ -vv - -# Copy wheel package -if [[ -d "${WHEELS_OUTPUT_FOLDER}" ]]; then - cp dist/dpctl*.whl "${WHEELS_OUTPUT_FOLDER[@]}" -fi diff --git a/conda-recipe/conda_build_config.yaml b/conda-recipe/conda_build_config.yaml index c5df1a7f2e..e15e9c816e 100644 --- a/conda-recipe/conda_build_config.yaml +++ b/conda-recipe/conda_build_config.yaml @@ -1,11 +1,20 @@ +# Python versions for ci/matrix.py, one per line, without selectors +python: + - "3.10" + - "3.11" + - "3.12" + - "3.13" + - "3.14.* *_cp314" + - "3.14.* *_cp314t" + - "3.15.* *_cp315" numpy: - - '1.26' + - '2' c_compiler: # [linux] - gcc # [linux] cxx_compiler: # [linux] - gxx # [linux] cxx_compiler_version: # [linux] - - '14' # [linux] + - '15' # [linux] c_stdlib: # [linux] - sysroot # [linux] c_stdlib_version: # [linux] diff --git a/conda-recipe/meta.yaml b/conda-recipe/meta.yaml index 83a2818ed0..10310faeb0 100644 --- a/conda-recipe/meta.yaml +++ b/conda-recipe/meta.yaml @@ -1,87 +1,78 @@ {% set required_compiler_version = "2026.0.0" %} - -{% set pyproject = load_file_data('pyproject.toml') %} -{% set py_build_deps = pyproject.get('build-system', {}).get('requires', []) %} - package: - name: dpctl - version: {{ GIT_DESCRIBE_TAG }} + name: dpctl + version: {{ GIT_DESCRIBE_TAG }} source: - path: .. + path: .. build: - number: {{ GIT_DESCRIBE_NUMBER }} - script_env: - - WHEELS_OUTPUT_FOLDER - - OVERRIDE_INTEL_IPO # [win] - ignore_run_exports: - - level-zero - missing_dso_whitelist: - - $RPATH/DPCTLSyclInterface.dll # [win] + number: {{ GIT_DESCRIBE_NUMBER }} + ignore_run_exports: + - level-zero + missing_dso_whitelist: + - $RPATH/DPCTLSyclInterface.dll # [win] + skip: true # [osx] requirements: - # TODO: keep in sync with /pyproject.toml - build: - - {{ compiler('cxx') }} - - {{ stdlib('c') }} - - {{ compiler('dpcpp') }} >={{ required_compiler_version }} - host: - - python - - pip >=24.0 - - level-zero-devel >=1.16 - - pybind11 >=2.13 - - {{ pin_compatible('intel-sycl-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }} - # Ensure we are using latest version of setuptools, since we don't need - # editable environments for release. - - setuptools >=69 - {% for dep in py_build_deps %} - {% if dep.startswith('ninja') %} - - {{ dep.split(';')[0] }} # [not win] - {% elif dep.startswith('cmake') %} - - {{ dep }} - {% elif dep.startswith('build>=') %} - - {{ 'python-' ~ dep }} - {% else %} - - {{ dep|replace('_','-') }} - {% endif %} - {% endfor %} - # versioneer dependency - - tomli # [py<311] - - opencl-headers >=2025.06.13 # [win] - run: - - python - - {{ pin_compatible('intel-sycl-rt', min_pin='x.x', max_pin='x') }} - - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }} - - numpy + # TODO: keep in sync with /pyproject.toml + build: + - {{ compiler('cxx') }} + - {{ stdlib('c') }} + - {{ compiler('dpcpp') }} >={{ required_compiler_version }} + - ninja >=1.11.1 # [linux] + - cmake >=3.29.0 + host: + - python + - pip >=24.0 + - level-zero-devel >=1.16 + - pybind11 >=3.0.2 + - intel-sycl-rt + - intel-cmplr-lib-rt + # Ensure we are using latest version of setuptools, since we don't need + # editable environments for release. + - setuptools >=69 + - wheel >=0.43 + - python-build >=1.1 + - scikit-build >=0.17.0,<1.0 + - cython >=3.1.0 # [py>=313] + - cython >=3.0.10 # [py<313] + - numpy + # WARNING: check with doc how to upgrade + - versioneer ==0.29 + # versioneer dependency + - tomli # [py<311] + run: + - python + - {{ pin_compatible('intel-sycl-rt', min_pin='x.x', max_pin='x') }} + - {{ pin_compatible('intel-cmplr-lib-rt', min_pin='x.x', max_pin='x') }} + - numpy test: - requires: - - {{ compiler('c') }} - - {{ compiler('cxx') }} - - {{ stdlib('c') }} - - cython - - setuptools - - pytest + requires: + - pip + commands: + - pip check + - python -c "import dpctl; print(dpctl.__version__)" + - python -m dpctl -f about: - home: https://github.com/IntelPython/dpctl.git - license: Apache-2.0 - license_file: LICENSE - summary: 'A lightweight Python wrapper for a subset of SYCL API.' - description: | - LEGAL NOTICE: Use of this software package is subject to the - software license agreement (as set forth above, in the license section of - the installed Conda package and/or the README file) and all notices, - disclaimers or license terms for third party or open source software - included in or with the software. -

- EULA: Apache-2.0 -

+ home: https://github.com/IntelPython/dpctl.git + license: Apache-2.0 + license_file: LICENSE + summary: 'A lightweight Python wrapper for a subset of SYCL API.' + description: | + LEGAL NOTICE: Use of this software package is subject to the + software license agreement (as set forth above, in the license section of + the installed Conda package and/or the README file) and all notices, + disclaimers or license terms for third party or open source software + included in or with the software. +

+ EULA: Apache-2.0 +

extra: - recipe-maintainers: - - ndgrigorian - - antonwolfy - - vlad-perevezentsev + recipe-maintainers: + - ndgrigorian + - antonwolfy + - vlad-perevezentsev diff --git a/conda-recipe/run_test.bat b/conda-recipe/run_test.bat deleted file mode 100644 index 42816416d3..0000000000 --- a/conda-recipe/run_test.bat +++ /dev/null @@ -1,10 +0,0 @@ -@echo on - -"%PYTHON%" -c "import dpctl; print(dpctl.__version__)" -if %ERRORLEVEL% neq 0 exit 1 - -"%PYTHON%" -m dpctl -f -if %ERRORLEVEL% neq 0 exit 1 - -python -m pytest -q -ra --disable-warnings --pyargs dpctl -vv -if %ERRORLEVEL% neq 0 exit 1 diff --git a/conda-recipe/run_test.sh b/conda-recipe/run_test.sh deleted file mode 100755 index de137e0caa..0000000000 --- a/conda-recipe/run_test.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/bash - -set -e - -${PYTHON} -c "import dpctl; print(dpctl.__version__)" -${PYTHON} -m dpctl -f -${PYTHON} -m pytest -q -ra --disable-warnings --pyargs dpctl -vv diff --git a/docs/_legacy/docfiles/user_guides/QuickStart.rst b/docs/_legacy/docfiles/user_guides/QuickStart.rst index 382528cc98..0daf02d77e 100644 --- a/docs/_legacy/docfiles/user_guides/QuickStart.rst +++ b/docs/_legacy/docfiles/user_guides/QuickStart.rst @@ -105,20 +105,18 @@ Build and Install Using Conda-Build You can use the conda-recipe included with the sources to build the dpctl package. The advantage of this approach is that all dependencies are pulled in -from oneAPI's intelpython conda channel that is installed as a part of oneAPI. +from the conda-forge channel. .. code-block:: bash - export ONEAPI_ROOT=/opt/intel/oneapi - conda build conda-recipe -c ${ONEAPI_ROOT}/conda_channel + conda build conda-recipe -c conda-forge --override-channels On Windows OS to cope with `long file names `_, use ``croot`` with a short folder path: .. code-block:: bat - set "ONEAPI_ROOT=C:\Program Files (x86)\Intel\oneAPI\" - conda build --croot=C:/tmp conda-recipe -c "%ONEAPI_ROOT%\conda_channel" + conda build --croot=C:/tmp conda-recipe -c conda-forge --override-channels After building the Conda package, install it by executing: diff --git a/docs/doc_sources/contributor_guides/building.rst b/docs/doc_sources/contributor_guides/building.rst index e492a42b4f..4dbece9aee 100644 --- a/docs/doc_sources/contributor_guides/building.rst +++ b/docs/doc_sources/contributor_guides/building.rst @@ -61,21 +61,19 @@ Build and Install Using Conda-Build ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ You can use the conda-recipe included with the sources to build the dpctl -package. The advantage of this approach is that all oneAPI library dependencies are -pulled in from oneAPI's local conda channel that is installed as a part of oneAPI. +package. The advantage of this approach is that all dependencies are pulled in +from the conda-forge channel. .. code-block:: bash - export ONEAPI_ROOT=/opt/intel/oneapi - conda build conda-recipe -c ${ONEAPI_ROOT}/conda_channel + conda build conda-recipe -c conda-forge --override-channels On Windows OS to cope with `long file names `_, use ``croot`` with a short folder path: .. code-block:: bat - set "ONEAPI_ROOT=C:\Program Files (x86)\Intel\oneAPI\" - conda build --croot=C:/tmp conda-recipe -c "%ONEAPI_ROOT%\conda_channel" + conda build --croot=C:/tmp conda-recipe -c conda-forge --override-channels After building the Conda package, install it by executing: diff --git a/pyproject.toml b/pyproject.toml index 833952dcce..dbdd3433b8 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -5,7 +5,7 @@ requires = [ "wheel>=0.43", "build>=1.1", "setuptools>=63.0.0", - "scikit-build>=0.17.0", + "scikit-build>=0.17.0,<1.0", "ninja>=1.11.1; platform_system!='Windows'", "cmake>=3.29.0", "cython>=3.1.0", @@ -89,6 +89,31 @@ Repository = "https://github.com/IntelPython/dpctl.git" exclude = "dpctl/_version.py" line-length = 80 +[tool.cibuildwheel] +build-verbosity = 1 + +[tool.cibuildwheel.linux] +# Intel's icx, as the conda recipe builds with, from Intel's repository, and the Level Zero headers +before-all = [ + "rpm --import https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB", + "printf '[oneAPI]\\nname=Intel oneAPI\\nbaseurl=https://yum.repos.intel.com/oneapi\\nenabled=1\\ngpgcheck=1\\nrepo_gpgcheck=1\\ngpgkey=https://yum.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB\\n' > /etc/yum.repos.d/oneAPI.repo", + "dnf install -y intel-oneapi-compiler-dpcpp-cpp-2026.2", + "git clone --depth 1 --branch v1.34.0 https://github.com/oneapi-src/level-zero /opt/level-zero" +] +environment = { CC = "/opt/intel/oneapi/compiler/latest/bin/icx", CXX = "/opt/intel/oneapi/compiler/latest/bin/icpx", CMAKE_GENERATOR = "Ninja", CMAKE_ARGS = "-DDPCTL_LEVEL_ZERO_INCLUDE_DIR=/opt/level-zero/include -DDPCTL_WITH_REDIST=ON" } +# The SYCL and icx runtime libraries come from wheels at run time, see [tool.gh-ci] +repair-wheel-command = "auditwheel repair --exclude libsycl.so.9 --exclude libintlc.so.5 --exclude libimf.so --exclude libsvml.so --exclude libirng.so -w {dest_dir} {wheel}" + +[tool.cibuildwheel.windows] +# icx from conda-forge, and the Level Zero headers +before-all = [ + "python {project}/ci/install_icx.py C:/icx", + "git clone --depth 1 --branch v1.34.0 https://github.com/oneapi-src/level-zero C:/level-zero" +] +environment = { CC = "C:/icx/Library/bin/icx.exe", CXX = "C:/icx/Library/bin/icx.exe", CMAKE_GENERATOR = "Ninja", CMAKE_ARGS = "-DDPCTL_LEVEL_ZERO_INCLUDE_DIR=C:/level-zero/include -DCMAKE_INTERPROCEDURAL_OPTIMIZATION:BOOL=FALSE", PATH = "C:/icx/Library/bin;C:/icx/Library/bin/compiler;$PATH", LIB = "C:/icx/Library/lib;$LIB", LINK = "/LIBPATH:C:/icx/Library/lib" } +# The SYCL and icx runtime libraries come from wheels at run time, see [tool.gh-ci] +repair-wheel-command = "delvewheel repair --no-dll DPCTLSyclInterface.dll;sycl9.dll;irc_msg.dll;libmmd.dll;svml_dispmd.dll;libircmd.dll;libirngmd.dll;ze_loader.dll;OpenCL.dll -w {dest_dir} {wheel}" + [tool.codespell] builtin = "clear,rare,informal,names" check-filenames = true @@ -119,6 +144,10 @@ source = [ [tool.cython-lint] max-line-length = 80 +[tool.gh-ci] +# What the wheels need at run time, which conda packages get from the recipe +wheel-requires = ["intel-sycl-rt >=2026.2,<2027", "intel-cmplr-lib-rt >=2026.2,<2027"] + [tool.isort] ensure_newline_before_comments = true force_grid_wrap = 0