python.yml

name: Python

on:
  push:
    branches:
      - main
      - release/**
  pull_request:
    branches:
      - main
      - release/**
    paths:
      - Cargo.*
      - .cargo/config.toml
      - clippy.toml
      - python/**
      - rust/**
      - protos/**
      - skills/lance-table-maintenance/scripts/**
      - .github/workflows/python.yml
      - .github/workflows/build_linux_wheel/**
      - .github/workflows/build_mac_wheel/**
      - .github/workflows/run_tests/**

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # This env var is used by Swatinem/rust-cache@v2 for the cache
  # key, so we set it to make sure it is always consistent.
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: "1"
  CI: "true"
  # Color output for pytest is off by default.
  PYTEST_ADDOPTS: "--color=yes"
  FORCE_COLOR: "1"
  # Change this to bust all caches (may be needed periodically if the caches accumulate
  # a lot of cruft).
  CACHE_PREFIX: "2"

jobs:
  # Rust-only PRs run a reduced set; pushes to main/release run everything.
  changes:
    runs-on: ubuntu-24.04
    permissions:
      contents: read
      pull-requests: read
    outputs:
      bindings: ${{ github.event_name != 'pull_request' || steps.filter.outputs.bindings == 'true' }}
    steps:
      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        if: github.event_name == 'pull_request'
        id: filter
        with:
          filters: |
            bindings:
              - python/**
              - .github/workflows/python.yml
              - .github/workflows/build_*_wheel/**
              - .github/workflows/run_tests/**

  lint:
    timeout-minutes: 45
    runs-on: "ubuntu-24.04-4x"
    defaults:
      run:
        shell: bash
        working-directory: python
    env:
      # Need up-to-date compilers for kernels
      CC: clang-18
      CXX: clang++-18
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: 3.11 # Ray does not support 3.12 yet.
      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
        with:
          workspaces: python
          prefix-key: ${{ env.CACHE_PREFIX }}
      - name: Install linting tools
        run: |
          pip install ruff==0.11.2 maturin tensorflow tqdm ray[data] pyright datasets polars[pyarrow,pandas] lance-namespace
          pip install torch --index-url https://download.pytorch.org/whl/cpu
      - name: Lint Python
        run: |
          ruff format --check --diff python
          ruff check python
          pyright
      - name: Install protoc
        uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc
        with:
          tool: protoc
      - name: Lint Rust
        run: |
          ALL_FEATURES=`cargo metadata --format-version=1 --no-deps | jq -r '.packages[] | .features | keys | .[]' | grep -v protoc | sort | uniq | paste -s -d "," -`
          cargo fmt --all -- --check
          cargo clippy  --profile ci --locked --features ${ALL_FEATURES} --tests -- -D warnings
      - name: Build
        run: |
          python -m venv venv
          source venv/bin/activate
          pip install torch tqdm --index-url https://download.pytorch.org/whl/cpu
          pip install maturin==1.13.3 uv
          maturin develop --uv --profile ci --locked --extras tests,ray
      - name: Run doctest
        run: |
          source venv/bin/activate
          pytest --doctest-modules python/lance

  python-rust-test:
    needs: changes
    if: needs.changes.outputs.bindings == 'true'
    timeout-minutes: 30
    name: pylance crate Rust tests
    runs-on: "ubuntu-24.04-4x"
    defaults:
      run:
        shell: bash
        working-directory: python
    env:
      # Need up-to-date compilers for kernels
      CC: clang-18
      CXX: clang++-18
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: 3.11
      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
        with:
          workspaces: python
          prefix-key: ${{ env.CACHE_PREFIX }}
      - name: Install protoc
        uses: taiki-e/install-action@66068bfca13dcb2ea07c3f613ca2836a37c755d5 # protoc
        with:
          tool: protoc
      # The default `extension-module` feature leaves libpython unlinked, which the
      # host interpreter resolves at load time but a test binary cannot. Disabling it
      # links libpython so these tests run instead of only compiling.
      - name: Test Rust
        run: cargo test --profile ci --locked --no-default-features

  linux-wheel:
    timeout-minutes: 45
    name: Python Linux x86_64 wheel
    runs-on: "ubuntu-24.04-4x"
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: "3.10"
      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
        with:
          workspaces: python
          prefix-key: ${{ env.CACHE_PREFIX }}
      # python/Cargo.toml enables abi3-py310, so this wheel is reusable by
      # every CPython version in the Linux test matrix.
      - name: Build cp310 ABI3 wheel
        uses: ./.github/workflows/build_linux_wheel
        with:
          python-minor-version: "10"
          args: "--profile ci"
      - name: Upload wheel
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: linux-x86_64-wheel
          path: python/target/wheels/pylance-*.whl
          if-no-files-found: error

  linux-memtest:
    timeout-minutes: 15
    name: Python Linux x86_64 memtest
    runs-on: ubuntu-24.04
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
      - uses: actions-rust-lang/setup-rust-toolchain@a0b538fa0b742a6aa35d6e2c169b4bd06d225a98 # v1
        with:
          cache: "false"
          rustflags: ""
      - name: Build memtest
        working-directory: memtest
        run: cargo build --release
      - name: Upload memtest
        uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4
        with:
          name: linux-x86_64-memtest
          path: memtest/target/release/libmemtest.so
          if-no-files-found: error

  linux:
    needs: [changes, linux-wheel, linux-memtest]
    timeout-minutes: 45
    strategy:
      matrix:
        python-minor-version: ${{ fromJSON(needs.changes.outputs.bindings == 'true' && '["10", "13", "14"]' || '["14"]') }}
    name: "Python Linux 3.${{ matrix.python-minor-version }} x86_64"
    runs-on: "ubuntu-24.04-4x"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: 3.${{ matrix.python-minor-version }}
      - name: Download wheel
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: linux-x86_64-wheel
          path: python/target/wheels
      - name: Download memtest
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: linux-x86_64-memtest
          path: memtest/python/memtest
      - uses: ./.github/workflows/run_tests
        with:
          memtest: true

  compat:
    needs: linux-wheel
    timeout-minutes: 60
    runs-on: ubuntu-24.04-8x
    name: Compatibility Tests
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: 3.14
      - name: Download wheel
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: linux-x86_64-wheel
          path: python/target/wheels
      - name: Install dependencies
        run: |
          pip install $(ls target/wheels/pylance-*.whl)[tests,ray]
      # Nearly every test here shells out to an old pylance in its own venv, so the
      # job spends most of its time waiting on subprocesses rather than on CPU.
      # Oversubscribing cores follows what nightly_run.yml already does with this
      # suite. Leaving COMPAT_TEMP_VENV unset is what makes that pay off: the venvs
      # then land in the shared cache directory, where VenvFactory's flock lets all
      # workers reuse one venv per version instead of each building its own.
      - name: Run compatibility tests
        run: |
          make compattest PYTEST_WORKERS=$(( $(nproc) * 2 ))

  linux-arm:
    needs: changes
    if: needs.changes.outputs.bindings == 'true'
    timeout-minutes: 45
    runs-on: ubuntu-24.04-arm64-4x
    name: Python Linux 3.14 ARM
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: 3.14
      - uses: Swatinem/rust-cache@779680da715d629ac1d338a641029a2f4372abb5 # v2
        with:
          workspaces: python
          prefix-key: ${{ env.CACHE_PREFIX }}
      - uses: ./.github/workflows/build_linux_wheel
        with:
          arm-build: "true"
          manylinux: "2_28"
          args: "--profile ci"
      - name: Install dependencies
        run: |
          sudo apt update -y -qq
          sudo apt install -y libhdf5-dev
      - uses: ./.github/workflows/run_tests
  aws-integtest:
    needs: [changes, linux-wheel]
    if: needs.changes.outputs.bindings == 'true'
    timeout-minutes: 45
    runs-on: "ubuntu-24.04-4x"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@34e114876b0b11c390a56381ad16ebd13914f8d5 # v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5
        with:
          python-version: "3.11" # TODO: upgrade when ray supports 3.12
      - name: Download wheel
        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4
        with:
          name: linux-x86_64-wheel
          path: python/target/wheels
      - name: Install dependencies
        run: |
          pip install ray[data]
          pip install torch --index-url https://download.pytorch.org/whl/cpu
      - uses: ./.github/workflows/run_integtests