diff --git a/.github/actions/setup-conda-arrow/action.yml b/.github/actions/setup-conda-arrow/action.yml new file mode 100644 index 000000000..6382063cc --- /dev/null +++ b/.github/actions/setup-conda-arrow/action.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +name: Set up conda Arrow +description: Install Arrow and Parquet from conda-forge for CMake builds. + +inputs: + extra-packages: + description: Extra conda packages to install, space separated. + required: false + default: "" + +runs: + using: composite + steps: + - name: Set up conda + uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1 + with: + miniforge-version: latest + channels: conda-forge + channel-priority: strict + activate-environment: arrow + - name: Install Arrow and Parquet + shell: bash -el {0} + env: + EXTRA_PACKAGES: ${{ inputs.extra-packages }} + run: '"${{ github.action_path }}/install-arrow.sh"' diff --git a/.github/actions/setup-conda-arrow/install-arrow.sh b/.github/actions/setup-conda-arrow/install-arrow.sh new file mode 100755 index 000000000..8590666f5 --- /dev/null +++ b/.github/actions/setup-conda-arrow/install-arrow.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +# Installs Arrow and Parquet into the active conda environment and exports the +# paths CMake and the test binaries need. Run from the setup-conda-arrow action. + +set -euo pipefail + +toolchain="${GITHUB_WORKSPACE}/cmake_modules/IcebergThirdpartyToolchain.cmake" + +# Keep conda Arrow in sync with the vendored build. +arrow_version=$(sed -nE 's/^set\(ICEBERG_ARROW_BUILD_VERSION "([0-9]+\.[0-9]+\.[0-9]+)"\)$/\1/p' "${toolchain}") +if [[ -z "${arrow_version}" ]]; then + echo "::error::Could not read ICEBERG_ARROW_BUILD_VERSION from ${toolchain}" + exit 1 +fi + +packages=("libarrow==${arrow_version}" "libparquet==${arrow_version}") +if [[ -n "${EXTRA_PACKAGES:-}" ]]; then + read -r -a extra_packages <<< "${EXTRA_PACKAGES}" + packages+=("${extra_packages[@]}") +fi +mamba install -y "${packages[@]}" + +echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "${GITHUB_ENV}" +# build_iceberg.sh reads this and fails if CMake configured vendored Arrow anyway. +echo "ICEBERG_REQUIRE_SYSTEM_ARROW=ON" >> "${GITHUB_ENV}" + +if [[ "${RUNNER_OS}" == "macOS" ]]; then + echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:+${DYLD_FALLBACK_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "${GITHUB_ENV}" +else + echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "${GITHUB_ENV}" +fi diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml index e59185617..1345804b3 100644 --- a/.github/workflows/aws_test.yml +++ b/.github/workflows/aws_test.yml @@ -66,7 +66,6 @@ jobs: s3: "ON" sigv4: "ON" bundle_awssdk: "OFF" - aws-sdk-features: core,config,s3,identity-management,sts,transfer - title: macOS 26 ARM64, S3, bundled AWS SDK runs-on: macos-26 s3: "ON" @@ -88,27 +87,6 @@ jobs: if: ${{ startsWith(matrix.runs-on, 'ubuntu') }} shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libjitterentropy3-dev - - name: Cache vcpkg packages - if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }} - uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 - id: vcpkg-cache - with: - path: /usr/local/share/vcpkg/installed - key: vcpkg-x64-linux-aws-sdk-cpp-s3-${{ matrix.s3 }}-sigv4-${{ matrix.sigv4 }}-${{ hashFiles('.github/workflows/aws_test.yml') }} - - name: Install AWS SDK via vcpkg - if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && steps.vcpkg-cache.outputs.cache-hit != 'true' }} - shell: bash - # Retry to ride out transient GitHub/mirror download failures (504s). - run: | - for attempt in 1 2 3; do - if vcpkg install "aws-sdk-cpp[${{ matrix.aws-sdk-features }}]:x64-linux"; then - exit 0 - fi - echo "::warning::vcpkg install failed (attempt ${attempt}/3), retrying in 30s" - sleep 30 - done - echo "::error::vcpkg install failed after 3 attempts" - exit 1 - name: Set Ubuntu Compilers if: ${{ startsWith(matrix.runs-on, 'ubuntu') }} run: | @@ -118,14 +96,18 @@ jobs: if: ${{ matrix.s3 == 'ON' }} shell: bash run: bash ci/scripts/start_minio.sh + - name: Set up conda Arrow and AWS SDK + if: ${{ matrix.bundle_awssdk == 'OFF' }} + uses: ./.github/actions/setup-conda-arrow + with: + # Use the same AWS SDK as Arrow to avoid incompatible symbols at runtime. + extra-packages: aws-sdk-cpp - name: Set up sccache uses: ./.github/actions/setup-sccache with: key-prefix: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }} - name: Build and test Iceberg shell: bash - env: - CMAKE_TOOLCHAIN_FILE: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }} run: ci/scripts/build_iceberg.sh "$(pwd)" OFF ON ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }} - name: Save sccache if: always() diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml index 198af3d67..234ada03a 100644 --- a/.github/workflows/cpp-linter.yml +++ b/.github/workflows/cpp-linter.yml @@ -61,6 +61,11 @@ jobs: run: | sudo apt-get update sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev + - name: Set up conda Arrow + uses: ./.github/actions/setup-conda-arrow + with: + # This leg builds Hive, whose Thrift runtime needs Boost headers. + extra-packages: libboost-devel - name: Set up sccache uses: ./.github/actions/setup-sccache with: @@ -79,7 +84,12 @@ jobs: -DICEBERG_SQL_SQLITE=ON \ -DICEBERG_SQL_POSTGRESQL=ON \ -DICEBERG_SQL_MYSQL=ON \ - -DICEBERG_BUILD_HIVE=ON + -DICEBERG_BUILD_HIVE=ON \ + -DICEBERG_BUNDLE_THRIFT=OFF + if [[ -d _deps/vendoredarrow-src ]]; then + echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow" + exit 1 + fi cmake --build . - name: Save sccache if: always() diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml index 343e661ef..d2c4ab85e 100644 --- a/.github/workflows/sanitizer_test.yml +++ b/.github/workflows/sanitizer_test.yml @@ -52,6 +52,8 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + - name: Set up conda Arrow + uses: ./.github/actions/setup-conda-arrow - name: Set up sccache uses: ./.github/actions/setup-sccache with: @@ -64,6 +66,10 @@ jobs: mkdir build && cd build cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON \ -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache + if [[ -d _deps/vendoredarrow-src ]]; then + echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow" + exit 1 + fi cmake --build . --verbose - name: Save sccache if: always() diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml index 91b5d60e9..ddb7af07c 100644 --- a/.github/workflows/sql_catalog_test.yml +++ b/.github/workflows/sql_catalog_test.yml @@ -97,6 +97,9 @@ jobs: shell: pwsh run: | vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows + - name: Set up conda Arrow + if: ${{ !startsWith(matrix.runs-on, 'windows') }} + uses: ./.github/actions/setup-conda-arrow - name: Set up sccache uses: ./.github/actions/setup-sccache with: @@ -115,6 +118,10 @@ jobs: -DCMAKE_C_COMPILER_LAUNCHER=sccache \ -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \ ${{ matrix.cmake_extra_args }} + if [[ "${RUNNER_OS}" != "Windows" && -d build/_deps/vendoredarrow-src ]]; then + echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow" + exit 1 + fi - name: Build SQL catalog tests shell: bash run: cmake --build build --target sql_catalog_test diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d260ef470..9055daad6 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -62,6 +62,8 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + - name: Set up conda Arrow + uses: ./.github/actions/setup-conda-arrow - name: Set up sccache uses: ./.github/actions/setup-sccache with: @@ -94,7 +96,9 @@ jobs: env: SCCACHE_DIR: ${{ github.workspace }}/.sccache SCCACHE_CACHE_SIZE: "2G" - ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_BUILD_HIVE=ON" + # Prebuilt Arrow exports no Thrift target, so resolve the Thrift runtime + # from conda instead of Arrow's bundled build. + ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_BUILD_HIVE=ON -DICEBERG_BUNDLE_THRIFT=OFF" steps: - name: Checkout iceberg-cpp uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -103,6 +107,12 @@ jobs: - name: Install dependencies shell: bash run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev + - name: Set up conda Arrow + uses: ./.github/actions/setup-conda-arrow + with: + # Thrift's public C++ headers include Boost, and libboost-devel (not + # libboost-headers) is the package carrying BoostConfig.cmake. + extra-packages: libboost-devel - name: Restore sccache cache uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 with: @@ -140,6 +150,8 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: persist-credentials: false + - name: Set up conda Arrow + uses: ./.github/actions/setup-conda-arrow - name: Set up sccache uses: ./.github/actions/setup-sccache with: diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh index 6a7dc607a..090dc2fe5 100755 --- a/ci/scripts/build_iceberg.sh +++ b/ci/scripts/build_iceberg.sh @@ -88,6 +88,11 @@ fi cmake "${CMAKE_ARGS[@]}" ${source_dir} +if [[ "${ICEBERG_REQUIRE_SYSTEM_ARROW:-OFF}" == "ON" && -d _deps/vendoredarrow-src ]]; then + echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow" + exit 1 +fi + cmake --build . --target install if [[ "${run_tests}" == "ON" ]]; then ctest --output-on-failure