From 9addcb4a8af52e37999d2f98ff8db868bf0204ec Mon Sep 17 00:00:00 2001
From: Abanoub Doss <abanoub.doss@gmail.com>
Date: Sun, 26 Jul 2026 14:40:50 -0500
Subject: [PATCH 1/5] ci: resolve Arrow and Parquet from conda-forge on Unix
 legs

---
 .github/actions/setup-conda-arrow/action.yml  | 41 ++++++++++++++++
 .../setup-conda-arrow/install-arrow.sh        | 49 +++++++++++++++++++
 .github/workflows/aws_test.yml                |  3 ++
 .github/workflows/sanitizer_test.yml          |  6 +++
 .github/workflows/sql_catalog_test.yml        |  7 +++
 .github/workflows/test.yml                    |  4 ++
 ci/scripts/build_iceberg.sh                   |  5 ++
 7 files changed, 115 insertions(+)
 create mode 100644 .github/actions/setup-conda-arrow/action.yml
 create mode 100755 .github/actions/setup-conda-arrow/install-arrow.sh

diff --git a/.github/actions/setup-conda-arrow/action.yml b/.github/actions/setup-conda-arrow/action.yml
new file mode 100644
index 000000000..6382063cc
--- /dev/null
+++ b/.github/actions/setup-conda-arrow/action.yml
@@ -0,0 +1,41 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: Set up conda Arrow
+description: Install Arrow and Parquet from conda-forge for CMake builds.
+
+inputs:
+  extra-packages:
+    description: Extra conda packages to install, space separated.
+    required: false
+    default: ""
+
+runs:
+  using: composite
+  steps:
+    - name: Set up conda
+      uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5 # v4.0.1
+      with:
+        miniforge-version: latest
+        channels: conda-forge
+        channel-priority: strict
+        activate-environment: arrow
+    - name: Install Arrow and Parquet
+      shell: bash -el {0}
+      env:
+        EXTRA_PACKAGES: ${{ inputs.extra-packages }}
+      run: '"${{ github.action_path }}/install-arrow.sh"'
diff --git a/.github/actions/setup-conda-arrow/install-arrow.sh b/.github/actions/setup-conda-arrow/install-arrow.sh
new file mode 100755
index 000000000..8590666f5
--- /dev/null
+++ b/.github/actions/setup-conda-arrow/install-arrow.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Installs Arrow and Parquet into the active conda environment and exports the
+# paths CMake and the test binaries need. Run from the setup-conda-arrow action.
+
+set -euo pipefail
+
+toolchain="${GITHUB_WORKSPACE}/cmake_modules/IcebergThirdpartyToolchain.cmake"
+
+# Keep conda Arrow in sync with the vendored build.
+arrow_version=$(sed -nE 's/^set\(ICEBERG_ARROW_BUILD_VERSION "([0-9]+\.[0-9]+\.[0-9]+)"\)$/\1/p' "${toolchain}")
+if [[ -z "${arrow_version}" ]]; then
+    echo "::error::Could not read ICEBERG_ARROW_BUILD_VERSION from ${toolchain}"
+    exit 1
+fi
+
+packages=("libarrow==${arrow_version}" "libparquet==${arrow_version}")
+if [[ -n "${EXTRA_PACKAGES:-}" ]]; then
+    read -r -a extra_packages <<< "${EXTRA_PACKAGES}"
+    packages+=("${extra_packages[@]}")
+fi
+mamba install -y "${packages[@]}"
+
+echo "CMAKE_PREFIX_PATH=${CONDA_PREFIX}" >> "${GITHUB_ENV}"
+# build_iceberg.sh reads this and fails if CMake configured vendored Arrow anyway.
+echo "ICEBERG_REQUIRE_SYSTEM_ARROW=ON" >> "${GITHUB_ENV}"
+
+if [[ "${RUNNER_OS}" == "macOS" ]]; then
+    echo "DYLD_FALLBACK_LIBRARY_PATH=${DYLD_FALLBACK_LIBRARY_PATH:+${DYLD_FALLBACK_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "${GITHUB_ENV}"
+else
+    echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH:+${LD_LIBRARY_PATH}:}${CONDA_PREFIX}/lib" >> "${GITHUB_ENV}"
+fi
diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml
index e59185617..84d628bef 100644
--- a/.github/workflows/aws_test.yml
+++ b/.github/workflows/aws_test.yml
@@ -118,6 +118,9 @@ jobs:
         if: ${{ matrix.s3 == 'ON' }}
         shell: bash
         run: bash ci/scripts/start_minio.sh
+      - name: Set up conda Arrow
+        if: ${{ matrix.bundle_awssdk == 'OFF' }}
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
diff --git a/.github/workflows/sanitizer_test.yml b/.github/workflows/sanitizer_test.yml
index 343e661ef..d2c4ab85e 100644
--- a/.github/workflows/sanitizer_test.yml
+++ b/.github/workflows/sanitizer_test.yml
@@ -52,6 +52,8 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
@@ -64,6 +66,10 @@ jobs:
           mkdir build && cd build
           cmake .. -G Ninja -DCMAKE_BUILD_TYPE=Debug -DICEBERG_ENABLE_ASAN=ON -DICEBERG_ENABLE_UBSAN=ON \
             -DCMAKE_C_COMPILER_LAUNCHER=sccache -DCMAKE_CXX_COMPILER_LAUNCHER=sccache
+          if [[ -d _deps/vendoredarrow-src ]]; then
+            echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow"
+            exit 1
+          fi
           cmake --build . --verbose
       - name: Save sccache
         if: always()
diff --git a/.github/workflows/sql_catalog_test.yml b/.github/workflows/sql_catalog_test.yml
index 91b5d60e9..ddb7af07c 100644
--- a/.github/workflows/sql_catalog_test.yml
+++ b/.github/workflows/sql_catalog_test.yml
@@ -97,6 +97,9 @@ jobs:
         shell: pwsh
         run: |
           vcpkg install zlib:x64-windows nlohmann-json:x64-windows nanoarrow:x64-windows roaring:x64-windows sqlite3:x64-windows
+      - name: Set up conda Arrow
+        if: ${{ !startsWith(matrix.runs-on, 'windows') }}
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
@@ -115,6 +118,10 @@ jobs:
             -DCMAKE_C_COMPILER_LAUNCHER=sccache \
             -DCMAKE_CXX_COMPILER_LAUNCHER=sccache \
             ${{ matrix.cmake_extra_args }}
+          if [[ "${RUNNER_OS}" != "Windows" && -d build/_deps/vendoredarrow-src ]]; then
+            echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow"
+            exit 1
+          fi
       - name: Build SQL catalog tests
         shell: bash
         run: cmake --build build --target sql_catalog_test
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index d260ef470..dc5d534aa 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -62,6 +62,8 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
@@ -140,6 +142,8 @@ jobs:
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
diff --git a/ci/scripts/build_iceberg.sh b/ci/scripts/build_iceberg.sh
index 6a7dc607a..090dc2fe5 100755
--- a/ci/scripts/build_iceberg.sh
+++ b/ci/scripts/build_iceberg.sh
@@ -88,6 +88,11 @@ fi
 
 cmake "${CMAKE_ARGS[@]}" ${source_dir}
 
+if [[ "${ICEBERG_REQUIRE_SYSTEM_ARROW:-OFF}" == "ON" && -d _deps/vendoredarrow-src ]]; then
+    echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow"
+    exit 1
+fi
+
 cmake --build . --target install
 if [[ "${run_tests}" == "ON" ]]; then
     ctest --output-on-failure

From 9ce88489a8f2d5109abdfe2f19f99d5334ba750c Mon Sep 17 00:00:00 2001
From: Abanoub Doss <abanoub.doss@gmail.com>
Date: Mon, 7 Sep 2026 18:41:37 -0500
Subject: [PATCH 2/5] ci: use conda Arrow for Hive and C++ linter

---
 .github/workflows/cpp-linter.yml | 12 +++++++++++-
 .github/workflows/test.yml       | 10 +++++++++-
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/cpp-linter.yml b/.github/workflows/cpp-linter.yml
index 198af3d67..234ada03a 100644
--- a/.github/workflows/cpp-linter.yml
+++ b/.github/workflows/cpp-linter.yml
@@ -61,6 +61,11 @@ jobs:
         run: |
           sudo apt-get update
           sudo apt-get install -y libcurl4-openssl-dev libsqlite3-dev libpq-dev default-libmysqlclient-dev
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
+        with:
+          # This leg builds Hive, whose Thrift runtime needs Boost headers.
+          extra-packages: libboost-devel
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
@@ -79,7 +84,12 @@ jobs:
             -DICEBERG_SQL_SQLITE=ON \
             -DICEBERG_SQL_POSTGRESQL=ON \
             -DICEBERG_SQL_MYSQL=ON \
-            -DICEBERG_BUILD_HIVE=ON
+            -DICEBERG_BUILD_HIVE=ON \
+            -DICEBERG_BUNDLE_THRIFT=OFF
+          if [[ -d _deps/vendoredarrow-src ]]; then
+            echo "::error::Expected Arrow from CMAKE_PREFIX_PATH, but CMake configured vendored Arrow"
+            exit 1
+          fi
           cmake --build .
       - name: Save sccache
         if: always()
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
index dc5d534aa..9055daad6 100644
--- a/.github/workflows/test.yml
+++ b/.github/workflows/test.yml
@@ -96,7 +96,9 @@ jobs:
     env:
       SCCACHE_DIR: ${{ github.workspace }}/.sccache
       SCCACHE_CACHE_SIZE: "2G"
-      ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_BUILD_HIVE=ON"
+      # Prebuilt Arrow exports no Thrift target, so resolve the Thrift runtime
+      # from conda instead of Arrow's bundled build.
+      ICEBERG_EXTRA_CMAKE_ARGS: "-DICEBERG_BUILD_HIVE=ON -DICEBERG_BUNDLE_THRIFT=OFF"
     steps:
       - name: Checkout iceberg-cpp
         uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
@@ -105,6 +107,12 @@ jobs:
       - name: Install dependencies
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev
+      - name: Set up conda Arrow
+        uses: ./.github/actions/setup-conda-arrow
+        with:
+          # Thrift's public C++ headers include Boost, and libboost-devel (not
+          # libboost-headers) is the package carrying BoostConfig.cmake.
+          extra-packages: libboost-devel
       - name: Restore sccache cache
         uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
         with:

From acaa38b5f3cf8517fb634359f594fd4a68c20f4f Mon Sep 17 00:00:00 2001
From: Abanoub Doss <abanoub.doss@gmail.com>
Date: Mon, 7 Sep 2026 19:28:54 -0500
Subject: [PATCH 3/5] ci: capture backtrace for REST Arrow test failure

---
 .github/workflows/aws_test.yml | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml
index 84d628bef..8ce9bed83 100644
--- a/.github/workflows/aws_test.yml
+++ b/.github/workflows/aws_test.yml
@@ -130,6 +130,17 @@ jobs:
         env:
           CMAKE_TOOLCHAIN_FILE: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }}
         run: ci/scripts/build_iceberg.sh "$(pwd)" OFF ON ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }}
+      - name: Diagnose REST Arrow test failure
+        if: ${{ failure() && startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }}
+        shell: bash
+        run: |
+          test -x build/src/iceberg/test/rest_arrow_file_io_test || exit 0
+          sudo apt-get install -y gdb
+          cd build
+          timeout 120s gdb --batch \
+            -ex 'set pagination off' -ex run -ex 'thread apply all bt' \
+            -ex 'info sharedlibrary' \
+            --args src/iceberg/test/rest_arrow_file_io_test
       - name: Save sccache
         if: always()
         uses: ./.github/actions/save-sccache

From e547ba74395608828dbb26812875ceef7daef112 Mon Sep 17 00:00:00 2001
From: Abanoub Doss <abanoub.doss@gmail.com>
Date: Mon, 7 Sep 2026 20:04:11 -0500
Subject: [PATCH 4/5] ci: use conda AWS SDK alongside conda Arrow

---
 .github/workflows/aws_test.yml | 29 ++++-------------------------
 1 file changed, 4 insertions(+), 25 deletions(-)

diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml
index 8ce9bed83..dba120a7e 100644
--- a/.github/workflows/aws_test.yml
+++ b/.github/workflows/aws_test.yml
@@ -66,7 +66,6 @@ jobs:
             s3: "ON"
             sigv4: "ON"
             bundle_awssdk: "OFF"
-            aws-sdk-features: core,config,s3,identity-management,sts,transfer
           - title: macOS 26 ARM64, S3, bundled AWS SDK
             runs-on: macos-26
             s3: "ON"
@@ -88,27 +87,6 @@ jobs:
         if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
         shell: bash
         run: sudo apt-get update && sudo apt-get install -y libcurl4-openssl-dev libjitterentropy3-dev
-      - name: Cache vcpkg packages
-        if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }}
-        uses: actions/cache@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0
-        id: vcpkg-cache
-        with:
-          path: /usr/local/share/vcpkg/installed
-          key: vcpkg-x64-linux-aws-sdk-cpp-s3-${{ matrix.s3 }}-sigv4-${{ matrix.sigv4 }}-${{ hashFiles('.github/workflows/aws_test.yml') }}
-      - name: Install AWS SDK via vcpkg
-        if: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && steps.vcpkg-cache.outputs.cache-hit != 'true' }}
-        shell: bash
-        # Retry to ride out transient GitHub/mirror download failures (504s).
-        run: |
-          for attempt in 1 2 3; do
-            if vcpkg install "aws-sdk-cpp[${{ matrix.aws-sdk-features }}]:x64-linux"; then
-              exit 0
-            fi
-            echo "::warning::vcpkg install failed (attempt ${attempt}/3), retrying in 30s"
-            sleep 30
-          done
-          echo "::error::vcpkg install failed after 3 attempts"
-          exit 1
       - name: Set Ubuntu Compilers
         if: ${{ startsWith(matrix.runs-on, 'ubuntu') }}
         run: |
@@ -118,17 +96,18 @@ jobs:
         if: ${{ matrix.s3 == 'ON' }}
         shell: bash
         run: bash ci/scripts/start_minio.sh
-      - name: Set up conda Arrow
+      - name: Set up conda Arrow and AWS SDK
         if: ${{ matrix.bundle_awssdk == 'OFF' }}
         uses: ./.github/actions/setup-conda-arrow
+        with:
+          # Use the same AWS SDK as Arrow to avoid incompatible symbols at runtime.
+          extra-packages: aws-sdk-cpp
       - name: Set up sccache
         uses: ./.github/actions/setup-sccache
         with:
           key-prefix: sccache-aws-${{ matrix.runs-on }}-bundle${{ matrix.bundle_awssdk }}-s3${{ matrix.s3 }}-sigv4${{ matrix.sigv4 }}
       - name: Build and test Iceberg
         shell: bash
-        env:
-          CMAKE_TOOLCHAIN_FILE: ${{ startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' && '/usr/local/share/vcpkg/scripts/buildsystems/vcpkg.cmake' || '' }}
         run: ci/scripts/build_iceberg.sh "$(pwd)" OFF ON ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }}
       - name: Diagnose REST Arrow test failure
         if: ${{ failure() && startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }}

From ceb88b64922729359432b7e7823d4020528e9b9a Mon Sep 17 00:00:00 2001
From: Abanoub Doss <abanoub.doss@gmail.com>
Date: Mon, 7 Sep 2026 20:38:14 -0500
Subject: [PATCH 5/5] ci: remove temporary REST Arrow failure diagnostics

---
 .github/workflows/aws_test.yml | 11 -----------
 1 file changed, 11 deletions(-)

diff --git a/.github/workflows/aws_test.yml b/.github/workflows/aws_test.yml
index dba120a7e..1345804b3 100644
--- a/.github/workflows/aws_test.yml
+++ b/.github/workflows/aws_test.yml
@@ -109,17 +109,6 @@ jobs:
       - name: Build and test Iceberg
         shell: bash
         run: ci/scripts/build_iceberg.sh "$(pwd)" OFF ON ${{ matrix.s3 }} ${{ matrix.sigv4 }} ${{ matrix.bundle_awssdk }}
-      - name: Diagnose REST Arrow test failure
-        if: ${{ failure() && startsWith(matrix.runs-on, 'ubuntu') && matrix.bundle_awssdk == 'OFF' }}
-        shell: bash
-        run: |
-          test -x build/src/iceberg/test/rest_arrow_file_io_test || exit 0
-          sudo apt-get install -y gdb
-          cd build
-          timeout 120s gdb --batch \
-            -ex 'set pagination off' -ex run -ex 'thread apply all bt' \
-            -ex 'info sharedlibrary' \
-            --args src/iceberg/test/rest_arrow_file_io_test
       - name: Save sccache
         if: always()
         uses: ./.github/actions/save-sccache