From ea11c9037d509c4c4b7014ad6e8ff891dc133a1e Mon Sep 17 00:00:00 2001 From: bodduv Date: Wed, 14 Oct 2026 10:46:48 +0200 Subject: [PATCH 1/2] BDX-1692: Package the Arrow C++ Parquet SDK as release artifacts Publish org.apache.arrow:arrow-parquet with linux-x86_64, linux-aarch_64, osx-x86_64 and osx-aarch_64 JARs. Each holds the static Arrow C++ Parquet libraries, headers and an arrow-parquet.pc built from the same Arrow C++ revision as the JNI libraries. A smoke test links and runs against every staged package. The Linux and macOS arm64 packages reuse the existing JNI build. The macOS Intel package comes from a Parquet-only job, because the full JNI build exceeds the runner time limit there; it targets macOS 13.0. Download ORC and Thrift sources from archive.apache.org, since Apache mirrors no longer host the pinned versions. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/jarbuild.yml | 78 +++++++++++++ ci/scripts/arrow_parquet_package.sh | 172 ++++++++++++++++++++++++++++ ci/scripts/jni_full_build.sh | 28 +++++ ci/scripts/jni_macos_build.sh | 5 + ci/scripts/jni_manylinux_build.sh | 5 + compose.yaml | 2 + 6 files changed, 290 insertions(+) create mode 100755 ci/scripts/arrow_parquet_package.sh diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index 28a9377944..001822a39e 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -149,6 +149,14 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing + - name: Use the Apache archive for Apache sources + # Apache mirrors keep only current releases; archive.apache.org keeps all of them. + run: | + versions=arrow/cpp/thirdparty/versions.txt + orc=$(sed -n 's/^ARROW_ORC_BUILD_VERSION=//p' "${versions}") + thrift=$(sed -n 's/^ARROW_THRIFT_BUILD_VERSION=//p' "${versions}") + echo "ARROW_ORC_URL=https://archive.apache.org/dist/orc/orc-${orc}/orc-${orc}.tar.gz" >> "${GITHUB_ENV}" + echo "ARROW_THRIFT_URL=https://archive.apache.org/dist/thrift/${thrift}/thrift-${thrift}.tar.gz" >> "${GITHUB_ENV}" - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: ghcr.io @@ -216,6 +224,14 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing + - name: Use the Apache archive for Apache sources + # Apache mirrors keep only current releases; archive.apache.org keeps all of them. + run: | + versions=arrow/cpp/thirdparty/versions.txt + orc=$(sed -n 's/^ARROW_ORC_BUILD_VERSION=//p' "${versions}") + thrift=$(sed -n 's/^ARROW_THRIFT_BUILD_VERSION=//p' "${versions}") + echo "ARROW_ORC_URL=https://archive.apache.org/dist/orc/orc-${orc}/orc-${orc}.tar.gz" >> "${GITHUB_ENV}" + echo "ARROW_THRIFT_URL=https://archive.apache.org/dist/thrift/${thrift}/thrift-${thrift}.tar.gz" >> "${GITHUB_ENV}" - name: Set up Python uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: @@ -365,12 +381,69 @@ jobs: with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz + parquet-macos-x86_64: + name: Parquet macos-15-intel x86_64 + runs-on: macos-15-intel + timeout-minutes: 120 + needs: + - source + env: + # Consumers link this package on Intel macOS 13 hosts + MACOSX_DEPLOYMENT_TARGET: "13.0" + steps: + - name: Download source archive + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: release-source + - name: Extract source archive + run: | + tar -xf apache-arrow-java-*.tar.gz --strip-components=1 + - name: Checkout Apache Arrow C++ + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + repository: ${{github.event.inputs.arrow_repo}} + ref: ${{github.event.inputs.arrow_branch}} + path: arrow + - name: Use the Apache archive for Apache sources + # Apache mirrors keep only current releases; archive.apache.org keeps all of them. + run: | + versions=arrow/cpp/thirdparty/versions.txt + thrift=$(sed -n 's/^ARROW_THRIFT_BUILD_VERSION=//p' "${versions}") + echo "ARROW_THRIFT_URL=https://archive.apache.org/dist/thrift/${thrift}/thrift-${thrift}.tar.gz" >> "${GITHUB_ENV}" + - name: Build + # Builds only the Arrow C++ Parquet libraries: the full JNI build on this + # runner exceeds the job time limit while building LLVM for Gandiva. + run: | + cmake -S arrow/cpp -B build/cpp -GNinja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="${PWD}/build/cpp-install" \ + -DARROW_BUILD_SHARED=OFF \ + -DARROW_DEPENDENCY_SOURCE=BUNDLED \ + -DARROW_DEPENDENCY_USE_SHARED=OFF \ + -DARROW_COMPUTE=ON \ + -DARROW_PARQUET=ON \ + -DARROW_WITH_LZ4=ON \ + -DARROW_WITH_SNAPPY=ON \ + -DARROW_WITH_UTF8PROC=ON \ + -DARROW_WITH_ZLIB=ON \ + -DARROW_WITH_ZSTD=ON \ + -DPARQUET_REQUIRE_ENCRYPTION=OFF + cmake --build build/cpp --target install + ci/scripts/arrow_parquet_package.sh arrow build/cpp-install jni + - name: Compress into single artifact to keep directory structure + run: tar -cvzf parquet-macos-x86_64.tar.gz jni/ + - name: Upload artifacts + uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 + with: + name: parquet-macos-x86_64 + path: parquet-macos-x86_64.tar.gz binaries: name: Binaries runs-on: ubuntu-latest needs: - jni-linux - jni-macos + - parquet-macos-x86_64 steps: - name: Download artifacts uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 @@ -383,6 +456,7 @@ jobs: tar -xvzf jni-linux-x86_64.tar.gz tar -xvzf jni-linux-aarch_64.tar.gz tar -xvzf jni-macos-aarch_64.tar.gz + tar -xvzf parquet-macos-x86_64.tar.gz - name: Test that shared libraries exist run: | set -x @@ -402,6 +476,10 @@ jobs: test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib + for platform in linux-x86_64 linux-aarch_64 osx-x86_64 osx-aarch_64; do + test -f jni/arrow_parquet/${platform}/lib/pkgconfig/arrow-parquet.pc + done + - name: Checkout apache/arrow-testing uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: diff --git a/ci/scripts/arrow_parquet_package.sh b/ci/scripts/arrow_parquet_package.sh new file mode 100755 index 0000000000..2feb7f4df8 --- /dev/null +++ b/ci/scripts/arrow_parquet_package.sh @@ -0,0 +1,172 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Stages the static Arrow C++ Parquet libraries, headers and pkg-config +# metadata of an Arrow C++ installation into +# /arrow_parquet/-, then links and runs a smoke test +# against the staged package. +# +# Usage: arrow_parquet_package.sh [] +# arrow_dir Arrow source checkout (LICENSE.txt, NOTICE.txt, git revision) +# arrow_install_dir Arrow C++ installation prefix +# dist_dir Directory that receives arrow_parquet/- +# pkgconfig_dir Optional pkg-config directory of external dependencies (e.g. vcpkg) + +set -euo pipefail + +# shellcheck source=ci/scripts/util_log.sh +. "$(dirname "${0}")/util_log.sh" + +github_actions_group_begin "Stage Arrow Parquet package" +arrow_dir=${1} +arrow_install_dir=${2} +dist_dir=${3} +dependency_pkgconfig_dir=${4:-} +cxx=${CXX:-c++} + +case "$(uname)" in +Linux) os=linux ;; +Darwin) os=osx ;; +*) + echo "Unsupported OS: $(uname)" >&2 + exit 1 + ;; +esac +arch=$(uname -m) +case "${arch}" in +arm64 | aarch64) arch=aarch_64 ;; +esac +package_dir="${dist_dir}/arrow_parquet/${os}-${arch}" +rm -rf "${package_dir}" +mkdir -p "${package_dir}/lib/pkgconfig" "${package_dir}/META-INF" +cp -R "${arrow_install_dir}/include" "${package_dir}/" +cp "${arrow_dir}/LICENSE.txt" "${package_dir}/META-INF/LICENSE" +cp "${arrow_dir}/NOTICE.txt" "${package_dir}/META-INF/NOTICE" +git -c safe.directory="${arrow_dir}" -C "${arrow_dir}" rev-parse HEAD >"${package_dir}/ARROW_CPP_REVISION" + +parquet_metadata=$(find "${arrow_install_dir}" -type f -name parquet.pc -print -quit) +if [ -z "${parquet_metadata}" ]; then + echo "Missing installed Parquet pkg-config metadata" >&2 + exit 1 +fi +PKG_CONFIG_PATH="$(dirname "${parquet_metadata}"):${dependency_pkgconfig_dir:+${dependency_pkgconfig_dir}:}${PKG_CONFIG_PATH:-}" +export PKG_CONFIG_PATH +packages=(parquet arrow-compute) +# Bundled builds carry these in libarrow_bundled_dependencies.a; only probe +# for them when the caller supplies an external dependency installation. +if [ -n "${dependency_pkgconfig_dir}" ]; then + for dependency in re2 thrift liblz4 libzstd libutf8proc; do + if pkg-config --exists "${dependency}"; then + packages+=("${dependency}") + fi + done +fi +read -ra library_dirs <<<"$(pkg-config --static --libs-only-L "${packages[@]}")" +read -ra libraries <<<"$(pkg-config --static --libs "${packages[@]}")" +cpp_version=$(pkg-config --modversion parquet) + +# Copy every static archive into the package and keep only system libraries +# as external link dependencies. +link_flags=() +for flag in "${libraries[@]}"; do + case "${flag}" in + -L*) continue ;; + -lc | -lm | -ldl | -lpthread | -lrt | -lstdc++ | -lc++ | -lresolv) + link_flags+=("${flag}") + continue + ;; + -l*) + archive="" + for directory in "${library_dirs[@]}"; do + candidate="${directory#-L}/lib${flag#-l}.a" + if [ -f "${candidate}" ]; then + archive="${candidate}" + break + fi + done + if [ -z "${archive}" ]; then + archive=$("${cxx}" -print-file-name="lib${flag#-l}.a") + fi + if [ ! -f "${archive}" ]; then + if [ "${os}" = osx ] && [[ "${flag}" = -lz || "${flag}" = -lcurl ]]; then + link_flags+=("${flag}") + continue + fi + echo "Missing static dependency: ${flag}" >&2 + exit 1 + fi + cp "${archive}" "${package_dir}/lib/" + ;; + *.a) + cp "${flag}" "${package_dir}/lib/" + archive_name=$(basename "${flag}" .a) + flag="-l${archive_name#lib}" + ;; + esac + link_flags+=("${flag}") +done + +cat >"${package_dir}/lib/pkgconfig/arrow-parquet.pc" <"${smoke_dir}/smoke.cc" <<'EOF' +#include +#include +#include +#include +#include + +int main() { + if (!arrow::compute::Initialize().ok()) return 1; + arrow::Int64Builder builder; + if (!builder.Append(42).ok()) return 1; + auto values = builder.Finish().ValueOrDie(); + auto table = arrow::Table::Make(arrow::schema({arrow::field("value", arrow::int64())}), {values}); + for (auto codec : {parquet::Compression::SNAPPY, parquet::Compression::GZIP, + parquet::Compression::ZSTD}) { + auto output = arrow::io::BufferOutputStream::Create().ValueOrDie(); + auto properties = parquet::WriterProperties::Builder().compression(codec)->build(); + if (!parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, 1, + properties).ok()) return 1; + auto input = std::make_shared(output->Finish().ValueOrDie()); + auto reader = parquet::arrow::OpenFile(input, arrow::default_memory_pool()).ValueOrDie(); + std::shared_ptr actual; + if (!reader->ReadTable(&actual).ok() || !table->Equals(*actual)) return 1; + } +} +EOF +# Resolve flags only from the staged package to prove it is self-contained. +read -ra smoke_options <<<"$(PKG_CONFIG_PATH='' PKG_CONFIG_LIBDIR="${package_dir}/lib/pkgconfig" \ + pkg-config --cflags --libs arrow-parquet)" +"${cxx}" -std=c++20 "${smoke_dir}/smoke.cc" "${smoke_options[@]}" -o "${smoke_dir}/smoke" +"${smoke_dir}/smoke" +github_actions_group_end diff --git a/ci/scripts/jni_full_build.sh b/ci/scripts/jni_full_build.sh index 5d0aee0555..ee2573fe8a 100755 --- a/ci/scripts/jni_full_build.sh +++ b/ci/scripts/jni_full_build.sh @@ -85,6 +85,34 @@ github_actions_group_end popd github_actions_group_begin "Prepare artifacts" +# Package the staged Arrow C++ Parquet SDK as platform-classified JARs +version=$(python3 -c 'import sys, xml.etree.ElementTree as ET; print(ET.parse(sys.argv[1]).getroot().find("{http://maven.apache.org/POM/4.0.0}version").text)' "${source_dir}/pom.xml") +parquet_artifact="${dist_dir}/arrow-parquet-${version}" +parquet_dir="${jni_build_dir}/arrow_parquet" +for platform in linux-x86_64 linux-aarch_64 osx-x86_64 osx-aarch_64; do + cmp "${parquet_dir}/linux-x86_64/ARROW_CPP_REVISION" "${parquet_dir}/${platform}/ARROW_CPP_REVISION" + jar cf "${parquet_artifact}-${platform}.jar" -C "${parquet_dir}/${platform}" . +done +jar cf "${parquet_artifact}.jar" \ + -C "${parquet_dir}/linux-x86_64" META-INF \ + -C "${parquet_dir}/linux-x86_64" ARROW_CPP_REVISION +cat >"${parquet_artifact}.pom" < + 4.0.0 + org.apache.arrow + arrow-parquet + ${version} + jar + Arrow Parquet + Static Arrow C++ Parquet libraries, headers and pkg-config metadata + + + Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + +EOF # copy all jar, zip and pom files to the distribution folder find ~/.m2/repository/org/apache/arrow \ "(" \ diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index 89846c2cb7..836582ca08 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -209,6 +209,11 @@ export JAVA_JNI_CMAKE_ARGS="${jni_cmake_args}" "${install_dir}" \ "${build_dir}" \ "${dist_dir}" +"${source_dir}/ci/scripts/arrow_parquet_package.sh" \ + "${arrow_dir}" \ + "${install_dir}" \ + "${dist_dir}" \ + "${VCPKG_ROOT_LOCAL:+${VCPKG_ROOT_LOCAL}/installed/${vcpkg_triplet}/lib/pkgconfig}" if [ "${ARROW_USE_CCACHE}" == "ON" ]; then github_actions_group_begin "ccache statistics after build" diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index c750031f35..3200b5cbb0 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -104,6 +104,11 @@ export JAVA_JNI_CMAKE_ARGS "${install_dir}" \ "${build_dir}" \ "${dist_dir}" +"${source_dir}/ci/scripts/arrow_parquet_package.sh" \ + "${arrow_dir}" \ + "${install_dir}" \ + "${dist_dir}" \ + "${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/lib/pkgconfig" if [ "${ARROW_USE_CCACHE}" == "ON" ]; then github_actions_group_begin "ccache statistics after build" diff --git a/compose.yaml b/compose.yaml index f5082a22aa..9acd46b9e2 100644 --- a/compose.yaml +++ b/compose.yaml @@ -107,6 +107,8 @@ services: - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated environment: ARROW_JAVA_CDATA: "ON" + ARROW_ORC_URL: + ARROW_THRIFT_URL: CCACHE_DIR: "/ccache" command: ["git config --global --add safe.directory /arrow-java && \ From 9282696fbdfda9e3cbdca9a446f4b4241f59ffb0 Mon Sep 17 00:00:00 2001 From: bodduv Date: Thu, 15 Oct 2026 16:33:52 +0200 Subject: [PATCH 2/2] BDX-1692: Drop the macOS x86_64 Arrow Parquet package Arrow Java ships no macOS x86_64 JNI libraries, so arrow-parquet follows the same platforms: linux-x86_64, linux-aarch_64 and osx-aarch_64. Remove the Parquet-only Intel job and the Thrift archive URL it needed; keep the ORC archive URL for the JNI builds. Assign pkg-config output before splitting it, so that a pkg-config failure stops the packaging script. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/jarbuild.yml | 74 ++--------------------------- ci/scripts/arrow_parquet_package.sh | 16 ++++--- ci/scripts/jni_full_build.sh | 2 +- compose.yaml | 1 - 4 files changed, 16 insertions(+), 77 deletions(-) diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index 001822a39e..384e61115c 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -149,14 +149,11 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - name: Use the Apache archive for Apache sources + - name: Use the Apache archive for the ORC source # Apache mirrors keep only current releases; archive.apache.org keeps all of them. run: | - versions=arrow/cpp/thirdparty/versions.txt - orc=$(sed -n 's/^ARROW_ORC_BUILD_VERSION=//p' "${versions}") - thrift=$(sed -n 's/^ARROW_THRIFT_BUILD_VERSION=//p' "${versions}") + orc=$(sed -n 's/^ARROW_ORC_BUILD_VERSION=//p' arrow/cpp/thirdparty/versions.txt) echo "ARROW_ORC_URL=https://archive.apache.org/dist/orc/orc-${orc}/orc-${orc}.tar.gz" >> "${GITHUB_ENV}" - echo "ARROW_THRIFT_URL=https://archive.apache.org/dist/thrift/${thrift}/thrift-${thrift}.tar.gz" >> "${GITHUB_ENV}" - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: ghcr.io @@ -224,14 +221,11 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing - - name: Use the Apache archive for Apache sources + - name: Use the Apache archive for the ORC source # Apache mirrors keep only current releases; archive.apache.org keeps all of them. run: | - versions=arrow/cpp/thirdparty/versions.txt - orc=$(sed -n 's/^ARROW_ORC_BUILD_VERSION=//p' "${versions}") - thrift=$(sed -n 's/^ARROW_THRIFT_BUILD_VERSION=//p' "${versions}") + orc=$(sed -n 's/^ARROW_ORC_BUILD_VERSION=//p' arrow/cpp/thirdparty/versions.txt) echo "ARROW_ORC_URL=https://archive.apache.org/dist/orc/orc-${orc}/orc-${orc}.tar.gz" >> "${GITHUB_ENV}" - echo "ARROW_THRIFT_URL=https://archive.apache.org/dist/thrift/${thrift}/thrift-${thrift}.tar.gz" >> "${GITHUB_ENV}" - name: Set up Python uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: @@ -381,69 +375,12 @@ jobs: with: name: jni-macos-${{ matrix.platform.arch }} path: jni-macos-${{ matrix.platform.arch }}.tar.gz - parquet-macos-x86_64: - name: Parquet macos-15-intel x86_64 - runs-on: macos-15-intel - timeout-minutes: 120 - needs: - - source - env: - # Consumers link this package on Intel macOS 13 hosts - MACOSX_DEPLOYMENT_TARGET: "13.0" - steps: - - name: Download source archive - uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 - with: - name: release-source - - name: Extract source archive - run: | - tar -xf apache-arrow-java-*.tar.gz --strip-components=1 - - name: Checkout Apache Arrow C++ - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - repository: ${{github.event.inputs.arrow_repo}} - ref: ${{github.event.inputs.arrow_branch}} - path: arrow - - name: Use the Apache archive for Apache sources - # Apache mirrors keep only current releases; archive.apache.org keeps all of them. - run: | - versions=arrow/cpp/thirdparty/versions.txt - thrift=$(sed -n 's/^ARROW_THRIFT_BUILD_VERSION=//p' "${versions}") - echo "ARROW_THRIFT_URL=https://archive.apache.org/dist/thrift/${thrift}/thrift-${thrift}.tar.gz" >> "${GITHUB_ENV}" - - name: Build - # Builds only the Arrow C++ Parquet libraries: the full JNI build on this - # runner exceeds the job time limit while building LLVM for Gandiva. - run: | - cmake -S arrow/cpp -B build/cpp -GNinja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX="${PWD}/build/cpp-install" \ - -DARROW_BUILD_SHARED=OFF \ - -DARROW_DEPENDENCY_SOURCE=BUNDLED \ - -DARROW_DEPENDENCY_USE_SHARED=OFF \ - -DARROW_COMPUTE=ON \ - -DARROW_PARQUET=ON \ - -DARROW_WITH_LZ4=ON \ - -DARROW_WITH_SNAPPY=ON \ - -DARROW_WITH_UTF8PROC=ON \ - -DARROW_WITH_ZLIB=ON \ - -DARROW_WITH_ZSTD=ON \ - -DPARQUET_REQUIRE_ENCRYPTION=OFF - cmake --build build/cpp --target install - ci/scripts/arrow_parquet_package.sh arrow build/cpp-install jni - - name: Compress into single artifact to keep directory structure - run: tar -cvzf parquet-macos-x86_64.tar.gz jni/ - - name: Upload artifacts - uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0 - with: - name: parquet-macos-x86_64 - path: parquet-macos-x86_64.tar.gz binaries: name: Binaries runs-on: ubuntu-latest needs: - jni-linux - jni-macos - - parquet-macos-x86_64 steps: - name: Download artifacts uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 @@ -456,7 +393,6 @@ jobs: tar -xvzf jni-linux-x86_64.tar.gz tar -xvzf jni-linux-aarch_64.tar.gz tar -xvzf jni-macos-aarch_64.tar.gz - tar -xvzf parquet-macos-x86_64.tar.gz - name: Test that shared libraries exist run: | set -x @@ -476,7 +412,7 @@ jobs: test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib - for platform in linux-x86_64 linux-aarch_64 osx-x86_64 osx-aarch_64; do + for platform in linux-x86_64 linux-aarch_64 osx-aarch_64; do test -f jni/arrow_parquet/${platform}/lib/pkgconfig/arrow-parquet.pc done diff --git a/ci/scripts/arrow_parquet_package.sh b/ci/scripts/arrow_parquet_package.sh index 2feb7f4df8..bca1ae12f7 100755 --- a/ci/scripts/arrow_parquet_package.sh +++ b/ci/scripts/arrow_parquet_package.sh @@ -68,8 +68,8 @@ fi PKG_CONFIG_PATH="$(dirname "${parquet_metadata}"):${dependency_pkgconfig_dir:+${dependency_pkgconfig_dir}:}${PKG_CONFIG_PATH:-}" export PKG_CONFIG_PATH packages=(parquet arrow-compute) -# Bundled builds carry these in libarrow_bundled_dependencies.a; only probe -# for them when the caller supplies an external dependency installation. +# Without an external dependency installation, Arrow bundles these into +# libarrow_bundled_dependencies.a; probing would pick up unrelated host copies. if [ -n "${dependency_pkgconfig_dir}" ]; then for dependency in re2 thrift liblz4 libzstd libutf8proc; do if pkg-config --exists "${dependency}"; then @@ -77,8 +77,11 @@ if [ -n "${dependency_pkgconfig_dir}" ]; then fi done fi -read -ra library_dirs <<<"$(pkg-config --static --libs-only-L "${packages[@]}")" -read -ra libraries <<<"$(pkg-config --static --libs "${packages[@]}")" +# Assign before splitting so that a pkg-config failure stops the script. +library_flags=$(pkg-config --static --libs-only-L "${packages[@]}") +read -ra library_dirs <<<"${library_flags}" +library_flags=$(pkg-config --static --libs "${packages[@]}") +read -ra libraries <<<"${library_flags}" cpp_version=$(pkg-config --modversion parquet) # Copy every static archive into the package and keep only system libraries @@ -165,8 +168,9 @@ int main() { } EOF # Resolve flags only from the staged package to prove it is self-contained. -read -ra smoke_options <<<"$(PKG_CONFIG_PATH='' PKG_CONFIG_LIBDIR="${package_dir}/lib/pkgconfig" \ - pkg-config --cflags --libs arrow-parquet)" +smoke_flags=$(PKG_CONFIG_PATH='' PKG_CONFIG_LIBDIR="${package_dir}/lib/pkgconfig" \ + pkg-config --cflags --libs arrow-parquet) +read -ra smoke_options <<<"${smoke_flags}" "${cxx}" -std=c++20 "${smoke_dir}/smoke.cc" "${smoke_options[@]}" -o "${smoke_dir}/smoke" "${smoke_dir}/smoke" github_actions_group_end diff --git a/ci/scripts/jni_full_build.sh b/ci/scripts/jni_full_build.sh index ee2573fe8a..0285f1e4ac 100755 --- a/ci/scripts/jni_full_build.sh +++ b/ci/scripts/jni_full_build.sh @@ -89,7 +89,7 @@ github_actions_group_begin "Prepare artifacts" version=$(python3 -c 'import sys, xml.etree.ElementTree as ET; print(ET.parse(sys.argv[1]).getroot().find("{http://maven.apache.org/POM/4.0.0}version").text)' "${source_dir}/pom.xml") parquet_artifact="${dist_dir}/arrow-parquet-${version}" parquet_dir="${jni_build_dir}/arrow_parquet" -for platform in linux-x86_64 linux-aarch_64 osx-x86_64 osx-aarch_64; do +for platform in linux-x86_64 linux-aarch_64 osx-aarch_64; do cmp "${parquet_dir}/linux-x86_64/ARROW_CPP_REVISION" "${parquet_dir}/${platform}/ARROW_CPP_REVISION" jar cf "${parquet_artifact}-${platform}.jar" -C "${parquet_dir}/${platform}" . done diff --git a/compose.yaml b/compose.yaml index 9acd46b9e2..a3a8654537 100644 --- a/compose.yaml +++ b/compose.yaml @@ -108,7 +108,6 @@ services: environment: ARROW_JAVA_CDATA: "ON" ARROW_ORC_URL: - ARROW_THRIFT_URL: CCACHE_DIR: "/ccache" command: ["git config --global --add safe.directory /arrow-java && \