diff --git a/.github/workflows/jarbuild.yml b/.github/workflows/jarbuild.yml index 28a9377944..384e61115c 100644 --- a/.github/workflows/jarbuild.yml +++ b/.github/workflows/jarbuild.yml @@ -149,6 +149,11 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing + - name: Use the Apache archive for the ORC source + # Apache mirrors keep only current releases; archive.apache.org keeps all of them. + run: | + orc=$(sed -n 's/^ARROW_ORC_BUILD_VERSION=//p' arrow/cpp/thirdparty/versions.txt) + echo "ARROW_ORC_URL=https://archive.apache.org/dist/orc/orc-${orc}/orc-${orc}.tar.gz" >> "${GITHUB_ENV}" - uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0 with: registry: ghcr.io @@ -216,6 +221,11 @@ jobs: with: repository: apache/parquet-testing path: arrow/cpp/submodules/parquet-testing + - name: Use the Apache archive for the ORC source + # Apache mirrors keep only current releases; archive.apache.org keeps all of them. + run: | + orc=$(sed -n 's/^ARROW_ORC_BUILD_VERSION=//p' arrow/cpp/thirdparty/versions.txt) + echo "ARROW_ORC_URL=https://archive.apache.org/dist/orc/orc-${orc}/orc-${orc}.tar.gz" >> "${GITHUB_ENV}" - name: Set up Python uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0 with: @@ -402,6 +412,10 @@ jobs: test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib + for platform in linux-x86_64 linux-aarch_64 osx-aarch_64; do + test -f jni/arrow_parquet/${platform}/lib/pkgconfig/arrow-parquet.pc + done + - name: Checkout apache/arrow-testing uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: diff --git a/ci/scripts/arrow_parquet_package.sh b/ci/scripts/arrow_parquet_package.sh new file mode 100755 index 0000000000..bca1ae12f7 --- /dev/null +++ b/ci/scripts/arrow_parquet_package.sh @@ -0,0 +1,176 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Stages the static Arrow C++ Parquet libraries, headers and pkg-config +# metadata of an Arrow C++ installation into +# /arrow_parquet/-, then links and runs a smoke test +# against the staged package. +# +# Usage: arrow_parquet_package.sh [] +# arrow_dir Arrow source checkout (LICENSE.txt, NOTICE.txt, git revision) +# arrow_install_dir Arrow C++ installation prefix +# dist_dir Directory that receives arrow_parquet/- +# pkgconfig_dir Optional pkg-config directory of external dependencies (e.g. vcpkg) + +set -euo pipefail + +# shellcheck source=ci/scripts/util_log.sh +. "$(dirname "${0}")/util_log.sh" + +github_actions_group_begin "Stage Arrow Parquet package" +arrow_dir=${1} +arrow_install_dir=${2} +dist_dir=${3} +dependency_pkgconfig_dir=${4:-} +cxx=${CXX:-c++} + +case "$(uname)" in +Linux) os=linux ;; +Darwin) os=osx ;; +*) + echo "Unsupported OS: $(uname)" >&2 + exit 1 + ;; +esac +arch=$(uname -m) +case "${arch}" in +arm64 | aarch64) arch=aarch_64 ;; +esac +package_dir="${dist_dir}/arrow_parquet/${os}-${arch}" +rm -rf "${package_dir}" +mkdir -p "${package_dir}/lib/pkgconfig" "${package_dir}/META-INF" +cp -R "${arrow_install_dir}/include" "${package_dir}/" +cp "${arrow_dir}/LICENSE.txt" "${package_dir}/META-INF/LICENSE" +cp "${arrow_dir}/NOTICE.txt" "${package_dir}/META-INF/NOTICE" +git -c safe.directory="${arrow_dir}" -C "${arrow_dir}" rev-parse HEAD >"${package_dir}/ARROW_CPP_REVISION" + +parquet_metadata=$(find "${arrow_install_dir}" -type f -name parquet.pc -print -quit) +if [ -z "${parquet_metadata}" ]; then + echo "Missing installed Parquet pkg-config metadata" >&2 + exit 1 +fi +PKG_CONFIG_PATH="$(dirname "${parquet_metadata}"):${dependency_pkgconfig_dir:+${dependency_pkgconfig_dir}:}${PKG_CONFIG_PATH:-}" +export PKG_CONFIG_PATH +packages=(parquet arrow-compute) +# Without an external dependency installation, Arrow bundles these into +# libarrow_bundled_dependencies.a; probing would pick up unrelated host copies. +if [ -n "${dependency_pkgconfig_dir}" ]; then + for dependency in re2 thrift liblz4 libzstd libutf8proc; do + if pkg-config --exists "${dependency}"; then + packages+=("${dependency}") + fi + done +fi +# Assign before splitting so that a pkg-config failure stops the script. +library_flags=$(pkg-config --static --libs-only-L "${packages[@]}") +read -ra library_dirs <<<"${library_flags}" +library_flags=$(pkg-config --static --libs "${packages[@]}") +read -ra libraries <<<"${library_flags}" +cpp_version=$(pkg-config --modversion parquet) + +# Copy every static archive into the package and keep only system libraries +# as external link dependencies. +link_flags=() +for flag in "${libraries[@]}"; do + case "${flag}" in + -L*) continue ;; + -lc | -lm | -ldl | -lpthread | -lrt | -lstdc++ | -lc++ | -lresolv) + link_flags+=("${flag}") + continue + ;; + -l*) + archive="" + for directory in "${library_dirs[@]}"; do + candidate="${directory#-L}/lib${flag#-l}.a" + if [ -f "${candidate}" ]; then + archive="${candidate}" + break + fi + done + if [ -z "${archive}" ]; then + archive=$("${cxx}" -print-file-name="lib${flag#-l}.a") + fi + if [ ! -f "${archive}" ]; then + if [ "${os}" = osx ] && [[ "${flag}" = -lz || "${flag}" = -lcurl ]]; then + link_flags+=("${flag}") + continue + fi + echo "Missing static dependency: ${flag}" >&2 + exit 1 + fi + cp "${archive}" "${package_dir}/lib/" + ;; + *.a) + cp "${flag}" "${package_dir}/lib/" + archive_name=$(basename "${flag}" .a) + flag="-l${archive_name#lib}" + ;; + esac + link_flags+=("${flag}") +done + +cat >"${package_dir}/lib/pkgconfig/arrow-parquet.pc" <"${smoke_dir}/smoke.cc" <<'EOF' +#include +#include +#include +#include +#include + +int main() { + if (!arrow::compute::Initialize().ok()) return 1; + arrow::Int64Builder builder; + if (!builder.Append(42).ok()) return 1; + auto values = builder.Finish().ValueOrDie(); + auto table = arrow::Table::Make(arrow::schema({arrow::field("value", arrow::int64())}), {values}); + for (auto codec : {parquet::Compression::SNAPPY, parquet::Compression::GZIP, + parquet::Compression::ZSTD}) { + auto output = arrow::io::BufferOutputStream::Create().ValueOrDie(); + auto properties = parquet::WriterProperties::Builder().compression(codec)->build(); + if (!parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, 1, + properties).ok()) return 1; + auto input = std::make_shared(output->Finish().ValueOrDie()); + auto reader = parquet::arrow::OpenFile(input, arrow::default_memory_pool()).ValueOrDie(); + std::shared_ptr actual; + if (!reader->ReadTable(&actual).ok() || !table->Equals(*actual)) return 1; + } +} +EOF +# Resolve flags only from the staged package to prove it is self-contained. +smoke_flags=$(PKG_CONFIG_PATH='' PKG_CONFIG_LIBDIR="${package_dir}/lib/pkgconfig" \ + pkg-config --cflags --libs arrow-parquet) +read -ra smoke_options <<<"${smoke_flags}" +"${cxx}" -std=c++20 "${smoke_dir}/smoke.cc" "${smoke_options[@]}" -o "${smoke_dir}/smoke" +"${smoke_dir}/smoke" +github_actions_group_end diff --git a/ci/scripts/jni_full_build.sh b/ci/scripts/jni_full_build.sh index 5d0aee0555..0285f1e4ac 100755 --- a/ci/scripts/jni_full_build.sh +++ b/ci/scripts/jni_full_build.sh @@ -85,6 +85,34 @@ github_actions_group_end popd github_actions_group_begin "Prepare artifacts" +# Package the staged Arrow C++ Parquet SDK as platform-classified JARs +version=$(python3 -c 'import sys, xml.etree.ElementTree as ET; print(ET.parse(sys.argv[1]).getroot().find("{http://maven.apache.org/POM/4.0.0}version").text)' "${source_dir}/pom.xml") +parquet_artifact="${dist_dir}/arrow-parquet-${version}" +parquet_dir="${jni_build_dir}/arrow_parquet" +for platform in linux-x86_64 linux-aarch_64 osx-aarch_64; do + cmp "${parquet_dir}/linux-x86_64/ARROW_CPP_REVISION" "${parquet_dir}/${platform}/ARROW_CPP_REVISION" + jar cf "${parquet_artifact}-${platform}.jar" -C "${parquet_dir}/${platform}" . +done +jar cf "${parquet_artifact}.jar" \ + -C "${parquet_dir}/linux-x86_64" META-INF \ + -C "${parquet_dir}/linux-x86_64" ARROW_CPP_REVISION +cat >"${parquet_artifact}.pom" < + 4.0.0 + org.apache.arrow + arrow-parquet + ${version} + jar + Arrow Parquet + Static Arrow C++ Parquet libraries, headers and pkg-config metadata + + + Apache License, Version 2.0 + https://www.apache.org/licenses/LICENSE-2.0.txt + + + +EOF # copy all jar, zip and pom files to the distribution folder find ~/.m2/repository/org/apache/arrow \ "(" \ diff --git a/ci/scripts/jni_macos_build.sh b/ci/scripts/jni_macos_build.sh index 89846c2cb7..836582ca08 100755 --- a/ci/scripts/jni_macos_build.sh +++ b/ci/scripts/jni_macos_build.sh @@ -209,6 +209,11 @@ export JAVA_JNI_CMAKE_ARGS="${jni_cmake_args}" "${install_dir}" \ "${build_dir}" \ "${dist_dir}" +"${source_dir}/ci/scripts/arrow_parquet_package.sh" \ + "${arrow_dir}" \ + "${install_dir}" \ + "${dist_dir}" \ + "${VCPKG_ROOT_LOCAL:+${VCPKG_ROOT_LOCAL}/installed/${vcpkg_triplet}/lib/pkgconfig}" if [ "${ARROW_USE_CCACHE}" == "ON" ]; then github_actions_group_begin "ccache statistics after build" diff --git a/ci/scripts/jni_manylinux_build.sh b/ci/scripts/jni_manylinux_build.sh index c750031f35..3200b5cbb0 100755 --- a/ci/scripts/jni_manylinux_build.sh +++ b/ci/scripts/jni_manylinux_build.sh @@ -104,6 +104,11 @@ export JAVA_JNI_CMAKE_ARGS "${install_dir}" \ "${build_dir}" \ "${dist_dir}" +"${source_dir}/ci/scripts/arrow_parquet_package.sh" \ + "${arrow_dir}" \ + "${install_dir}" \ + "${dist_dir}" \ + "${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/lib/pkgconfig" if [ "${ARROW_USE_CCACHE}" == "ON" ]; then github_actions_group_begin "ccache statistics after build" diff --git a/compose.yaml b/compose.yaml index f5082a22aa..a3a8654537 100644 --- a/compose.yaml +++ b/compose.yaml @@ -107,6 +107,7 @@ services: - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated environment: ARROW_JAVA_CDATA: "ON" + ARROW_ORC_URL: CCACHE_DIR: "/ccache" command: ["git config --global --add safe.directory /arrow-java && \