Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions .github/workflows/jarbuild.yml
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,11 @@ jobs:
with:
repository: apache/parquet-testing
path: arrow/cpp/submodules/parquet-testing
- name: Use the Apache archive for the ORC source
# Apache mirrors keep only current releases; archive.apache.org keeps all of them.
run: |
orc=$(sed -n 's/^ARROW_ORC_BUILD_VERSION=//p' arrow/cpp/thirdparty/versions.txt)
echo "ARROW_ORC_URL=https://archive.apache.org/dist/orc/orc-${orc}/orc-${orc}.tar.gz" >> "${GITHUB_ENV}"
- uses: docker/login-action@9780b0c442fbb1117ed29e0efdff1e18412f7567 # v3.3.0
with:
registry: ghcr.io
Expand Down Expand Up @@ -216,6 +221,11 @@ jobs:
with:
repository: apache/parquet-testing
path: arrow/cpp/submodules/parquet-testing
- name: Use the Apache archive for the ORC source
# Apache mirrors keep only current releases; archive.apache.org keeps all of them.
run: |
orc=$(sed -n 's/^ARROW_ORC_BUILD_VERSION=//p' arrow/cpp/thirdparty/versions.txt)
echo "ARROW_ORC_URL=https://archive.apache.org/dist/orc/orc-${orc}/orc-${orc}.tar.gz" >> "${GITHUB_ENV}"
- name: Set up Python
uses: actions/setup-python@0b93645e9fea7318ecaed2b359559ac225c90a2b # v5.3.0
with:
Expand Down Expand Up @@ -402,6 +412,10 @@ jobs:
test -f jni/arrow_orc_jni/aarch_64/libarrow_orc_jni.dylib
test -f jni/gandiva_jni/aarch_64/libgandiva_jni.dylib

for platform in linux-x86_64 linux-aarch_64 osx-aarch_64; do
test -f jni/arrow_parquet/${platform}/lib/pkgconfig/arrow-parquet.pc
done

- name: Checkout apache/arrow-testing
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
Expand Down
176 changes: 176 additions & 0 deletions ci/scripts/arrow_parquet_package.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
#!/usr/bin/env bash
#
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.

# Stages the static Arrow C++ Parquet libraries, headers and pkg-config
# metadata of an Arrow C++ installation into
# <dist_dir>/arrow_parquet/<os>-<arch>, then links and runs a smoke test
# against the staged package.
#
# Usage: arrow_parquet_package.sh <arrow_dir> <arrow_install_dir> <dist_dir> [<pkgconfig_dir>]
# arrow_dir Arrow source checkout (LICENSE.txt, NOTICE.txt, git revision)
# arrow_install_dir Arrow C++ installation prefix
# dist_dir Directory that receives arrow_parquet/<os>-<arch>
# pkgconfig_dir Optional pkg-config directory of external dependencies (e.g. vcpkg)

set -euo pipefail

# shellcheck source=ci/scripts/util_log.sh
. "$(dirname "${0}")/util_log.sh"

github_actions_group_begin "Stage Arrow Parquet package"
arrow_dir=${1}
arrow_install_dir=${2}
dist_dir=${3}
dependency_pkgconfig_dir=${4:-}
cxx=${CXX:-c++}

case "$(uname)" in
Linux) os=linux ;;
Darwin) os=osx ;;
*)
echo "Unsupported OS: $(uname)" >&2
exit 1
;;
esac
arch=$(uname -m)
case "${arch}" in
arm64 | aarch64) arch=aarch_64 ;;
esac
package_dir="${dist_dir}/arrow_parquet/${os}-${arch}"
rm -rf "${package_dir}"
mkdir -p "${package_dir}/lib/pkgconfig" "${package_dir}/META-INF"
cp -R "${arrow_install_dir}/include" "${package_dir}/"
cp "${arrow_dir}/LICENSE.txt" "${package_dir}/META-INF/LICENSE"
cp "${arrow_dir}/NOTICE.txt" "${package_dir}/META-INF/NOTICE"
git -c safe.directory="${arrow_dir}" -C "${arrow_dir}" rev-parse HEAD >"${package_dir}/ARROW_CPP_REVISION"

parquet_metadata=$(find "${arrow_install_dir}" -type f -name parquet.pc -print -quit)
if [ -z "${parquet_metadata}" ]; then
echo "Missing installed Parquet pkg-config metadata" >&2
exit 1
fi
PKG_CONFIG_PATH="$(dirname "${parquet_metadata}"):${dependency_pkgconfig_dir:+${dependency_pkgconfig_dir}:}${PKG_CONFIG_PATH:-}"
export PKG_CONFIG_PATH
packages=(parquet arrow-compute)
# Without an external dependency installation, Arrow bundles these into
# libarrow_bundled_dependencies.a; probing would pick up unrelated host copies.
if [ -n "${dependency_pkgconfig_dir}" ]; then
for dependency in re2 thrift liblz4 libzstd libutf8proc; do
if pkg-config --exists "${dependency}"; then
packages+=("${dependency}")
fi
done
fi
# Assign before splitting so that a pkg-config failure stops the script.
library_flags=$(pkg-config --static --libs-only-L "${packages[@]}")
read -ra library_dirs <<<"${library_flags}"
library_flags=$(pkg-config --static --libs "${packages[@]}")
read -ra libraries <<<"${library_flags}"
cpp_version=$(pkg-config --modversion parquet)

# Copy every static archive into the package and keep only system libraries
# as external link dependencies.
link_flags=()
for flag in "${libraries[@]}"; do
case "${flag}" in
-L*) continue ;;
-lc | -lm | -ldl | -lpthread | -lrt | -lstdc++ | -lc++ | -lresolv)
link_flags+=("${flag}")
continue
;;
-l*)
archive=""
for directory in "${library_dirs[@]}"; do
candidate="${directory#-L}/lib${flag#-l}.a"
if [ -f "${candidate}" ]; then
archive="${candidate}"
break
fi
done
if [ -z "${archive}" ]; then
archive=$("${cxx}" -print-file-name="lib${flag#-l}.a")
fi
if [ ! -f "${archive}" ]; then
if [ "${os}" = osx ] && [[ "${flag}" = -lz || "${flag}" = -lcurl ]]; then
link_flags+=("${flag}")
continue
fi
echo "Missing static dependency: ${flag}" >&2
exit 1
fi
cp "${archive}" "${package_dir}/lib/"
;;
*.a)
cp "${flag}" "${package_dir}/lib/"
archive_name=$(basename "${flag}" .a)
flag="-l${archive_name#lib}"
;;
esac
link_flags+=("${flag}")
done

cat >"${package_dir}/lib/pkgconfig/arrow-parquet.pc" <<EOF
prefix=\${pcfiledir}/../..
libdir=\${prefix}/lib
includedir=\${prefix}/include

Name: Arrow Parquet
Description: Static Arrow C++ Parquet libraries (C++20)
Version: ${cpp_version}
Libs: -L\${libdir} ${link_flags[*]}
Cflags: -I\${includedir} -DARROW_STATIC -DARROW_COMPUTE_STATIC -DPARQUET_STATIC
EOF
github_actions_group_end

github_actions_group_begin "Verify Arrow Parquet package"
smoke_dir=$(mktemp -d)
trap 'rm -rf "${smoke_dir}"' EXIT
cat >"${smoke_dir}/smoke.cc" <<'EOF'
#include <arrow/api.h>
#include <arrow/compute/initialize.h>
#include <arrow/io/api.h>
#include <parquet/arrow/reader.h>
#include <parquet/arrow/writer.h>

int main() {
if (!arrow::compute::Initialize().ok()) return 1;
arrow::Int64Builder builder;
if (!builder.Append(42).ok()) return 1;
auto values = builder.Finish().ValueOrDie();
auto table = arrow::Table::Make(arrow::schema({arrow::field("value", arrow::int64())}), {values});
for (auto codec : {parquet::Compression::SNAPPY, parquet::Compression::GZIP,
parquet::Compression::ZSTD}) {
auto output = arrow::io::BufferOutputStream::Create().ValueOrDie();
auto properties = parquet::WriterProperties::Builder().compression(codec)->build();
if (!parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), output, 1,
properties).ok()) return 1;
auto input = std::make_shared<arrow::io::BufferReader>(output->Finish().ValueOrDie());
auto reader = parquet::arrow::OpenFile(input, arrow::default_memory_pool()).ValueOrDie();
std::shared_ptr<arrow::Table> actual;
if (!reader->ReadTable(&actual).ok() || !table->Equals(*actual)) return 1;
}
}
EOF
# Resolve flags only from the staged package to prove it is self-contained.
smoke_flags=$(PKG_CONFIG_PATH='' PKG_CONFIG_LIBDIR="${package_dir}/lib/pkgconfig" \
pkg-config --cflags --libs arrow-parquet)
read -ra smoke_options <<<"${smoke_flags}"
"${cxx}" -std=c++20 "${smoke_dir}/smoke.cc" "${smoke_options[@]}" -o "${smoke_dir}/smoke"
"${smoke_dir}/smoke"
github_actions_group_end
28 changes: 28 additions & 0 deletions ci/scripts/jni_full_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -85,6 +85,34 @@ github_actions_group_end
popd

github_actions_group_begin "Prepare artifacts"
# Package the staged Arrow C++ Parquet SDK as platform-classified JARs
version=$(python3 -c 'import sys, xml.etree.ElementTree as ET; print(ET.parse(sys.argv[1]).getroot().find("{http://maven.apache.org/POM/4.0.0}version").text)' "${source_dir}/pom.xml")
parquet_artifact="${dist_dir}/arrow-parquet-${version}"
parquet_dir="${jni_build_dir}/arrow_parquet"
for platform in linux-x86_64 linux-aarch_64 osx-aarch_64; do
cmp "${parquet_dir}/linux-x86_64/ARROW_CPP_REVISION" "${parquet_dir}/${platform}/ARROW_CPP_REVISION"
jar cf "${parquet_artifact}-${platform}.jar" -C "${parquet_dir}/${platform}" .
done
jar cf "${parquet_artifact}.jar" \
-C "${parquet_dir}/linux-x86_64" META-INF \
-C "${parquet_dir}/linux-x86_64" ARROW_CPP_REVISION
cat >"${parquet_artifact}.pom" <<EOF
<project xmlns="http://maven.apache.org/POM/4.0.0">
<modelVersion>4.0.0</modelVersion>
<groupId>org.apache.arrow</groupId>
<artifactId>arrow-parquet</artifactId>
<version>${version}</version>
<packaging>jar</packaging>
<name>Arrow Parquet</name>
<description>Static Arrow C++ Parquet libraries, headers and pkg-config metadata</description>
<licenses>
<license>
<name>Apache License, Version 2.0</name>
<url>https://www.apache.org/licenses/LICENSE-2.0.txt</url>
</license>
</licenses>
</project>
EOF
# copy all jar, zip and pom files to the distribution folder
find ~/.m2/repository/org/apache/arrow \
"(" \
Expand Down
5 changes: 5 additions & 0 deletions ci/scripts/jni_macos_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -209,6 +209,11 @@ export JAVA_JNI_CMAKE_ARGS="${jni_cmake_args}"
"${install_dir}" \
"${build_dir}" \
"${dist_dir}"
"${source_dir}/ci/scripts/arrow_parquet_package.sh" \
"${arrow_dir}" \
"${install_dir}" \
"${dist_dir}" \
"${VCPKG_ROOT_LOCAL:+${VCPKG_ROOT_LOCAL}/installed/${vcpkg_triplet}/lib/pkgconfig}"

if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
github_actions_group_begin "ccache statistics after build"
Expand Down
5 changes: 5 additions & 0 deletions ci/scripts/jni_manylinux_build.sh
Original file line number Diff line number Diff line change
Expand Up @@ -104,6 +104,11 @@ export JAVA_JNI_CMAKE_ARGS
"${install_dir}" \
"${build_dir}" \
"${dist_dir}"
"${source_dir}/ci/scripts/arrow_parquet_package.sh" \
"${arrow_dir}" \
"${install_dir}" \
"${dist_dir}" \
"${VCPKG_ROOT}/installed/${VCPKG_TARGET_TRIPLET}/lib/pkgconfig"

if [ "${ARROW_USE_CCACHE}" == "ON" ]; then
github_actions_group_begin "ccache statistics after build"
Expand Down
1 change: 1 addition & 0 deletions compose.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -107,6 +107,7 @@ services:
- ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
environment:
ARROW_JAVA_CDATA: "ON"
ARROW_ORC_URL:
CCACHE_DIR: "/ccache"
command:
["git config --global --add safe.directory /arrow-java && \
Expand Down
Loading