diff --git a/.github/actions/java-test/action.yaml b/.github/actions/java-test/action.yaml
index 78eebf40e87..0b586292288 100644
--- a/.github/actions/java-test/action.yaml
+++ b/.github/actions/java-test/action.yaml
@@ -81,6 +81,10 @@ runs:
# cache) by setup-builder / setup-macos-builder via maven-bootstrap, which
# every caller of this action runs first.
+ - name: Resolve Maven model before tests
+ shell: bash
+ run: dev/ci/prepare-maven-model.sh ${{ inputs.maven_opts }}
+
- name: Run all tests
shell: bash
if: ${{ inputs.suites == '' }}
diff --git a/.github/actions/rust-test/action.yaml b/.github/actions/rust-test/action.yaml
index ab5af0b68ca..cf65e10f804 100644
--- a/.github/actions/rust-test/action.yaml
+++ b/.github/actions/rust-test/action.yaml
@@ -37,13 +37,16 @@ runs:
shell: bash
run: |
cd native
- cargo clippy --color=never --all-targets --workspace -- -D warnings
+ # Published DataFusion 55.2 uses fetch_update, deprecated by Rust 1.100.
+ # Preserve its sources and lint every Comet member strictly; vendor unit
+ # tests below remain explicit and mandatory.
+ cargo clippy --locked --color=never --all-targets --workspace --exclude datafusion-physical-plan --no-deps -- -D warnings
- name: Check native Python UDF feature
shell: bash
run: |
cd native
- cargo clippy --color=never -p datafusion-comet --all-targets --features python-udf -- -D warnings
+ cargo clippy --locked --color=never -p datafusion-comet --all-targets --features python-udf --no-deps -- -D warnings
- name: Check compilation
shell: bash
@@ -74,8 +77,7 @@ runs:
- name: Build common module (pre-requisite for Rust tests)
shell: bash
run: |
- cd common
- ../mvnw -B clean compile -DskipTests
+ ./mvnw -B compile -DskipTests
- name: Install nextest
shell: bash
@@ -106,6 +108,16 @@ runs:
export LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH}
RUST_BACKTRACE=1 cargo nextest run -p datafusion-comet --lib --features python-udf python_udf
+ - name: Test vendored DataFusion with the Comet workspace graph
+ shell: bash
+ run: |
+ cd native
+ export LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH}
+ # Building both packages preserves Comet's feature unification; the filter
+ # runs the vendor tests explicitly instead of relying on default-members.
+ cargo nextest run --locked -p datafusion-comet -p datafusion-physical-plan --lib -E 'package(datafusion-physical-plan)'
+ cargo nextest run --locked -p datafusion-comet -p datafusion-physical-plan --lib --features datafusion-physical-plan/test_utils -E 'package(datafusion-physical-plan) & test(statistics::)'
+
# The steps above lint and test the accounting allocator over the system allocator. Lint and
# test it over jemalloc too, since that is a different allocator backend and nothing else in CI
# builds it.
@@ -114,5 +126,5 @@ runs:
run: |
cd native
export LD_LIBRARY_PATH=${JAVA_HOME}/lib/server:${LD_LIBRARY_PATH}
- cargo clippy --color=never -p datafusion-comet --all-targets --features jemalloc -- -D warnings
+ cargo clippy --locked --color=never -p datafusion-comet --all-targets --features jemalloc --no-deps -- -D warnings
RUST_BACKTRACE=1 cargo nextest run -p datafusion-comet --lib --features jemalloc alloc_accounting
diff --git a/.github/workflows/README.md b/.github/workflows/README.md
index 1976ca2632a..2cb1888b0a6 100644
--- a/.github/workflows/README.md
+++ b/.github/workflows/README.md
@@ -91,7 +91,7 @@ tiers partition the list and that the `pr` tier is exactly the default profile.
|
v
+-----------------------+
- | preflight | ubuntu-slim
+ | preflight | ubuntu-24.04
| (RAT, prettier, |
| missing-suites, |
| actionlint) |
@@ -99,7 +99,7 @@ tiers partition the list and that the `pr` tier is exactly the default profile.
| on success
v
+-----------------------+
- | changes | ubuntu-slim
+ | changes | ubuntu-24.04
| (compute-changes.py: |
| one boolean per |
| heavy job) |
@@ -137,14 +137,14 @@ tiers partition the list and that the `pr` tier is exactly the default profile.
+-----------------------------------+-----------------------------------+
v
+-----------------------+
- | required_checks | ubuntu-slim
+ | required_checks | ubuntu-24.04
| one flat name that |
| is safe to require |
+-----------+-----------+
| schedule only, when red
v
+-----------------------+
- | nightly_report | ubuntu-slim
+ | nightly_report | ubuntu-24.04
| opens or comments on |
| a ci-nightly-failure |
| issue |
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 936b804272d..0673bd87a20 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -124,14 +124,14 @@ jobs:
# required-status-check evaluation read. Guarding this job on the label name
# therefore let any non-gating label overwrite the commit run's real
# `Preflight` verdict with `skipped` (issue #5007). Running it unconditionally
- # costs about a minute of ubuntu-slim time per label event and keeps the
+ # costs about a minute of runner time per label event and keeps the
# reported verdict truthful. Skipping the redundant work is the heavy jobs'
# job; POLICY in dev/ci/compute-changes.py drops every job a `labeled` event
# does not gate.
# ---------------------------------------------------------------------------
preflight:
name: Preflight
- runs-on: ubuntu-slim
+ runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
@@ -164,10 +164,9 @@ jobs:
- name: Install mermaid-cli
run: |
set -eux
- # ubuntu-slim carries none of Chrome's shared libraries, so the browser downloads
- # fine and then dies with `libatk-1.0.so.0: cannot open shared object file`. These
- # are what chrome-headless-shell actually links against; libgtk-3 pulls in most of
- # the rest (atk, cairo, pango, gdk-pixbuf) as dependencies.
+ # Install Chrome's runtime libraries explicitly so the browser setup does not
+ # depend on which packages happen to be preinstalled on the runner.
+ # libgtk-3 pulls in atk, cairo, pango and gdk-pixbuf as dependencies.
sudo apt-get update -qq
sudo apt-get install -y -qq --no-install-recommends \
libgtk-3-0t64 libnss3 libasound2t64 libgbm1 libatk-bridge2.0-0t64 \
@@ -200,6 +199,9 @@ jobs:
- name: Check Iceberg write report summary
run: python3 dev/ci/test-summarize-iceberg-writes.py
+ - name: Check model-only Maven retry
+ run: python3 dev/ci/test-maven-model.py
+
- name: Check CI config invariants
run: python3 dev/ci/check-ci-config.py
@@ -237,7 +239,7 @@ jobs:
changes:
name: Detect changes
needs: preflight
- runs-on: ubuntu-slim
+ runs-on: ubuntu-24.04
# `actions: read` is for dev/ci/nightly-base.py, which lists previous
# scheduled runs to pick the nightly's diff base.
permissions:
@@ -266,6 +268,7 @@ jobs:
delta_3_5: ${{ steps.compute.outputs.delta_3_5 }}
delta_4_0: ${{ steps.compute.outputs.delta_4_0 }}
delta_4_1: ${{ steps.compute.outputs.delta_4_1 }}
+ delta_4_2: ${{ steps.compute.outputs.delta_4_2 }}
steps:
- uses: actions/checkout@v7
with:
@@ -501,14 +504,25 @@ jobs:
needs: changes
permissions:
contents: read
- # Nightly by default; PRs need the `run-delta-tests` label. No Spark 4.2
- # caller: the contrib profile is dormant until a Delta release supports it.
+ # Nightly by default; PRs need the `run-delta-tests` label.
if: needs.changes.outputs.delta_4_1 == 'true'
uses: ./.github/workflows/delta_contrib_test.yml
with:
spark: '4.1'
java: '17'
+ delta_4_2:
+ name: Delta Contrib Tests (Spark 4.2)
+ needs: changes
+ permissions:
+ contents: read
+ # Delta 4.4.1 publishes a dedicated Spark 4.2 artifact.
+ if: needs.changes.outputs.delta_4_2 == 'true'
+ uses: ./.github/workflows/delta_contrib_test.yml
+ with:
+ spark: '4.2'
+ java: '17'
+
spark_4_0:
name: Spark SQL Tests (Spark 4.0)
needs: changes
@@ -552,7 +566,7 @@ jobs:
uses: ./.github/workflows/spark_sql_test_reusable.yml
with:
spark-short: '4.2'
- spark-full: '4.2.0'
+ spark-full: '4.2.1'
java: 17
iceberg_1_8:
@@ -659,6 +673,7 @@ jobs:
- delta_3_5
- delta_4_0
- delta_4_1
+ - delta_4_2
- spark_4_0
- spark_4_1
- spark_4_2
@@ -666,7 +681,7 @@ jobs:
- iceberg_1_9
- iceberg_1_10
- iceberg_1_11
- runs-on: ubuntu-slim
+ runs-on: ubuntu-24.04
steps:
- name: Summarize upstream results
env:
@@ -703,7 +718,7 @@ jobs:
permissions:
actions: read
issues: write
- runs-on: ubuntu-slim
+ runs-on: ubuntu-24.04
steps:
- uses: actions/github-script@v9
with:
diff --git a/.github/workflows/delta_build_gate.yml b/.github/workflows/delta_build_gate.yml
index caf8a37ae3f..45574c2c8e9 100644
--- a/.github/workflows/delta_build_gate.yml
+++ b/.github/workflows/delta_build_gate.yml
@@ -64,6 +64,19 @@ jobs:
rust-version: ${{ env.RUST_VERSION }}
jdk-version: 17
+ - name: Restore Maven dependencies
+ uses: actions/cache/restore@v6
+ with:
+ path: |
+ ~/.m2/repository
+ /root/.m2/repository
+ key: ${{ runner.os }}-java-maven-${{ hashFiles('**/pom.xml') }}-delta-build-gate
+ restore-keys: |
+ ${{ runner.os }}-java-maven-
+
+ - name: Resolve Maven model before build-gate checks
+ run: dev/ci/prepare-maven-model.sh
+
- name: Run dev/verify-contrib-delta-gate.sh
run: |
dev/verify-contrib-delta-gate.sh
diff --git a/.github/workflows/delta_contrib_test.yml b/.github/workflows/delta_contrib_test.yml
index 10e6b9f1029..6cfff239568 100644
--- a/.github/workflows/delta_contrib_test.yml
+++ b/.github/workflows/delta_contrib_test.yml
@@ -113,11 +113,11 @@ jobs:
- name: Install Comet core jars
run: |
# The delta profile is what produces the spark test-jar the contrib suites depend on.
- ./mvnw -B -q -Prelease -Pspark-${{ inputs.spark }},delta install -pl common,spark -DskipTests -Dspotless.check.skip=true
+ ./mvnw -B -q -Prelease -Pspark-${{ inputs.spark }},delta install -DskipTests -Dspotless.check.skip=true
- name: Run Delta contrib test suites
run: |
- SPARK_HOME=$(pwd) COMET_CONF_DIR=$(pwd)/conf ./mvnw -B -Prelease -Pspark-${{ inputs.spark }},delta test -pl contrib/delta-spark
+ SPARK_HOME=$(pwd) COMET_CONF_DIR=$(pwd)/conf ./mvnw -B -Prelease -Pspark-${{ inputs.spark }},delta verify -Dtest=none -DwildcardSuites=org.apache.comet.contrib.delta,org.apache.spark.sql.comet.DeltaPlanDataInjectorSuite
# Saved only on main: an entry written from a pull request or a
# `gh-readonly-queue/*` branch cannot be restored by any later run, and
diff --git a/.github/workflows/iceberg_spark_test_reusable.yml b/.github/workflows/iceberg_spark_test_reusable.yml
index faa7ae610e5..4c682b0f48a 100644
--- a/.github/workflows/iceberg_spark_test_reusable.yml
+++ b/.github/workflows/iceberg_spark_test_reusable.yml
@@ -184,7 +184,7 @@ jobs:
needs: iceberg-spark
if: ${{ !cancelled() }}
name: iceberg-spark-shard-coverage/iceberg-${{ inputs.iceberg-full }}/spark-${{ inputs.spark-full }}/scala-${{ inputs.scala }}/java-${{ inputs.java }}
- runs-on: ubuntu-slim
+ runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
- name: Download Iceberg shard inventories
diff --git a/.github/workflows/pr_build_linux.yml b/.github/workflows/pr_build_linux.yml
index 995267bc60e..8ae4e0e64c6 100644
--- a/.github/workflows/pr_build_linux.yml
+++ b/.github/workflows/pr_build_linux.yml
@@ -310,6 +310,9 @@ jobs:
restore-keys: |
${{ runner.os }}-java-maven-
+ - name: Resolve Maven model before strict compilation
+ run: dev/ci/prepare-maven-model.sh -Pspark-3.5,delta -Pstrict-warnings
+
- name: Compile with strict warnings
run: ./mvnw -B test-compile -Pspark-3.5,delta -Pstrict-warnings -DskipTests
diff --git a/.github/workflows/pr_title_check.yml b/.github/workflows/pr_title_check.yml
index 65c27465234..66ac8662f5d 100644
--- a/.github/workflows/pr_title_check.yml
+++ b/.github/workflows/pr_title_check.yml
@@ -27,7 +27,7 @@ on:
jobs:
check-pr-title:
- runs-on: ubuntu-slim
+ runs-on: ubuntu-24.04
steps:
- uses: actions/checkout@v7
- name: Check PR title
diff --git a/.github/workflows/pyarrow_udf_test.yml b/.github/workflows/pyarrow_udf_test.yml
index f29caa052ff..e92f2c3d869 100644
--- a/.github/workflows/pyarrow_udf_test.yml
+++ b/.github/workflows/pyarrow_udf_test.yml
@@ -51,7 +51,7 @@ jobs:
native_python_udf: true
- name: Spark 4.2
maven_profiles: "-Pspark-4.2"
- pyspark: "4.2.0"
+ pyspark: "4.2.1"
native_python_udf: true
container:
# Pinned to the Debian 12 (bookworm) base so the system `python3` is 3.11. The default
diff --git a/.github/workflows/rc_build_smoke.yml b/.github/workflows/rc_build_smoke.yml
new file mode 100644
index 00000000000..3042272d9bf
--- /dev/null
+++ b/.github/workflows/rc_build_smoke.yml
@@ -0,0 +1,250 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+name: RC Build & Smoke
+
+on:
+ workflow_dispatch:
+ inputs:
+ expected_commit:
+ description: Full source SHA; must match the selected workflow branch or tag
+ required: true
+ type: string
+
+permissions:
+ contents: read
+
+concurrency:
+ group: rc-build-smoke-${{ inputs.expected_commit }}
+ cancel-in-progress: false
+
+jobs:
+ source:
+ name: Pin source and release inputs
+ runs-on: ubuntu-24.04
+ timeout-minutes: 10
+ outputs:
+ commit: ${{ steps.source.outputs.commit }}
+ lock: ${{ steps.source.outputs.lock }}
+ steps:
+ - name: Validate requested source SHA
+ env:
+ EXPECTED_COMMIT: ${{ inputs.expected_commit }}
+ WORKFLOW_COMMIT: ${{ github.sha }}
+ run: |
+ python3 - <<'PY'
+ import os, re
+ expected = os.environ['EXPECTED_COMMIT']
+ if not re.fullmatch('[0-9a-f]{40}', expected):
+ raise SystemExit('expected_commit must be a full lowercase Git SHA')
+ if expected != os.environ['WORKFLOW_COMMIT']:
+ raise SystemExit('Selected workflow ref does not match expected_commit')
+ PY
+ - uses: actions/checkout@v7
+ with:
+ ref: ${{ inputs.expected_commit }}
+ - uses: actions/setup-java@v6
+ with:
+ distribution: temurin
+ java-version: "17"
+ - name: Check release-stage boundaries before starting builds
+ run: |
+ export RC_TEST_TMPDIR="$GITHUB_WORKSPACE/target/rc-unit-tests"
+ mkdir -p "$RC_TEST_TMPDIR"
+ export PYTHONDONTWRITEBYTECODE=1
+ python3 dev/release/joom/test-native-build-provenance.py --work-dir "$RC_TEST_TMPDIR"
+ python3 dev/release/joom/test-packaged-runtime.py
+ python3 dev/release/joom/test-rc-result.py
+ - name: Record source and lockfile
+ id: source
+ env:
+ EXPECTED_COMMIT: ${{ inputs.expected_commit }}
+ run: |
+ test "$(git rev-parse HEAD)" = "$EXPECTED_COMMIT"
+ test -z "$(git status --porcelain --untracked-files=all)"
+ echo "commit=$EXPECTED_COMMIT" >> "$GITHUB_OUTPUT"
+ echo "lock=$(sha256sum native/Cargo.lock | cut -d' ' -f1)" >> "$GITHUB_OUTPUT"
+
+ native:
+ name: Release native (${{ matrix.arch }})
+ needs: source
+ strategy:
+ fail-fast: true
+ matrix:
+ include:
+ - arch: amd64
+ runner: ubuntu-24.04
+ - arch: arm64
+ runner: ubuntu-24.04-arm
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 120
+ env:
+ CARGO_JOBS: 4
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ ref: ${{ needs.source.outputs.commit }}
+ - name: Check native architecture and disk
+ env:
+ ARCH: ${{ matrix.arch }}
+ run: |
+ test "$(uname -m)" = "$(case "$ARCH" in amd64) echo x86_64 ;; arm64) echo aarch64 ;; esac)"
+ df -h "$GITHUB_WORKSPACE"
+ mkdir -p "$HOME/.cargo/rc-registry-$ARCH" "$HOME/.cargo/rc-git"
+ - name: Restore release source dependencies
+ uses: actions/cache/restore@v6
+ with:
+ path: |
+ ~/.cargo/rc-registry-${{ matrix.arch }}
+ ~/.cargo/rc-git
+ key: rc-cargo-${{ matrix.arch }}-${{ hashFiles('native/Cargo.lock', 'dev/release/joom/Dockerfile') }}
+ - name: Build release native library and provenance
+ env:
+ ARCH: ${{ matrix.arch }}
+ run: |
+ export CARGO_REGISTRY="$HOME/.cargo/rc-registry-"
+ export CARGO_GIT="$HOME/.cargo/rc-git"
+ dev/release/joom/build-release.sh --native "$ARCH" "$GITHUB_WORKSPACE/target/rc-build"
+ - name: Upload native stage
+ uses: ./.github/actions/upload-artifact-retry
+ with:
+ name: rc-native-${{ matrix.arch }}-${{ needs.source.outputs.commit }}
+ path: target/rc-build/linux/
+ if-no-files-found: error
+ retention-days: 3
+
+ package:
+ name: Package final dualarch JARs
+ needs: [source, native]
+ runs-on: ubuntu-24.04
+ timeout-minutes: 60
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ ref: ${{ needs.source.outputs.commit }}
+ - name: Restore Maven dependencies
+ uses: actions/cache/restore@v6
+ with:
+ path: ~/.m2/repository
+ key: rc-maven-${{ hashFiles('**/pom.xml') }}
+ - name: Download native stages
+ uses: ./.github/actions/download-artifact-retry
+ with:
+ pattern: rc-native-*-${{ needs.source.outputs.commit }}
+ path: target/rc-build/linux
+ merge-multiple: "true"
+ - name: Package once and prepare portable smoke dependencies
+ env:
+ PREPARE_SMOKE: "1"
+ run: |
+ df -h "$GITHUB_WORKSPACE"
+ dev/release/joom/build-release.sh --package amd64 "$GITHUB_WORKSPACE/target/rc-build"
+ - name: Upload immutable candidate
+ uses: ./.github/actions/upload-artifact-retry
+ with:
+ name: rc-candidate-${{ needs.source.outputs.commit }}
+ path: |
+ target/rc-build/*.jar
+ target/rc-build/*.json
+ target/rc-build/*.txt
+ target/rc-build/effective-pom.xml
+ target/rc-build/linux/
+ target/rc-build/test-support/
+ if-no-files-found: error
+ retention-days: 7
+
+ smoke:
+ name: Packaged smoke (${{ matrix.arch }})
+ needs: [source, package]
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - arch: amd64
+ runner: ubuntu-24.04
+ - arch: aarch64
+ runner: ubuntu-24.04-arm
+ runs-on: ${{ matrix.runner }}
+ timeout-minutes: 30
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ ref: ${{ needs.source.outputs.commit }}
+ - uses: actions/setup-java@v6
+ with:
+ distribution: temurin
+ java-version: "17"
+ - name: Download final candidate without rebuilding
+ uses: ./.github/actions/download-artifact-retry
+ with:
+ name: rc-candidate-${{ needs.source.outputs.commit }}
+ path: target/rc-build
+ - name: Run existing tests from final JARs
+ env:
+ ARCH: ${{ matrix.arch }}
+ SOURCE_COMMIT: ${{ needs.source.outputs.commit }}
+ LOCK_SHA: ${{ needs.source.outputs.lock }}
+ run: |
+ python3 dev/release/joom/verify-packaged-runtime.py run \
+ --repo "$GITHUB_WORKSPACE" --release "$GITHUB_WORKSPACE/target/rc-build" \
+ --architecture "$ARCH" --commit "$SOURCE_COMMIT" --lock-sha256 "$LOCK_SHA" \
+ --timeout-seconds 180
+ - name: Upload smoke evidence
+ if: ${{ always() }}
+ uses: ./.github/actions/upload-artifact-retry
+ with:
+ name: rc-smoke-${{ matrix.arch }}-${{ needs.source.outputs.commit }}
+ path: target/rc-build/smoke/${{ matrix.arch }}/
+ if-no-files-found: error
+ retention-days: 14
+
+ qualified:
+ name: Verify both receipts and export release artifacts
+ needs: [source, package, smoke]
+ runs-on: ubuntu-24.04
+ timeout-minutes: 15
+ steps:
+ - uses: actions/checkout@v7
+ with:
+ ref: ${{ needs.source.outputs.commit }}
+ - name: Download final candidate
+ uses: ./.github/actions/download-artifact-retry
+ with:
+ name: rc-candidate-${{ needs.source.outputs.commit }}
+ path: target/rc-build
+ - name: Download both smoke receipts
+ uses: ./.github/actions/download-artifact-retry
+ with:
+ pattern: rc-smoke-*-${{ needs.source.outputs.commit }}
+ path: target/rc-smoke-reports
+ - name: Verify the same final bytes passed on both architectures
+ env:
+ SOURCE_COMMIT: ${{ needs.source.outputs.commit }}
+ LOCK_SHA: ${{ needs.source.outputs.lock }}
+ run: |
+ python3 dev/release/joom/verify-rc-result.py \
+ --release "$GITHUB_WORKSPACE/target/rc-build" \
+ --smoke-reports "$GITHUB_WORKSPACE/target/rc-smoke-reports" \
+ --commit "$SOURCE_COMMIT" --lock-sha256 "$LOCK_SHA" \
+ --output "$GITHUB_WORKSPACE/target/rc-qualified"
+ - name: Upload verified release artifacts
+ uses: ./.github/actions/upload-artifact-retry
+ with:
+ name: rc-build-smoke-${{ needs.source.outputs.commit }}
+ path: target/rc-qualified/
+ if-no-files-found: error
+ retention-days: 14
diff --git a/Makefile b/Makefile
index 78c6435ae39..72a4fcf80f2 100644
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@
# specific language governing permissions and limitations
# under the License.
-.PHONY: all core jvm test clean rat release-linux release bench print-benchmark-args
+.PHONY: all core jvm test test-rust test-vendor clean rat release-linux release bench print-benchmark-args
define spark_jvm_17_extra_args
$(shell ./mvnw help:evaluate -q -DforceStdout -Dexpression=extraJavaTestArgs $(PROFILES))
@@ -41,9 +41,15 @@ core:
cd native && cargo build $(FEATURES_ARG)
test-rust:
# We need to compile CometException so that the cargo test can pass
- ./mvnw compile -pl common -DskipTests $(PROFILES)
+ ./mvnw compile -DskipTests $(PROFILES)
cd native && cargo build $(FEATURES_ARG) && \
RUST_BACKTRACE=1 cargo test $(FEATURES_ARG)
+test-vendor:
+ ./mvnw compile -DskipTests $(PROFILES)
+ cargo build --locked --manifest-path native/Cargo.toml -p comet-test-udfs
+ # Include Comet when resolving test features, not a standalone vendor graph.
+ cargo test --locked --manifest-path native/Cargo.toml -p datafusion-comet -p datafusion-physical-plan --lib $(FEATURES_ARG)
+ cargo test --locked --manifest-path native/Cargo.toml -p datafusion-comet -p datafusion-physical-plan --lib --features datafusion-physical-plan/test_utils statistics::
jvm:
./mvnw clean package -DskipTests $(PROFILES)
test-jvm: core
diff --git a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala
index 96c18a85de0..8c66492db6a 100644
--- a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala
+++ b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala
@@ -388,6 +388,12 @@ object DeltaScanSupport {
return encryptionReason
}
+ val contextReason =
+ unsupportedEncryptionContextReason(hadoopConf, dataFileUris ++ dvUris, propagatedConfCache)
+ if (contextReason.isDefined) {
+ return contextReason
+ }
+
// Always zero-I/O (plain propagated-conf read, no keystore): native's S3 client has no
// HTTP proxy support at all (no fs.s3a.proxy.* key is read anywhere in s3.rs), so a bucket
// requiring a proxy for S3 egress must decline here rather than claim and then connect
@@ -1155,6 +1161,42 @@ object DeltaScanSupport {
}
}
+ /**
+ * Decline S3 encryption context because native does not reproduce Hadoop's setup-time parsing.
+ * A local Delta log can reference S3 data or DVs, so metadata reads need not initialize S3A.
+ * Resolve the same bucket aliases and credential providers as Hadoop, without exposing values.
+ */
+ private[delta] def unsupportedEncryptionContextReason(
+ hadoopConf: Configuration,
+ uris: Seq[URI],
+ propagatedConfCache: MutableMap[String, Configuration] = MutableMap.empty)
+ : Option[String] = {
+ val key = "fs.s3a.encryption.context"
+ uris
+ .flatMap(s3Bucket)
+ .distinct
+ .iterator
+ .map { bucket =>
+ try {
+ val propagatedConf = propagatedConfCache.getOrElseUpdate(
+ bucket,
+ propagateBucketOptions(hadoopConf, bucket))
+ hadoopLookupPasswordEffective(propagatedConf, bucket, key) match {
+ case Left(reason) => Some(reason)
+ case Right(Some(_)) =>
+ Some(
+ s"Native Delta scan does not support $key for $bucket " +
+ "(Hadoop validates the context during S3A initialization)")
+ case Right(None) => None
+ }
+ } catch {
+ case e @ (_: IOException | _: RuntimeException) =>
+ Some(unverifiableValueReason(bucket, key, e))
+ }
+ }
+ .collectFirst { case Some(reason) => reason }
+ }
+
private def encryptionAlgorithmReason(conf: Configuration, bucket: String): Option[String] =
effectiveEncryptionAlgorithm(conf, bucket) match {
case Left(reason) => Some(reason)
@@ -1522,22 +1564,22 @@ object DeltaScanSupport {
* for the Arm A/B/C dispatch this delegates to) -- or `Left(reason)` when the value cannot be
* safely verified.
*
- * USED ONLY for keys whose real consumer IS `lookupPassword` -- the [[LookupPasswordConsumer]]
- * entries of [[S3ConfigKeyConsumers]], via [[s3KeyDivergenceReason]]. An earlier version ran
- * EVERY compared key through this function, reasoning that a wider read could only ever
- * over-decline; for a value-EQUALITY comparator that reasoning is half-true: the long-form
- * alias this function consults FIRST can hold exactly the value native resolves while Hadoop's
- * true propagate-then-plain-get value differs (e.g. a `${...}` reference whose referent
- * propagation redirects), producing a false EQUALITY that admits a really-diverging scan. A
- * [[PropagatedOptionConsumer]] key must therefore resolve like its actual consumer instead --
- * see [[s3KeyDivergenceReason]].
+ * Used for keys whose real consumer is `lookupPassword`: the [[LookupPasswordConsumer]] entries
+ * of [[S3ConfigKeyConsumers]] and the dedicated [[unsupportedEncryptionContextReason]] gate. An
+ * earlier version ran EVERY compared key through this function, reasoning that a wider read
+ * could only ever over-decline; for a value-EQUALITY comparator that reasoning is half-true:
+ * the long-form alias this function consults FIRST can hold exactly the value native resolves
+ * while Hadoop's true propagate-then-plain-get value differs (e.g. a `${...}` reference whose
+ * referent propagation redirects), producing a false EQUALITY that admits a really-diverging
+ * scan. A [[PropagatedOptionConsumer]] key must therefore resolve like its actual consumer
+ * instead -- see [[s3KeyDivergenceReason]].
*
* Honors [[ClearTextFallbackKey]] (via [[resolveViaCredentialAliases]]'s [[NoProvider]] arm),
* matching `getPassword`'s real refusal to read plaintext conf when the flag is off.
*
- * `hadoopConf` must be a [[propagateBucketOptions]] result (the caller,
- * [[s3KeyDivergenceReason]], always passes one) so that `${...}` references embedded in any
- * alias resolve exactly like `S3AFileSystem#initialize`'s real propagate-then-resolve order.
+ * `hadoopConf` must be a [[propagateBucketOptions]] result (both callers pass one) so that
+ * `${...}` references embedded in any alias resolve exactly like `S3AFileSystem#initialize`'s
+ * real propagate-then-resolve order.
*/
private def hadoopLookupPasswordEffective(
hadoopConf: Configuration,
@@ -1849,6 +1891,17 @@ object DeltaScanSupport {
private def isGcsAuthKey(key: String): Boolean =
(key.startsWith("fs.gs.") || key.startsWith("google.cloud.")) && key.contains("auth")
+ private val GcsServiceAccountEnableKeys =
+ Set("fs.gs.auth.service.account.enable", "google.cloud.auth.service.account.enable")
+
+ private val GcsAdcAuthTypes = Set("COMPUTE_ENGINE", "APPLICATION_DEFAULT")
+
+ private def isGcsAdcEquivalent(key: String, value: String): Boolean = {
+ val v = value.trim
+ (GcsServiceAccountEnableKeys.contains(key) && v.equalsIgnoreCase("true")) ||
+ (key == "fs.gs.auth.type" && GcsAdcAuthTypes.contains(v.toUpperCase(java.util.Locale.ROOT)))
+ }
+
/**
* True when `uri`'s scheme is `gs` (case-insensitive) -- the ONLY scheme object_store's
* `ObjectStoreScheme::parse` (parquet_support.rs) routes to `GoogleCloudStorage`; `gcs` is not
@@ -1887,7 +1940,7 @@ object DeltaScanSupport {
.collect {
case entry
if isGcsAuthKey(entry.getKey) && entry.getValue != null &&
- entry.getValue.nonEmpty =>
+ entry.getValue.nonEmpty && !isGcsAdcEquivalent(entry.getKey, entry.getValue) =>
entry.getKey
}
.toSeq
diff --git a/contrib/delta-spark/src/main/scala/org/apache/spark/sql/comet/CometDeltaNativeScanExec.scala b/contrib/delta-spark/src/main/scala/org/apache/spark/sql/comet/CometDeltaNativeScanExec.scala
index bcd1d82f906..23661a7fab1 100644
--- a/contrib/delta-spark/src/main/scala/org/apache/spark/sql/comet/CometDeltaNativeScanExec.scala
+++ b/contrib/delta-spark/src/main/scala/org/apache/spark/sql/comet/CometDeltaNativeScanExec.scala
@@ -23,7 +23,7 @@ import org.apache.spark.rdd.RDD
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.QueryPlan
import org.apache.spark.sql.catalyst.plans.physical.{Partitioning, UnknownPartitioning}
-import org.apache.spark.sql.execution.{FileSourceScanExec, InSubqueryExec, ReusedSubqueryExec, ScalarSubquery, SparkPlan, SubqueryAdaptiveBroadcastExec}
+import org.apache.spark.sql.execution.{FileSourceScanExec, InSubqueryExec, SparkPlan}
import org.apache.spark.sql.execution.datasources.HadoopFsRelation
import org.apache.spark.sql.execution.metric.SQLMetric
import org.apache.spark.sql.types.StructType
@@ -45,14 +45,14 @@ import org.apache.comet.serde.OperatorOuterClass.Operator
* `TreeNode.makeCopy` on MERGE re-planning (the CometIcebergNativeScanExec lesson).
*/
case class CometDeltaNativeScanExec(
- override val nativeOp: Operator,
+ @transient override val nativeOp: Operator,
override val output: Seq[Attribute],
requiredSchema: StructType,
runtimeFilters: Seq[Expression],
dataFilters: Seq[Expression],
@transient relation: HadoopFsRelation,
originalPlan: FileSourceScanExec,
- override val serializedPlanOpt: SerializedPlan,
+ @transient override val serializedPlanOpt: SerializedPlan,
sourceKey: String)
extends CometLeafExec
with CometScanWithPlanData {
@@ -75,8 +75,8 @@ case class CometDeltaNativeScanExec(
//
// Forcing invariant: this lazy val is forced by the `metrics` override below, and AQE's UI
// plan-walk calls `.metrics` on every node MID-PLANNING, including while a DPP subquery is
- // still an adaptive placeholder or a partition filter holds an unresolved ScalarSubquery (see
- // `hasUnevaluableSubqueryFilter` below). That's safe ONLY because constructing `scanHelper` is a
+ // still an adaptive placeholder or a filter holds an unresolved ScalarSubquery. That's safe
+ // ONLY because constructing `scanHelper` is a
// cheap case-class build with no file listing, and core's `CometScanExec.metrics` touches only
// `wrapped.driverMetrics` (populated by Spark's own planning) plus a static metric-node
// constructor -- neither file listing nor subquery resolution. If core's `metrics` ever touches
@@ -85,54 +85,11 @@ case class CometDeltaNativeScanExec(
@transient private lazy val scanHelper: CometScanExec =
CometDeltaNativeScanExec.planningHelper(originalPlan, runtimeFilters)
- // NOT lazy val: while a DPP subquery is still an adaptive placeholder, or a partition filter
- // holds an unresolved scalar subquery, this returns a temporary value that must not be
- // memoized -- after CometPlanAdaptiveDynamicPruningFilters rewrites the filters (DPP case) or
- // AQE resolves the subquery (scalar case), later reads must see the real post-pruning
- // partition count.
- override def outputPartitioning: Partitioning =
- if (hasUnevaluableSubqueryFilter) UnknownPartitioning(0)
- else UnknownPartitioning(perPartitionData.length)
-
- // runtimeFilters IS scanHelper.partitionFilters element-for-element, so checking runtimeFilters
- // here avoids constructing/forcing the derived scanHelper just to read partitioning. The
- // InSubqueryExec placeholder shapes mirror
- // CometPlanAdaptiveDynamicPruningFilters.extractSABData + hasWrappedSAB -- keep in sync. The
- // ScalarSubquery case is probed rather than treated as permanently unevaluable: Spark exposes no
- // public finished/updated flag on ExecSubqueryExpression, but `eval()` doubles as one -- it only
- // reads the cached `result` behind a `require(updated, ...)` guard, while the subquery is
- // actually run by `updateResult()` (invoked separately during prepare/AQE), never by `eval()`.
- // Once resolved, outputPartitioning below reports the real perPartitionData.length instead of
- // staying at zero -- a fused native parent's buildNativeContext requires that count to match.
- private def hasUnevaluableSubqueryFilter: Boolean =
- runtimeFilters.exists(_.exists {
- // Match `e: InSubqueryExec` and dispatch on e.plan rather than unapplying InSubqueryExec
- // directly: its unapply arity differs across Spark versions and this module ships no
- // version shim.
- case e: InSubqueryExec => isAdaptivePlaceholder(e.plan)
- case s: ScalarSubquery => !isScalarSubqueryResolved(s)
- case _ => false
- })
-
- // `eval()` never triggers the subquery's execution: on a resolved subquery it is a pure cached
- // read of `result` (verified against bytecode: `Predef.require(updated(), ...)` then a plain
- // field read), so this probe is safe to call repeatedly, including from AQE's mid-planning plan
- // walks. Pre-resolution, the ONLY throw is `require`'s `IllegalArgumentException`; catch exactly
- // that, since anything else escaping is a genuine bug we must not mask as unpartitioned.
- private def isScalarSubqueryResolved(s: ScalarSubquery): Boolean =
- try {
- s.eval()
- true
- } catch {
- case _: IllegalArgumentException => false
- }
-
- private def isAdaptivePlaceholder(p: SparkPlan): Boolean = p match {
- case ReusedSubqueryExec(inner) => isAdaptivePlaceholder(inner)
- case _: CometSubqueryAdaptiveBroadcastExec => true
- case _: SubqueryAdaptiveBroadcastExec => true
- case _ => false
- }
+ // Planning metadata must not list files or execute subqueries. Checking runtimeFilters alone
+ // misses scalar subqueries harvested into dataFilters and can deadlock in AQE planning.
+ // CometScanWithPlanData's execution path sizes fused parents from perPartitionData directly,
+ // so it does not depend on this planning-time placeholder for the actual partition count.
+ override lazy val outputPartitioning: Partitioning = UnknownPartitioning(0)
override lazy val outputOrdering: Seq[SortOrder] = originalPlan.outputOrdering
diff --git a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala
index a4dec885391..b658ecee0f5 100644
--- a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala
+++ b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala
@@ -24,10 +24,10 @@ import java.io.File
import scala.collection.mutable
import scala.collection.mutable.ListBuffer
-import org.apache.spark.CometListenerBusUtils
+import org.apache.spark.{CometListenerBusUtils, SparkEnv}
import org.apache.spark.scheduler.{SparkListener, SparkListenerTaskEnd}
import org.apache.spark.sql.{DataFrame, Row}
-import org.apache.spark.sql.catalyst.expressions.{AttributeReference, DynamicPruningExpression, DynamicPruningSubquery, Expression, NamedExpression, StructsToJson}
+import org.apache.spark.sql.catalyst.expressions.{AttributeReference, DynamicPruningExpression, DynamicPruningSubquery, Expression, GreaterThan, NamedExpression, StructsToJson}
import org.apache.spark.sql.comet.{CometDeltaNativeScanExec, CometSubqueryBroadcastExec}
import org.apache.spark.sql.execution.{FileSourceScanExec, InSubqueryExec, QueryExecution, ReusedSubqueryExec, ScalarSubquery, SparkPlan, SubqueryBroadcastExec, SubqueryExec}
import org.apache.spark.sql.execution.datasources.v2.V2TableWriteExec
@@ -819,7 +819,8 @@ class CometDeltaNativeScanSuite extends CometDeltaTestBase {
assert(scans.size == 1)
val numFiles = scans.head.metrics.get("numFiles").map(_.value).getOrElse(0L)
assert(numFiles == 1, s"expected a single data file, got $numFiles")
- val numPartitions = scans.head.outputPartitioning.numPartitions
+ val numPartitions =
+ scans.head.asInstanceOf[CometDeltaNativeScanExec].perPartitionData.length
assert(
numPartitions > 1,
"expected the single file to be split into more than one native partition, " +
@@ -2129,9 +2130,8 @@ class CometDeltaNativeScanSuite extends CometDeltaTestBase {
"parent") {
// Crash shape: a scalar subquery used as a partition filter (`p = (SELECT max(p) ...)`)
// lands in runtimeFilters. Once execution resolves it, a native aggregate sitting
- // directly on top of the scan (no intervening exchange) reads the scan's
- // outputPartitioning to size its own execution context; that getter must report the
- // real post-pruning partition count, not a value stuck from before resolution.
+ // directly on top of the scan (no intervening exchange) must size its execution context
+ // from the real post-pruning perPartitionData, independently of planning partitioning.
withTempPath { factDir =>
withTempPath { thresholdsDir =>
val factPath = factDir.getAbsolutePath
@@ -2185,6 +2185,60 @@ class CometDeltaNativeScanSuite extends CometDeltaTestBase {
}
}
+ test("planning partitioning does not execute a scalar subquery harvested into data filters") {
+ withTempPath { dir =>
+ val path = dir.getAbsolutePath
+ spark.range(0, 50).write.format("delta").save(path)
+ val scan = deltaNativeScans(spark.read.format("delta").load(path)).collect {
+ case s: CometDeltaNativeScanExec => s
+ }.head
+ val innerPlan = spark.range(1).selectExpr("id AS c").queryExecution.executedPlan
+ val scalar = ScalarSubquery(
+ SubqueryExec("planning-data-filter-subquery", innerPlan),
+ NamedExpression.newExprId)
+ // No runtime partition filter: the old runtimeFilters-only guard missed this shape
+ // and started the data-filter subquery while AQE was only inspecting plan metadata.
+ val filtered = scan.copy(
+ runtimeFilters = Seq.empty,
+ dataFilters = Seq(GreaterThan(scan.output.head, scalar)))
+ intercept[IllegalArgumentException](scalar.eval())
+ assert(filtered.outputPartitioning.numPartitions == 0)
+ assert(filtered.metrics.nonEmpty)
+ intercept[IllegalArgumentException](scalar.eval())
+ }
+ }
+
+ test(
+ "Delta scan copies keep the native plan and Java serialization drops driver-only payloads") {
+ withTempPath { dir =>
+ val path = dir.getAbsolutePath
+ spark.range(0, 50).write.format("delta").save(path)
+ val df = spark.read.format("delta").load(path)
+ checkDeltaNativeScanAnswer(df)
+ val scan = deltaNativeScans(df).collect { case s: CometDeltaNativeScanExec => s }.head
+ assert(scan.nativeOp != null)
+ assert(scan.serializedPlanOpt != null)
+ val copies =
+ Seq(scan.clone(), scan.makeCopy(scan.productIterator.map(_.asInstanceOf[AnyRef]).toArray))
+ copies.foreach { plan =>
+ val copied = plan.asInstanceOf[CometDeltaNativeScanExec]
+ assert(copied.nativeOp == scan.nativeOp)
+ assert(
+ copied.serializedPlanOpt.plan.map(_.toSeq) ==
+ scan.serializedPlanOpt.plan.map(_.toSeq))
+ assert(copied.sameResult(scan))
+ }
+ // Spark serializes task closures with Java serialization even when its row/shuffle
+ // serializer is Kryo. Driver-only protobufs must not inflate those task binaries.
+ val serializer = SparkEnv.get.closureSerializer.newInstance()
+ val restored =
+ serializer.deserialize[CometDeltaNativeScanExec](serializer.serialize(scan))
+ assert(restored.output == scan.output)
+ assert(restored.nativeOp == null)
+ assert(restored.serializedPlanOpt == null)
+ }
+ }
+
test(
"metrics evaluates without throwing when runtimeFilters holds a ScalarSubquery " +
"placeholder (pins the invariant documented on CometDeltaNativeScanExec.scanHelper: " +
@@ -2199,7 +2253,7 @@ class CometDeltaNativeScanSuite extends CometDeltaTestBase {
}.head
// A real execution.ScalarSubquery instance (the exec-time class CometDeltaNativeScanExec
- // itself matches against in hasUnevaluableSubqueryFilter), wrapping a never-executed
+ // carries in its filters), wrapping a never-executed
// SubqueryExec -- deliberately never run, so this is unresolved exactly as it would be
// when AQE's mid-planning walk reaches this node ahead of subquery execution.
val innerPlan = spark.range(1).selectExpr("id AS c").queryExecution.executedPlan
diff --git a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaTestBase.scala b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaTestBase.scala
index 50fbcc23fe2..f57cd54b4db 100644
--- a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaTestBase.scala
+++ b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaTestBase.scala
@@ -19,6 +19,7 @@
package org.apache.comet.contrib.delta
+import org.apache.hadoop.fs.s3a.Constants
import org.apache.spark.SparkConf
import org.apache.spark.sql.{CometTestBase, DataFrame}
import org.apache.spark.sql.execution.SparkPlan
@@ -29,6 +30,9 @@ import org.apache.spark.sql.execution.adaptive.AdaptiveSparkPlanHelper
*/
abstract class CometDeltaTestBase extends CometTestBase with AdaptiveSparkPlanHelper {
+ protected def supportsS3EncryptionContext: Boolean =
+ classOf[Constants].getFields.exists(_.getName == "S3_ENCRYPTION_CONTEXT")
+
override protected def sparkConf: SparkConf = {
val conf = super.sparkConf
conf.set("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
diff --git a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala
index 1d240ba3e22..c47e85e7f33 100644
--- a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala
+++ b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala
@@ -21,6 +21,7 @@ package org.apache.comet.contrib.delta
import java.io.{File, IOException}
import java.net.URI
+import java.nio.charset.StandardCharsets.UTF_8
import java.nio.file.Files
import java.util.{Locale, UUID}
import java.util.concurrent.atomic.AtomicBoolean
@@ -32,6 +33,7 @@ import org.apache.hadoop.fs.{Path, RawLocalFileSystem}
import org.apache.hadoop.fs.s3a.{S3AFileSystem, S3AUtils}
import org.apache.hadoop.security.alias.CredentialProviderFactory
import org.apache.spark.sql.comet.CometDeltaNativeScanExec
+import org.apache.spark.sql.delta.DeltaLog
import org.apache.spark.sql.delta.actions.DeletionVectorDescriptor
import org.apache.spark.sql.execution.SparkPlan
@@ -46,6 +48,72 @@ import org.apache.comet.rules.CometScanRule
*/
class DeltaScanContribSuite extends CometDeltaTestBase {
+ if (supportsS3EncryptionContext) {
+ // This pins metadata-setup parity; the MinIO local-clone test exercises the admission gate.
+ test("malformed S3 encryption context fails on the driver before Delta scan planning") {
+ val conf = spark.sparkContext.hadoopConfiguration
+ val settings = Map(
+ "fs.s3a.impl" -> classOf[S3AFileSystem].getName,
+ "fs.s3a.impl.disable.cache" -> "true",
+ "fs.s3a.endpoint" -> "http://127.0.0.1:1",
+ "fs.s3a.endpoint.region" -> "us-east-1",
+ "fs.s3a.path.style.access" -> "true",
+ "fs.s3a.connection.ssl.enabled" -> "false",
+ "fs.s3a.bucket.probe" -> "0",
+ "fs.s3a.aws.credentials.provider" ->
+ "org.apache.hadoop.fs.s3a.SimpleAWSCredentialsProvider",
+ "fs.s3a.access.key" -> "synthetic-access",
+ "fs.s3a.secret.key" -> "synthetic-secret",
+ "fs.s3a.encryption.algorithm" -> "",
+ "fs.s3a.encryption.context" -> "synthetic-malformed-pair")
+ val original = settings.keys.map(key => key -> Option(conf.getRaw(key))).toMap
+ try {
+ settings.foreach { case (key, value) => conf.set(key, value) }
+ for (algorithm <- Seq("", "AES256", "SSE-KMS", "DSSE-KMS")) {
+ conf.set("fs.s3a.encryption.algorithm", algorithm)
+ val valid = new Configuration(conf)
+ valid.set("fs.s3a.encryption.context", "purpose=synthetic")
+ val fs = new S3AFileSystem()
+ try {
+ fs.initialize(new java.net.URI("s3a://valid-context/"), valid)
+ assert(fs.getUri.getHost == "valid-context")
+ } finally {
+ fs.close()
+ }
+ val errors = scala.collection.mutable.ArrayBuffer.empty[(String, String, Seq[String])]
+ Seq(false, true).foreach { enabled =>
+ withSQLConf(CometConf.COMET_ENABLED.key -> enabled.toString) {
+ val group = s"malformed-context-${UUID.randomUUID()}"
+ spark.sparkContext.setJobGroup(group, group)
+ try {
+ val error = intercept[IllegalArgumentException] {
+ spark.read.format("delta").load("s3a://malformed-context/table").collect()
+ }
+ val frames = error.getStackTrace.map(_.toString).toSeq
+ assert(frames.exists(_.contains("S3AEncryption.getS3EncryptionContext")))
+ assert(frames.exists(_.contains("S3AFileSystem.initialize")))
+ assert(spark.sparkContext.statusTracker.getJobIdsForGroup(group).isEmpty)
+ errors += ((
+ error.getClass.getName,
+ error.getMessage,
+ frames.filter(_.startsWith("org.apache.hadoop."))))
+ } finally {
+ spark.sparkContext.clearJobGroup()
+ }
+ }
+ }
+ assert(errors.size == 2)
+ assert(errors.head == errors.last)
+ }
+ } finally {
+ original.foreach {
+ case (key, Some(value)) => conf.set(key, value)
+ case (key, None) => conf.unset(key)
+ }
+ }
+ }
+ }
+
test("contrib is inert when comet exec is disabled") {
// The COMET_EXEC_ENABLED gate lives in DeltaScanContrib.tryTransformV1; this pins it there.
withTempPath { dir =>
@@ -879,6 +947,51 @@ class DeltaScanContribSuite extends CometDeltaTestBase {
.isEmpty)
}
+ test(
+ "gcsHadoopOnlyAuthReason passes ADC-equivalent service-account enable and auth type keys") {
+ val conf = new Configuration(false)
+ conf.set("google.cloud.auth.service.account.enable", "true")
+ conf.set("fs.gs.auth.service.account.enable", "TRUE")
+ conf.set("fs.gs.auth.type", "COMPUTE_ENGINE")
+ assert(
+ DeltaScanSupport
+ .gcsHadoopOnlyAuthReason(conf, Seq(new URI("gs://mybucket/part-0.parquet")))
+ .isEmpty)
+ conf.set("fs.gs.auth.type", "APPLICATION_DEFAULT")
+ assert(
+ DeltaScanSupport
+ .gcsHadoopOnlyAuthReason(conf, Seq(new URI("gs://mybucket/part-0.parquet")))
+ .isEmpty)
+ }
+
+ test(
+ "gcsHadoopOnlyAuthReason still declines a service-account keyfile next to enable=true, " +
+ "and declines enable=false or a non-ADC auth type") {
+ val withKeyfile = new Configuration(false)
+ withKeyfile.set("google.cloud.auth.service.account.enable", "true")
+ withKeyfile.set("google.cloud.auth.service.account.json.keyfile", "/secret/svc-key.json")
+ val reason = DeltaScanSupport.gcsHadoopOnlyAuthReason(
+ withKeyfile,
+ Seq(new URI("gs://mybucket/part-0.parquet")))
+ assert(reason.isDefined)
+ assert(reason.get.contains("google.cloud.auth.service.account.json.keyfile"))
+ assert(!reason.get.contains("google.cloud.auth.service.account.enable"))
+
+ val disabled = new Configuration(false)
+ disabled.set("google.cloud.auth.service.account.enable", "false")
+ assert(
+ DeltaScanSupport
+ .gcsHadoopOnlyAuthReason(disabled, Seq(new URI("gs://mybucket/part-0.parquet")))
+ .exists(_.contains("google.cloud.auth.service.account.enable")))
+
+ val keyfileType = new Configuration(false)
+ keyfileType.set("fs.gs.auth.type", "SERVICE_ACCOUNT_JSON_KEYFILE")
+ assert(
+ DeltaScanSupport
+ .gcsHadoopOnlyAuthReason(keyfileType, Seq(new URI("gs://mybucket/part-0.parquet")))
+ .exists(_.contains("fs.gs.auth.type")))
+ }
+
test(
"gcsHadoopOnlyAuthReason does not fire for s3a/file URIs even when fs.gs.auth.* is set " +
"(scheme-scoped)") {
@@ -2050,6 +2163,172 @@ class DeltaScanContribSuite extends CometDeltaTestBase {
assert(!reason.get.contains("realSecretValue"))
}
+ test("S3 deletion-vector context gates the real claim path after encryption algorithms") {
+ withTempPath { dir =>
+ val path = dir.getAbsolutePath
+ spark
+ .range(4)
+ .coalesce(1)
+ .write
+ .format("delta")
+ .option("delta.enableDeletionVectors", "true")
+ .save(path)
+ val log = DeltaLog.forTable(spark, path)
+ val snapshot = log.update()
+ val files = snapshot.allFiles.collect()
+ assert(files.length == 1)
+ val dv = DeletionVectorDescriptor
+ // A real sidecar starts with a version byte. Older Delta serializers require
+ // the bitmap offset even though this plan-only test never opens the sidecar.
+ .onDiskWithAbsolutePath("s3a://context-dv-bucket/deletion_vector.bin", 40, 1, Some(1))
+ val replaced = files.head.copy(deletionVector = dv).withoutTightBoundStats
+ val commit = new File(dir, f"_delta_log/${snapshot.version + 1}%020d.json")
+ Files.write(
+ commit.toPath,
+ (files.head.remove.json + "\n" + replaced.json + "\n").getBytes(UTF_8))
+ val selected = log.update().allFiles.collect()
+ assert(selected.length == 1)
+ assert(selected.head.absolutePath(log).toUri.getScheme == "file")
+ assert(
+ selected.head.deletionVector
+ .absolutePath(new Path(path))
+ .toUri
+ .getHost == "context-dv-bucket")
+
+ val conf = spark.sparkContext.hadoopConfiguration
+ val contextKey = "fs.s3a.bucket.context-dv-bucket.encryption.context"
+ val algorithmKey = "fs.s3a.bucket.context-dv-bucket.encryption.algorithm"
+ val original = Seq(contextKey, algorithmKey).map(key => key -> Option(conf.getRaw(key)))
+ try {
+ conf.unset(contextKey)
+ conf.unset(algorithmKey)
+ // Plan only: the descriptor intentionally names no real sidecar and must never be read.
+ val baseline = spark.read.format("delta").load(path)
+ assert(deltaNativeScans(baseline).nonEmpty, baseline.queryExecution.executedPlan.toString)
+
+ conf.set(contextKey, "synthetic-malformed-pair")
+ val declined = spark.read.format("delta").load(path)
+ assert(deltaNativeScans(declined).isEmpty)
+ val reasons =
+ new ExtendedExplainInfo().getFallbackReasons(declined.queryExecution.executedPlan)
+ assert(
+ reasons.exists(_.contains("does not support fs.s3a.encryption.context")),
+ reasons.toString)
+ Seq("SSE-C", "CSE-KMS", "CSE-CUSTOM").foreach { algorithm =>
+ conf.set(algorithmKey, algorithm)
+ val encrypted = spark.read.format("delta").load(path)
+ assert(deltaNativeScans(encrypted).isEmpty)
+ val encryptionReasons =
+ new ExtendedExplainInfo().getFallbackReasons(encrypted.queryExecution.executedPlan)
+ assert(
+ encryptionReasons.exists(_.contains(s"fs.s3a.encryption.algorithm=$algorithm")),
+ encryptionReasons.toString)
+ assert(
+ !encryptionReasons.exists(_.contains("does not support fs.s3a.encryption.context")))
+ }
+ } finally {
+ original.foreach {
+ case (key, Some(value)) => conf.set(key, value)
+ case (key, None) => conf.unset(key)
+ }
+ }
+ }
+ }
+
+ test("unsupportedEncryptionContextReason passes absent/empty and non-S3 contexts") {
+ val conf = new Configuration(false)
+ assert(DeltaScanSupport.unsupportedEncryptionContextReason(conf, DataBucketUris).isEmpty)
+ conf.set("fs.s3a.encryption.context", "")
+ assert(DeltaScanSupport.unsupportedEncryptionContextReason(conf, DataBucketUris).isEmpty)
+ conf.set("fs.s3a.encryption.context", "synthetic-malformed-pair")
+ assert(
+ DeltaScanSupport
+ .unsupportedEncryptionContextReason(
+ conf,
+ Seq(new URI("file:///tmp/data.parquet"), new URI("gs://data-bucket/data.parquet")))
+ .isEmpty)
+ }
+
+ test(
+ "unsupportedEncryptionContextReason declines global/short/long values without printing them") {
+ val keys = Seq(
+ "fs.s3a.encryption.context",
+ "fs.s3a.bucket.data-bucket.encryption.context",
+ "fs.s3a.bucket.data-bucket.fs.s3a.encryption.context")
+ for (key <- keys; value <- Seq("purpose=synthetic", "synthetic-malformed-pair")) {
+ val conf = new Configuration(false)
+ conf.set(key, value)
+ val reason = DeltaScanSupport.unsupportedEncryptionContextReason(conf, DataBucketUris)
+ assert(reason.exists(_.contains("fs.s3a.encryption.context")))
+ assert(reason.exists(_.contains("data-bucket")))
+ assert(!reason.get.contains(value))
+ }
+ val unrelated = new Configuration(false)
+ unrelated.set("fs.s3a.bucket.other-bucket.encryption.context", "synthetic-malformed-pair")
+ assert(DeltaScanSupport.unsupportedEncryptionContextReason(unrelated, DataBucketUris).isEmpty)
+ }
+
+ test(
+ "unsupportedEncryptionContextReason resolves propagated references and preserves fallback") {
+ val conf = new Configuration(false)
+ conf.set("fs.s3a.encryption.context", s"$${fs.s3a.context.reference}")
+ conf.set("fs.s3a.context.reference", "")
+ conf.set("fs.s3a.bucket.data-bucket.context.reference", "synthetic-malformed-pair")
+ val reason = DeltaScanSupport.unsupportedEncryptionContextReason(conf, DataBucketUris)
+ assert(reason.isDefined)
+ assert(!reason.get.contains("synthetic-malformed-pair"))
+ conf.setBoolean("hadoop.security.credential.clear-text-fallback", false)
+ assert(DeltaScanSupport.unsupportedEncryptionContextReason(conf, DataBucketUris).isEmpty)
+ val unresolved = new Configuration(false)
+ unresolved.set("fs.s3a.encryption.context", s"$${unresolved.synthetic.context}")
+ val unresolvedReason =
+ DeltaScanSupport.unsupportedEncryptionContextReason(unresolved, DataBucketUris)
+ assert(unresolvedReason.isDefined)
+ assert(!unresolvedReason.get.contains("unresolved.synthetic.context"))
+ }
+
+ test("unsupportedEncryptionContextReason finds context stored only in JCEKS bucket aliases") {
+ val keys = Seq(
+ "fs.s3a.encryption.context",
+ "fs.s3a.bucket.data-bucket.encryption.context",
+ "fs.s3a.bucket.data-bucket.fs.s3a.encryption.context")
+ keys.foreach { key =>
+ withJceks(Map(key -> "purpose=synthetic")) { conf =>
+ assert(conf.get(key) == null)
+ conf.setBoolean("hadoop.security.credential.clear-text-fallback", false)
+ val reason = DeltaScanSupport.unsupportedEncryptionContextReason(conf, DataBucketUris)
+ assert(reason.isDefined)
+ assert(!reason.get.contains("purpose=synthetic"))
+ }
+ }
+ }
+
+ test(
+ "unsupportedEncryptionContextReason declines unverifiable providers without exposing paths") {
+ for ((key, value) <- Seq(
+ "fs.s3a.security.credential.provider.path" -> "jceks://file/no/synthetic-store.jceks",
+ "hadoop.security.credential.provider.path" -> "jceks://s3a@secret-bucket/secret.jceks")) {
+ val conf = new Configuration(false)
+ conf.set(key, value)
+ val reason = DeltaScanSupport.unsupportedEncryptionContextReason(conf, DataBucketUris)
+ assert(reason.isDefined)
+ assert(!reason.get.contains(value))
+ }
+ val corruptFile = File.createTempFile("comet-context-corrupt", ".jceks")
+ try {
+ Files.write(corruptFile.toPath, Array[Byte](1, 2, 3))
+ val conf = new Configuration(false)
+ conf.set(
+ "hadoop.security.credential.provider.path",
+ "jceks://file" + corruptFile.getAbsolutePath)
+ val reason = DeltaScanSupport.unsupportedEncryptionContextReason(conf, DataBucketUris)
+ assert(reason.isDefined)
+ assert(!reason.get.contains(corruptFile.getAbsolutePath))
+ } finally {
+ corruptFile.delete()
+ }
+ }
+
test(
"unsupportedEncryptionAlgorithmReason declines a bucket configured with global SSE-C, " +
"naming the algorithm key and the algorithm but never the customer-provided key value") {
@@ -2894,6 +3173,10 @@ class DeltaScanContribSuite extends CometDeltaTestBase {
"effectiveEncryptionAlgorithm's doc)"),
"fs.s3a.server-side-encryption.key" ->
"deprecated alias of fs.s3a.encryption.key, same reasoning",
+ "fs.s3a.encryption.context" ->
+ ("handled by unsupportedEncryptionContextReason: declines non-empty or unverifiable " +
+ "Hadoop-effective context for every S3 data/DV bucket, including when a local " +
+ "Delta log did not initialize that bucket's S3A filesystem"),
"fs.s3a.encryption.cse.kms.region" ->
("CSE tuning, newer Hadoop only: consulted solely when the algorithm resolves to " +
"a CSE variant, and the allowlist gate declines every CSE algorithm outright, " +
diff --git a/dev/ci/check-ci-config.py b/dev/ci/check-ci-config.py
index c752a50c266..f4bdf3fd1b1 100644
--- a/dev/ci/check-ci-config.py
+++ b/dev/ci/check-ci-config.py
@@ -133,6 +133,10 @@
([".mvn/maven.config"], MVN_JOBS),
([".mvn/wrapper/maven-wrapper.properties"], MVN_JOBS),
(["mvnw"], MVN_JOBS),
+ (["dev/ci/prepare-maven-model.sh"], {
+ "build_linux", "build_linux_full", "build_linux_all_profiles",
+ "build_macos", "build_macos_full", "delta_gate",
+ }),
# The artifact wrappers are used by every producer and consumer of a
# shared artifact. Without these, an edit confined to one of them routes
# to nothing at all and merges having been exercised by no consumer.
@@ -178,6 +182,7 @@
"delta_3_5",
"delta_4_0",
"delta_4_1",
+ "delta_4_2",
},
),
(["contrib/delta-spark/README.md"], set()),
@@ -227,6 +232,7 @@
"delta_3_5",
"delta_4_0",
"delta_4_1",
+ "delta_4_2",
"build_linux_all_profiles",
}
# Spark 3.4 is deprecated and sits outside the queue and nightly tiers
@@ -239,7 +245,7 @@
# each sits in.
ICEBERG_OPT_IN = {"iceberg_1_8", "iceberg_1_9", "iceberg_1_10", "iceberg_1_11"}
# Likewise one label for every Delta profile.
-DELTA_OPT_IN = {"delta_3_5", "delta_4_0", "delta_4_1"}
+DELTA_OPT_IN = {"delta_3_5", "delta_4_0", "delta_4_1", "delta_4_2"}
ALL_JOBS = QUEUE_TIER | NIGHTLY_TIER | SPARK_DEPRECATED | {"docs"}
assert not QUEUE_TIER & NIGHTLY_TIER, "a job is queue or nightly, never both"
diff --git a/dev/ci/check-suites.py b/dev/ci/check-suites.py
index 96a49cf4900..ce93369c540 100644
--- a/dev/ci/check-suites.py
+++ b/dev/ci/check-suites.py
@@ -52,6 +52,10 @@ def file_to_class_name(path: Path) -> str | None:
root = Path(".")
for path in root.rglob("*Suite.scala"):
+ # Build outputs may contain a patched Spark test checkout. Match the
+ # forward declaration check below: inventory project sources only.
+ if "target" in path.parts:
+ continue
# contrib suites run via their own module-level workflows
# (e.g. delta_contrib_test.yml), not the main PR build matrix
if path.parts[0] == "contrib":
diff --git a/dev/ci/compute-changes.py b/dev/ci/compute-changes.py
index 2e91d4c91a0..5520e5fffd3 100644
--- a/dev/ci/compute-changes.py
+++ b/dev/ci/compute-changes.py
@@ -126,6 +126,7 @@
"Makefile",
"rust-toolchain.toml",
"dev/verify-contrib-delta-gate.sh",
+ "dev/ci/prepare-maven-model.sh",
".github/workflows/ci.yml",
".github/workflows/delta_build_gate.yml",
".github/actions/setup-builder/**",
@@ -325,6 +326,7 @@
"!spark/src/main/scala/org/apache/comet/GenerateDocs.scala",
"spark/pom.xml",
"dev/diffs/4.2.0.diff",
+ "dev/diffs/4.2.1.diff",
"pom.xml",
"rust-toolchain.toml",
".github/workflows/ci.yml",
@@ -464,10 +466,11 @@
".github/actions/setup-builder/**",
],
# Same inputs as delta_3_5: the Delta contrib workflow is called once per
- # Spark profile, and the three calls share one path filter. Populated
+ # Spark profile, and the four calls share one path filter. Populated
# below, after the dict, so the lists cannot drift.
"delta_4_0": [],
"delta_4_1": [],
+ "delta_4_2": [],
}
FILTERS["spark_4_1_hive"] = FILTERS["spark_4_1"]
FILTERS["build_linux_full"] = FILTERS["build_linux"]
@@ -475,6 +478,7 @@
FILTERS["build_linux_all_profiles"] = FILTERS["build_linux"]
FILTERS["delta_4_0"] = FILTERS["delta_3_5"]
FILTERS["delta_4_1"] = FILTERS["delta_3_5"]
+FILTERS["delta_4_2"] = FILTERS["delta_3_5"]
# Which events may run each job, independent of the path filters above.
#
@@ -615,10 +619,11 @@
# The Delta contrib suites all run nightly. The PR tier's strict warnings
# job compiles the contrib against Spark 3.5, so a change that breaks its
# build on Spark 3.5 fails on its own pull request. One label opts a pull
- # request into all three.
+ # request into all four.
"delta_3_5": ["nightly", "label:run-delta-tests"],
"delta_4_0": ["nightly", "label:run-delta-tests"],
"delta_4_1": ["nightly", "label:run-delta-tests"],
+ "delta_4_2": ["nightly", "label:run-delta-tests"],
}
diff --git a/dev/ci/prepare-maven-model.sh b/dev/ci/prepare-maven-model.sh
new file mode 100755
index 00000000000..2f65a7a1dbb
--- /dev/null
+++ b/dev/ci/prepare-maven-model.sh
@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Resolve the root Maven model before a compile/test command. Only a transient
+# parent-POM transfer failure is retried. This command never compiles or tests:
+# the nonrecursive validate lifecycle and the allowed arguments are fixed here.
+set -euo pipefail
+for argument in "$@"; do
+ case "$argument" in
+ -P?*|-D?*) ;;
+ *) echo "Only attached -P/-D options are accepted; no Maven goals: $argument" >&2
+ exit 2 ;;
+ esac
+done
+root_dir=$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)
+log=$(mktemp "${RUNNER_TEMP:-${TMPDIR:-/tmp}}/comet-maven-model.XXXXXX")
+trap 'rm -f "$log"' EXIT
+cd "$root_dir"
+for attempt in 1 2 3; do
+ if ./mvnw -B -N -U -Dstyle.color=never "$@" validate 2>&1 | tee "$log"; then
+ exit 0
+ else
+ statuses=("${PIPESTATUS[@]}")
+ status=${statuses[0]}
+ if [ "${statuses[1]}" -ne 0 ]; then
+ echo "Could not record Maven model output" >&2
+ exit "${statuses[1]}"
+ fi
+ fi
+ # A resolved model can execute validation/enforcer plugins. Never retry
+ # those failures, even if a later diagnostic mentions a repository error.
+ if grep -qE '^\[INFO\] (Building |Reactor Build Order:|--- )' "$log" ||
+ ! grep -qE 'Non-resolvable parent POM.*Could not transfer artifact.*status code: (408|429|500|502|503|504)([^0-9]|$)' "$log"; then
+ exit "$status"
+ fi
+ if [ "$attempt" -eq 3 ]; then
+ echo "Maven parent POM remains unavailable after $attempt model-only attempts" >&2
+ exit "$status"
+ fi
+ # Resolver already honors Retry-After and has its own bounded retry budget.
+ # Leave a pause between cold-start attempts instead of immediately hammering
+ # the repository. -U above avoids reusing a cached failed transfer.
+ delay=$((60 * (1 << (attempt - 1)) + RANDOM % 5))
+ echo "Transient parent-POM transfer failed; retrying model resolution in ${delay}s" >&2
+ sleep "$delay"
+done
diff --git a/dev/ci/test-maven-model.py b/dev/ci/test-maven-model.py
new file mode 100644
index 00000000000..e5148e2fb2f
--- /dev/null
+++ b/dev/ci/test-maven-model.py
@@ -0,0 +1,110 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Failure-injection coverage for the model-only Maven startup retry."""
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import tempfile
+import unittest
+
+SCRIPT = Path(os.environ.get("COMET_MAVEN_MODEL_SCRIPT",
+ Path(__file__).resolve().with_name("prepare-maven-model.sh")))
+
+
+class MavenModelTest(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory(prefix="maven-model-test-")
+ self.root = Path(self.tmp.name)
+ scripts = self.root / "dev/ci"
+ scripts.mkdir(parents=True)
+ shutil.copyfile(SCRIPT, scripts / SCRIPT.name)
+ (self.root / "logs").mkdir()
+ (self.root / "bin").mkdir()
+ sleep = self.root / "bin/sleep"
+ sleep.write_text("#!/bin/sh\nexit 0\n")
+ sleep.chmod(0o755)
+ wrapper = self.root / "mvnw"
+ wrapper.write_text("""#!/usr/bin/env python3
+import json, os, sys
+from pathlib import Path
+log = Path('calls.json')
+calls = json.loads(log.read_text()) if log.exists() else []
+calls.append(sys.argv[1:])
+log.write_text(json.dumps(calls))
+mode = os.environ['MODEL_TEST_MODE']
+if mode == 'recover' and len(calls) > 1:
+ print('[INFO] BUILD SUCCESS'); sys.exit(0)
+if mode == 'invalid':
+ print('[ERROR] Non-parseable POM'); sys.exit(17)
+if mode == 'enforcer':
+ print('[INFO] Building parent 1.2.0')
+ print('[ERROR] Enforcer rule failed'); sys.exit(17)
+if mode == 'started':
+ print('[INFO] Building parent 1.2.0')
+code = '404' if mode == 'not-found' else '429'
+print('[FATAL] Non-resolvable parent POM: Could not transfer artifact parent:pom:1 '
+ + 'status code: ' + code + ', reason phrase: Unavailable')
+sys.exit(17)
+""")
+ wrapper.chmod(0o755)
+
+ def tearDown(self):
+ self.tmp.cleanup()
+
+ def run_model(self, mode, *args):
+ env = dict(os.environ, MODEL_TEST_MODE=mode,
+ RUNNER_TEMP=str(self.root / "logs"),
+ PATH=str(self.root / "bin") + os.pathsep + os.environ["PATH"])
+ result = subprocess.run(["bash", str(self.root / "dev/ci" / SCRIPT.name), *args],
+ cwd=self.root, env=env, capture_output=True, text=True)
+ calls = self.root / "calls.json"
+ return result, json.loads(calls.read_text()) if calls.exists() else []
+
+ def test_transient_parent_failure_recovers_only_model(self):
+ result, calls = self.run_model("recover", "-Pspark-4.2", "-Dexample=value")
+ self.assertEqual(result.returncode, 0, result.stdout + result.stderr)
+ self.assertEqual(len(calls), 2)
+ for call in calls:
+ self.assertEqual(call, ["-B", "-N", "-U", "-Dstyle.color=never",
+ "-Pspark-4.2", "-Dexample=value", "validate"])
+
+ def test_retry_budget_is_bounded_and_preserves_failure(self):
+ result, calls = self.run_model("exhaust")
+ self.assertEqual(result.returncode, 17)
+ self.assertEqual(len(calls), 3)
+
+ def test_other_failures_are_not_retried(self):
+ for mode in ("not-found", "invalid", "enforcer", "started"):
+ with self.subTest(mode=mode):
+ calls_path = self.root / "calls.json"
+ calls_path.unlink(missing_ok=True)
+ result, calls = self.run_model(mode)
+ self.assertEqual(result.returncode, 17)
+ self.assertEqual(len(calls), 1)
+
+ def test_compile_or_test_goals_are_rejected(self):
+ for goal in ("compile", "test", "install", "help:effective-pom"):
+ with self.subTest(goal=goal):
+ result, calls = self.run_model("recover", goal)
+ self.assertEqual(result.returncode, 2)
+ self.assertEqual(calls, [])
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/dev/diffs/4.2.1.diff b/dev/diffs/4.2.1.diff
new file mode 100644
index 00000000000..df87c0e2a98
--- /dev/null
+++ b/dev/diffs/4.2.1.diff
@@ -0,0 +1,6030 @@
+diff --git a/core/src/test/scala/org/apache/spark/storage/FallbackStorageSuite.scala b/core/src/test/scala/org/apache/spark/storage/FallbackStorageSuite.scala
+index 6df8bc85b51..dabb75e2b75 100644
+--- a/core/src/test/scala/org/apache/spark/storage/FallbackStorageSuite.scala
++++ b/core/src/test/scala/org/apache/spark/storage/FallbackStorageSuite.scala
+@@ -268,6 +268,11 @@ class FallbackStorageSuite extends SparkFunSuite with LocalSparkContext {
+ }
+
+ test("Upload from all decommissioned executors") {
++ // Comet replaces Spark's shuffle with its own native shuffle, which is incompatible with
++ // the fallback storage migration path used by BlockManagerDecommissioner.
++ val cometEnv = System.getenv("ENABLE_COMET")
++ assume(cometEnv == null || cometEnv == "0" || cometEnv == "false",
++ "Skipped when Comet is enabled: incompatible with Comet native shuffle storage")
+ sc = new SparkContext(getSparkConf(2, 2))
+ withSpark(sc) { sc =>
+ TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
+@@ -298,6 +303,11 @@ class FallbackStorageSuite extends SparkFunSuite with LocalSparkContext {
+ }
+
+ test("Upload multi stages") {
++ // Comet replaces Spark's shuffle with its own native shuffle, which is incompatible with
++ // the fallback storage migration path used by BlockManagerDecommissioner.
++ val cometEnv = System.getenv("ENABLE_COMET")
++ assume(cometEnv == null || cometEnv == "0" || cometEnv == "false",
++ "Skipped when Comet is enabled: incompatible with Comet native shuffle storage")
+ sc = new SparkContext(getSparkConf())
+ withSpark(sc) { sc =>
+ TestUtils.waitUntilExecutorsUp(sc, 1, 60000)
+@@ -332,6 +342,11 @@ class FallbackStorageSuite extends SparkFunSuite with LocalSparkContext {
+
+ CompressionCodec.shortCompressionCodecNames.keys.foreach { codec =>
+ test(s"$codec - Newly added executors should access old data from remote storage") {
++ // Comet replaces Spark's shuffle with its own native shuffle, which is incompatible with
++ // the fallback storage migration path used by BlockManagerDecommissioner.
++ val cometEnv = System.getenv("ENABLE_COMET")
++ assume(cometEnv == null || cometEnv == "0" || cometEnv == "false",
++ "Skipped when Comet is enabled: incompatible with Comet native shuffle storage")
+ sc = new SparkContext(getSparkConf(2, 0).set(IO_COMPRESSION_CODEC, codec))
+ withSpark(sc) { sc =>
+ TestUtils.waitUntilExecutorsUp(sc, 2, 60000)
+diff --git a/pom.xml b/pom.xml
+index ac1bac9d649..c1f1d8b8b33 100644
+--- a/pom.xml
++++ b/pom.xml
+@@ -154,6 +154,8 @@
+ 4.0.3
+ 2.5.3
+ 2.0.8
++ 4.2
++ 1.2.0
+
+
+ org.apache.datasketches
+diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
+index 7c5e49f3a69..bfa0bce303d 100644
+--- a/project/SparkBuild.scala
++++ b/project/SparkBuild.scala
+@@ -455,9 +455,11 @@ object SparkBuild extends PomBuild {
+
+ /* Spark SQL Core settings */
+ enable(SQL.settings)(sql)
++ enable(CometTestSettings.settings)(sql)
+
+ /* Hive console settings */
+ enable(Hive.settings)(hive)
++ enable(CometTestSettings.settings)(hive)
+
+ enable(HiveThriftServer.settings)(hiveThriftServer)
+
+@@ -1426,6 +1428,37 @@ object SQL {
+ }
+ }
+
++/**
++ * Comet disables itself when its shuffle manager is not registered. When Comet is enabled, make
++ * that shuffle manager the default for every test SparkConf in the modules that have Comet on
++ * their classpath, so that suites which build their own SparkSession or SparkContext run Comet
++ * too, not just the ones that go through SharedSparkSession or TestHive. Comet's in-memory
++ * cache, which is disabled by default, is turned on the same way, and its serializer made the
++ * default. Spark keeps the first cache serializer it loads for the rest of the JVM, so a suite
++ * that caches through a session of its own would otherwise leave the suites after it in Spark's
++ * cache format.
++ */
++object CometTestSettings {
++ private val cometEnabled =
++ sys.env.get("ENABLE_COMET").forall(v => v == "1" || v.equalsIgnoreCase("true"))
++
++ private val defaults = Seq(
++ "spark.shuffle.manager" ->
++ "org.apache.spark.sql.comet.execution.shuffle.CometShuffleManager",
++ "spark.comet.exec.inMemoryCache.enabled" -> "true",
++ "spark.sql.cache.serializer" ->
++ "org.apache.spark.sql.comet.execution.arrow.ArrowCachedBatchSerializer")
++
++ lazy val settings: Seq[Setting[_]] =
++ if (cometEnabled) {
++ defaults.filterNot { case (key, _) => sys.props.contains(key) }.map {
++ case (key, value) => (Test / javaOptions) += s"-D$key=$value"
++ }
++ } else {
++ Seq.empty
++ }
++}
++
+ object Hive {
+
+ lazy val settings = Seq(
+diff --git a/sql/core/pom.xml b/sql/core/pom.xml
+index f3edcddc7e6..0947c343f68 100644
+--- a/sql/core/pom.xml
++++ b/sql/core/pom.xml
+@@ -97,6 +97,10 @@
+ org.apache.spark
+ spark-tags_${scala.binary.version}
+
++
++ org.apache.datafusion
++ comet-spark-spark${spark.version.short}_${scala.binary.version}
++
+
+
+
# Instructions for migration documentation
Keep migration execution guidance, decisions, handoffs, progress and cleanup
@@ -5,10 +24,10 @@ instructions inside this directory. Do not add migration-specific guidance to
the repository root AGENTS.md or other project documentation.
Before work, read README.md, decisions.json, STATE.json and approved-plan.json.
-The latest direct user instruction is: **do not write implementation code until
-the user explicitly asks**. Local documentation updates and read-only preparation
-are allowed. Creating/managing worker chats is authorized; it does not authorize
-starting implementation while the code hold is active.
+The latest direct user instruction authorizes implementation of **G01 only**.
+Do not start G02 or any later group without a new explicit user instruction after
+review of G01. Session orchestration, local Git and the approved network scope
+remain authorized; publishing/production actions remain excluded.
If a required tool, permission or environment capability is missing, report the
specific blocker to the coordinator/user rather than spend time building a long
@@ -30,4 +49,15 @@ The main chat is the default implementation/integration owner; internal helpers
perform bounded research and independent review. The coordinator chooses delegation
and occasional user-visible sessions autonomously within the authorized scope.
Nine feature branches do not require nine separate chats. See HANDOFF.md and
-decisions.json#execution_model. The current code hold remains active.
+decisions.json#execution_model. The stage gate remains active for G02–G10; only G01 is authorized.
+
+## G01 finalization and committed language
+
+The user authorized final English documentation, service-history consolidation,
+exact-SHA RC qualification, and conditional G01 integration/tagging on 2026-11-18.
+See decisions.json#g01_finalization_authorization and G01-CLOSEOUT.md. Earlier
+no-merge/no-tag statements are historical for G01. G02-G10 and Java 21 work remain
+closed. Commit messages and committed documentation must be English; user quotes
+originally written in Russian are translated, with originals preserved in the
+pre-finalization archive. A final post-build source commit would change provenance;
+record release acceptance in the immutable annotated tag and artifact receipts.
diff --git a/docs/joom-migration/CHECKPOINT-VERIFICATION.json b/docs/joom-migration/CHECKPOINT-VERIFICATION.json
new file mode 100644
index 00000000000..6c7304ea535
--- /dev/null
+++ b/docs/joom-migration/CHECKPOINT-VERIFICATION.json
@@ -0,0 +1,15 @@
+{
+ "status": "G01_PR_PREPARED_AMD64_GHA_PENDING",
+ "source_before_documentation_commit": "235f3d306e3ec4be2d571b93a6c7dc99fb0af512",
+ "base_unchanged": "9691385f35e4c5bd29514dca78d2044dd20e4bed",
+ "alignment_receipt": "G01-PR-preparation.json",
+ "all_source_docs_identical_during_alignment": true,
+ "intentional_followup_changes": [
+ "3ANSIassume removals",
+ "approvedpolicyandPRhandoffdocs"
+ ],
+ "active_sessions": {},
+ "merge_rc1_g02_authorized": false,
+ "new_ansi_tests_passed": false,
+ "restart_entry_point": "PAUSED.md"
+}
diff --git a/docs/joom-migration/CLEANUP.md b/docs/joom-migration/CLEANUP.md
index 21f15ff404d..7413e52da3b 100644
--- a/docs/joom-migration/CLEANUP.md
+++ b/docs/joom-migration/CLEANUP.md
@@ -1,48 +1,122 @@
-# Учёт диска и очистка после миграции
+
-Для каждого ресурса: тип; точный path/image ID/tag/volume/builder/cache ID;
-группа и команда создания; существовал ли раньше; принадлежит ли только этой
-работе; размер до/после; надо ли сохранить; точечная команда очистки и условия.
+> **G01 finalization authorized:** documentation and history cleanup, followed by exact-SHA RC qualification, are in progress. Integration into `branch-1.2` and `joom-1.2-1-rc1` are conditional on successful checks. G02 and Java 21 work remain on hold.
-Учитывать:
+# Cleanup after the migration
-- Maven module `target`, Cargo `native/target` и release `cargo-target/{arm64,amd64}`;
-- release outputs: JAR, native `.so`, checksums, commit/build manifests;
-- скачанные archives/tools и одноразовые каталоги `/Users/skotlov/d/temp`;
-- Maven wrapper/JVM dependency cache, Cargo registry/git cache, Rustup toolchains;
-- Docker images/layers, containers, named volumes, buildx builders/build cache;
-- benchmark datasets/reports, Spark event logs/warehouse/spill и test outputs.
+All commands below are for manual execution AFTER the entire migration and preservation of the required artifacts. Caches are still needed by subsequent groups and RCs. Nothing has been deleted automatically.
-Не создавать отдельные копии штатных shared caches ради упрощения учёта.
-В них различать «использовано» и «достоверно создано этой работой». Сам по себе
-mtime не доказывает принадлежность. Если точное разделение невозможно, явно
-пометить shared cache и не рекомендовать его полное удаление.
+Keep the final JARs, both `.so` files, sha256.txt, commit.txt, build-manifest.json, effective-pom.xml, linux/* metadata/toolchain/ELF imports, smoke evidence and docs/joom-migration. Do not remove the entire root target directory: it contains release artifacts and the Spark fixture.
-## Что сохранить
+## Sizes
-Итоговые JAR, SHA256, Git SHA/build manifests и результаты qualification.
-Для воспроизведения — утверждённые решения/план и нужные коммиты.
-Документация миграции целиком находится в `docs/joom-migration`; пользователь
-решит отдельно, оставлять ли этот каталог в итоговых исходниках.
+Measurements use du (allocated KiB); see cleanup-ledger.json for the detailed snapshot and timestamp. Nested sizes are included in their parent and must not be counted twice.
-## Что не считать нашими disposable outputs
+- `/Users/skotlov/d/projects/datafusion-comet/native/target`: 26.01 GiB.
+- `/Users/skotlov/d/projects/datafusion-comet/target/joom-spark-4.2.1`: 0.80 GiB.
+- `/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1`: 5.49 GiB.
+- `/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1/cargo-target`: 4.94 GiB.
+- `/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1/.work`: 0.00 GiB.
+- `/Users/skotlov/d/projects/datafusion-comet/target/joom-release-validator-cases`: 0.00 GiB.
+- `/Users/skotlov/d/projects/datafusion-comet/common/target`: 0.00 GiB.
+- `/Users/skotlov/d/projects/datafusion-comet/contrib/delta-spark/target`: 0.00 GiB.
+- `/Users/skotlov/d/projects/datafusion-comet/spark/target`: 0.39 GiB.
+- `/Users/skotlov/d/projects/datafusion-comet/spark-integration/target`: 0.00 GiB.
-- Пользовательские checkouts `/Users/skotlov/d/temp/src/*` и Joom repositories.
-- Rust/Rustup/Homebrew, которые пользователь установил сам, и существующие JDK.
-- Предсуществующие Docker images/volumes/builders и shared caches.
+## Targeted cleanup after the last RC
-Не предлагать общий `docker system prune -a --volumes`, полную очистку Maven/Cargo/
-Homebrew caches или shared builder. Финальная инструкция должна указывать только
-проверенные ресурсы и точечные команды, сохраняя финальные артефакты.
+Native host build outputs, not source/vendor
-Старые материалы аудита в temp перечисляются как исторические кандидаты на
-ручную очистку. Они не должны удаляться только на основании имени каталога.
+```bash
+rm -rf -- /Users/skotlov/d/projects/datafusion-comet/native/target
+```
+
+Generated Spark qualification checkout; original user Spark checkout remains untouched
+
+```bash
+rm -rf -- /Users/skotlov/d/projects/datafusion-comet/target/joom-spark-4.2.1
+```
+
+Release compiler intermediates only; keeps final JARs/manifests/native files/smoke
+
+```bash
+rm -rf -- /Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1/cargo-target /Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1/.work
+```
+
+Synthetic negative-validator fixtures
+
+```bash
+rm -rf -- /Users/skotlov/d/projects/datafusion-comet/target/joom-release-validator-cases
+```
+
+Only these task-exclusive builder tags, after no task containers remain; no force and no prune
+
+```bash
+docker --context desktop-linux image rm comet-joom-builder-focal:1.2-arm64 comet-joom-builder-focal:1.2-amd64
+```
+
+Only these task-created dependency volumes, after no task containers remain; no force and no prune
+
+```bash
+docker --context desktop-linux volume rm comet-joom-1.2-cargo-arm64 comet-joom-1.2-cargo-amd64 comet-joom-1.2-cargo-git
+```
+
+## Shared caches and sources
+
+`~/.m2`, `~/.cargo`, `~/.rustup`, `~/.sbt`, `~/.ivy2`, Coursier and ProtocBridge were measured only as shared usage. Their entire size cannot be attributed to this migration; do not clear them wholesale. Preserve the user-installed JDK/Rust/Homebrew and source checkouts, including `/Users/skotlov/d/temp/src`.
+
+Docker builder images: ARM de016db98def..., AMD92564ee97ddb...; exact IDs/sizes are in the ledger. The three named volumes did not exist in the baseline and were created by this task. Docker build cache/shared Desktop disk usage cannot be precisely attributed by owner. Do not use system/builder/volume prune, forced removal, root Maven clean or git clean -fdx.
+
+## Auxiliary materials
+
+Current logs/scripts/receipts: `/Users/skotlov/d/temp/20261117_comet12-g01-final-qualification`; previous: `/Users/skotlov/d/temp/20261117_comet12-g01-resumed`, `/Users/skotlov/d/temp/20261117_comet12-g01`. Historical audit directories and their purposes are listed in cleanup-ledger.json. After preserving results, a selected exact task directory may be removed manually; the name temp does not itself authorize deletion.
+
+## G01 corrections after the first full CI
+
+Auxiliary patches, local test logs, checkpoint and new GHA snapshots: `/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes`. This directory was created for the current task; preserve it until acceptance. The temporary pristine copy of 121 Spark files was already removed after equality was confirmed. Rebuilds reused existing `native/target`, `spark/target`, `contrib/delta-spark/target`, `target/joom-spark-4.2.1` and standard caches; do not remove them automatically. No final JARs were released by this iteration.
+
+Snapshot after targeted checks: auxiliary directory ~6.3MiB; existing `native/target` ~39.7GiB total; Spark fixture with outputs ~831MiB. These are entire directory sizes, not the increase caused by the iteration.
+
+The next follow-up GHA is stored separately in `/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha`; the first run is frozen in the parent directory. Standard Maven caches and existing targets were also used for Scala2.12/Spark3.5 and Spark4.1 compatibility controls. No new Rust/Docker/release builds occurred in the follow-up. Full large SQL console logs were not retained: only selected lines, SHA receipts and bounded tails.
+
+Final snapshot on 2026-11-18: G01 CI-fix auxiliary data ~21.4MiB, existing native/target ~39.7GiB, Spark fixture ~829MiB (entire sizes, not increases). Run44205897075 and all three attempts are preserved in `20261117_followup-gha`; only small API/log/report receipts were saved, without extracting large archives. Continued use may slightly increase the directory size. No automatic cleanup occurred; old release JARs are not the current qualified RC.
+
+## Manual RC Build & Smoke: 2026-11-18
+
+Currently checked runtime JARs/checksums/provenance/smoke: `/Users/skotlov/d/projects/datafusion-comet/target/joom-release/gha-44388252264` (~410MiB for the entire directory). Keep these until the migration ends; two JARs are used by the image. The older `target/joom-release/joom-1.2-1-rc1` belongs to source2b8f80b3f and is historical, not this checked artifact set.
+
+Raw API/logs/control scripts/fault tests/actionlint: `/Users/skotlov/d/temp/20261118_comet12-rc-workflow`. It also contains the GitHub ZIP `qualified-artifact-44388252264.zip` (~181MiB); after archiving evidence and acceptance, that specifically selected task directory may be removed manually. No new local release/native builds occurred; Docker probes used existing images and --rm containers. Native builds and multi-gigabyte test dependencies ran on GHA; their caches were not downloaded to the laptop. Local RAT checks may have updated the ordinary M2 cache and `target/rat.txt`; preserve the shared cache.
+
+GitHub artifact retention: native3 days, candidate/test support7 days, smoke/qualified14 days. Current qualified artifact ID14469302529; expiry is configured by the workflow, while the local copy does not expire. No JFrog/Docker-registry publication occurred.
+
+## Final G01 translation, history and RC1 acceptance
+
+- Auxiliary archive, mappings, checks, API snapshots, final ZIP and completion receipt: `/Users/skotlov/d/temp/20261118_comet12-g01-finalization`. Retain `documentation-before.zip` and `completion.json` until the handoff is verified.
+- The final qualified output is `target/joom-release/gha-`, where the final run ID is recorded in the annotated `joom-1.2-1-rc1` tag and `completion.json`. Expect approximately 410 MiB unpacked plus a 190 MiB downloaded ZIP. No local native rebuild is needed for this finalization.
+- Preserve the final JARs, checksum/build/provenance and smoke receipts before any optional cleanup. Old `gha-44388252264` artifacts remain historical and are not the final rewritten RC1.
+- Recovery refs `refs/joom-migration/checkpoints/20261118-before-finalization`, `refs/joom-migration/checkpoints/20261118-rebuilt-code` and the existing earlier checkpoint refs are local recovery evidence. Do not delete them automatically. No shared cache or user checkout is made disposable by this step.
+
+After retaining the final handoff, optional deletion of only the final downloaded ZIP is:
+
+```bash
+rm /Users/skotlov/d/temp/20261118_comet12-g01-finalization/qualified-artifact-.zip
+```
+
+Replace `` with the verified final run ID. Consult `completion.json` for exact paths, archive/JAR sizes and hashes. Nothing is automatically removed.
diff --git a/docs/joom-migration/G01-CLOSEOUT.md b/docs/joom-migration/G01-CLOSEOUT.md
new file mode 100644
index 00000000000..546093c0d1c
--- /dev/null
+++ b/docs/joom-migration/G01-CLOSEOUT.md
@@ -0,0 +1,58 @@
+
+
+# G01 closeout and RC1 identity
+
+G01 implements the Spark 4.2.1 baseline. The remaining agreed Joom features in G02-G09 have **not** been ported. G02 and Java 21 qualification remain explicitly on hold. This is an internal migration candidate, not the completed production replacement for `joom-1.1-13`.
+
+## Source freeze and release acceptance
+
+This document is committed before the final release build. Its own commit SHA and the future Actions run ID cannot be embedded here without changing the source being qualified. The immutable annotated tag `joom-1.2-1-rc1` is created **only after** exact-source RC qualification succeeds. Its annotation records the final run URL, source SHA and JAR checksums. The downloaded `release-qualification.json`, `commit.txt`, `build-manifest.json`, `sha256.txt` and both smoke receipts provide machine-readable acceptance evidence for those exact bytes. Do not infer successful release acceptance from this document alone or use a historical B93 artifact as RC1.
+
+```bash
+git rev-parse joom-1.2-1-rc1^{commit}
+git show --no-patch joom-1.2-1-rc1
+git merge-base --is-ancestor joom-1.2-1-rc1 branch-1.2
+```
+
+Integration uses a fast-forward, preserving the qualified SHA. No post-build documentation or merge commit is added to the release. The final run's artifacts are downloaded to `target/joom-release/gha-`. An additional local completion receipt is saved at `/Users/skotlov/d/temp/20261118_comet12-g01-finalization/completion.json`. The Actions run can be located from the tag annotation or [RC Build & Smoke](https://github.com/joomcode/datafusion-comet/actions/workflows/rc_build_smoke.yml).
+
+## Retained behavior and validation
+
+- Exact versions: Comet 1.2.0 base; Spark 4.2.1; Scala 2.13.18; Delta 4.4.1 using `io.delta:delta-spark_4.2_2.13`; DataFusion 55.2.0; native Arrow/Parquet 59.3.2; Java 17; Rust 1.100.0.
+- The upstream Delta scanner retains the agreed GCP ADC admission, safe planning, driver-payload serialization and DPP/empty-scan behavior. A conservative encryption-context admission guard preserves fallback.
+- The published pristine DataFusion physical-plan import stays a separate commit. G01 changes workspace integration and four IPC statistics test assertions, not vendor production algorithms.
+- Spark patch-level decimal casts, mandatory duplicate-Parquet errors and immutable Union/AQE partition-layout handling have targeted regression coverage. Existing Spark SQL fixture assertions are retained with native counterparts.
+- Full Comet CI [44205897075](https://github.com/joomcode/datafusion-comet/actions/runs/44205897075), attempt 3, qualified runtime source `26590264b878fb8eae2ca439a8bcc0dfb513cb36`: 127 successful jobs, 17 conditional skips, no failures. Runtime sources, tests, Spark fixture and dependency locks are byte-identical after finalization. This is preserved source-based evidence, not a claim that the rewritten SHA ran full CI again. Automatic full CI is canceled as instructed by the user.
+- The final rewritten SHA must independently pass the manual seven-job RC workflow: native AMD64 and ARM64 release builds; one dualarch packaging; 12/12 existing cases on each native architecture (Delta 4, DPP 1, reload 1, ANSI 6); strict receipt binding and artifact export. Canceled or ignored smoke tests are not accepted.
+- Loaded bundled native SHA, JVM class origins, actual Spark/Java versions, source/lockfile/graph identity and archive/JAR checksums must agree. Source archive RAT, formatting, CI configuration, Bash syntax and 22 release fault tests must pass.
+
+Four ignored HDFS-cluster tests are inapplicable to Joom GCP; the fifth ignored Rust test is an existing benchmark. Upstream vendor deprecation warnings are documented and were not removed by editing production code. Real GCS/GKE ADC reads, production performance and the remaining feature-group correctness work are not qualified by G01.
+
+## Translation and history preservation
+
+All committed migration documentation is English. User quotations originally written in Russian are translated quotations; the original snapshot is retained in `/Users/skotlov/d/temp/20261118_comet12-g01-finalization/documentation-before.zip`. The full 58 active feature cards plus historical H01, approved nine-group plan and tenth reserve, dependencies, source references, correctness guards and authorization boundaries are preserved. The English text snapshots are `approved-plan.txt` and `correctness-test-plan.txt`; external historical research paths retain their original names.
+
+`G01-HISTORY.json` maps the original G01 commits to the rewritten history. All G01 service/checkpoint documentation is consolidated into one final documentation commit. Eighteen substantive code commits remain; RC implementation and its follow-up fixes are combined. Existing integration-base ancestry is retained. Backup ref `refs/joom-migration/checkpoints/20261118-before-finalization` and the original documentation archive preserve recovery evidence.
+
+The final non-documentation tree must equal pre-finalization source `b93d553856a0339af7468bcff2070e735a1a3c71` byte for byte, including file modes. JSON structures, numbers, booleans, card membership, dependencies and untranslated technical identifiers are checked against the archived originals; changed prose is reviewed for semantic preservation. `MANIFEST.json` covers every document except itself. Final integration verifies the feature, base and tag identities, the qualified release source, checksums and a clean checkout.
+
+Use `README.md`, `STATE.json` and `decisions.json` for continuation. Historical stages and receipts retain their original source-specific conclusions. This closeout and the latest finalization authorization override historical merge/tag holds for G01 only. Do not begin G02 until the user explicitly authorizes it.
+
+For disk accounting and targeted optional cleanup, see `CLEANUP.md` and `cleanup-ledger.json`. Do not delete shared caches, user checkouts or recovery evidence automatically.
diff --git a/docs/joom-migration/G01-HISTORY.json b/docs/joom-migration/G01-HISTORY.json
new file mode 100644
index 00000000000..2ace2f04247
--- /dev/null
+++ b/docs/joom-migration/G01-HISTORY.json
@@ -0,0 +1,191 @@
+{
+ "old_head": "b93d553856a0339af7468bcff2070e735a1a3c71",
+ "base": "9691385f35e4c5bd29514dca78d2044dd20e4bed",
+ "code_head": "cced071d47aa0fb4c1e896f901f4d4cff671fa97",
+ "code_tree_unchanged": true,
+ "mapping": [
+ {
+ "old": "57bcfbb6a94c29ea6592b9cbdd7e17840a0f94c8",
+ "title": "spark: establish Joom Spark 4.2.1 and Delta baseline",
+ "new": "57cc4cb6243692ef00da22428c9441c70aaf1239",
+ "tree": "2ff9e74202695c6208b56aad110309f4e03e5308",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "b7f25aa37569ad50b1dee2f8204f57ac219ddc91",
+ "title": "native: vendor pristine datafusion-physical-plan 55.2.0",
+ "new": "5a9cbf8569721ce18733cf0cf18c73bc57abd48f",
+ "tree": "8782f890fc8da0fb699a9bf9464df98ebfc84041",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "a7b044913fa411cf8130876b9f2e1cbe7f734f35",
+ "title": "native: test vendored DataFusion in the Comet workspace",
+ "new": "1322919571ef70a4f454ea41b545a6efc9d3d301",
+ "tree": "457710e58b82070739c42769d48fef12743d2be5",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "525658d82e634af16b1f2212957f03ca9e16c657",
+ "title": "ci: exclude generated Spark trees from suite inventory",
+ "new": "d354026be7e18a6a2e8fed90f3772e3b078eeea5",
+ "tree": "de135f5a112b7737e08ee67a2c88968db69c950a",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "b58bd65b24a8f2a20c576143ceafd17c42e25b22",
+ "title": "release: build and validate Joom Comet for Spark 4.2.1",
+ "new": "0710508c40bca0caee88b489da78e03bd00db922",
+ "tree": "978a02fdd5028001d35e105c93c0ea322c96ea59",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "b20edd3a1f58ad22aac76a6baf6bf0227c5d1aef",
+ "title": "test: import shuffle exchange in Spark 4.2.1 fixture",
+ "new": "92e97e46fa6de19c10824b3a1aef9908e5193633",
+ "tree": "f1c7f1133b4c5136c4fdb992a4a9e421a7dfa67d",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "56129e487b4404f1835dcb6e6b2df32cc295bd6f",
+ "title": "release: handle optional Maven flags in macOS Bash",
+ "new": "ed5d78ed9d1ed22236d7060fe03989d6da232b62",
+ "tree": "cbddb6045f8f3fa0bf1827779ad48827d4e400a5",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "fecf2b29051ec5b67e6ef81b08844710d5ef53ac",
+ "title": "docs: preserve complete G01 migration decisions and evidence",
+ "disposition": "CONSOLIDATED_DOCUMENTATION"
+ },
+ {
+ "old": "235f3d306e3ec4be2d571b93a6c7dc99fb0af512",
+ "title": "test: run legacy ANSI overflow cases on Spark 4.2",
+ "new": "87f58a83cef6c13e36815664a335e4fc2afcee17",
+ "tree": "9ffafc4beb8f705a9a6942ecf87941e453f7c9c1",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "ac48332d4f76e64d5835d0dd0e31daaa13a2110f",
+ "title": "docs: record G01 PR handoff and verification policy",
+ "disposition": "CONSOLIDATED_DOCUMENTATION"
+ },
+ {
+ "old": "b406dccadc420b5896c0702083b60fe7da5c85db",
+ "title": "ci: use standard Ubuntu runners for Joom PR gates",
+ "new": "a41d3e9116437b6d63a8d14beec7f085d6d5ed9b",
+ "tree": "19900db998d051ad2de658e1aad946f2479e6b22",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "db2bd16584c1381853cc170abab7cd87f289c85c",
+ "title": "docs: format G01 migration records for PR checks",
+ "disposition": "CONSOLIDATED_DOCUMENTATION"
+ },
+ {
+ "old": "8ffa31a01a32da0d8846da4482febfca0cdbef62",
+ "title": "cast: match Spark tiny-decimal rescaling fixes",
+ "new": "f8223f0f325c76a272e2eef63e28eb61daa67e98",
+ "tree": "3047c0fab6021bd8ff9e191c86672fd028590260",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "0b7078ca49e27ec2d95bb0e1b7041db51c157814",
+ "title": "parquet: reject duplicate projected field selection",
+ "new": "3c1e8b71e2a523776329aefe3193f0fb735caec4",
+ "tree": "8b6c1cac93815320d3108a417cc0cf861af2c6de",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "2be57cf8afe2d70f4190af2bf607a46f4890ca17",
+ "title": "union: enforce known partition layouts during execution",
+ "new": "99ed2f61a9e92d765b3f0d09099c61c1e44ff20b",
+ "tree": "6b2c8a9097d7695998e6cb879736691881b5cd36",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "16c46b14fcdab2b878e68f74af6c68fadf0c48bc",
+ "title": "test: preserve Spark SQL contracts with Comet plans",
+ "new": "b642976a191e96d03896f61ec89b01fd4e915b47",
+ "tree": "ac6ab53bb0c55680901ef493298735217ba8b63c",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "e3fa097f4dae3340ab8586526286fc5a1393c990",
+ "title": "test(delta): give deletion-vector fixtures an explicit offset",
+ "new": "72758d302c62f2560c8abce755ae76296a66cbcc",
+ "tree": "2b0fe5f39a52640dee3825b3a65f6f2db52a4059",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "6ae961dcbae949db09e349b05c91fc1cd6464d87",
+ "title": "ci: retry transient parent POM resolution before builds",
+ "new": "d1c85348c8d38c8743a27f077715863eb0938278",
+ "tree": "f7d99f8e08adbd098cb6599fa92e71e9544b7df3",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "ac3694504de3d5e3b93d19d8be87e48d6346b804",
+ "title": "docs: record G01 CI corrections and local validation",
+ "disposition": "CONSOLIDATED_DOCUMENTATION"
+ },
+ {
+ "old": "6fc14d57d378d147d65bd53b5fe28bcb0ca1265f",
+ "title": "union: preserve partitioning policy and bindings through AQE copies",
+ "new": "43af9beaf9dab627064758e84dbfd71035c24d6f",
+ "tree": "9987bb43713809c325091f3922ac49dfab2ffbe4",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "30b6c4d7f531fad7b6550c5d3f1c92cefd096e58",
+ "title": "parquet: assert typed read errors from duplicate-name prevalidation",
+ "new": "6277197a7c760f775f782f76a1d8aabcf8a01ce3",
+ "tree": "c5a9a6e0515f23e8d52d8d25040661fcabd3ee08",
+ "disposition": "RETAINED_CODE_COMMIT"
+ },
+ {
+ "old": "26590264b878fb8eae2ca439a8bcc0dfb513cb36",
+ "title": "docs: record G01 follow-up correctness evidence",
+ "disposition": "CONSOLIDATED_DOCUMENTATION"
+ },
+ {
+ "old": "c6e9d375b0f9511eac09934e44861eb666bc6f1d",
+ "title": "release: build and smoke-test RC artifacts on native Linux runners",
+ "disposition": "CONSOLIDATED_RC_TOOLING",
+ "new": "cced071d47aa0fb4c1e896f901f4d4cff671fa97"
+ },
+ {
+ "old": "548ca55ab573b6d8e52b3c40a849775ba058ea24",
+ "title": "docs: record G01 qualification and manual RC workflow authorization",
+ "disposition": "CONSOLIDATED_DOCUMENTATION"
+ },
+ {
+ "old": "966b8529b4a0e32b272af9a21aebe29b721069ba",
+ "title": "test: isolate duplicate receipt rejection from filesystem order",
+ "disposition": "CONSOLIDATED_RC_TOOLING",
+ "new": "cced071d47aa0fb4c1e896f901f4d4cff671fa97"
+ },
+ {
+ "old": "ee6bddb07e95af4bdb22ff801230871f913a6018",
+ "title": "release: resolve smoke classpaths within the packaging reactor",
+ "disposition": "CONSOLIDATED_RC_TOOLING",
+ "new": "cced071d47aa0fb4c1e896f901f4d4cff671fa97"
+ },
+ {
+ "old": "d7f0df5bd4750b204aedcbdb0a69efb1e984287e",
+ "title": "release: scope container Git trust through protected system config",
+ "new": "cced071d47aa0fb4c1e896f901f4d4cff671fa97",
+ "tree": "dd4c53dedd3689ee1e5ed4e19b4552824599b33b",
+ "disposition": "CONSOLIDATED_RC_TOOLING"
+ },
+ {
+ "old": "b93d553856a0339af7468bcff2070e735a1a3c71",
+ "title": "docs: restore ASF headers on migration reports",
+ "disposition": "CONSOLIDATED_DOCUMENTATION"
+ }
+ ],
+ "documentation_commit": "The final English documentation commit follows this code history; resolve joom-1.2-1-rc1 for the final source SHA.",
+ "policy": "All G01 service/checkpoint documentation changes are consolidated into one final English documentation commit; the pristine vendor import remains separate. RC tooling fixes are folded into its substantive commit.",
+ "recovery_ref": "refs/joom-migration/checkpoints/20261118-before-finalization",
+ "scope": "Commits after the existing integration base only. Existing branch-1.2 ancestry is retained."
+}
diff --git a/docs/joom-migration/G01-PR-preparation.json b/docs/joom-migration/G01-PR-preparation.json
new file mode 100644
index 00000000000..8dd8fea732d
--- /dev/null
+++ b/docs/joom-migration/G01-PR-preparation.json
@@ -0,0 +1,121 @@
+{
+ "checked_at": "2026-11-17T16:15:12.584396+00:00",
+ "alignment": {
+ "checked_on": "2026-11-17T16:12:25.610654+00:00",
+ "old_head": "e074b53db5f0f6323894f0edf12c44c8ea10c94e",
+ "new_head": "fecf2b29051ec5b67e6ef81b08844710d5ef53ac",
+ "base_unchanged": "9691385f35e4c5bd29514dca78d2044dd20e4bed",
+ "backup_ref": "refs/joom-migration/checkpoints/20261117-before-pr-alignment",
+ "old_tree": "756536ebd29f0533080f3c56b8dc8aa50ab5cdc1",
+ "new_tree": "756536ebd29f0533080f3c56b8dc8aa50ab5cdc1",
+ "entire_tree_identical": true,
+ "documentation_checksums_before": {
+ "AGENTS.md": "8435cd2ea2ac72cc4e7e18d22e861ed7aefbef727dfbb9941fb238c588c7b6e5",
+ "CHECKPOINT-VERIFICATION.json": "c3653af569dacff7ae23be6019454e87720729198edf1b14f8b3e730e5e92a18",
+ "CLEANUP.md": "ec5aacc998f7de72deee6d61963c33f380e2fb09f51e8001fc69ccf37496e918",
+ "G01-preparation.json": "349f0178f81805e018b5ed9343564eb8dec9e2f5d01ed724dfc32394091a5804",
+ "G01-progress.md": "44eda958149f960999839871e7e2231b876dc38f8e7984d4d02f609c2ce236ee",
+ "G01-report.md": "78981202fb10ad2750dcf361695ad8050eebc8c3fba4ae0dcb2da115edfa99f7",
+ "G01-results.json": "2332de040a2d5e9e2146245e784625212f3cd1db203eeb0de2aeb853303ebf33",
+ "G01-review-notes.json": "4bdcdfe191a2ebecea10e037ebf0ebe271e760356d77f30b0f15016ec154e476",
+ "HANDOFF.md": "9eca47867bcf32595dc9e82df5029f35299462e3936cd3f976d9282a083a3c50",
+ "NETWORK.md": "4c8d3b084363eb8e49b51eac6c620e9977ea3a2994824e0566e3ac0a149d46b3",
+ "PAUSED.md": "66d44e9078d8511cf92f5ac36c8b525f5d2e65e7f2e95ef82fd8e09a6cc8cbc4",
+ "PERFORMANCE.md": "f01d946951bb4c1e54ea608518d6379f0d38f4e7c9990d6593bba0e2aad96344",
+ "README.md": "762bc59e5bfaa357b1cbd93550e8194b216578b52f2868576e6bf533087b4cde",
+ "STATE.json": "12faf5ab9e98172afaa8963f947c24629bc4f4cab963dec630485b5c61c95816",
+ "approved-plan-ru.txt": "8da8494f2bb7df176e9f39377d6e00de7053f3d9cbfa0b2e3411588a427b69bc",
+ "approved-plan.json": "f994753b5a9378b9e9783ab4eba748cd4f6423e4591cdb5f1f9079b096531a8d",
+ "cleanup-ledger.json": "c6e0ae6660359afe1d50437d6afacec4f010543cf24d7ee6140fa59b3b9fd8fd",
+ "correctness-test-plan-ru.txt": "6efd6d143ef8e0e5cebc22cb79b779c08de6ea8920ead49c448313129590cc9c",
+ "created-files.txt": "2ca179f2550c953afc41af96c365aaec215572b39362e6c7461ad0bc15f6dcbf",
+ "decisions.json": "91c27d6f993cf19bcfe75b82e36cbfa71cc223a9f5008989c2487588ff42775e",
+ "feature-register.json": "8b376bedd5b723e60e960412db27b01644e4fd10d75f4f63679bc4cf4affecff",
+ "resource-baseline.json": "107defecf4a734ac5a4d52d237fa2b1ebd3945aea7da5c724dc56cb5e15cd0e9",
+ "target-native-dependencies.json": "d4d0fc916e083af2e114f21f7e410da14f4debfe7b7937758b74d148c1b27200",
+ "vendor-provenance.json": "e69ec40d794663cf181a1adad202bafed0a2f7b3ac92ffca60737a6eaca8224f"
+ },
+ "documentation_checksums_after": {
+ "AGENTS.md": "8435cd2ea2ac72cc4e7e18d22e861ed7aefbef727dfbb9941fb238c588c7b6e5",
+ "CHECKPOINT-VERIFICATION.json": "c3653af569dacff7ae23be6019454e87720729198edf1b14f8b3e730e5e92a18",
+ "CLEANUP.md": "ec5aacc998f7de72deee6d61963c33f380e2fb09f51e8001fc69ccf37496e918",
+ "G01-preparation.json": "349f0178f81805e018b5ed9343564eb8dec9e2f5d01ed724dfc32394091a5804",
+ "G01-progress.md": "44eda958149f960999839871e7e2231b876dc38f8e7984d4d02f609c2ce236ee",
+ "G01-report.md": "78981202fb10ad2750dcf361695ad8050eebc8c3fba4ae0dcb2da115edfa99f7",
+ "G01-results.json": "2332de040a2d5e9e2146245e784625212f3cd1db203eeb0de2aeb853303ebf33",
+ "G01-review-notes.json": "4bdcdfe191a2ebecea10e037ebf0ebe271e760356d77f30b0f15016ec154e476",
+ "HANDOFF.md": "9eca47867bcf32595dc9e82df5029f35299462e3936cd3f976d9282a083a3c50",
+ "NETWORK.md": "4c8d3b084363eb8e49b51eac6c620e9977ea3a2994824e0566e3ac0a149d46b3",
+ "PAUSED.md": "66d44e9078d8511cf92f5ac36c8b525f5d2e65e7f2e95ef82fd8e09a6cc8cbc4",
+ "PERFORMANCE.md": "f01d946951bb4c1e54ea608518d6379f0d38f4e7c9990d6593bba0e2aad96344",
+ "README.md": "762bc59e5bfaa357b1cbd93550e8194b216578b52f2868576e6bf533087b4cde",
+ "STATE.json": "12faf5ab9e98172afaa8963f947c24629bc4f4cab963dec630485b5c61c95816",
+ "approved-plan-ru.txt": "8da8494f2bb7df176e9f39377d6e00de7053f3d9cbfa0b2e3411588a427b69bc",
+ "approved-plan.json": "f994753b5a9378b9e9783ab4eba748cd4f6423e4591cdb5f1f9079b096531a8d",
+ "cleanup-ledger.json": "c6e0ae6660359afe1d50437d6afacec4f010543cf24d7ee6140fa59b3b9fd8fd",
+ "correctness-test-plan-ru.txt": "6efd6d143ef8e0e5cebc22cb79b779c08de6ea8920ead49c448313129590cc9c",
+ "created-files.txt": "2ca179f2550c953afc41af96c365aaec215572b39362e6c7461ad0bc15f6dcbf",
+ "decisions.json": "91c27d6f993cf19bcfe75b82e36cbfa71cc223a9f5008989c2487588ff42775e",
+ "feature-register.json": "8b376bedd5b723e60e960412db27b01644e4fd10d75f4f63679bc4cf4affecff",
+ "resource-baseline.json": "107defecf4a734ac5a4d52d237fa2b1ebd3945aea7da5c724dc56cb5e15cd0e9",
+ "target-native-dependencies.json": "d4d0fc916e083af2e114f21f7e410da14f4debfe7b7937758b74d148c1b27200",
+ "vendor-provenance.json": "e69ec40d794663cf181a1adad202bafed0a2f7b3ac92ffca60737a6eaca8224f"
+ },
+ "all_24_documentation_hashes_match": true,
+ "substantive_commits": [
+ {
+ "old": "ea4b3228ec5346181f74d8632d6af13b4441a359",
+ "new": "57bcfbb6a94c29ea6592b9cbdd7e17840a0f94c8",
+ "subject": "spark: establish Joom Spark 4.2.1 and Delta baseline"
+ },
+ {
+ "old": "0a76d68a33f6b4bcbce9fbb412b90350c5fed80b",
+ "new": "b7f25aa37569ad50b1dee2f8204f57ac219ddc91",
+ "subject": "native: vendor pristine datafusion-physical-plan 55.2.0"
+ },
+ {
+ "old": "15a466a09abf6106bbd80402e1967ba96076fbcc",
+ "new": "a7b044913fa411cf8130876b9f2e1cbe7f734f35",
+ "subject": "native: test vendored DataFusion in the Comet workspace"
+ },
+ {
+ "old": "857d2de1fa021969f6b2efd7d47c5b680daa578b",
+ "new": "525658d82e634af16b1f2212957f03ca9e16c657",
+ "subject": "ci: exclude generated Spark trees from suite inventory"
+ },
+ {
+ "old": "73efe84b4e21f53a4da054504e0dbd2a6d693bea",
+ "new": "b58bd65b24a8f2a20c576143ceafd17c42e25b22",
+ "subject": "release: build and validate Joom Comet for Spark 4.2.1"
+ },
+ {
+ "old": "d725b298e9380587a3331117bb64e7fa8391674a",
+ "new": "b20edd3a1f58ad22aac76a6baf6bf0227c5d1aef",
+ "subject": "test: import shuffle exchange in Spark 4.2.1 fixture"
+ },
+ {
+ "old": "2b8f80b3f253559bc999f73769b05c88ebcd5fc2",
+ "new": "56129e487b4404f1835dcb6e6b2df32cc295bd6f",
+ "subject": "release: handle optional Maven flags in macOS Bash"
+ }
+ ],
+ "consolidated_documentation_commit": "fecf2b29051ec5b67e6ef81b08844710d5ef53ac",
+ "base_is_ancestor": true,
+ "pr_add_add_conflicts_removed": true
+ },
+ "independent_review": "PASSED; all7 substantive patches byte-identical; pristine import150files; all24 originaldocs hash-identical during alignment",
+ "intentional_after_alignment_changes": [
+ "Removed exactly3 ANSI assume guards in235f3d306",
+ "Recorded newly approved verification/CI/HDFS/vendor/doc-retention policy and PR handoff"
+ ],
+ "local_checks": {
+ "git_diff_check": "PASSED",
+ "spotless": "PASSED2.628s",
+ "broad_local_suites_repeated": false
+ },
+ "base_ref_unchanged": "9691385f35e4c5bd29514dca78d2044dd20e4bed",
+ "native_amd64_gha": "PENDING_USER_PR_CREATION",
+ "rc_tag_created": false,
+ "g02_started": false,
+ "old_local_artifacts": "Historical build source2b8f80b3f; not rebuilt or represented as artifacts of the new PR SHA."
+}
diff --git a/docs/joom-migration/G01-PR.md b/docs/joom-migration/G01-PR.md
new file mode 100644
index 00000000000..604a0acca73
--- /dev/null
+++ b/docs/joom-migration/G01-PR.md
@@ -0,0 +1,53 @@
+
+
+> **Final source-freeze note:** G01 implementation and documentation are complete. RC1 acceptance is a post-commit gate recorded by the immutable `joom-1.2-1-rc1` tag and the delivered artifact receipts; see `G01-CLOSEOUT.md`. Earlier source SHAs, runs and pending states below are historical evidence. G02 and Java 21 work remain unauthorized.
+
+## Which issue does this PR close?
+
+Related to apache/datafusion-comet#4967. This is migration G01; it does not claim to close the full upstream ANSI issue.
+
+## Rationale for this change
+
+Prepare the Joom Comet 1.2 baseline for Spark 4.2.1 and Delta 4.4.1, retain the agreed Delta/GCP behavior, and provide a tested vendor and release foundation for the remaining Joom feature ports.
+
+## What changes are included in this PR?
+
+- Pin Spark 4.2.1, Scala 2.13.18 and the Spark-4.2-specific Delta 4.4.1 artifact. Adapt the Spark SQL fixture and CI lanes.
+- Preserve GCP ADC admission, safe Delta planning, driver-only payload serialization and DPP/empty scans. Add the conservative Hadoop encryption-context admission guard.
+- Import pristine published datafusion-physical-plan 55.2.0 separately, connect it through the root workspace/patch/lockfile, and explicitly test vendor with the Comet feature graph. Vendor production algorithms remain unchanged; four spill tests are adapted to Arrow 59.3.2.
+- Match tiny-decimal casts to Spark's patch-level behavior and reject ambiguous projected Parquet field selection with Spark's structured error envelope.
+- Preserve Union partition layouts and immutable planning policy through AQE copies, bind layouts to the rewritten output, and fail before an incompatible RDD layout can silently corrupt aggregate results.
+- Retain original Spark fixture contracts and add native result/error/layout counterparts. Enable the three existing ANSI arithmetic tests, keep ADD JAR checks on isolated local Ivy fixtures, and correct the synthetic Delta DV offset.
+- Provide dualarch release build, integrity and publication scripts. Add a separate manual **RC Build & Smoke** workflow: two native release builds, one final packaging, two exact-JAR smoke jobs and a final receipt check. It exports downloadable artifacts and performs no JFrog/Nexus/production publication.
+- Use available Ubuntu runners and retry only transient parent-POM transfers before any compilation/test lifecycle. Migration decisions, evidence and cleanup guidance remain under docs/joom-migration.
+
+## How are these changes tested?
+
+The full native AMD64 Comet CI [44205897075](https://github.com/joomcode/datafusion-comet/actions/runs/44205897075) passed on `26590264b878fb8eae2ca439a8bcc0dfb513cb36`: 127 successful jobs, 17 conditional skips and no failures. It includes native 2649, vendor 1695, statistics 42 and Delta 294 tests, all nine Spark 4.2.1 SQL shards, and the other enabled Spark/Iceberg/macOS matrices. Artifact inspection confirms actual execution of the three formerly canceled ANSI arithmetic cases, both decimal rescaling cases, all ten duplicate-reader regressions and both Union regressions on Spark 4.1/4.2. Existing cancellations and ignored tests are reported separately, never counted as passed.
+
+Subsequent commits change only release/CI tools and migration documentation; native, JVM, Delta runtime sources, Spark fixture and dependency locks remain byte-identical to that CI-qualified commit. Full Comet CI for these commits was intentionally stopped at the owner's request. Exact-source Apache RAT for `b93d553856a0339af7468bcff2070e735a1a3c71` passed separately.
+
+The new manual [RC Build & Smoke 44388252264](https://github.com/joomcode/datafusion-comet/actions/runs/44388252264) passed all seven jobs on `b93d553856a0339af7468bcff2070e735a1a3c71`. Both native architectures built with the release profile. The same final dualarch JARs passed **12/12 existing smoke cases per architecture**: Delta 4, DPP 1, reload 1 and ANSI 6, with no cancellations/ignores. The harness checks actual Spark 4.2.1, Java 17, class origins and the SHA256 of the loaded bundled native library. The final job binds both receipts to the same source, lockfile, package, support and JAR hashes and exports those original bytes without repackaging.
+
+The qualified artifact was downloaded and its GitHub ZIP digest, runtime JAR hashes, embedded Git revision, native graph/ELF provenance and both receipts were verified locally. Local tool validation includes 22 fault tests (9 native-stage, 7 portable-smoke, 6 receipt-binding), actionlint, CI configuration and Bash syntax. The Focal Git ownership issue was reproduced with UID 1001 in the actual builder and fixed with a container-only, checkout-specific system config.
+
+The old source2b8f80b3f JARs and incomplete Rosetta runs are historical. Current release-byte AMD64 qualification is complete on a native runner.
+
+This PR is G01 only. G02-G09, real GCS/GKE reads and production performance are not qualified here. The user has authorized G01 history/documentation finalization and conditional fast-forward integration plus RC1 tagging after exact-SHA RC qualification. The annotated tag and artifact receipts identify final acceptance. No external publication or production operation is included.
diff --git a/docs/joom-migration/G01-preparation.json b/docs/joom-migration/G01-preparation.json
index 38a217f6d41..076c66307e2 100644
--- a/docs/joom-migration/G01-preparation.json
+++ b/docs/joom-migration/G01-preparation.json
@@ -3,10 +3,10 @@
"group": "G01",
"thread_id": "01a1e716-379a-7d92-b6bd-cbd9c8500afc",
"host_id": "local",
- "title_requested": "Comet G01: подготовка без кода",
+ "title_requested": "Comet G01: preparation without code",
"status": "READ_ONLY_PREPARATION_COMPLETED",
"verified_head": "5e2f39b8a581d2bb055a4981cbf649c4bf059222",
"implementation_started": false,
- "worker_report": "Ограничения получены. Указанные документы и все 12 карточек G01 прочитаны; локальные исходники Comet, DataFusion и Spark доступны. Объекты source 981e26df… и upstream base 93cc6f9d… доступны. Текущая ветка: branch-1.2. HEAD: 5e2f39b8a581d2bb055a4981cbf649c4bf059222. Рабочее дерево чистое; joom/1.2-dev отсутствует. Code hold действует: реализацию не начинаю. Файлы и Git не изменял; загрузки, установки, сборки и тесты не запускал. Порядок G01 подтверждён: Точные Spark 4.2.1, Scala 2.13.18, Delta 4.4.1 на registry DataFusion 55.2.0, runtime Arrow/Parquet 59.3.2; исходные correctness-проверки. Pristine published datafusion-physical-plan 55.2.0 отдельным механическим коммитом. Root patch, общий lockfile/workspace и явный vendor unit/regression target с проверкой единого dependency graph. Повторные проверки, адаптация release scripts и упаковки amd64/arm64, проверка native-library reload; внутренний RC1. Сохраняются ANSI=true, обязательные ошибки и mixed SUM/AVG overflow guards. PR5889 исключён; performance — штатными средствами проекта. Технических препятствий для следующего шага read-only подготовки не выявлено. Недостающих данных или инструментов, блокирующих такую подготовку, сейчас нет. Начало реализации ожидает прямого указания пользователя. Следующий разрешённый шаг по новому поручению — read-only сопоставление G01.1 с локальными POM, CI и Spark checkout. Сейчас останавливаюсь и жду поручения; G02 не начинаю. Новых файлов и сохранённых результатов нет.",
- "coordinator_review": "Соответствует утверждённому G01, включает все12карточек, сохраняет code hold/ANSI/PR5889. Координатор отдельно проверил доступность JDK/Rust и отсутствие изменений исходников."
+ "worker_report": "Restrictions received. The specified documents and all 12 G01 cards have been read; local Comet, DataFusion and Spark sources are available. Source object 981e26df… and upstream base 93cc6f9d… are available. Current branch: branch-1.2. HEAD: 5e2f39b8a581d2bb055a4981cbf649c4bf059222. The worktree is clean; joom/1.2-dev does not exist. The code hold remains active: implementation is not being started. No files or Git were changed; no downloads, installations, builds or tests were run. G01 order confirmed: exact Spark 4.2.1, Scala 2.13.18, Delta 4.4.1 on registry DataFusion 55.2.0, runtime Arrow/Parquet 59.3.2; initial correctness checks. Pristine published datafusion-physical-plan 55.2.0 in a separate mechanical commit. Root patch, shared lockfile/workspace and explicit vendor unit/regression target, verifying a single dependency graph. Repeat checks, adapt release scripts and amd64/arm64 packaging, check native-library reload; internal RC1. ANSI=true, mandatory errors and mixed SUM/AVG overflow guards remain required. PR5889 is excluded; performance uses established project tools. No technical obstacle to the next read-only preparation step was identified. No missing data or tool currently blocks that preparation. Implementation awaits the user's direct instruction. The next authorized step under a new task is read-only mapping of G01.1 to local POMs, CI and Spark checkout. Stopping now and awaiting instructions; G02 is not being started. No new files or saved results.",
+ "coordinator_review": "Matches approved G01, covers all 12 cards, preserves the code hold/ANSI/PR5889 boundaries. The coordinator separately verified JDK/Rust availability and absence of source changes."
}
diff --git a/docs/joom-migration/G01-progress.md b/docs/joom-migration/G01-progress.md
new file mode 100644
index 00000000000..4de4053ada6
--- /dev/null
+++ b/docs/joom-migration/G01-progress.md
@@ -0,0 +1,100 @@
+
+
+> **G01 finalization authorized:** documentation and history cleanup, followed by exact-SHA RC qualification, are in progress. Integration into `branch-1.2` and `joom-1.2-1-rc1` are conditional on successful checks. G02 and Java 21 work remain on hold.
+
+# G01 implementation history
+
+The entries below are historical checkpoints, not current authorization. See `STATE.json`, `G01-report.md` and `decisions.json` for the latest state and decisions.
+
+> **Historical PR preparation:** history aligned without changing the tree/24 documents or branch-1.2; three ANSI guards removed for actual AMD64 GHA execution. Local checks were fast, full scope was delegated to GHA. The user creates the PR using G01-PR.md. No merge/RC1/G02 was authorized then.
+
+> **Historical status:** G01 implemented; AMD64 runtime qualification incomplete. `branch-1.2` unchanged; user preview/confirmation required first. G02 unauthorized. Results: G01-report.md / G01-results.json.
+
+> **Pause on 2026-11-17:** all our builds/tests stopped at user request. Code saved in Git; G01 incomplete. Authoritative continuation point: `PAUSED.md`; no restart without explicit resume. G02 unauthorized.
+
+**G01 resumed on2026-11-17; new sandbox filesystem/TCP/Docker-socket probes passed. Historical checkpoint: PAUSED.md; current status: STATE.json.**
+
+Status at the initial2026-11-17 checkpoint: registry baseline passed; G01.2 pristine vendoring starting. G02–G10 prohibited pending separate user authorization. Main executor: coordinator chat. Branch joom/1.2-dev created from branch-1.2; upstream rechecked through GitHub API:93cc6f9d71136bd0d87f77dd901989144d8c09fb. Registry baseline committed as 71435d6f1; pristine vendor as 98c9f2f19. Unfinished integration/release changes preserved in dirty tree and checkpoint.
+
+## Completed at that checkpoint
+
+- Spark4.2.1/Scala2.13.18/Java17, Delta delta-spark_4.2_2.13:4.4.1 in the profile.
+- Maven wrapper3.9.6 works; effective POM confirmed. Log and XML in the task directory.
+- Native registry build passed: cargo build --locked -j4 -p datafusion-comet, preserving native/Cargo.lock. DF55.2.0/Arrow-Parquet59.3.2; Mac arm64 libcomet.dylib created in native/target/debug. Duration4m07; a compilation check, not Rust unit tests.
+- ADC whitelist and two original Joom tests ported from a43523692.
+- Delta partitioning does not invoke file planning/subqueries; core obtains actual execution count from perPartitionData. Added scalar DATA-filter metadata regression.
+- Delta nativeOp/serializedPlanOpt transient; splits test reads actual data. Full query-text interning and its regression belong to G08/J17 and are not claimed here.
+- Added zero-partition pruning regression from 45fad9eea (production equivalent already upstream).
+- CI Spark4.2.1 + Delta4.2 lane/routing; python3 dev/ci/check-ci-config.py passed.
+- Removed -pl from touched Makefile/Delta-workflow paths.
+- Local Spark fixture created WITHOUT network from the user's checkout in target/joom-spark-4.2.1. Original /Users/skotlov/d/temp/src/spark unchanged. Diff4.2.0 applied three-way to v4.2.1; six import-only conflicts resolved, preserving new Spark imports and Comet imports. dev/diffs/4.2.1.diff generated from an actual Spark Git diff,283345bytes; clean-index git apply --check passed.
+
+## Registry-baseline results
+
+- Local RPC environment corrected using standard SPARK_LOCAL_IP=127.0.0.1. DPP/NativeBase smoke:10/10 + ParquetEncryptionITCase, BUILD SUCCESS.
+- First full Delta run:283/284. ADC, scanner, deletion vectors, DML, planning-deadlock and driver-payload serialization regressions passed.
+- Sole failure: Hadoop4.2 added fs.s3a.encryption.context, correctly detected by the discovery harness without an admission guard.
+- Adapted the production+unit portion of upstream draftPR7593 (not the whole PR): reject native execution for nonempty/unverifiable Hadoop-effective S3 context, including sidecar DV with a local Delta log; aliases/JCEKS/redaction retained. No AWS network use; GCS/file bypass without provider read. DeltaScanContribSuite:164/164 passed, BUILD SUCCESS1m56s. An explicit gs:// control remained to be checked on the next Delta compile.
+- ANSI baseline52647 BUILD SUCCESS:65 passed,3 standard upstream cancellations of old generic arithmetic tests on Spark4.2 (issue4967); not full ANSI proof. Existing CometExpressionSuite ANSI, CometMathExpressionSuite and SQL fixtures ansi/case_when_lazy checked. A nonexistent CometAnsiExpressionSuite was NOT used as evidence.
+
+Logs of all successful and failed runs are preserved in the task directory below. The first failed run selected suites in the wrong Maven modules; the next failed on local RPC; both were diagnosed and fixed. Maven/Cargo used standard caches through then-approved escalated executions. Persistent sandbox permissions were unchanged. ps was unavailable; no workaround was attempted.
+
+## Planned next steps at that checkpoint
+
+1. Complete the ANSI registry baseline and fix G01 compilation/test failures. Do not hide new failures or claim unrun suites passed.
+2. Only after the registry baseline: import pristine published physical-plan55.2 in a separate mechanical commit, then root patch/workspace/shared lock/test target. Cargo already downloaded the archive to ~/.cargo; verify checksum b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97.
+3. Repeat Comet/Delta/ANSI/native/vendor checks and SparkSQL4.2.1 suites.
+4. Adapt release builder/publisher, locked graph, GLIBC<=2.31, both architectures, local validate-only publisher checks, actual smoke tests. RC1 is internal and an incomplete port. No publication or production operations.
+5. Actual GCP ADC reads require an authorized isolated test resource; current network scope does not permit productionGCP. An auth-gate unit test does not constitute such a read.
+
+Main logs/download directory: `/Users/skotlov/d/temp/20261117_comet12-g01`. Add target/joom-spark-4.2.1, native/target and actual Maven/Cargo caches to the cleanup ledger. When changing sessions, first check actual Git status/HEAD and active tool sessions.
+
+## At the pause
+
+Registry baseline and pristine vendor committed. Workspace integration and release scripts still dirty/unqualified. Native core961pass/12UDF-path fail/5ignored; vendor unit tests not run. All commands ended. See PAUSED.md.
+
+## After resuming
+
+- New sandbox roots, local TCP and Docker socket checked successfully without escalation.
+- Cargo1.100 OUT_DIR lookup corrected;12 UDF failures resolved.
+- Workspace unit tests:4326 passed,5 ignored; vendor1695/1695.
+- Four vendor test-only spill-size assumptions replaced by comparison with the physical flushed/finalized file size; production algorithms unchanged. Cause: Arrow59.3.2 omits all-valid bitmaps, whereas the published vendor lock used59.2.0.
+- Strict Clippy for all Comet crates passed; pristine vendor excluded from lint because Rust1.100 deprecates fetch_update. Its unit gate retained.
+ -19 negative/model release-validator/publisher checks passed using mock HTTP/Git.
+- Linux arm64 builder created; amd64 builder in progress. Release cargo builds not yet started.
+- Vendor-linked root Delta install:291/291 PASSED8m20s, including gs control.
+- Vendor-linked ANSI/runtime:75passed,3upstreamcanceled, BUILD SUCCESS1m40s.
+- SparkSQL dependency setup: POM-only artifact holes filled selectively with standard Maven; shared cache neither moved nor cleared. Installed-fork SHA recorded.
+- Both Linux builder images prepared; amd64 CPUID confirmsx86-64-v3.
+- Release-script review closed: fresh module outputs, actual GitSHA, SPI providers, actual graph/toolchain/ELF imports, PUT2xx, optimizer-safe cleanup guards. Dirty-source refusal checked before Docker/output creation.
+- I03 clarified against core/Spark4.2.1: Unknown0 matches the contract; a possible extra exchange versus obsolete contribUnknown1 was not measured.
+- Next major run was the release builder; source/HEAD must remain fixed while it runs. Save live process IDs/statuses in external active-commands.json.
+
+## Resuming after the second pause
+
+Checked inside sandbox: ProtocBridge cache write, loopback bind/connect, Docker27.4.0. The user confirmed both architectures must be completed in G01 and service checkpoint commits combined. Implementation/checks only G01; current raw outputs: /Users/skotlov/d/temp/20261117_comet12-g01-final-qualification. Service history is combined before the final build, preserving substantive commits; mapping and backup ref are in this task directory.
+
+## PR28 runner allocation fix
+
+GitHubAPI confirmed Preflight/RequiredChecks had ubuntu-slim labels, empty runner_name,0 steps. Replace6 runs-on labels in ci.yml/PRtitle/Icebergcoverage with ubuntu-24.04; update sourceCI diagram/comments. Tests/commands unchanged. LocalCIconfigcheckpassed. RAT found4migrationdocumentswithoutheaders; standardheadersadded. pull_request_target labeler reads unchangedbase and remains outsidefeaturefix. Rawlogs: /Users/skotlov/d/temp/20261117_comet12-g01-runner-fix.
+
+## PR28 Markdown preflight fix
+
+Runner replacement confirmed by actual allocated ubuntu-24.04 runner in run44135599020. RAT and separate PR title check passed; Preflight stopped at four migration Markdown files. User installed Homebrew Prettier3.10.1. Only the four flagged files were formatted; all repository Markdown now passes. Text preserved after ignoring whitespace/table separator alignment; receipt is in the runner-fix task directory. No runtime/test coverage/base branch changes. Push and new GHA verdict pending at that historical checkpoint.
diff --git a/docs/joom-migration/G01-report.md b/docs/joom-migration/G01-report.md
new file mode 100644
index 00000000000..d6ca1b1b107
--- /dev/null
+++ b/docs/joom-migration/G01-report.md
@@ -0,0 +1,195 @@
+
+
+> **Final source-freeze note:** G01 implementation and documentation are complete. RC1 acceptance is a post-commit gate recorded by the immutable `joom-1.2-1-rc1` tag and the delivered artifact receipts; see `G01-CLOSEOUT.md`. Earlier source SHAs, runs and pending states below are historical evidence. G02 and Java 21 work remain unauthorized.
+
+> **G01 finalization authorized:** documentation and history cleanup, followed by exact-SHA RC qualification, are in progress. Integration into `branch-1.2` and `joom-1.2-1-rc1` are conditional on successful checks. G02 and Java 21 work remain on hold.
+
+> **Latest completed result, 2026-11-18:** [RC Build & Smoke 44388252264](https://github.com/joomcode/datafusion-comet/actions/runs/44388252264) on `b93d553856a0339af7468bcff2070e735a1a3c71` — **SUCCESS,7/7jobs;12/12packaged smoke cases on AMD64 and ARM64**. Artifacts were downloaded and locally verified. The reports below preserve historical stages; see “Manual RC Build & Smoke” and `G01-results.json#rc_build_smoke` for this completed qualification. Earlier merge/tag holds are historical and are superseded by the current conditional finalization authorization.
+
+# G01: results and review proposal
+
+Status: **G01 release JARs built, checked on both architectures and downloaded; final English documentation/history cleanup and exact-SHA RC qualification in progress**.
+The pre-integration `branch-1.2` is `9691385f35e4c5bd29514dca78d2044dd20e4bed`; G02 has not started.
+Planned candidate: `joom-1.2-1-rc1` (tag not yet created; finalization must bind it to the qualified final source). Historical artifacts were built from `2b8f80b3f253559bc999f73769b05c88ebcd5fc2` and are not presented as the final RC of the new PR.
+Apache base: `93cc6f9d71136bd0d87f77dd901989144d8c09fb`.
+
+## Proposed contents for branch-1.2
+
+1. Exact Spark4.2.1 / Scala2.13.18 / Java17 and Delta4.4.1 profile; CI and `dev/diffs/4.2.1.diff` generated from the Spark fixture.
+2. Native Delta using the upstream scanner: GCP ADC whitelist, planning partitioning without scans/scalar-subquery execution, transient driver payloads during serialization, DPP/empty-scan regression. Hadoop encryption-context admission guards preserve fallback and do not change ordinary GCS behavior.
+3. Pristine published `datafusion-physical-plan55.2.0` in a separate commit, root patch, shared Cargo.lock/workspace, explicit vendor test target. Runtime DataFusion55.2 / Arrow-Parquet59.3.2.
+4. Test-only corrections to four vendor IPC statistics assertions for actual Arrow59.3.2 sizes; no DataFusion algorithms changed in G01. Correct test-UDF library lookup for the new Cargo1.100 layout.
+5. Release Dockerfile/build/publish/validator for Linux ARM64 and AMD64 x86-64-v3, GLIBC<=2.31; checks for version/graph/source-and-lock cleanliness, Git SHA, ELF, two bundled `.so` files, Delta SPI and SHA256. Optional Maven flags now work with Bash3.2.
+6. CI inventory excludes generated Spark fixtures in target; explicit vendor tests and exact Spark4.2.1 are included in CI.
+7. Documentation and decisions only in `docs/joom-migration`.
+
+## Substantive commits at the earlier qualification stage
+
+```text
+ea4b3228e spark: establish Joom Spark 4.2.1 and Delta baseline
+0a76d68a3 native: vendor pristine datafusion-physical-plan 55.2.0
+15a466a09 native: test vendored DataFusion in the Comet workspace
+857d2de1f ci: exclude generated Spark trees from suite inventory
+73efe84b4 release: build and validate Joom Comet for Spark 4.2.1
+13cfe918c docs: preserve approved migration plan and G01 execution record
+d725b298e test: import shuffle exchange in Spark 4.2.1 fixture
+2b8f80b3f release: handle optional Maven flags in macOS Bash
+```
+
+Service planning/checkpoint commits were combined into one documentation commit before that build. Full original commits are retained in local backup ref `refs/joom-migration/checkpoints/20261117-before-service-cleanup`; old/new SHA mapping is in `G01-results.json`. The pristine vendor import remained separate: `0a76d68a3`; workspace integration: `15a466a09`.
+
+## Checks at the earlier local qualification stage
+
+| Check | Result |
+| ------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------ |
+| Native workspace | 4326 passed,0 failed,5 ignored; includes1695 vendor tests |
+| Vendor statistics with test_utils | 42 passed |
+| Full Delta | 291 passed |
+| ANSI/runtime | 75 passed,3 previous upstream cancellations (#4967) |
+| Spark4.2.1 SQL | MathFunctions62 + DPP AQE on2/off1 =65 passed |
+| Packaged Linux JARs | ARM64 12/12; AMD64:2 Delta cases passed, then stalled. FORK also did not finish within300s. Remaining AMD64 cases were not run |
+| Release archive/provenance | ELF,GLIBC<=2.31,Git SHA,lock,versions,two `.so` files,Delta SPI,SHA256 passed |
+| Negative validator/publisher fixtures | 19 passed; no real publication |
+| Bash3.2 regression | Online/offline flags passed; original failure reproduced |
+| Formatting/lint | Cargo fmt and strict Comet Clippy passed; pristine-vendor deprecation warnings documented |
+
+Five ignored Rust tests:4 require an HDFS cluster,1 is an existing benchmark. Unit/Delta/ANSI checks ran before the history squash; their runtime source was preserved. Subsequent changes concerned only the test-fixture import and release Bash helper. A new full-classpath Spark compile and65 SQL tests passed after the fixture fix.
+
+The SHA of our installed Joom JAR was checked on the Spark SQL classpath. Packaged smoke puts the final JARs first, excludes other Comet/main classes, checks NativeBase/Delta-scan class origins and loads bundled `.so`; network none, only local fixtures/loopback. It checks complete results, expected ANSI errors and actual native execution.
+
+## Historical artifacts
+
+Directory: `/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1`.
+
+| File | SHA256 |
+| ------------------------------------------- | ------------------------------------------------------------------ |
+| comet-spark-spark4.2_2.13-1.2.0.jar | `40182026a96ccbf4958d2f741d19268e518173ff2960f937f1d3b700172defff` |
+| comet-contrib-delta-spark4.2_2.13-1.2.0.jar | `0c50187c4624b46b31a47326b7b281d6cde96d1b458697845d74debc0b07ae8c` |
+
+Alongside them: `sha256.txt`, `commit.txt`, `build-manifest.json`, `effective-pom.xml`, both `.so` files, actual toolchain/Cargo metadata/ELF imports and `smoke/` evidence. The internal JAR base version is1.2.0; the Joom Maven publication version corresponds to the tag. No push/JFrog/production operation occurred at this stage.
+
+Measured native build: ARM64 5m31s; AMD64 12m26s. After the Bash-only fix: ARM cache hit1.39s, AMD partial rebuild9m26s. These are Docker build measurements on this Mac, not Spark-job performance.
+
+## Boundaries of the historical result
+
+**Blocker to full qualification at that stage:** standard AMD64 smoke under Rosetta stopped after2 Delta tests. Almost all JVM threads froze in Rosetta rt_mutex_schedule; child jspawnhelper awaited a handshake; jcmd/SIGQUIT produced no dump. FORK also failed to finish. A simple Java-only probe with100 spawns passed; that did not prove absence of Comet bugs. The user reported that only a Mac was available and requested the result with that limitation. No additional workarounds were attempted; reliable closure required a native AMD64 runner. Details/logs are in G01-results.json and raw evidence. RC1 was not tagged, and actual publisher validate-only using that tag did not run.
+
+The candidate contains G01 scope. Remaining agreed capabilities belong to G02–G09, including full ANSI qualification P03/I10. Do not treat RC1 as the completed replacement for `joom-1.1-13`.
+
+Actual GCS/GKE ADC reads and production performance were not checked. Full BoundaryFormats deadlock reproduction remains G09; query interning/task-binary regression belongs to G08. The three upstream generic ANSI cancellations were disclosed; typed mandatory-overflow checks passed, including ARM64 packaged tests; AMD64 ANSI smoke had not run at that stage. Spark teardown reported possible daemon-thread-leak warnings; selected tests passed and later-group lifecycle checks remain required.
+
+## Preview and integration sequence at that stage
+
+Full diff and separate implementation/vendor previews are in `/Users/skotlov/d/temp/20261117_comet12-g01-final-qualification`. Tree comparison: `git diff branch-1.2 joom/1.2-dev`; vendor residual: `git diff 0a76d68a3 joom/1.2-dev -- native/vendor/datafusion-physical-plan`.
+
+The feature branch resolved add/add conflicts introduced by history cleanup: branch-1.2 became its ancestor without changing branch-1.2 itself. The entire file tree and24 documents matched the pre-alignment candidate; seven substantive patches and a separate vendor import were preserved. The PR could then be prepared without moving or overwriting the base ref. **The historical integration gate required GHA/review and direct user confirmation; the user has now authorized integration after finalization checks.**
+
+The user first reviews the composition and diff. G02 still requires a separate direct authorization even after G01 integration.
+
+Full machine-readable results: `G01-results.json`. Raw logs and auxiliary scripts: `/Users/skotlov/d/temp/20261117_comet12-g01-final-qualification` and previous `/Users/skotlov/d/temp/20261117_comet12-g01-resumed`. Cleanup: `CLEANUP.md`; new-file list: `created-files.txt`.
+
+## First native AMD64 GHA result for PR28
+
+[Run44137489889, attempt2](https://github.com/joomcode/datafusion-comet/actions/runs/44137489889) ended in **FAILURE** on `db2bd16584c1381853cc170abab7cd87f289c85c`. 117 jobs passed,9 independently failed, Required Checks also failed,17 skipped. No jobs remained running; five-minute monitoring ended when the run completed.
+
+Logs confirmed2639 passed Comet native tests (5 previous skips),1695 passed vendor tests without skips,42 additional passed statistics tests; Delta/Spark4.2.1 —294 passed without failures/cancellations. Comet scans/shuffle/exec and both macOS suites passed. The previously canceled ANSI add/subtract/multiply **actually passed** on AMD64. Comet expressions:2284 passed,2 failed,2 canceled,12 ignored. SparkSQL4.2.1:6 shards passed,3 failed.
+
+Groups unresolved at that stage:
+
+- **Correctness priority — SPARK-60262:** two Parquet tests with ambiguous fieldID/name mapping failed to produce mandatory `PARQUET_FIELD_REQUESTED_MORE_THAN_ONCE`. Spark added the guard in commit `59f657f2b059b28b0c5df2e5c6e28a35ebd25369`; no equivalent was found in Comet's independent field resolution. Returned rows were not checked in those negative tests, so absence of silent incorrect results was not established. Protection/fallback and both regressions were required, not test exclusion.
+- **ANSI casts:** Spark4.2.1/SPARK-60119 (`eaed2f091c8dd4f86e92a7a87a6a28f424110897`) rounds extremely small `1e-2147483647` to zero. One test expected an obsolete literal exception; native string→decimal with negative scale still returned an error instead of successful zero. Result/error parity and adaptation were required, not cancellation.
+- **SparkSQL core-1:**33 operator/fusion/partitioning assertions expected Spark-specific plans. Some preceding data checks passed;12 Union cases and1 DataFrame case failed before subsequent result checks. Simply disabling33 tests was unacceptable.
+- **Other test-compatibility cases:**3 corrupted-Parquet diagnostic mismatches (a read error did occur); one synthetic DV-fixture failure each on Spark3.5/4.0 (Delta serializer assertion; target Spark4.2.1 passed).
+- **Dependency infrastructure:**3 jobs failed on Maven Central HTTP429 before tests; SparkSQL core-2 `SPARK-33084: Add jar support Ivy URI in SQL` could not resolve `spring-core:6.1.6` through Ivy. The Ivy log did not state an HTTP code.
+
+Full diagnostics and exact tests: `/Users/skotlov/d/temp/20261117_comet12-g01-runner-fix/gha-final-summary.json` and JSON/log files listed there. About43MiB were saved in the task directory and recorded in the cleanup ledger. Sources and Git refs did not change during monitoring. Final documents were saved locally without another commit/push to avoid starting CI again. `branch-1.2`/RC1/G02 were unchanged. CI-profile execution did not replace final release-JAR smoke at RC checkpoints.
+
+## Corrections after the first full GHA
+
+Corrected source package: `6ae961dcbae949db09e349b05c91fc1cd6464d87`. At this historical stage, a full repeat GHA had not started. The nine independent failures of the preceding run were analyzed; changes were matched to exact Spark code and independently reviewed.
+
+- Decimal casts match SPARK-60119 through a behavioral serde flag with exact patch-version boundaries; older Spark retains its old behavior.
+- Native Parquet rejects ambiguous projected-field mapping before reading data. Pruning, exact physical-name keys, nested-error order and Spark error envelope/cause/parameters are preserved.
+- UNION with a known layout can no longer silently use concat when partition counts are wrong; an empty layout preserves partitions and a zero result for COUNT-like consumers.
+- Spark-specific assertions remain on the Spark path; separate native counterparts check complete results, actual native operators, layout and extension observers. Hooks account for an already converted native projection. Corrupt Parquet checks both reader paths and requires an error.
+- Only the synthetic Delta DV fixture changed: an explicit bitmap offset. Maven parent-POM resolution retries are bounded and occur only before compilation/tests; checks themselves are neither retried nor weakened. ADD JAR uses a local Ivy fixture preserving transitive/exclude and class-loading assertions.
+
+| Targeted check | Result |
+| ---------------------------------------------------------------- | ---------------------------------------------------------------------------------- |
+| Rust parser / tiny values / field resolver / error serialization | 5 + 2 + 10 + 1 passed |
+| Comet JVM | 30 passed; after envelope-converter fix,19/19 rerun |
+| Spark SQL | 223 passed,0 failed;3 previous ignored Union cases |
+| Delta changed fixture | 1/1 passed |
+| Maven retry fault injection | 4/4 passed |
+| Native build, Maven install, format, CI config | passed |
+| Spark patch | regenerated from v4.2.1; forward/reverse apply and equality of all121 files passed |
+
+Early local compilation/assertion failures are preserved in logs; the table reflects the final successful checks of those scenarios. No correctness checks were disabled. Full Rust/vendor/Comet/Delta/Spark SQL suites were delegated to the upcoming GHA. Local results do not prove the complete absence of bugs.
+
+No new release build/smoke or RC1 tag occurred at that stage. Earlier JARs referred to the old source SHA and were not final artifacts of the corrected PR. `branch-1.2` remained unchanged; G02 had not started. Raw evidence: `/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes`; machine summary: `G01-results.json#ci_corrections`.
+
+## Follow-up after run44180095887
+
+[Full GHA on ac3694504](https://github.com/joomcode/datafusion-comet/actions/runs/44180095887) ended in **FAILURE:124success/3failure/17skipped**. All nine original failed jobs passed, including9/9 Spark4.2.1 SQL shards. Selected lines of full logs confirmed the original33 SQL failures,21 native Union and9 native-extension counterparts,3 SPARK-60262 and6 SPARK-59832, both SPARK-33084. The expression artifact confirmed3 ANSI arithmetic and2 decimal-cast cases; total2287passed,2 previous cancellations,12ignored. Canceled/ignored cases were not reported as passes. Native2649passed/5skipped; vendor1695passed; statistics42; Delta4.2 —294passed without cancellations.
+
+The run discovered two additional issues:10 reader assertions expecting the old exception class after early footer validation and **incorrect Union/AQE results on Spark4.1** when the configuration changed:10 rows `(k,4)` instead of5 `(k,8)`. The third failure was the Required Checks aggregator. Those verdicts apply only to the old SHA, not to the following correction.
+
+New source commit `30b6c4d7f531fad7b6550c5d3f1c92cefd096e58` contains:
+
+- Immutable policy in CometUnion constructor state; AQE/case-class copy preserves the planning choice. Layout is derived from the children of the specific copy; per-instance laziness and early evaluation before starting children are retained, and incompatible RDD counts still fail fast. Snapshot-capable Spark4.2.1 continues to manage policy itself; legacy capability is detected by presence of `UnionConfSnapshot`, accounting for4.2.0 and backports.
+- Correct partitioning namespace after child reordering/replacement: remap Spark layout to fixed Comet output. A new strict test found this case; `references.subsetOf(outputSet)` remains asserted.
+- Reader checks require the exact structured file-read error, actual file URI and direct native cause. Native-plan checks, mandatory rejection, diagnostics and absence of INTERNAL_ERROR remain. Older Spark retains original CometNativeException; production code was not changed merely for this assertion. `ignoreCorruptFiles`/`ignoreMissingFiles` still exclude full native scan; no new silent-skipping path was added.
+
+The local negative control reproduced wrong groups before the fix. Afterwards: Spark4.1 union12passed; Spark4.2 union12passed; Spark3.5/Scala2.12 legacy controls35passed; reader42 —27passed; converter19passed; SparkSQL with the newly installed JAR —21 native Union+9 extension+3 field guards, all passed without cancellations/ignored. Coverage includes both directions of configuration change, AQE on/off, Hash/Single, copy/withNewChildren, live child-layout replacement, late conversion, complete data and RDD counts. Maven install/format and independent review passed. Rust/vendor did not change in this follow-up.
+
+A new full GHA was pending a single follow-up push at that stage. Source-commit files matched the locally tested worktree byte for byte (`followup-source-commit-verification.json`). Evidence of the first corrected run is frozen in `/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes`; the next run is stored separately in `20261117_followup-gha`. Old Linux release JARs had not been updated and RC1 had not been created; merge/base/G02 were then subject to the direct user gate.
+
+## Final full GHA on the qualified runtime-source PR HEAD
+
+[Run44205897075,attempt3](https://github.com/joomcode/datafusion-comet/actions/runs/44205897075) completed **SUCCESS** on `26590264b878fb8eae2ca439a8bcc0dfb513cb36`: **127successful/0failed/17conditional-skipped jobs**, Required Checks passed. The verdict applies to those sources, not the historical release JARs.
+
+The first attempt had only8 Maven parent-POM HTTP429 failures before compilation/tests,2 dependent coverage failures and the aggregator. A selective rerun preserved116successful executions; after the second attempt, only Spark3.4 expressions HTTP429 and the aggregator remained. The third selective rerun preserved125successful executions and completed the remaining2. Timestamp equality was confirmed; no full-pipeline rerun, new commits or weaker checks were involved. A temporary502 jobs API error was handled using the standard read-only check-runs API; records from other workflows were excluded from the count.
+
+On that SHA, confirmed:9/9 Spark4.2.1 SQL shards,2649native tests (5 previous ignored),1695vendor tests,42statistics tests and294Delta tests without cancellations. The expression artifact contains2287passed,0failed,2 previous canceled,12ignored; **the three ANSI add/subtract/multiply and both decimal-cast cases actually passed**. All10 duplicate-reader cases passed; both Union regression tests actually passed on4.1 and4.2. Canceled/ignored cases are not represented as successful tests; full ANSI qualification in later groups remains planned.
+
+This closes the native AMD64 source/runtime GHA gap for G01. **Smoke of the exact bytes of final release JARs remains separate RC acceptance.** Previous dualarch JARs from source2b8f80b3f lack the latest corrections; no new release build/RC1 tag had occurred at this stage. GCS/GKE production reads and production performance are not qualified.
+
+PR28 was updated with the result without pushing another commit; local progress documents were saved without another GHA. `branch-1.2` remained `9691385f35e4c5bd29514dca78d2044dd20e4bed`; no merge/RC1/G02 occurred. The next step then was user review.
+
+Evidence: `/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha`, including immutable attempt1/2/3 snapshots, failure classifications and `java-artifact-*-verification.json`. The cleanup ledger records final size (~21.4MiB auxiliary;39.7GiB total existing native target;829MiB Spark fixture). Nothing was automatically removed.
+
+## Manual RC Build & Smoke
+
+Qualified source before finalization: `b93d553856a0339af7468bcff2070e735a1a3c71`. [Run44388252264](https://github.com/joomcode/datafusion-comet/actions/runs/44388252264) completed SUCCESS: **7/7 jobs**, zero failed/skipped.
+
+Native AMD64 and ARM64 were built on native runners with the release profile. Final Comet and Delta JARs were packaged once. Each architecture actually passed **12/12 cases**: Delta4, DPP1, reload1, ANSI6. Checks verified Spark4.2.1, Java17, final-JAR class origins and SHA256 of the bundled `.so` actually loaded. The final job matched both receipts to identical source/lock/JAR/package/support/native hashes.
+
+Artifacts were downloaded to `/Users/skotlov/d/projects/datafusion-comet/target/joom-release/gha-44388252264`. ZIP SHA256 matched GitHub: `1c633ecec930ce8a2e4f95a2fe859ddefcd89f3a2898fc77105775448ad6035b`. Local checks verified checksums, embedded Git revision, versions, ELF/dependency graph, provenance and both receipts.
+
+| Runtime JAR | SHA256 |
+| ------------------------------------------- | ------------------------------------------------------------------ |
+| comet-spark-spark4.2_2.13-1.2.0.jar | `9f7d57bfb88efceeecce15d96ca320a2d38f81aacc95f4bbf67d46915bfc0686` |
+| comet-contrib-delta-spark4.2_2.13-1.2.0.jar | `5581281ff5bf826f04a038318d802be13d9f66cb8084e3d8a40846b5175bd632` |
+
+Workflow preparation corrected three observed failures: an ambiguous negative fixture, a separate Maven classpath invocation without reactor artifacts on clean M2, and Git2.25.1 ignoring command-scope safe.directory. The latter was reproduced in the same builder with UID1001 and fixed using configuration trusting only the checkout inside the container. No failure was hidden by weakened checks or blind retry.
+
+The workflow was registered in main by adding only its YAML file under separate user authorization. Branch-1.2 did not change. Automatic Comet CI runs were stopped as instructed; one delayed run finished on two missing license headers before it could be stopped. Both were fixed; Apache RAT for exact source b93 passed. Runtime code matches previously qualified 26590264b. The new workflow adds release-profile and packaged-JAR checks to the prior full CI qualification.
+
+Old JARs from2b8f80b3f remain historical. The native AMD64 packaged-smoke gap is closed for the newer JARs. RC1 tagging, JFrog publication, merge and G02 had not occurred at the end of that run. This is a G01 candidate; remaining Joom capabilities belong to G02–G09. Guide: `RC-BUILD-SMOKE.md`; cleanup: `CLEANUP.md`. The user has subsequently authorized G01 finalization and conditional integration/RC1 tagging; publication and G02 remain excluded.
diff --git a/docs/joom-migration/G01-results.json b/docs/joom-migration/G01-results.json
new file mode 100644
index 00000000000..83e16ec5783
--- /dev/null
+++ b/docs/joom-migration/G01-results.json
@@ -0,0 +1,779 @@
+{
+ "status": "G01_IMPLEMENTATION_COMPLETE_FINAL_RC1_GATE",
+ "checked_at": "2026-11-18T22:39:06.361313+00:00",
+ "source_commit": "b93d553856a0339af7468bcff2070e735a1a3c71",
+ "source_branch": "joom/1.2-dev",
+ "integration_branch": "branch-1.2",
+ "integration_branch_unchanged": "9691385f35e4c5bd29514dca78d2044dd20e4bed",
+ "integration_gate": "User authorized G01 finalization, conditional integration and RC1 tagging on 2026-11-18; no G02 authorization.",
+ "candidate_tag": "joom-1.2-1-rc1",
+ "versions": {
+ "comet": "1.2.0",
+ "spark": "4.2.1",
+ "scala": "2.13.18",
+ "delta": "4.4.1",
+ "datafusion": "55.2.0",
+ "arrow_parquet": "59.3.2",
+ "rust": "1.100.0",
+ "java": "17"
+ },
+ "release_directory": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/gha-44388252264",
+ "jar_sha256": {
+ "comet-spark-spark4.2_2.13-1.2.0.jar": "9f7d57bfb88efceeecce15d96ca320a2d38f81aacc95f4bbf67d46915bfc0686",
+ "comet-contrib-delta-spark4.2_2.13-1.2.0.jar": "5581281ff5bf826f04a038318d802be13d9f66cb8084e3d8a40846b5175bd632"
+ },
+ "validation": {
+ "native_workspace_passed": 4326,
+ "native_ignored": 5,
+ "vendor_included_in_native_passed": 1695,
+ "additional_vendor_statistics_passed": 42,
+ "delta_passed": 291,
+ "ansi_runtime_passed": 75,
+ "ansi_upstream_canceled": 3,
+ "spark_sql_passed": 65,
+ "packaged_smoke": [
+ {
+ "architecture": "aarch64",
+ "cases": [
+ {
+ "case": "delta",
+ "suite": "org.apache.comet.contrib.delta.CometDeltaNativeScanSuite",
+ "passed": 4,
+ "classpath_and_bundled_native_selection": "VERIFIED"
+ },
+ {
+ "case": "dpp",
+ "suite": "org.apache.spark.sql.comet.CometDppFallbackRepro3949Suite",
+ "passed": 1,
+ "classpath_and_bundled_native_selection": "VERIFIED"
+ },
+ {
+ "case": "reload",
+ "suite": "org.apache.comet.CometSparkSessionExtensionsSuite",
+ "passed": 1,
+ "classpath_and_bundled_native_selection": "VERIFIED"
+ },
+ {
+ "case": "ansi",
+ "suite": "org.apache.comet.CometExpressionSuite",
+ "passed": 6,
+ "classpath_and_bundled_native_selection": "VERIFIED"
+ }
+ ],
+ "network": "none; loopback only"
+ }
+ ],
+ "synthetic_validator_publisher_negative_passed": 19,
+ "bash32_online_offline_passed": 2,
+ "elf_glibc_spi_git_sha_and_archive_validator": "PASSED"
+ },
+ "limitations": [
+ "G01 baseline only: remaining agreed Joom-specific groups G02-G09 are not ported.",
+ "Real GCS/GKE ADC reads and production performance are not qualified here.",
+ "Java 17 qualification only; Java 21 work is explicitly deferred.",
+ "No external artifact publication or production rollout is authorized.",
+ "Full runtime CI is retained by exact runtime/test/lockfile equality; the final rewritten SHA requires its own release build and packaged smoke.",
+ "Earlier validation counters below are historical local runs; use final_full_gha and the RC receipt for the latest qualification."
+ ],
+ "raw_evidence_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-final-qualification",
+ "history_cleanup": {
+ "old_head": "3cae82c3f75a7c3aa85356515e85d0f6d984da24",
+ "backup_ref": "refs/joom-migration/checkpoints/20261117-before-service-cleanup",
+ "new_head": "13cfe918cf22c72b2034fa73761ba481a583d0f3",
+ "substantive_commits": [
+ {
+ "old": "71435d6f1add745cb3fa4b2072977ec301e52ad1",
+ "new": "ea4b3228ec5346181f74d8632d6af13b4441a359",
+ "message": "spark: establish Joom Spark 4.2.1 and Delta baseline"
+ },
+ {
+ "old": "98c9f2f195c52d1b291e933be0c91624afece67a",
+ "new": "0a76d68a33f6b4bcbce9fbb412b90350c5fed80b",
+ "message": "native: vendor pristine datafusion-physical-plan 55.2.0"
+ },
+ {
+ "old": "5640cc13cbebaba78056d0b3a83cb8a745e8cc8b",
+ "new": "15a466a09abf6106bbd80402e1967ba96076fbcc",
+ "message": "native: test vendored DataFusion in the Comet workspace"
+ },
+ {
+ "old": "f8a00fd4b357ac92bdc588d8a93bb4bc624fd55d",
+ "new": "857d2de1fa021969f6b2efd7d47c5b680daa578b",
+ "message": "ci: exclude generated Spark trees from suite inventory"
+ },
+ {
+ "old": "b94c150bbe32e8a6ccb420ebe72607d087f9b4ea",
+ "new": "73efe84b4e21f53a4da054504e0dbd2a6d693bea",
+ "message": "release: build and validate Joom Comet for Spark 4.2.1"
+ }
+ ],
+ "final_documentation_commit": "13cfe918cf22c72b2034fa73761ba481a583d0f3",
+ "non_documentation_tree_unchanged": true,
+ "before_release_build": true
+ },
+ "independent_reviews": {
+ "scope_review": {
+ "reviewer": "g01_scope_final_review",
+ "status": "NO_CONFIRMED_IMPLEMENTATION_BLOCKER",
+ "g01_cards": 12,
+ "pending_acceptance": [
+ "Spark SQL with fork classpath",
+ "Real final JAR execution on both Linux architectures"
+ ],
+ "scope_boundaries": [
+ "RC1 is G01 only; P03/I10 full Joom qualification continues across later groups",
+ "BoundaryFormats deadlock reproduction remains G09",
+ "Real GCS/GKE ADC read not qualified"
+ ]
+ },
+ "packaged_smoke_review": {
+ "reviewer": "g01_final_smoke_review",
+ "status": "NO_CONFIRMED_STATIC_BLOCKER",
+ "expected_cases_per_arch": 12,
+ "checks": [
+ "Final JAR first; remove all other Comet/main project classes",
+ "Empty native search path forces bundled library",
+ "NativeBase/Delta scanner class origin verified",
+ "Native plans and Spark result/error parity in selected tests",
+ "Network none; local fixtures only"
+ ]
+ },
+ "cargo_cache_investigation": {
+ "status": "BOUNDED_REVIEW_NO_ROOT_CAUSE_ESTABLISHED",
+ "facts": [
+ "Same builder images/compiler/config/flags",
+ "Part of Cargo cache reused",
+ "Serde build-script fingerprint rerun propagated downstream"
+ ],
+ "action": "Do not clear caches or change environment. On any future independently necessary rerun, fingerprint diagnostic logging may identify precise dirty reason."
+ }
+ },
+ "amd64_qualification_gap": {
+ "status": "CLOSED_FOR_CURRENT_RELEASE_ON_NATIVE_AMD64",
+ "run_id": 44388252264,
+ "actual_packaged_cases_passed": 12
+ },
+ "candidate_tag_status": "NOT_CREATED_USER_HOLD",
+ "pr_preparation": {
+ "status": "PENDING_NATIVE_AMD64_GHA",
+ "preparation_report": "G01-PR-preparation.json",
+ "body": "G01-PR.md",
+ "historical_local_results_not_current_pr_gha": true,
+ "three_ansi_cases_now_enabled_but_pending": true
+ },
+ "github_qualification": {
+ "run_id": 44205897075,
+ "url": "https://github.com/joomcode/datafusion-comet/actions/runs/44205897075",
+ "attempt": 3,
+ "head_sha": "26590264b878fb8eae2ca439a8bcc0dfb513cb36",
+ "checked_at": "2026-11-18T03:16:00.841276+00:00",
+ "conclusion": "SUCCESS",
+ "job_counts": {
+ "success": 127,
+ "failure": 0,
+ "skipped": 17
+ },
+ "required_checks": "PASSED",
+ "retry_history": [
+ {
+ "attempt": 1,
+ "success": 116,
+ "failure": 11,
+ "skipped": 17,
+ "causes": "8parentPOM429+2dependentcoverage+aggregate"
+ },
+ {
+ "attempt": 2,
+ "success": 125,
+ "failure": 2,
+ "skipped": 17,
+ "causes": "Spark34parentPOM429+aggregate"
+ },
+ {
+ "attempt": 3,
+ "success": 127,
+ "failure": 0,
+ "skipped": 17
+ }
+ ],
+ "successful_jobs_preserved": true,
+ "source_unchanged_during_retries": true,
+ "target_spark42": {
+ "sql_shards": "9/9 PASSED",
+ "native": "2649passed/5existingignored",
+ "vendor": "1695passed/0ignored;statistics42passed",
+ "delta": "294passed/0failed/0canceled/0ignored",
+ "expressions": "2287passed/0failed/2existingcanceled/12ignored",
+ "ansi_three_arithmetic_cases": "ACTUALLY_EXECUTED_AND_PASSED",
+ "two_decimal_cast_cases": "ACTUALLY_EXECUTED_AND_PASSED",
+ "duplicate_reader_cases": "10ACTUALLY_EXECUTED_AND_PASSED",
+ "union_regressions": "Both conf-flip/native-grouping and live-copy/layout-binding tests passed on Spark4.1 and4.2"
+ },
+ "artifact_verification_receipts": [
+ "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/java-artifact-14386009386-verification.json",
+ "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/java-artifact-14386336295-verification.json",
+ "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/java-artifact-14386457154-verification.json",
+ "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/java-artifact-14386965743-verification.json"
+ ],
+ "raw_evidence_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha",
+ "qualification": "SOURCE_CI_PASSED; final exact-byte release JAR build/smoke remains RC gate; production/GCP validation not claimed",
+ "final_receipt": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/final-qualification-run-44205897075-head-26590264b.json",
+ "spark34_expressions": "2230passed/0failed/37canceled/12ignored"
+ },
+ "ci_corrections": {
+ "status": "PUSHED_NATIVE_GHA_PENDING",
+ "source_commit": "6ae961dcbae949db09e349b05c91fc1cd6464d87",
+ "previous_pr_head": "db2bd16584c1381853cc170abab7cd87f289c85c",
+ "base_unchanged": "9691385f35e4c5bd29514dca78d2044dd20e4bed",
+ "fixes": [
+ "Spark SPARK-60119 tiny string-to-decimal rounding: version-contract flag, both Decimal128/256 paths, result/error and serde regressions",
+ "SPARK-60262: projected-field duplicate guard before data pages, exact physical names, nested error priority, structured factory cause and version-correct file-read envelope",
+ "Known-layout UNION: reject mismatched partition counts before unsafe concat; preserve declared partitions for all-empty inputs",
+ "Spark test fixtures retain original Spark contracts and add real native data/error/partition/observer checks; no new ignore/cancel",
+ "Delta synthetic DV fixture carries explicit bitmap offset compatible with older serializers",
+ "Maven model-only bounded transient parent-POM retry; actual compile/tests still run once; isolated Ivy local-repository fixture keeps transitive/exclude/class-loading checks"
+ ],
+ "checks": {
+ "rust": {
+ "parser": 5,
+ "tiny_decimal": 2,
+ "parquet_resolver": 10,
+ "structured_error": 1,
+ "failed": 0
+ },
+ "jvm_initial": {
+ "passed": 30,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "log": "jvm-regressions-concrete.log"
+ },
+ "jvm_converter_after_envelope_fix": {
+ "passed": 19,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "log": "jvm-error-envelope-final.log",
+ "note": "Rerun of converter tests after envelope correction; do not add to initial count as unique tests"
+ },
+ "spark_sql": {
+ "passed": 223,
+ "failed": 0,
+ "ignored": 3,
+ "canceled": 0,
+ "parts": {
+ "union": 189,
+ "extensions": 20,
+ "pushdown": 2,
+ "set_operations": 1,
+ "ivy": 2,
+ "field_selection": 3,
+ "corrupt_parquet": 6
+ },
+ "note": "Latest successful result per scenario, across targeted2/3/4/final logs. Three ignored Union tests are unchanged upstream ignores."
+ },
+ "delta_fixture": {
+ "passed": 1,
+ "failed": 0,
+ "log": "delta-fixture-final.log"
+ },
+ "ci_model_fault_injection": {
+ "passed": 4,
+ "failed": 0
+ },
+ "native_build": "PASSED",
+ "maven_install_spark42_delta": "PASSED",
+ "cargo_fmt": "PASSED",
+ "ci_config": "PASSED",
+ "spotless": "PASSED",
+ "spark_patch": "121 files; forward/reverse apply and exact patched-file equality passed",
+ "whitespace": "Actual Spark sources pass git diff --check v4.2.1; Comet sources excluding generated patch pass git diff --check. Generated patch preserves required single-space context lines."
+ },
+ "raw_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes",
+ "independent_review": "Runtime and CI reviews: no outstanding substantive finding after followups",
+ "push_strategy": "Single batched feature-branch push after local checks",
+ "gha_interval_seconds": 300,
+ "release_artifacts": "Previously built 2b8f80b3f JARs are historical; current corrections not release-packaged or RC tagged",
+ "scope": "G01 only; merge/base/RC1/G02 held",
+ "pr_head_commit": "ac3694504de3d5e3b93d19d8be87e48d6346b804",
+ "run_id": 44180095887,
+ "run_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44180095887",
+ "github_observed_successes": {
+ "native": {
+ "passed": 2649,
+ "skipped": 5
+ },
+ "vendor": {
+ "passed": 1695,
+ "skipped": 0
+ },
+ "vendor_statistics": {
+ "passed": 42
+ },
+ "delta40": {
+ "passed": 293,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "changed_fixture_actually_executed": true
+ },
+ "strict_scala35": "PASSED",
+ "delta_build_gate": "PASSED"
+ },
+ "github_final": {
+ "run_id": 44180095887,
+ "head_sha": "ac3694504de3d5e3b93d19d8be87e48d6346b804",
+ "html_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44180095887",
+ "status": "completed",
+ "conclusion": "failure",
+ "updated_at": "2026-11-17T23:26:56Z",
+ "job_counts": {
+ "success": 124,
+ "skipped": 17,
+ "failure": 3
+ },
+ "failed_jobs": [
+ {
+ "id": 134397478464,
+ "name": "PR Build (Linux) / Spark 4.1, JDK 17 [exec]",
+ "conclusion": "failure",
+ "html_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44180095887/job/134397478464"
+ },
+ {
+ "id": 134397478471,
+ "name": "PR Build (Linux) / Spark 4.2, JDK 17 [scans]",
+ "conclusion": "failure",
+ "html_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44180095887/job/134397478471"
+ },
+ {
+ "id": 134449352914,
+ "name": "Required Checks",
+ "conclusion": "failure",
+ "html_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44180095887/job/134449352914"
+ }
+ ],
+ "all_nine_previous_failed_jobs_passed": [
+ {
+ "id": 134382784825,
+ "name": "Delta Contrib Build Gate / Build-gate verification (default builds exclude Delta)",
+ "conclusion": "success"
+ },
+ {
+ "id": 134382785209,
+ "name": "Delta Contrib Tests (Spark 4.0) / Delta contrib (Spark 4.0)",
+ "conclusion": "success"
+ },
+ {
+ "id": 134382785727,
+ "name": "Delta Contrib Tests (Spark 3.5) / Delta contrib (Spark 3.5)",
+ "conclusion": "success"
+ },
+ {
+ "id": 134383206040,
+ "name": "PR Build (Linux) / Strict Scala warnings (Spark 3.5, JDK 17)",
+ "conclusion": "success"
+ },
+ {
+ "id": 134397478372,
+ "name": "PR Build (Linux) / Spark 4.0, JDK 21 [exec]",
+ "conclusion": "success"
+ },
+ {
+ "id": 134397478694,
+ "name": "PR Build (Linux) / Spark 4.2, JDK 17 [expressions]",
+ "conclusion": "success"
+ },
+ {
+ "id": 134403256068,
+ "name": "Spark SQL Tests (Spark 4.2) / spark-sql-sql_core-4/spark-4.2.1-jdk17",
+ "conclusion": "success"
+ },
+ {
+ "id": 134403256288,
+ "name": "Spark SQL Tests (Spark 4.2) / spark-sql-sql_core-1/spark-4.2.1-jdk17",
+ "conclusion": "success"
+ },
+ {
+ "id": 134403256457,
+ "name": "Spark SQL Tests (Spark 4.2) / spark-sql-sql_core-2/spark-4.2.1-jdk17",
+ "conclusion": "success"
+ }
+ ],
+ "tests_evidence": {
+ "native": {
+ "passed": 2649,
+ "skipped": 5,
+ "log": "job-134383205998-tail.log"
+ },
+ "vendor": {
+ "passed": 1695,
+ "skipped": 0,
+ "statistics_passed": 42,
+ "log": "job-134383205998-tail.log"
+ },
+ "delta42": {
+ "passed": 294,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "log": "job-134382785838-tail.log"
+ },
+ "delta40": {
+ "passed": 293,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "previous_case_verified": true,
+ "log": "job-134382785209-tail.log"
+ },
+ "delta35": {
+ "passed": 292,
+ "failed": 0,
+ "canceled": 1,
+ "ignored": 0,
+ "previous_case_verified": true,
+ "canceled_case_name_not_in_tail": true,
+ "log": "job-134382785727-tail.log"
+ },
+ "expressions42": {
+ "passed": 2287,
+ "failed": 0,
+ "canceled": 2,
+ "ignored": 12,
+ "three_ansi_and_two_cast_cases_executed_passed": true,
+ "receipt": "spark42-expressions-artifact-verification.json"
+ },
+ "sql42": {
+ "shards_passed": 9,
+ "shards_failed": 0,
+ "core1_counts": {
+ "passed": 10481,
+ "failed": 0,
+ "canceled": 64,
+ "ignored": 1155
+ },
+ "core2_counts": {
+ "passed": 1809,
+ "failed": 0,
+ "canceled": 1,
+ "ignored": 3440
+ },
+ "core4_counts": {
+ "passed": 7542,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 356
+ },
+ "previous_individual_cases_evidence": "Collected separately by coordinator helper under20261117_ci-reliability; see parent report"
+ }
+ },
+ "remaining_failures_classification": {
+ "134397478464": "Spark4.1 AQE union conf-flip query returns10rows(k,4) rather than5rows(k,8); actual result mismatch, CometExecSuite.scala4140",
+ "134397478471": "Ten duplicate-Parquet rejection tests reach expected read error, fail only cause-chain CometNativeException class assertion at CometNativeReaderSuite.scala123",
+ "134449352914": "Aggregate Required Checks reports the two failed jobs"
+ },
+ "monitor_actions": "Read-only API/logs/reports; no source/docs/Git changes, no rerun/push/cancel. Auxiliary JSON/log files only.",
+ "qualification_limit": "This verdict applies only to ac3694504. Later local uncommitted follow-up fixes are not covered by this run."
+ }
+ },
+ "g01_ci_followup": {
+ "job_id": 134397478471,
+ "profile": "Spark4.2 scans",
+ "cases": 10,
+ "kind": "asserted legacy CometNativeException envelope after new footer prevalidation",
+ "data_contract": "Native scan still rejects file; no successful wrong data observed; ignoreCorruptFiles routes away from full native",
+ "fix": "Strict test-only assertion of actual structured Spark file-read envelope, exact path and native cause; old-profile assertion retained",
+ "patch": "20261117_parquet/native-reader-duplicate-envelope.patch",
+ "local_result": "PASSED; see native-reader-duplicate-envelope.log",
+ "source_changes": "NONE; test source only; not committed/pushed until current full CI results collected",
+ "independent_review": "No weakening found"
+ },
+ "g01_union_followup": {
+ "status": "PASSED_LOCAL_AND_FINAL_GHA",
+ "current_code_location": "Committed and pushed on joom/1.2-dev; PR28 HEAD 26590264b878fb8eae2ca439a8bcc0dfb513cb36",
+ "production_changes": [
+ "Immutable legacy Union partitioning policy survives AQE/case-class copies; Spark4.2.1 delegates to authoritative snapshot",
+ "Preserve per-instance lazy layout, infer from live children of each copy; capture layout before executing children",
+ "Rebind snapshot-derived partitioning attributes to fixed Comet output after child reorder/replacement"
+ ],
+ "reader_test_change": "Strict typed file-read envelope/path/direct-cause; legacy CometNativeException assertion retained when guard absent",
+ "checks": {
+ "spark41_before_fix": "Reproduced wrong results:10rows(k,4) instead5rows(k,8); expected negative-control failure",
+ "spark41_after_fix": {
+ "passed": 12,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0
+ },
+ "spark42_union_after_fix": {
+ "passed": 12,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0
+ },
+ "spark42_reader_after_assertion_fix": {
+ "passed": 27,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0
+ },
+ "spark42_converter": {
+ "passed": 19,
+ "failed": 0
+ },
+ "spark42_native_spark_fixtures": {
+ "union": 21,
+ "extensions": 9,
+ "field_selection": 3,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "log": "spark-union-policy-followup.log"
+ },
+ "fresh_installed_test_jar_sha256": "4d448f7d253483dc92043d05564127544066290d21a72ad0d4fa704bce618f1b",
+ "maven42_install": "PASSED",
+ "format": "PASSED",
+ "spark35_scala212_legacy": {
+ "passed": 35,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "log": "union-reader-spark35-legacy.log",
+ "purpose": "Cross-compile changed shared API and preserve old raw duplicate-error contract"
+ }
+ },
+ "independent_review": "No outstanding finding; namespace followup matches recommended remap; tests not weakened",
+ "no_rust_or_vendor_change": true,
+ "raw_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes",
+ "source_patch_sha256": "403deeae1ebde3e20511e54c593beeae682f230c14677b034f471eaf98c1c8b1",
+ "pending": "Only user review/integration and final release-JAR acceptance; no unresolved GHA failures",
+ "active_local_builds": [],
+ "source_commit": "30b6c4d7f531fad7b6550c5d3f1c92cefd096e58",
+ "new_ci_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha",
+ "pr_head_commit": "26590264b878fb8eae2ca439a8bcc0dfb513cb36",
+ "run_id": 44205897075,
+ "run_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44205897075"
+ },
+ "source_commit_scope": "Current downloaded RC Build & Smoke artifact; original2b8f80b3frelease preserved under historical_release_artifacts",
+ "qualified_pr_head_commit": "26590264b878fb8eae2ca439a8bcc0dfb513cb36",
+ "github_qualification_history": [
+ {
+ "run_id": 44137489889,
+ "url": "https://github.com/joomcode/datafusion-comet/actions/runs/44137489889",
+ "attempt": 2,
+ "head_sha": "db2bd16584c1381853cc170abab7cd87f289c85c",
+ "completed_at_utc": "2026-11-17T19:26:09Z",
+ "conclusion": "FAILURE",
+ "job_counts": {
+ "success": 117,
+ "failure": 10,
+ "skipped": 17
+ },
+ "independent_failed_jobs": 9,
+ "aggregate_failure": "Required Checks134305657089 follows the nine failedjobs",
+ "target_spark42": {
+ "native_vendor": "PASSED;2639Comet+1695vendor+42targetedstatistics;5existingnative skips",
+ "comet_suites": {
+ "scans": "PASSED",
+ "shuffle": "PASSED",
+ "exec": "PASSED",
+ "expressions": "2284passed/2failed/2canceled/12ignored"
+ },
+ "delta": "294passed/0failed/0canceled/0ignored",
+ "sql_shards": "6passed/3failed(core-1,core-2,core-4)",
+ "ansi_add_subtract_multiply": "ACTUALLY_PASSED_NATIVE_AMD64"
+ },
+ "macos": "Both scans andexpressionsPASSED",
+ "qualification": "G01_NOT_QUALIFIED; mandatoryerror/castcompatibility/testcontract issues unresolved",
+ "highest_priority": "MissingSPARK-60262 PARQUET_FIELD_REQUESTED_MORE_THAN_ONCE guard forflat/nested fieldID/namecollisions. Actualreturnedrows were notasserted; do not claimabsenceofsilentresult risk.",
+ "failure_groups": {
+ "dependency_resolution": "4jobs:3MavenHTTP429 +SQLcore2Ivydependency org.springframework:spring-core:6.1.6; IvyHTTPstatusnotreported",
+ "delta_fixture": "2oldprofilejobs Spark3.5/4.0, syntheticDVdescriptor serializerassertion; target4.2passes",
+ "ansi_casts": "2cases:obsolete literalexceptionexpectation andnative negative-scale errorwhereSpark4.2.1returnszero",
+ "sql_core1": "33operator/fusion/partitioningcontractassertions; some laterdata checksunreached; cannotblanketskip",
+ "sql_core4": "2missingmandatorymappingerrors +3corruptedParquetexceptiondiagnosticmismatches"
+ },
+ "raw_evidence_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-runner-fix",
+ "diagnosis_files": [
+ "first-test-job-failures.json",
+ "sql42-failure-diagnosis.json",
+ "parquet-core4-failures.json",
+ "sql42-core1-diagnosis.json",
+ "sql42-core1-failure-cases.json",
+ "union-core1-failures.json"
+ ],
+ "operations": "Monitoring/read-only diagnosis only; no sourceedits/commits/push/reruns duringmonitoring. Localmigrationdocumentsupdated; branch-1.2/RC1/G02unchanged."
+ }
+ ],
+ "historical_release_artifacts": {
+ "source_commit": "2b8f80b3f253559bc999f73769b05c88ebcd5fc2",
+ "source_commit_scope": "Historical release JARs only; source_commit and jar_sha256 below refer to2b8f80b3f, not the qualified PR",
+ "release_directory": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1",
+ "jar_sha256": {
+ "comet-spark-spark4.2_2.13-1.2.0.jar": "40182026a96ccbf4958d2f741d19268e518173ff2960f937f1d3b700172defff",
+ "comet-contrib-delta-spark4.2_2.13-1.2.0.jar": "0c50187c4624b46b31a47326b7b281d6cde96d1b458697845d74debc0b07ae8c"
+ },
+ "amd64_qualification_gap": {
+ "status": "DEFAULT_AMD64_SMOKE_INCOMPLETE_STALLED",
+ "stage": "After2 Delta cases passed; remaining group did not complete",
+ "runtime": "Rosetta /run/rosetta/rosetta",
+ "observed_by_independent_reviewer": {
+ "java_threads": 351,
+ "threads_blocked_in_rosetta_rt_mutex_schedule": 349,
+ "cpu_ticks_no_progress": true,
+ "direct_java_child": "PID727 jspawnhelper awaiting4-byte parent handshake on fd438",
+ "jcmd_attach": "No response in10.5s",
+ "sigquit": "No Java dump emitted"
+ },
+ "interpretation": "Strong evidence of process-spawn/emulation suspension; does not independently prove absence of Comet bug. Native AMD64 default-mode execution is the most reliable follow-up.",
+ "java_only_probe": {
+ "spark_or_comet_loaded": false,
+ "background_threads": 32,
+ "process_spawns": 100,
+ "status": "PASSED; simple probe does not reproduce Spark JVM stall"
+ },
+ "environment_variation": {
+ "jdk.lang.Process.launchMechanism": "FORK",
+ "status": "ALSO_INCOMPLETE; Delta group timed out at300s after same2 cases; no further tests started"
+ },
+ "official_jdk_mode_reference": "https://bugs.java.com/bugdatabase/view_bug?bug_id=8357180",
+ "source_and_jars_unchanged": true,
+ "native_runner_question": "Requested asynchronously; awaiting user information",
+ "user_response": "Only Mac available; show result with this limitation",
+ "next_action": "No further Mac experiments; report qualification gap. Native AMD64 follow-up needed to complete default runtime verification."
+ },
+ "candidate_tag_status": "NOT_CREATED; full AMD64 runtime qualification incomplete",
+ "validation": {
+ "native_workspace_passed": 4326,
+ "native_ignored": 5,
+ "vendor_included_in_native_passed": 1695,
+ "additional_vendor_statistics_passed": 42,
+ "delta_passed": 291,
+ "ansi_runtime_passed": 75,
+ "ansi_upstream_canceled": 3,
+ "spark_sql_passed": 65,
+ "packaged_smoke": [
+ {
+ "architecture": "aarch64",
+ "cases": [
+ {
+ "case": "delta",
+ "suite": "org.apache.comet.contrib.delta.CometDeltaNativeScanSuite",
+ "passed": 4,
+ "classpath_and_bundled_native_selection": "VERIFIED"
+ },
+ {
+ "case": "dpp",
+ "suite": "org.apache.spark.sql.comet.CometDppFallbackRepro3949Suite",
+ "passed": 1,
+ "classpath_and_bundled_native_selection": "VERIFIED"
+ },
+ {
+ "case": "reload",
+ "suite": "org.apache.comet.CometSparkSessionExtensionsSuite",
+ "passed": 1,
+ "classpath_and_bundled_native_selection": "VERIFIED"
+ },
+ {
+ "case": "ansi",
+ "suite": "org.apache.comet.CometExpressionSuite",
+ "passed": 6,
+ "classpath_and_bundled_native_selection": "VERIFIED"
+ }
+ ],
+ "network": "none; loopback only"
+ }
+ ],
+ "synthetic_validator_publisher_negative_passed": 19,
+ "bash32_online_offline_passed": 2,
+ "elf_glibc_spi_git_sha_and_archive_validator": "PASSED"
+ },
+ "limitations": [
+ "Full AMD64 packaged runtime smoke did not complete in either default Rosetta launch or bounded FORK variation. Two Delta cases passed in each attempt. Remaining AMD64 cases unexecuted. No RC1 tag created.",
+ "RC1 covers G01 only; full Joom migration and ANSI qualification continue in G02-G09",
+ "Three existing upstream generic ANSI cancellations (#4967); targeted mandatory-overflow error parity passes",
+ "Real GCS/GKE ADC read and production performance not qualified",
+ "Possible daemon thread leak warnings in Spark suite teardown; no failed tests; lifecycle qualification continues in later groups",
+ "Isolated network-none containers logged caught hostname-resolution diagnostics; selected tests completed successfully"
+ ]
+ },
+ "rc_build_smoke": {
+ "run_id": 44388252264,
+ "run_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44388252264",
+ "head_sha": "b93d553856a0339af7468bcff2070e735a1a3c71",
+ "conclusion": "SUCCESS",
+ "success_jobs": 7,
+ "failed_jobs": 0,
+ "skipped_jobs": 0,
+ "source_gate_fault_tests": 22,
+ "actual_packaged_cases": {
+ "amd64": 12,
+ "aarch64": 12
+ },
+ "groups_per_arch": {
+ "delta": 4,
+ "dpp": 1,
+ "reload": 1,
+ "ansi": 6
+ },
+ "download": {
+ "run_id": 44388252264,
+ "source_commit": "b93d553856a0339af7468bcff2070e735a1a3c71",
+ "artifact_id": 14469302529,
+ "artifact_name": "rc-build-smoke-b93d553856a0339af7468bcff2070e735a1a3c71",
+ "artifact_api_digest": "sha256:1c633ecec930ce8a2e4f95a2fe859ddefcd89f3a2898fc77105775448ad6035b",
+ "downloaded_zip_sha256": "1c633ecec930ce8a2e4f95a2fe859ddefcd89f3a2898fc77105775448ad6035b",
+ "archive": "/Users/skotlov/d/temp/20261118_comet12-rc-workflow/qualified-artifact-44388252264.zip",
+ "output_directory": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/gha-44388252264",
+ "artifact_bytes": 189616637,
+ "unpacked_bytes": 429572457,
+ "jar_sha256": {
+ "comet-spark-spark4.2_2.13-1.2.0.jar": "9f7d57bfb88efceeecce15d96ca320a2d38f81aacc95f4bbf67d46915bfc0686",
+ "comet-contrib-delta-spark4.2_2.13-1.2.0.jar": "5581281ff5bf826f04a038318d802be13d9f66cb8084e3d8a40846b5175bd632"
+ },
+ "smoke_architectures": {
+ "amd64": {
+ "passed": 12,
+ "receipt_sha256": "43daf2cd5fa45b6823d9440ecf01d014f41d4b5c768ed3ad47f1da85bf592bde"
+ },
+ "aarch64": {
+ "passed": 12,
+ "receipt_sha256": "7d65b78eea3bd86777d4adc151a342611b5cb0a5d560e21b484fb44c9f68f3d1"
+ }
+ },
+ "archive_runtime_provenance_and_smoke_receipts_verified": true
+ },
+ "qualification": {
+ "source_commit": "b93d553856a0339af7468bcff2070e735a1a3c71",
+ "cargo_lock_sha256": "8067744e33e7a5adfa94b22d47a6178285083e27ffda4258fe4d70aba355209a",
+ "qualification": "RELEASE_BUILD_AND_PACKAGED_SMOKE_PASSED",
+ "scope": "G01 baseline; later Joom feature groups and production GCP qualification remain",
+ "release_jar_sha256": {
+ "comet-spark-spark4.2_2.13-1.2.0.jar": "9f7d57bfb88efceeecce15d96ca320a2d38f81aacc95f4bbf67d46915bfc0686",
+ "comet-contrib-delta-spark4.2_2.13-1.2.0.jar": "5581281ff5bf826f04a038318d802be13d9f66cb8084e3d8a40846b5175bd632"
+ },
+ "checksum_manifest_sha256": "07b34e42c9f38aeb8c9f331bd7914b24f6091fb71135f8919af570945cef74b1",
+ "test_support_manifest_sha256": "87351ce516521f54db6e4ba95d303632e4849f3f0d6596ef6c9dd1a7da25761d",
+ "architectures": {
+ "amd64": {
+ "passed": 12,
+ "receipt_sha256": "43daf2cd5fa45b6823d9440ecf01d014f41d4b5c768ed3ad47f1da85bf592bde"
+ },
+ "aarch64": {
+ "passed": 12,
+ "receipt_sha256": "7d65b78eea3bd86777d4adc151a342611b5cb0a5d560e21b484fb44c9f68f3d1"
+ }
+ },
+ "github_run_id": "44388252264",
+ "github_run_attempt": "1"
+ },
+ "source_rat": "EXACT_B93_PASSED",
+ "runtime_source_equal_to_ci_qualified_26590264b": true
+ },
+ "source_commit_role": "Historical B93 qualified artifact; final RC1 source is resolved from the annotated release tag.",
+ "final_rc1_identity": {
+ "tag": "joom-1.2-1-rc1",
+ "receipt": "release-qualification.json delivered beside the JARs",
+ "closeout": "G01-CLOSEOUT.md"
+ }
+}
diff --git a/docs/joom-migration/G01-review-notes.json b/docs/joom-migration/G01-review-notes.json
new file mode 100644
index 00000000000..c3692ca0d36
--- /dev/null
+++ b/docs/joom-migration/G01-review-notes.json
@@ -0,0 +1,71 @@
+{
+ "recorded_on": "2026-11-17",
+ "scope": "Durable continuation notes from completed internal read-only reviews. These are findings and remaining checks, not proof of release qualification.",
+ "delta_review": {
+ "scanner_partitioning": "Planning outputPartitioning must never force perPartitionData or execute scalar subqueries. Current UnknownPartitioning(0) is metadata-only: upstream core actual fused execution derives partition count from perPartitionData.length. Preserve that execution invariant and live runtimeFilters on sparkFallback.",
+ "serialization": "nativeOp and serializedPlanOpt are driver-only transient fields. JVM regression passed clone/copy retention and Java serialization omission. Query-text interning is J17/G08, not completed by these changes. Full J17 interning requires the related internScan/internCommons/internScans core APIs and the task-binary regression; transient fields alone do not complete that old contract.",
+ "old_deadlock_reproducer": "Original full Joom end-to-end reproducer depends on BoundaryFormats (G09). G01 adds direct scalar data-filter planning regression; preserve full reproduction for G09 rather than silently treating it as already covered.",
+ "s3_encryption_context": {
+ "source": "Selected production/unit portions of upstream draftPR7593, local original JSON /Users/skotlov/d/temp/20261115_comet-upstream-branches-pr-audit/pr-7593-detail.json",
+ "why": "Hadoop3.5 validates context during S3A initialization; native reader does not. Local Delta logs may reference S3 data/DV without metadata initializing that bucket, so missing guard permits success where Spark must fail.",
+ "required_behavior": "After encryption-algorithm gate, inspect dataFileUris plus DV URIs. Reject nonempty/unverifiable Hadoop-effective context, preserve global/short/long aliases and credential-provider rules, redact values/paths. Pure GCS is unaffected.",
+ "io": "Conf-only with no provider; may read local global JCEKS. Reject nonlocal and S3A-specific providers without opening them. Do not describe helper as unconditionally zero-I/O.",
+ "tests": "Keep plan-only local-log/S3-DV real-claim test: cold metadata error test fails before Comet admission and is insufficient alone. Do not collect the synthetic remote-DV dataframe. Cold metadata test pins loopback endpoint127.0.0.1:1 and SimpleAWSCredentialsProvider.",
+ "do_not_weaken": "Do not simply trim final effective context: whitespace long alias plus malformed short alias could admit a scan that Hadoop rejects. Current conservative fallback is deliberate.",
+ "gcp_control": "PASSED in repeated full291 Delta tests"
+ },
+ "actual_gcp_validation": "Real GCS/GKE ADC reading has not been tested. Local auth-whitelist/unit tests are not that qualification.",
+ "partitioning_followup": {
+ "recommendation": "Retain Unknown0. Direct upstream core contract and Spark4.2.1 nonbucketed scan behavior justify it; extending only filter guards still violates no-planning perPartitionData contract.",
+ "refs": [
+ "spark/src/main/scala/org/apache/spark/sql/comet/operators.scala:1382",
+ "spark/src/main/scala/org/apache/spark/sql/comet/operators.scala:1135",
+ "spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala:131",
+ "target/joom-spark-4.2.1/sql/core/src/main/scala/org/apache/spark/sql/execution/DataSourceScanExec.scala:482",
+ "target/joom-spark-4.2.1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/plans/physical/partitioning.scala:246"
+ ],
+ "difference": "Unknown1 satisfies AllTuples, Unknown0 does not; possible extra SinglePartition exchange relative to oldcontrib, not measured. Same as sourceJoom and standardSpark. No new product approval needed."
+ }
+ },
+ "vendor_review": {
+ "pristine_import": "Published archive55.2.0, not a Git workspace-inheriting manifest. Separate commit98c9f2f19 preserves150 archive files. No .cargo_vcs_info.json in this archive; do not claim it exists.",
+ "resolution": "Use root native/Cargo.lock and root crates-io patch. Vendor nested Cargo.lock remains pristine provenance only; native workspace_root and single local physical-plan identity were checked. Existing registry versions/checksums unchanged.",
+ "unit_tests": "Default-members deliberately remain9 Comet packages. Explicit vendor test target is required. Current commands select core and vendor together for Comet feature unification; statistics test_utils run is additional, never --all-features (force_hash_collisions).",
+ "formatting": "Vendor-local rustfmt.toml from DataFusion55.2.0 (edition2024,width90); do not reformat published Rust sources using Comet edition2021,width100.",
+ "clippy": "Strict Comet workspace clippy PASSED with vendor excluded/no-deps; pristine vendor deprecated warning retained and documented.",
+ "compiler_versions": "Upstream CI uses stable; draft release builder now pins1.100.0 to host. Compiler parity must be verified including lint jobs. Old Joom pin was1.99.0."
+ },
+ "release_review": {
+ "status": "BUILD_BOTHARCH_AND_ARM64_SMOKE_PASSED; AMD64_SMOKE_INCOMPLETE",
+ "contract": "BaseJAR1.2.0 names/contents; publication GAV version equals immutable joom-1.2-* tag. No publish authorized. --validate-only must complete before credentials, staging or HTTP.",
+ "tag_validation": "Use refs/tags/^{commit}, not arbitrary Git ref; compare tag commit with commit.txt, lock SHA and artifact hashes.",
+ "packaging": "Preserve Linux arm64/aarch64 and amd64/x86-64-v3, GLIBC<=2.31, exactly two libcomet.so entries, no libcomet.dylib, byte-compare libraries and verify all3 Delta SPI resources.",
+ "stale_resource": "Host Mac dylib is staged under spark/target/classes/org/apache/comet/darwin/aarch64. Removing only old JAR is insufficient. Draft cleans only generated native resources; final archive validation must reject stale macOS libraries.",
+ "do_not_root_clean": "Root Maven clean would delete target/joom-spark-4.2.1 fixture and release outputs under target. Preserve them.",
+ "provenance": "Before/after builds verify clean tracked+untracked source tree, stable HEAD and root Cargo.lock. Cache Cargo registry and git sources (pinned Iceberg git dependency). Record actual compiler/images/flags/versions/hashes.",
+ "qualification": "uname/container execution probes are not native/JAR execution proof, especially amd64 x86-64-v3 under emulation. Both packaged architectures require real smoke tests. JFrog no-overwrite concurrency depends on server policy and is not established by local dry-run.",
+ "remaining_runtime_qualification": "AMD64 runtime qualification incomplete under Rosetta; native runner unavailable. Real GCP/production performance and later groups remain."
+ },
+ "performance": "Use existing project benchmarks only, no special performance framework; no performance or production GCP qualification has been claimed.",
+ "pause": "G01 implementation/build and ARM64 smoke passed; AMD64 runtime verification incomplete under Rosetta/FORK. User requested report with limitation. Review/confirmation before integration; G02 not authorized.",
+ "sparksql_resolution": {
+ "repository_config": "Task-local SBT config uses Maven Local first then approved Central; no new source checkout or cache relocation.",
+ "artifact_identity": "Compare selected sql/Test/dependencyClasspath and fullClasspath Comet JAR SHA to installed-fork-identity.json; sameGAV upstreamexists.",
+ "from_url_warning": "SBT ModuleID.from URL is fallback only; do not rely on it to force fork if artifact is published upstream.",
+ "cache_repairs": "Fetched exact missing Parquet1.17.0 tests classifiers, Log4j api/core2.25.3, util-interface1.11.5 via ordinary Maven. Only incomplete task-created SBT1.12.8 boot reset; shared/unrelated caches preserved."
+ },
+ "packaged_smoke_recipe": {
+ "runner": "Existing org.scalatest.tools.Runner, four separate JVMs perarchitecture.",
+ "classpath": "Final core+DeltaJAR first, freshly built coretestsJAR and Delta target/test-classes, filtered externaltestdeps. Remove every other CometJAR and allprojectmain target/classes.",
+ "native_selection": "Set java.library.path to emptydir and LD_LIBRARY_PATH onlyJAVA_HOME/lib/server. Confirm NativeBase andDeltaNativeScanExec classorigin finalJARs via class-load log.",
+ "classpath_generation": "Root Maven -Prelease,spark-4.2,delta dependency:build-classpath -DincludeScope=test -Dmdep.outputFile=target/joom-release-smoke-classpath.txt insidebuilder; combine spark andDelta module outputs.",
+ "flags": "Use effectivePOM properties/argLine with shlex, add headless/tmpdir/empty-native librarypath/log4j/spark.test.home/testing; SPARK_LOCAL_IP127.0.0.1,SPARK_LOCAL_HOSTNAMElocalhost,COMET_CONF_DIR/SPARK_HOME repo.",
+ "cases": [
+ "CometDeltaNativeScanSuite: plain native read, DELETE DV read, planning scalar-datafilter, clone/serialization fields",
+ "CometDppFallbackRepro3949Suite: native context preserves zero partitions after file pruning (bothAQE)",
+ "CometSparkSessionExtensionsSuite: reloading NativeBase does not load a second copy",
+ "CometExpressionSuite: -z ANSI integral overflow fidelity (6cases)"
+ ],
+ "isolation": "--network none; localfixtures/loopback only; no GCS/MinIO."
+ }
+}
diff --git a/docs/joom-migration/HANDOFF.md b/docs/joom-migration/HANDOFF.md
index 3715f8d187e..8b791e7318b 100644
--- a/docs/joom-migration/HANDOFF.md
+++ b/docs/joom-migration/HANDOFF.md
@@ -1,71 +1,106 @@
-# Передача работы между сессиями
+
+
+> **Current authorization (2026-11-18):** finish G01, finalize English documentation, consolidate service/checkpoint history, repeat exact-source RC Build & Smoke if needed, then integrate into branch-1.2 and create joom-1.2-1-rc1 after successful verification. Check completeness afterwards. Do not start G02 or do further Java 21 work. STATE.json records actual progress; decisions.json records the latest instruction.
+
+> **Historical G01 PR preparation:** history was aligned without changing the tree/24 documents or branch-1.2; three ANSI guards were removed for actual native AMD64 GHA execution. Only fast checks locally, full coverage in GHA. The user creates the PR using G01-PR.md. At that point: no merge/RC1/G02. See decisions.json for newer decisions.
+
+> **Historical status before native RC qualification:** G01 implemented; AMD64 runtime qualification NOT complete. branch-1.2 NOT changed; user preview/confirmation first. G02 not authorized. Results: G01-report.md / G01-results.json. These limitations describe that earlier checkpoint, not the current RC result.
+
+> **G01 resumed by the user on 2026-11-17:** ProtocBridge cache writes, loopback and Docker were verified inside the sandbox. The historical pause below is retained for traceability; current status is in STATE.json. Only G01 was authorized.
+
+> **Historical pause, 2026-11-17:** all our builds/tests were stopped at the user's request. Code saved in Git; G01 unfinished. PAUSED.md was the authoritative continuation point; no restart before explicit resume. G02 not authorized.
+
+**G01 was resumed by the user on 2026-11-17. STATE.json is the current status; PAUSED.md records the continuation point. G02 is not authorized.**
+
+# Handoff between work sessions
+
+The local datafusion-comet project is registered as
+`79ec4996-572a-4070-96f7-2344b555eb2c`, with primary directory
`/Users/skotlov/d/projects/datafusion-comet`.
-Пользователь прямо разрешил координатору создавать/вести сессии, передавать задания
-и уточнения и проверять результаты. Ручной перенос контекста пользователем не нужен.
-
-## Выбранная рабочая схема
-
-Один основной чат — исполнитель и координатор. Девять feature branches остаются,
-отдельный пользовательский чат на каждую группу не требуется. Координатор сам
-выбирает задачи и момент делегирования внутренним помощникам: source/compatibility
-исследования и независимый review. Реализация и интеграция по умолчанию остаются
-в основном чате, с одним пишущим исполнителем на checkout.
-
-Новую пользовательскую сессию координатор создаёт при конкретной пользе от
-длительной изолированной задачи/ограниченного контекста или необходимости передачи
-продолжения. Новый PR, большой diff или compaction не являются автоматическим
-основанием. Причину сообщить пользователю, checkpoint/контекст передать самому;
-повторное согласие в рамках уже данного scope не требуется. Распределение задач
-пользователь вручную не организует.
-
-## Текущая фаза
-
-**Код не писать до нового прямого указания пользователя.** Подготовительная сессия G01
-завершила read-only проверку и остаётся idle; реализация ей не назначена. До нового
-разрешения не создавать feature branch, не менять исходники/тесты/build scripts,
-не начинать builds/downloads автоматически. Создание чата не является стартом кода.
-
-## Работа координатора
-
-1. По умолчанию выполнять работу в основном чате с внутренними помощниками.
- Отдельные пользовательские сессии создавать по указанным выше критериям.
-2. В prompt явно указывать repo и `docs/joom-migration/README.md`, `decisions.json`,
- `STATE.json`, `approved-plan.json`, `PERFORMANCE.md`, cleanup ledger и карточки группы.
- Из корневого AGENTS.md миграционный раздел удалён по просьбе пользователя.
-3. Передавать точную группу/branch/base SHA, допустимые действия и последние решения.
- Текущий прямой code hold имеет приоритет над общим разрешением Git.
-4. Читать ответ worker, проверять доказательства и сохранять отчёт здесь.
- Отдельные чаты не синхронизируют последующие сообщения автоматически.
-5. После разрешения реализации: один пишущий исполнитель на checkout; review и
- исследования можно параллельно. Интеграцию/общие STATE/решения ведёт координатор.
-6. Передавать уточнения самостоятельно. Реальный environment blocker или новое
- требуемое решение сообщать пользователю.
-
-## Обязательное содержимое задания
-
-- группа и все её карточки из реестра; цель/границы, branch, точный base SHA;
-- последний запрет/разрешение кода, сеть с условием безопасности, запрет публикации;
-- источники approved-plan и пользователем предоставленные checkouts;
-- ANSI=true correctness/mandatory errors; PR5889 не включать;
-- performance по существующим Comet conventions без специальной инфраструктуры;
-- точный scope записи; процессная документация только в docs/joom-migration;
-- формат отчёта; следующую группу автоматически не начинать.
-
-## Отчёт worker
-
-При подготовке достаточно ответа в чате: координатор прочитает и сохранит его здесь.
-Разрешённая связь task-to-task — только по этой работе; сообщений людям не отправлять.
-При реализации отчёт сохранять в этом каталоге, raw logs — в датированном temp.
-
-Фиксировать group/branch/base/head/cleanliness, сделанные и оставшиеся карточки,
-upstream replacements, изменённые shared APIs, тестовые команды/exit codes/log paths,
-native coverage, performance результаты и ограничения, созданные paths/Docker IDs/
-volumes/размеры для cleanup, артефакты/checksums, блокеры и точный следующий шаг.
-Непроведённые проверки не объявлять успешными.
-
-Смена сессии/compaction не является основанием повторять готовые стадии.
-STATE и решения отражают фактический прогресс и прямые пользовательские разрешения;
-старые snapshots не должны перезаписывать текущее состояние.
+The user explicitly authorized the coordinator to create/manage sessions, pass tasks
+and clarifications, and verify results. The user does not need to copy context manually.
+
+## Selected execution model
+
+One main chat executes and coordinates. The nine feature branches remain;
+a separate user-visible chat per group is unnecessary. The coordinator chooses
+bounded source/compatibility research and independent review tasks for internal
+helpers, and when to delegate them. Implementation and integration stay in the main
+chat by default, with a single writer per checkout.
+
+The coordinator creates a new user-visible session when a lengthy isolated task,
+a bounded separate context or a continuation handoff offers concrete benefit.
+A new PR, large diff or compaction is not an automatic reason. Explain the reason
+to the user and transfer the checkpoint/context independently; repeated consent is
+not required within the existing scope. The user does not manually allocate tasks.
+
+## Current phase
+
+**The user has authorized G01, implemented by the main chat in joom/1.2-dev.**
+The G01 preparation chat remains idle. Do not proceed to G02-G10 without separate
+explicit user authorization after the previous group's report. The latest G01
+finalization instruction includes integration and RC1 after successful checks.
+
+## Coordinator responsibilities
+
+1. Execute in the main chat with internal helpers by default. Create separate
+ user-visible sessions only under the criteria above.
+2. Explicitly include the repo and docs/joom-migration/README.md, decisions.json,
+ STATE.json, approved-plan.json, PERFORMANCE.md, cleanup ledger and group cards in
+ each task prompt. The migration section was removed from root AGENTS.md at the user's request.
+3. Pass the exact group/branch/base SHA, allowed actions and latest decisions.
+ A current explicit code hold takes precedence over general Git permission.
+4. Read the worker response, verify evidence and save the report here.
+ Separate chats do not automatically synchronize later messages.
+5. Once implementation is authorized, keep one writer per checkout; review and
+ research may run in parallel. The coordinator owns integration/shared STATE/decisions.
+ Any explicit assignment of disjoint documentation files must remain nonoverlapping.
+6. Forward clarifications independently. Report real environment blockers or a new
+ required decision to the user.
+
+## Required task contents
+
+- Group and all its register cards; goal/boundaries, branch and exact base SHA.
+- Latest code hold/authorization, network safety condition and publication prohibition.
+- Approved-plan sources and user-provided checkouts.
+- ANSI=true correctness/mandatory errors; do not include PR5889.
+- Performance using established Comet conventions, without special infrastructure.
+- Exact write scope; process documentation only in docs/joom-migration.
+- Report format; do not automatically begin the next group.
+
+## Worker report
+
+For preparation, a chat response is sufficient: the coordinator reads and records it here.
+Task-to-task communication is authorized only for this work; do not message people.
+During implementation, save reports in this directory and raw logs in dated temp directories.
+
+Record group/branch/base/head/cleanliness, completed and remaining cards,
+upstream replacements, changed shared APIs, test commands/exit codes/log paths,
+native coverage, performance results and limitations, created paths/Docker IDs/
+volumes/sizes for cleanup, artifacts/checksums, blockers and exact next step.
+Do not claim unperformed checks passed.
+
+Changing sessions/compaction is not a reason to repeat completed stages.
+STATE and decisions reflect actual progress and direct user authorization;
+old snapshots must not overwrite the current state.
+
+Translation note: historical Russian statements above have been translated into
+English. The original pre-translation documents are archived at
+`/Users/skotlov/d/temp/20261118_comet12-g01-finalization/documentation-before.zip`.
diff --git a/docs/joom-migration/MANIFEST.json b/docs/joom-migration/MANIFEST.json
index bb81e1751a6..52f9a71c07b 100644
--- a/docs/joom-migration/MANIFEST.json
+++ b/docs/joom-migration/MANIFEST.json
@@ -4,67 +4,111 @@
"files": [
{
"path": "AGENTS.md",
- "sha256": "1ca923cbe1a2fef82674d2177b18c151a5722939ddb906a08a010de581d173c9"
+ "sha256": "5b5ab8d6ef26484342e1e7ee9c1e6c6896e9581120d1900b5ea8a36b0b2d4346"
+ },
+ {
+ "path": "CHECKPOINT-VERIFICATION.json",
+ "sha256": "554091614ff69409a97be397afd9c7c534b75d848f490ffdc2fe16449d92ab80"
},
{
"path": "CLEANUP.md",
- "sha256": "500d782de7be2e3aafb2a670e8d010ac8f1ead449a369592eacdfdbc4b79d98f"
+ "sha256": "7e6420f6be4e1aa475376523134bcb04e7c4755a1c156f07fb7ea0774a917da2"
+ },
+ {
+ "path": "G01-CLOSEOUT.md",
+ "sha256": "d7038e27674f621c1777ccda5e9c2d34711c5be1aa08c711cf09715e1a00d694"
+ },
+ {
+ "path": "G01-HISTORY.json",
+ "sha256": "873d878f4dd076fa3e4824473ef624d87309a0d59a7ed5bf2c9fba2b2a08bad2"
+ },
+ {
+ "path": "G01-PR-preparation.json",
+ "sha256": "72b74d5040bf7ccab56ec1976723232a1d8a516e825d0aff6634c19d35c4b6ec"
+ },
+ {
+ "path": "G01-PR.md",
+ "sha256": "74346e87932762c161da75d25cf0bd94498375685cb19146b1c8cef451a61f5f"
},
{
"path": "G01-preparation.json",
- "sha256": "349f0178f81805e018b5ed9343564eb8dec9e2f5d01ed724dfc32394091a5804"
+ "sha256": "6f1fe3dcb88eb6a3dfe9e2432e11ef2013b06657ffffd61e1dba50fc2dd9107d"
+ },
+ {
+ "path": "G01-progress.md",
+ "sha256": "f4dfcc1bd9f8b852c3fa86cef0e0a63e11598637cb099e8085da6d1c2ba07e0d"
+ },
+ {
+ "path": "G01-report.md",
+ "sha256": "db3f5edd55e256667eb01aff7193c0672ee20a20388c5d34269df0f97ec1475f"
+ },
+ {
+ "path": "G01-results.json",
+ "sha256": "0457b55f6c2c4d61b7181bbfdfe807633c6c1ae355e6805f0ff4e18685a3b30d"
+ },
+ {
+ "path": "G01-review-notes.json",
+ "sha256": "4bdcdfe191a2ebecea10e037ebf0ebe271e760356d77f30b0f15016ec154e476"
},
{
"path": "HANDOFF.md",
- "sha256": "010ee6afa1d06b3b5c1abf803d5ebe39bf46cb0e2b56707a4f560e7f15935367"
+ "sha256": "26bc20e331ca3c090e0b0edafeb2cb59ad9190020096a76a131bbf2a33ab0d06"
},
{
"path": "NETWORK.md",
- "sha256": "4c8d3b084363eb8e49b51eac6c620e9977ea3a2994824e0566e3ac0a149d46b3"
+ "sha256": "3c3d5ed6b9c3397f724829022a78841f4c761ced1acbea152cf67716efd95b7d"
+ },
+ {
+ "path": "PAUSED.md",
+ "sha256": "109aa3ba1f1471a99f6751e64469100cd340172a02df73b9c5cb26daf10c6a9d"
},
{
"path": "PERFORMANCE.md",
- "sha256": "f01d946951bb4c1e54ea608518d6379f0d38f4e7c9990d6593bba0e2aad96344"
+ "sha256": "cc746bde47be1f6b6a6e09d3be04aa0b911f85e35530e6ef5f8ceffae5f9bb54"
+ },
+ {
+ "path": "RC-BUILD-SMOKE.md",
+ "sha256": "c0f2f525c089ecfc200579918b2742c68461c5909005152b99894859c0fdc97d"
},
{
"path": "README.md",
- "sha256": "afe578ef3523bc3c467b9883a69721f0b2a124c2236cf4245d4dc484c4c5f5af"
+ "sha256": "8b5bf12dee398e3a6df50719ec7ad51d63581d7f3619c5e63f96f2202e4280cf"
},
{
"path": "STATE.json",
- "sha256": "cae42887ddf9e5604b0f42c3e9f3bb81cf649d718f52fee547f8e142f63be1b9"
+ "sha256": "fdf5dee24d2cc00e242bf0cf4b190fbb7819862e8b6fb70901beea0c7464ada3"
},
{
- "path": "approved-plan-ru.txt",
- "sha256": "8da8494f2bb7df176e9f39377d6e00de7053f3d9cbfa0b2e3411588a427b69bc"
+ "path": "approved-plan.json",
+ "sha256": "bcd0b2d7c2dd77348ae0ad436f5dbcb48815d21b31a72490b9a51d1e9ddc092c"
},
{
- "path": "approved-plan.json",
- "sha256": "f994753b5a9378b9e9783ab4eba748cd4f6423e4591cdb5f1f9079b096531a8d"
+ "path": "approved-plan.txt",
+ "sha256": "179acffd06b75b8a917345debeae35cca4313fa2b86e8b45fd05b62fbf3604b4"
},
{
"path": "cleanup-ledger.json",
- "sha256": "9c175326f11c48e2cb4ef8b22f6ff1c7451dc723d6aaf22e0740f8fc313732b5"
+ "sha256": "74d672765aa6817ea01bddcd0a4fb70dcf6a270ca60f8bfa789c59fea09e4f2a"
},
{
- "path": "correctness-test-plan-ru.txt",
- "sha256": "6efd6d143ef8e0e5cebc22cb79b779c08de6ea8920ead49c448313129590cc9c"
+ "path": "correctness-test-plan.txt",
+ "sha256": "41a610cd15d805e6fd31ad420899274e63f328a2d392b87efa82a2f79ba69f73"
},
{
"path": "created-files.txt",
- "sha256": "c73c53dd3527acb3182c36169bf818572ff6746c87fe413aece5988820c07a5a"
+ "sha256": "a1065a82a3d1eace55b9be3b7a256be1c9a63853cf49239ad7c68a9d71dc1808"
},
{
"path": "decisions.json",
- "sha256": "fe5f801e3e0cddab42ff3ea798b31f198f8475bb21394b32187aa0fbf37c9517"
+ "sha256": "0ba242a36f365eabdd092582bae0204b722f3de7d13d9e083710189cccea0f0a"
},
{
"path": "feature-register.json",
- "sha256": "a0059e076e32aceaa68112140bc1c7a9dba27cb977d391b168e45b4e2a84b393"
+ "sha256": "1753641cb7a48bec5adbb4d74be14644512e2ad2a3d1531d701258f88d1020bf"
},
{
"path": "resource-baseline.json",
- "sha256": "107defecf4a734ac5a4d52d237fa2b1ebd3945aea7da5c724dc56cb5e15cd0e9"
+ "sha256": "e32d57857f61310d54421da31041fb6f2546aa2b33bac0ec2912e7ca3b61a957"
},
{
"path": "target-native-dependencies.json",
@@ -72,7 +116,7 @@
},
{
"path": "vendor-provenance.json",
- "sha256": "c6496befb4c54ba281a977d28a573f60e0bb44615d72f34be96d1a1f5cd4f45a"
+ "sha256": "d5a02802ab679f9e23381db70f860dc409dc66500b0d1fd2be1a832886433e5b"
}
],
"original_research_directories": [
@@ -81,5 +125,5 @@
],
"upstream_base_commit": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
"source_commit": "981e26dff6b64110a8b3949a0ed065523d9012e4",
- "updated_on": "2026-11-17"
+ "updated_on": "2026-11-18T22:44:25.789804+00:00"
}
diff --git a/docs/joom-migration/NETWORK.md b/docs/joom-migration/NETWORK.md
index a6e121fd05d..d670b053e36 100644
--- a/docs/joom-migration/NETWORK.md
+++ b/docs/joom-migration/NETWORK.md
@@ -1,29 +1,49 @@
-# Согласованный сетевой scope
+
-| Цель | Семейства команд | Ожидаемые hosts |
-|---|---|---|
-| Maven wrapper/JVM dependencies/plugins, Spark test build | `./mvnw ...`, Maven compile/test/verify/package/install и Spark `build/sbt ...` | `repo.maven.apache.org`, `repo1.maven.org`; при необходимости `repo.scala-sbt.org`, `repo.typesafe.com` и их штатные redirects |
-| Rust compiler/components и crates | `rustup toolchain/component ...`, `cargo fetch/build/test/bench/fmt/clippy ...` | `sh.rustup.rs`, `static.rust-lang.org`, `index.crates.io`, `static.crates.io`, `crates.io` |
-| Закреплённые Cargo git dependencies и бинарные build tools | Cargo fetch/build, скачивание Protobuf release binary; read-only GitHub API | `github.com` (в частности `apache/iceberg-rust` на закреплённом commit), `api.github.com`, `raw.githubusercontent.com`, `codeload.github.com`, `release-assets.githubusercontent.com`, `objects.githubusercontent.com` и их штатные CDN redirects |
-| Linux builder images и build/run с перечисленными загрузками внутри контейнера | `docker pull/build/buildx build/run ...` | Docker Hub: `registry-1.docker.io`, `auth.docker.io`, `production.cloudflare.docker.com` и штатные blob/CDN endpoints |
-| Ubuntu build packages внутри builder | `apt-get update/install ...` | `archive.ubuntu.com`, `security.ubuntu.com`, `ports.ubuntu.com` и штатные Ubuntu mirrors/redirects |
+# Approved network scope
-Согласование включает обращения для разрешения зависимостей, загрузку archives,
-metadata/checksums/signatures, а также публичных build tools и fixtures, являющихся
-зависимостями выбранной сборки/тестов. Фиксированные зависимости и Cargo.lock сохраняются;
-произвольное обновление зависимостей в объём разрешения не входит.
-Результаты, логи и скачанные исполнителем вспомогательные файлы — в разрешённых
-каталогах; обычные Maven/Cargo/Rustup/Docker caches могут использовать штатные пути.
+Status: APPROVED_WITH_USER_SAFETY_CONDITION, 2026-11-17.
+Explicit consent and its condition are recorded in decisions.json.
-Примеры ожидаемых команд после подготовки нужных файлов (выбирать по текущей стадии; это согласование сети, а не команда запустить все примеры):
+The user gave one authorization for the entire approved G01-G09 migration and
+necessary G10 qualification through delivery of local JARs, including repeated
+builds and tests. It covers the listed downloads and necessary child-process
+requests on the host and inside Docker, not just one literal command.
+
+| Purpose | Command families | Expected hosts |
+| ------------------------------------------------------------------------------ | --------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Maven wrapper/JVM dependencies/plugins, Spark test build | `./mvnw ...`, Maven compile/test/verify/package/install and Spark `build/sbt ...` | `repo.maven.apache.org`, `repo1.maven.org`; if necessary `repo.scala-sbt.org`, `repo.typesafe.com` and their normal redirects |
+| Rust compiler/components and crates | `rustup toolchain/component ...`, `cargo fetch/build/test/bench/fmt/clippy ...` | `sh.rustup.rs`, `static.rust-lang.org`, `index.crates.io`, `static.crates.io`, `crates.io` |
+| Pinned Cargo Git dependencies and binary build tools | Cargo fetch/build, Protobuf release binary download; read-only GitHub API | `github.com` (in particular `apache/iceberg-rust` at the pinned commit), `api.github.com`, `raw.githubusercontent.com`, `codeload.github.com`, `release-assets.githubusercontent.com`, `objects.githubusercontent.com` and their normal CDN redirects |
+| Linux builder images and build/run with the listed downloads inside containers | `docker pull/build/buildx build/run ...` | Docker Hub: `registry-1.docker.io`, `auth.docker.io`, `production.cloudflare.docker.com` and standard blob/CDN endpoints |
+| Ubuntu build packages inside the builder | `apt-get update/install ...` | `archive.ubuntu.com`, `security.ubuntu.com`, `ports.ubuntu.com` and standard Ubuntu mirrors/redirects |
+
+Authorization includes dependency-resolution requests, downloads of archives,
+metadata/checksums/signatures, and public build tools and fixtures that are
+dependencies of the selected build/tests. Preserve pinned dependencies and
+Cargo.lock; arbitrary dependency upgrades are outside this permission.
+Save results, logs and executor-downloaded auxiliary files in the authorized
+locations; normal Maven/Cargo/Rustup/Docker caches may use standard paths.
+
+Examples of expected commands after the necessary files are prepared (select for
+the current stage; network authorization is not an instruction to run all examples):
```sh
./mvnw -B -Pspark-4.2,delta -Dspark.version=4.2.1 -Ddelta.spark.version=4.4.1 verify
@@ -34,21 +54,47 @@ docker pull --platform linux/amd64 ubuntu:20.04
docker pull --platform linux/arm64 ubuntu:20.04
```
-Точные flags/выбор тестовых пакетов уточняются по исходникам и текущей стадии;
-пример не заменяет обязательный native-before-JVM build order. Отдельные vendor-unit
-tests должны использовать тот же workspace/lock/patch graph, что release.
-Действующий wrapper закрепляет Maven3.9.6. Старый Joom Linux builder закрепляет
-Rust1.99.0, Ubuntu20.04/GCC10/Java17/Protobuf30.2; его адаптация — часть G01.
+Refine exact flags/test package selection from the sources and current stage;
+examples do not replace the mandatory native-before-JVM build order. Separate
+vendor unit tests must use the same workspace/lock/patch graph as the release.
+The current wrapper pins Maven3.9.6. The old Joom Linux builder pins Rust1.99.0,
+Ubuntu20.04/GCC10/Java17/Protobuf30.2; adapting it is part of G01. These are historical
+baseline details, not permission to override the versions recorded in final manifests.
+
+The original scope excludes Git push/PR publication, Maven deploy/JFrog publish,
+Docker push, production GCP queries/jobs/writes, Spark image changes, messages to
+people, host Homebrew/SDKMAN installations, privileged binfmt/system setup, shared
+cache cleanup and arbitrary new source clones. Ask the user for a missing research
+checkout. If another private registry or other source is actually needed, identify
+it and obtain a scope extension first. Later scoped Git permissions are below.
+
+The user's explicit consent (English translation from Russian):
+
+> I authorize, for the entire Comet1.2/Spark4.2.1 migration, the network operations listed in docs/joom-migration/NETWORK.md: Maven/SBT, Rustup/Cargo, pinned Git dependencies and build tools from GitHub, Docker Hub and Ubuntu packages, including child processes inside containers. This is subject to the condition that everything downloaded from the network to my laptop and executed is a known and widely used artifact whose safety for my machine is not reasonably in doubt. No push, external publication or production operations.
+
+Execution condition: official sources, known widely used artifacts, pinned
+versions/commits and available publisher checksums; inspect external installer/build
+scripts before execution. Registry/lockfile inclusion does not override the user's
+safety condition. Obtain separate agreement for unusual or questionable tools/sources.
+This translation does not expand the original list of operations or hosts.
-Не входит: Git push/публикация PR, Maven deploy/JFrog publish, Docker push,
-production GCP queries/jobs/writes, изменение Spark image, отправка сообщений людям,
-host Homebrew/SDKMAN installations, privileged binfmt/system setup, очистка общих
-caches, произвольное клонирование новых исходников. Недостающий checkout для
-исследования просить у пользователя. Если реально понадобится иной private registry
-или иной источник, сначала назвать его и получить расширение scope.
+## Scoped addition: G01 PR preparation, 2026-11-17
-Прямое согласие пользователя:
+That user request authorized preparing/pushing the G01 branch to
+github.com/joomcode/datafusion-comet and providing a manual PR handoff. An absent
+base branch could be published only at its exact unchanged SHA; existing base
+contents/ref could not be changed. This was an exception to the historical no-push
+clause above, not authorization for merge, RC1/G02, artifact publication or production.
+Read-only GitHub checks/runs for the PR were in scope. Expected hosts:
+github.com/api.github.com and standard GitHub redirects. The user would create the
+PR manually; gh installation was not required.
-> Разрешаю на всю миграцию Comet1.2/Spark4.2.1 сетевые операции, перечисленные в docs/joom-migration/NETWORK.md: Maven/SBT, Rustup/Cargo, закреплённые git-зависимости и build tools с GitHub, Docker Hub и Ubuntu packages, включая дочерние процессы внутри контейнеров. При условии, что все скачиваемое на мой ноут из сети и запускаемое - является известными и широкоиспользуемыми артефактами, в безопасности которых для моей машины не приходится сомневаться. Без push, внешней публикации и production операций.
+## Scoped addition: G01 finalization, 2026-11-18
-Условие исполнения: официальные источники, известные широко используемые артефакты, закреплённые версии/commit и доступные publisher checksums; внешние installer/build scripts проверять до исполнения. Registry/lockfile не отменяют пользовательское условие безопасности. Нестандартные или сомнительные инструменты/источники согласовывать отдельно. Исходный перечень операций и hosts этим обновлением не расширен.
+The latest explicit instruction authorizes G01 history cleanup, updating its PR,
+repeating RC Build & Smoke for the resulting source when necessary, integration
+into branch-1.2 and creation of joom-1.2-1-rc1 after successful verification, followed
+by completeness checks. It supersedes the earlier G01 no-merge/no-RC1 hold.
+The already authorized GitHub operations include workflow control/status/logs and
+artifact download. This does not authorize JFrog/Maven/Docker artifact publication,
+production operations, further Java 21 work or G02-G10 implementation.
diff --git a/docs/joom-migration/PAUSED.md b/docs/joom-migration/PAUSED.md
new file mode 100644
index 00000000000..a771861978a
--- /dev/null
+++ b/docs/joom-migration/PAUSED.md
@@ -0,0 +1,34 @@
+
+
+> **Final source-freeze note:** G01 implementation and documentation are complete. RC1 acceptance is a post-commit gate recorded by the immutable `joom-1.2-1-rc1` tag and the delivered artifact receipts; see `G01-CLOSEOUT.md`. Earlier source SHAs, runs and pending states below are historical evidence. G02 and Java 21 work remain unauthorized.
+
+> **G01 finalization authorized:** documentation and history cleanup, followed by exact-SHA RC qualification, are in progress. Integration into `branch-1.2` and `joom-1.2-1-rc1` are conditional on successful checks. G02 and Java 21 work remain on hold.
+
+# G01 continuation checkpoint
+
+Latest completed qualification: **RC Build & Smoke 44388252264 on b93d553856a0339af7468bcff2070e735a1a3c71 succeeded. Final JARs were downloaded and locally verified.** All 7 jobs and 12/12 packaged tests on AMD64 and ARM64 passed. Directory: `target/joom-release/gha-44388252264`. Apache RAT for source b93 passed; runtime code has not changed since full CI on 26590264b. Monitoring and builds ended. Final progress documents were saved locally without another push. The user has now authorized G01 finalization, English documentation, service-history cleanup, an exact-SHA RC rerun as needed, integration and RC1 tagging after successful checks. Earlier merge/tag holds below are historical. Publication, G02 and Java 21 work remain excluded.
+
+- Feature `joom/1.2-dev`; PR https://github.com/joomcode/datafusion-comet/pull/28. The abbreviated historical base was 9691385f. The pre-integration base `branch-1.2` is `9691385f35e4c5bd29514dca78d2044dd20e4bed`. Verify actual refs and the current finalization state before resuming.
+- Earlier run44180095887/ac3694504 ended in FAILURE:124success/3failure/17skipped. All nine original failed jobs passed; two additional cases were discovered and have since been fixed in the source above. Do not attribute an old verdict to a new source.
+- Source35/Scala2.12 controls35passed;41union12passed;42union12passed; reader27passed/converter19passed; fresh-JAR SparkSQL21+9+3passed. The pre-fix negative control on41 did return incorrect groups. All source files after commit matched the tested worktree byte for byte.
+- Run44205897075 completed successfully on its exact HEAD; polling ended. New auxiliary directory: `/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha`; check-pr-gha.py/fetch-job-log.py are available there. Helper g01_gha_monitor_five_minutes finished and all final receipts were saved; do not overwrite immutable receipts of the previous run.
+- Older Linux release JARs use source2b8f80b3f and **do not contain the fixes**. Do not treat them as a final RC or production-ready. Host/M2 JARs with the new Scala code were used only for local tests. Dualarch packaged smoke remains an RC gate; it passed for the newer b93 artifact set.
+- Explicitly select Java17; use direct stable-toolchain Cargo bin and JNI-library PATH when needed. Working official protoc33.5 is already in the ProtocBridge cache. Global SDK/PATH/host tools were not changed.
+- All execution documentation is here; preserve caches/user checkouts. New test-only/network outputs are in the cleanup ledger. Git feature push/PR28 updates are authorized; artifact publication/production operations are excluded.
diff --git a/docs/joom-migration/PERFORMANCE.md b/docs/joom-migration/PERFORMANCE.md
index 7d47fe328de..e35810838a1 100644
--- a/docs/joom-migration/PERFORMANCE.md
+++ b/docs/joom-migration/PERFORMANCE.md
@@ -1,50 +1,69 @@
-# Проверка производительности в принятом стиле Comet
+
+
+# Performance validation in the established Comet style
+
+User decision, 2026-11-17: follow the project's established practice;
+do not create a special testing method/infrastructure solely for the migration.
+
+## Existing tools
- Scala microbenchmarks: `spark/src/test/scala/org/apache/spark/sql/benchmark`,
- `CometBenchmarkBase` на Spark `SqlBasedBenchmark`; Makefile `benchmark-%`.
-- Rust microbenchmarks: существующие Criterion benches в
+ `CometBenchmarkBase` on Spark `SqlBasedBenchmark`; Makefile `benchmark-%`.
+- Rust microbenchmarks: existing Criterion benches in
`native/{core,operators,shuffle,spark-expr}/benches`; Makefile `bench` / Cargo bench.
-- Macro benchmarks: существующие TPC-H/TPC-DS runners в `benchmarks/tpc`.
-- Общая инструкция: `docs/source/contributor-guide/benchmarking.md`;
- учитывать macOS-ограничения из `benchmarking_macos.md`.
-
-Для каждого изменения выбирать подходящий существующий suite/case, проверять,
-что действительно исполняется изменённый путь, и сохранять команды, параметры,
-версии, результаты и ограничения. При необходимости расширять соответствующий
-существующий benchmark обычным для проекта способом; отдельный новый framework,
-нестандартный генератор или специальную систему сравнения не создавать.
-
-| Группа | Подходящие существующие suites/benches |
-|---|---|
-| G01 | Штатные read suites и доступный baseline; packaging не является benchmark |
-| G02 | `CometStringFfiImportBenchmark`, temporal suites и Rust cast/timestamp benches |
-| G03 | `CometExecBenchmark`, `CometTopKBenchmark`, Rust `sort_row_partitions` |
-| G04 | `CometAggregateExpressionBenchmark`, Rust `aggregate`, `collect`, `aggregate_misc`, `min_max` |
-| G05 | `CometSortMergeJoinBenchmark` |
-| G06 | `CometWindowBenchmark`, Rust operators `window` |
-| G07 | `CometShuffleBenchmark`, `CometColumnarShuffleWriteBenchmark`, `CometColumnarToRowBenchmark`, `CometC2RIsolatedBench`, Rust shuffle benches |
-| G08 | `CometCodegenDispatchBenchmark`, `CometScalaUDFDispatchBenchmark`, `CometOperatorSerdeBenchmark` |
-| G09 | Доступные TPC-H/DS и operator suites; учитывать фактические планы/выбор engines |
-
-## Границы проверки
-
-`CometBenchmarkBase` устанавливает ANSI=false из-за invalid-input fixtures.
-Эти измерения не заменяют обязательные ANSI=true correctness/regression tests.
-Разрешённые ранее новые correctness-тесты остаются частью реализации; ограничение
-на новую performance-инфраструктуру их не отменяет.
-
-Focused suite/bench предпочтительнее бесконтрольного `setup/all/publish`:
-`benchmarks/micro/run.py` может устанавливать пакеты, клонировать/обновлять исходники,
-строить release и публиковать изменения. Не запускать эти побочные действия.
-Перед запуском смотреть фактический Makefile/runner: некоторые targets сначала
-делают release; старые README примеры не гарантируют актуальность флагов/версий.
-
-Не копировать механически пример генерации100GB из macOS guide. Выбирать
-ограниченный репрезентативный размер и фиксировать созданные данные в cleanup ledger.
-AWS provisioning не входит в GCP-only работу. Полные измерения на реальных GCP jobs
-пользователь проведёт позже. Непроверенное ускорение не объявлять подтверждённым.
+- Macro benchmarks: existing TPC-H/TPC-DS runners in `benchmarks/tpc`.
+- General guide: `docs/source/contributor-guide/benchmarking.md`;
+ respect the macOS limitations in `benchmarking_macos.md`.
+
+For each change, choose an appropriate existing suite/case, verify that the changed
+path actually executes, and record commands, parameters, versions, results and
+limitations. If necessary, extend the relevant existing benchmark in the project's
+normal style; do not create a separate new framework, unusual generator or special
+comparison system.
+
+| Group | Suitable existing suites/benches |
+| ----- | ------------------------------------------------------------------------------------------------------------------------------------------- |
+| G01 | Standard read suites and available baseline; packaging is not a benchmark |
+| G02 | `CometStringFfiImportBenchmark`, temporal suites and Rust cast/timestamp benches |
+| G03 | `CometExecBenchmark`, `CometTopKBenchmark`, Rust `sort_row_partitions` |
+| G04 | `CometAggregateExpressionBenchmark`, Rust `aggregate`, `collect`, `aggregate_misc`, `min_max` |
+| G05 | `CometSortMergeJoinBenchmark` |
+| G06 | `CometWindowBenchmark`, Rust operators `window` |
+| G07 | `CometShuffleBenchmark`, `CometColumnarShuffleWriteBenchmark`, `CometColumnarToRowBenchmark`, `CometC2RIsolatedBench`, Rust shuffle benches |
+| G08 | `CometCodegenDispatchBenchmark`, `CometScalaUDFDispatchBenchmark`, `CometOperatorSerdeBenchmark` |
+| G09 | Available TPC-H/DS and operator suites; inspect actual plans/engine selection |
+
+## Validation boundaries
+
+`CometBenchmarkBase` sets ANSI=false because its fixtures include invalid inputs.
+Those measurements do not replace mandatory ANSI=true correctness/regression tests.
+Previously authorized new correctness tests remain part of implementation;
+restrictions on new performance infrastructure do not cancel them.
+
+Prefer a focused suite/bench over an uncontrolled `setup/all/publish`:
+`benchmarks/micro/run.py` may install packages, clone/update sources, build a release
+and publish changes. Do not run these side effects. Inspect the actual Makefile/runner
+before execution: some targets build a release first; old README examples do not
+guarantee current flags/versions.
+
+Do not mechanically copy the 100GB generation example from the macOS guide. Select
+a bounded representative size and record generated data in the cleanup ledger.
+AWS provisioning is outside this GCP-only work. The user will run full measurements
+on real GCP jobs later. Do not present unmeasured acceleration as confirmed.
diff --git a/docs/joom-migration/RC-BUILD-SMOKE.md b/docs/joom-migration/RC-BUILD-SMOKE.md
new file mode 100644
index 00000000000..33005bbbdcf
--- /dev/null
+++ b/docs/joom-migration/RC-BUILD-SMOKE.md
@@ -0,0 +1,69 @@
+
+
+> **Final source-freeze note:** G01 implementation and documentation are complete. RC1 acceptance is a post-commit gate recorded by the immutable `joom-1.2-1-rc1` tag and the delivered artifact receipts; see `G01-CLOSEOUT.md`. Earlier source SHAs, runs and pending states below are historical evidence. G02 and Java 21 work remain unauthorized.
+
+> **G01 finalization authorized:** documentation and history cleanup, followed by exact-SHA RC qualification, are in progress. Integration into `branch-1.2` and `joom-1.2-1-rc1` are conditional on successful checks. G02 and Java 21 work remain on hold.
+
+# RC Build & Smoke
+
+Separate manual workflow: `.github/workflows/rc_build_smoke.yml`, display name **RC Build & Smoke**. It does not trigger Comet CI or publish to JFrog/Nexus/production. The entire build runs in GitHub Actions, without transferring JARs from the laptop.
+
+## Running it
+
+In Actions, select RC Build & Smoke → Run workflow → the desired branch/tag. Set `expected_commit` to the full lowercase Git SHA of that ref. It must match the workflow ref SHA; another source will be rejected before builds. The new file is already registered by bootstrap commit `fa7347e5ea549a01c77e325dbd3449d93c9d5700` on default `main`; it was the only file changed there. G01 sources were not moved into main.
+
+Previously qualified run: https://github.com/joomcode/datafusion-comet/actions/runs/44388252264, source `b93d553856a0339af7468bcff2070e735a1a3c71`. All 7 jobs passed, including 12/12 existing smoke cases on each architecture. History cleanup changes source identity; the final RC requires matching exact-SHA qualification.
+
+## Stages and acceptance criteria
+
+1. Source gate: exact SHA/cleanliness/lockfile and 22 fast tooling tests.
+2. Native matrix: Linux AMD64 and ARM64 on native runners; Cargo `--locked --release`, Rust1.100.0, Ubuntu20.04, GCC10, JDK17. AMD64 flags `-Ctarget-cpu=x86-64-v3`; ARM64 baseline flags are empty. Commit/lock/compiler/image/ELF/SHA256 are recorded.
+3. Package: validate both native provenance records; one Maven reactor `package dependency:build-classpath` for Spark4.2.1/Scala2.13.18/Delta4.4.1. Produce exactly two runtime JARs with both `.so` files, matching portable test support and manifest/checksums. Use the same release script/validator as local builds.
+4. Smoke matrix: download the same candidate without Maven/rebuilding; run Delta4+DPP1+reload1+ANSI6. All12 must pass with zero failed/canceled/ignored. Verify actual Spark4.2.1, Java17, native architecture, final-JAR class origins and SHA256 of the bundled `.so` actually loaded.
+5. Final: both receipts must refer to the same JAR/source/lock/package/support hashes. Only then produce a qualified artifact. JARs are not repackaged after smoke.
+
+The strict binding of receipts to bytes prevents reuse of an older successful smoke after replacing a package artifact during retry. Partial native/candidate artifacts are not a qualified release.
+
+Focal Git2.25.1 ignores command-scope `safe.directory`: the release builder mounts a separate staging configuration trusting only the container checkout path at `/etc/gitconfig:ro`. Host Git configuration is unchanged. Maven classpaths are computed in the same reactor so a clean runner does not need a prior install of internal Comet modules.
+
+## Versions and use
+
+| Component | Version |
+| -------------------- | -------------------------------------------------- |
+| Spark / Scala binary | 4.2.1 / 2.13 |
+| Scala / JDK | 2.13.18 / 17 |
+| Comet upstream base | 1.2.0, pinned branch-1.2 source93cc6f9d |
+| Delta | 4.4.1, `io.delta:delta-spark_4.2_2.13:4.4.1` |
+| DataFusion | 55.2.0, local physical-plan vendor |
+| Arrow/Parquet native | 59.3.2 |
+| Arrow/Parquet JVM | 18.3.0 / 1.17.0 |
+| Platforms | Linux AMD64(x86-64-v3), ARM64; GLIBC imports<=2.31 |
+
+Runtime files: `comet-spark-spark4.2_2.13-1.2.0.jar` and `comet-contrib-delta-spark4.2_2.13-1.2.0.jar`. The second is an adapter: the official Delta Spark4.2 artifact4.4.1 and its dependencies are required separately. DataFusion/native Arrow/Parquet are embedded in `.so`.
+
+Future Maven coordinates: `org.apache.datafusion:comet-spark-spark4.2_2.13:` and `org.apache.datafusion:comet-contrib-delta-spark4.2_2.13:`. Planned RC1 `joom-1.2-1-rc1`, stable `joom-1.2-1`; this workflow creates neither tags nor publications. The G01 candidate does not yet include the later G02–G09 ports.
+
+## Artifacts and retention
+
+Qualified artifact: `rc-build-smoke-`; previously qualified ID14469302529. It contains runtime JARs, source/lock/toolchain/native metadata, checksums, both smoke receipts/logs and `release-qualification.json`. Native intermediate artifacts are retained for3 days, candidate with test support for7 days, qualified/smoke for14 days. Caches are restore-only; feature runs do not write large cache entries.
+
+After downloading, verify the artifact-archive SHA256, `sha256.txt`, embeddedGit/version/ELF/provenance and binding of both receipts to final JARs. The local result of the previously qualified run is `target/joom-release/gha-44388252264`. Removal and sizes are documented in `CLEANUP.md`; preserve shared caches and user checkouts.
+
+Full Comet CI for release-only commits was canceled at the user's direct instruction. Runtime/native/JVM code matches qualified 26590264b (CometCI44205897075:127success/17conditional skips); the new workflow additionally checked the release profile and final JARs. Source-tarball Apache RAT for b93d55385 passed separately. This does not constitute production GCP/performance qualification. G01 integration/tagging has subsequently been authorized conditional on finalization checks; G02 remains separately gated.
diff --git a/docs/joom-migration/README.md b/docs/joom-migration/README.md
index b1146b6fc99..a9461c29f2d 100644
--- a/docs/joom-migration/README.md
+++ b/docs/joom-migration/README.md
@@ -1,121 +1,86 @@
-# Joom Comet 1.2: план и передача работы
-
-Этот каталог — постоянная точка входа для реализации миграции в Joom Comet.
-История чата не является единственным источником решений. Состояние на 2026-11-17:
-План утверждён, Java/Rust и Docker проверены, сеть разрешена с условием безопасности.
-**Код не писать до нового прямого указания пользователя.** Сейчас только документация
-и read-only подготовка рабочих сессий; реализация групп не начата.
-
-## Что читать перед работой
-
-1. `decisions.json`: действующие пользовательские решения и разрешения.
-2. `STATE.json`: текущая группа, ветки, проверки, ограничения и следующий шаг.
-3. `approved-plan.json`: утверждённые девять групп и резерв десятой, зависимости, RC.
-4. `feature-register.json`: полный реестр 58 действующих пунктов + историческая H01;
- выбрать все карточки своей группы из `approved-plan.json`, не исключать их молча.
-5. `correctness-test-plan-ru.txt`, `vendor-provenance.json`,
- `target-native-dependencies.json`: требования корректности и происхождение vendor.
-6. `NETWORK.md`: согласованный сетевой scope с условием безопасности.
- Действующий статус и буквальное согласие брать из `decisions.json`.
-7. `HANDOFF.md`: правила задания и отчёта для рабочей сессии.
-8. `PERFORMANCE.md`, `cleanup-ledger.json`, `CLEANUP.md`: штатные benchmarks и учёт диска.
-
-## Неизменные решения
+
+
+> **Final source-freeze note:** G01 implementation and documentation are complete. RC1 acceptance is a post-commit gate recorded by the immutable `joom-1.2-1-rc1` tag and the delivered artifact receipts; see `G01-CLOSEOUT.md`. Earlier source SHAs, runs and pending states below are historical evidence. G02 and Java 21 work remain unauthorized.
+
+> **G01 finalization authorized:** documentation and history cleanup, followed by exact-SHA RC qualification, are in progress. Integration into `branch-1.2` and `joom-1.2-1-rc1` are conditional on successful checks. G02 and Java 21 work remain on hold.
+
+> **Latest completed G01 qualification:** RC Build & Smoke passed on `b93d55385`. All7 jobs and12 packaged smoke cases on each architecture passed. Final JARs were downloaded to `target/joom-release/gha-44388252264`; SHA256 and provenance were verified. Full runtime CI on 26590264b is retained; runtime code has not changed. Results: `RC-BUILD-SMOKE.md`, `G01-report.md`, `G01-results.json`, `STATE.json`. The subsequent finalization instruction authorizes history cleanup, English documentation, matching final RC qualification, integration and RC1 tagging after successful checks. Publication and G02 remain excluded.
+
+# Joom Comet 1.2: plan and handoff
+
+This directory is the durable entry point for implementing the Joom Comet migration. Chat history is not the only source of decisions. Initial environment/preparation state on2026-11-17: plan approved, Java/Rust and Docker checked, network approved subject to safety conditions. **G01 implementation is authorized.** The user separately authorizes each subsequent group after evaluating the previous one; do not start G02–G10 now.
+
+## Read before working
+
+1. `decisions.json`: current user decisions and permissions.
+2. `STATE.json`: current group, branches, checks, limitations and next step.
+3. `approved-plan.json`: approved nine groups and tenth-group reserve, dependencies, RCs.
+4. `feature-register.json`: full register of58 active items plus historical H01; select every card of your group from `approved-plan.json`, excluding none silently.
+5. `correctness-test-plan.txt`, `vendor-provenance.json`, `target-native-dependencies.json`: correctness requirements and vendor provenance.
+6. `NETWORK.md`: approved network scope with safety condition. Read the active authorization and its translated user statement in `decisions.json`.
+7. `HANDOFF.md`: worker-session task and reporting rules.
+8. `PERFORMANCE.md`, `cleanup-ledger.json`, `CLEANUP.md`: established benchmarks and disk accounting.
+
+## Fixed decisions
- Source: `joom-1.1-13`, commit `981e26dff6b64110a8b3949a0ed065523d9012e4`.
-- Apache `branch-1.2` зафиксирована на `93cc6f9d71136bd0d87f77dd901989144d8c09fb`.
- Не заменять базу более новым upstream без отдельного обсуждения.
-- Spark4.2.1 / Scala2.13.18; DataFusion55.2.0, Arrow/Parquet59.3.2;
- целевая Delta integration4.4.1. Точная совместимость проверяется при реализации.
-- Максимально использовать согласованные upstream-реализации и сохранять все
- недостающие Joom-возможности. Наличие похожего upstream-кода не доказывает эквивалентность.
-- ANSI=true. Сравнивать Spark4.2.1 Comet-off/on: результаты И обязательные ошибки;
- сохранить mixed SUM/AVG overflow guards. Известный unsafe путь исправить или
- ограничить до исполнения. Успешная сборка сама по себе не доказывает корректность.
-- PR5889 не трогать и не включать. Доработки после source tag13 не добавлять автоматически.
-- Итог: локальные Comet/Delta JAR для Linux amd64+arm64 и Spark4.2 image,
- SHA256, Git SHA, параметры сборки и результаты проверок. Push/PR/JFrog publish/
- изменение datasci image/production rollout сейчас не выполняются.
-- Performance проверять штатными средствами Comet: Scala microbenchmarks, Rust
- Criterion и доступные TPC-H/DS runners; специальную инфраструктуру не создавать.
- Полные production GCP performance tests пользователь проведёт позже;
- этот перенос не следует объявлять прошедшим такие тесты.
-
-## Организация работы
-
-Основной чат реализует и координирует G01–G09 последовательно в утверждённых
-feature branches (G10 — резерв). Координатор сам делегирует внутренним помощникам
-ограниченные исследования и независимый review, передаёт контекст и проверяет результат.
-Девять веток не требуют девяти чатов. В общем checkout один пишущий исполнитель;
-тяжёлые builds не запускать все параллельно.
-
-Отдельную пользовательскую сессию создавать по решению координатора при конкретной
-пользе от длительной изолированной задачи/отдельного контекста либо необходимости
-передачи продолжения. Новый PR, большой diff или compaction сами по себе не повод.
-Сначала объяснить причину и сохранить HEAD/tree/tests/next step, затем передать
-задание и проверить результат. Права управления сессиями уже даны пользователем.
-Существующий G01 prep-чат оставить idle; реализацию там автоматически не начинать.
-
-После каждой законченной стадии сохранять коммиты, проверки и следующий шаг в
-`STATE.json` и отчёте внутри `docs/joom-migration`.
-Одноразовые raw logs/tool outputs сохранять в датированных temp-каталогах и учитывать
-в cleanup ledger. В репозитории миграционные инструкции хранить только здесь.
-Перед началом следующей группы проверить реальный Git HEAD/чистоту и зависимости;
-файлы состояния не заменяют эту проверку. Интеграцией управляет координатор.
-Не коммитить заведомо не относящиеся к своей группе изменения другого исполнителя.
-
-RC1: после G01. RC2: после интеграции G02–G06. RC3: после G07–G09 и необходимых
-итоговых исправлений. При дополнительных попытках расходовать следующий свободный
-номер; теги не двигать. Stable `joom-1.2-1` — на квалифицированном последнем RC.
-Финальные проверки обязательны независимо от наличия отдельной ветки G10.
-
-## Передача разрешений
-
-В `decisions.json` записано прямое пользовательское разрешение необходимых локальных
-операций в этом checkout, включая `.git`. Старый запрет Git для этой работы заменён.
-Сетевые операции `NETWORK.md` разрешены на всю миграцию с прямым условием:
-скачиваемые и запускаемые артефакты должны быть известными и широко используемыми,
-без обоснованных сомнений в их безопасности. При сомнениях или необходимости
-нестандартного инструмента/источника согласовать отдельно. Push, внешняя публикация
-и production-операции не входят в это разрешение.
-Пользователь добавил локальный Comet-проект и прямо разрешил координатору создавать/
-вести рабочие сессии, передавать им задания/уточнения и проверять результаты.
-Передачу между сессиями организует координатор без ручного копирования пользователем.
-Это не снимает последний запрет писать код до отдельного поручения.
-
-Все одноразовые скрипты, raw logs и tool outputs сохранять только в подкаталогах
-`/Users/skotlov/d/temp`; каждая новая директория там имеет имя `YYYYMMDD_description`.
-Не использовать `/tmp` для выбираемых исполнителем output/temp файлов. Стандартные
-tool-managed caches сохраняют обычное расположение. Не очищать общий Maven/SBT cache
-из `dev/local-ci.sh` автоматически; сначала проверить поведение и использовать
-имеющийся Spark checkout вместо бесконтрольных новых скачиваний исходников.
-Дополнительные исходники для исследования просить скачать/checkout у пользователя;
-не использовать UI для получения кода. Cargo git dependencies — отдельная часть
-согласованного сетевого scope, а не разрешение клонировать любые проекты.
-
-## Сохранность и исходный аудит
-
-Этот пакет предназначен для хранения в Git. `MANIFEST.json` фиксирует SHA256 файлов
-на момент передачи; при намеренном обновлении файла обновить и его checksum.
-Прежние материалы сохранены в:
+- Apache `branch-1.2` pinned to `93cc6f9d71136bd0d87f77dd901989144d8c09fb`. Do not replace the base with newer upstream without separate discussion.
+- Spark4.2.1 / Scala2.13.18; DataFusion55.2.0, Arrow/Parquet59.3.2; target Delta integration4.4.1. Exact compatibility is checked during implementation.
+- Maximize use of agreed upstream implementations while retaining all missing Joom capabilities. Similar upstream code does not prove equivalence.
+- ANSI=true. Compare Spark4.2.1 Comet-off/on: results AND mandatory errors; retain mixed SUM/AVG overflow guards. Fix or restrict a known unsafe path before execution. A successful build alone does not prove correctness.
+- Do not touch/include PR5889. Do not automatically add changes after source tag13.
+- Delivery: local Comet/Delta JARs for Linux amd64+arm64 and Spark4.2 image, SHA256, Git SHA, build parameters and test results. Feature Git push/PR updates and conditional G01 integration/RC1 tagging are now authorized as recorded in `decisions.json`; JFrog publication, datasci image changes and production rollout remain excluded.
+- Check performance using established Comet tools: Scala microbenchmarks, Rust Criterion and available TPC-H/DS runners; do not create special infrastructure. The user will later run full production GCP performance checks; do not claim this migration has passed them.
+
+## Work organization
+
+The main chat implements and coordinates G01–G09 sequentially in approved feature branches (G10 is a reserve). The coordinator independently delegates bounded research and independent review to internal helpers, passes context and verifies results. Nine branches do not require nine chats. There is one implementation writer in the shared checkout; do not run all heavy builds concurrently. Bounded disjoint documentation/review assignments may be delegated explicitly by the coordinator.
+
+Create a separate user-visible session at the coordinator's discretion when a long isolated task/separate context or continuation handoff offers a concrete benefit. A new PR, large diff or compaction alone is not a reason. First explain why and preserve HEAD/tree/tests/next step, then pass the task and verify the result. Session-management permissions have already been granted. Leave the existing G01 preparation chat idle; do not automatically start implementation there.
+
+After each completed stage, record commits, checks and next step in `STATE.json` and the report inside `docs/joom-migration`. Save one-off raw logs/tool outputs in dated temp directories and account for them in the cleanup ledger. Repository migration instructions belong only here. Before the next group, verify actual Git HEAD/cleanliness and dependencies; state files do not replace this check. The coordinator owns integration. Do not commit known unrelated changes from another executor.
+
+RC1: after G01. RC2: after integrating G02–G06. RC3: after G07–G09 and necessary final fixes. Additional attempts consume the next unused number; tags are immutable. Stable `joom-1.2-1` uses the last qualified RC. Final checks remain mandatory whether or not a separate G10 branch is used.
+
+## Transferring permissions
+
+`decisions.json` records explicit user authorization for necessary local operations in this checkout, including `.git`. The previous Git prohibition was superseded for this work. Network operations in `NETWORK.md` are authorized for the entire migration with an explicit condition: downloaded/executed artifacts must be known and widely used, with no reasonable safety concerns. Seek separate agreement if an unusual tool/source is needed or safety is uncertain. That dependency-download authorization alone does not include push, external publication or production operations; later explicit feature-push/PR and G01 integration/tag permissions are recorded separately in `decisions.json`.
+
+The user added the local Comet project and explicitly authorized the coordinator to create/manage working sessions, send tasks/clarifications and verify results. The coordinator handles inter-session transfer without requiring the user to copy context manually. Current implementation authorization covers only G01; later groups remain gated.
+
+Keep all one-off scripts, raw logs and tool outputs only in subdirectories of `/Users/skotlov/d/temp`; each new directory there is named `YYYYMMDD_description`. Do not use `/tmp` for executor-selected output/temp files. Standard tool-managed caches retain their usual locations. Do not automatically clear the shared Maven/SBT cache through `dev/local-ci.sh`; inspect its behavior first and use the existing Spark checkout instead of downloading source without control. Ask the user to download/checkout additional research sources; do not obtain source through UI. Cargo Git dependencies are a separate approved network scope, not permission to clone arbitrary projects.
+
+## Preservation and original audit
+
+This package is intended for Git storage. `MANIFEST.json` records file SHA256 at handoff; update its checksum when intentionally changing a file. Previous materials are preserved in:
- `/Users/skotlov/d/temp/20261116_comet12-migration-register`
- `/Users/skotlov/d/temp/20261116_comet12-consolidated-plan`
- `/Users/skotlov/d/temp/20261116_comet12-execution-setup`
-Снимки `approved-plan.json`/`feature-register.json` фиксируют утверждённый состав
-до реализации; прогресс вести в `STATE.json`. Не запускать старый генератор реестра
-для обновления прогресса: он намеренно выставляет NOT_STARTED/NOT_RUN и не является
-трекером выполнения. Подтверждённые новые пользовательские решения заносить прежде
-всего в `decisions.json` и отражать в текущем задании; старые temp-отчёты могут быть
-историческими снимками. Не рассчитывать на автоматическую синхронизацию истории чатов.
-
-## Окружение и блокеры
-
-Java17.0.19-tem подходит и проверена вместе с javac; глобально сейчас выбрана Java21.
-Координатор задаёт JAVA_HOME на17.0.19-tem для сборочной команды, не меняя глобальную Java.
-Rust/Cargo1.100.0 установлены и проверены; Homebrew proxies находятся в
-`/opt/homebrew/opt/rustup/bin`. Для своих команд можно добавить этот путь в PATH;
-редактирование пользовательского `~/.zshrc` не требуется.
-Если реально отсутствует tool/access/capability, сообщить конкретный блокер
-пользователю и решить его вместе; не тратить время на длинные обходные пути.
+`approved-plan.json`/`feature-register.json` snapshots preserve approved pre-implementation scope; track progress in `STATE.json`. Do not run the old register generator to update progress: it deliberately sets NOT_STARTED/NOT_RUN and is not an execution tracker. Record new confirmed user decisions primarily in `decisions.json` and reflect them in current tasks; old temp reports may be historical snapshots. Do not rely on automatic synchronization of chat histories.
+
+All newly committed documentation and commit messages must be English. Original Russian evidence and pre-translation content are retained in `/Users/skotlov/d/temp/20261118_comet12-g01-finalization/documentation-before.zip`. Translated user statements in the documentation preserve the decisions rather than claiming to be original English quotations.
+
+## Environment and blockers
+
+Java17.0.19-tem is suitable and verified with javac; the global selection at the preparation checkpoint was Java21. The coordinator sets JAVA_HOME to17.0.19-tem per build command without changing global Java. Java21 qualification is currently on hold at the user's request.
+Rust/Cargo1.100.0 are installed and checked; Homebrew proxies are in `/opt/homebrew/opt/rustup/bin`. Add that path to PATH for individual commands if needed; editing the user's `~/.zshrc` is unnecessary.
+If a required tool/access/capability is actually missing, report the concrete blocker to the user and resolve it together; do not spend time on long workarounds.
diff --git a/docs/joom-migration/STATE.json b/docs/joom-migration/STATE.json
index c1d4d3ce8b5..d8e4ffef128 100644
--- a/docs/joom-migration/STATE.json
+++ b/docs/joom-migration/STATE.json
@@ -1,18 +1,18 @@
{
- "updated_on": "2026-11-17",
- "phase": "READ_ONLY_PREPARATION_USER_CODE_HOLD",
+ "updated_on": "2026-11-18T22:39:06.361313+00:00",
+ "phase": "G01_IMPLEMENTATION_COMPLETE_EXACT_SHA_RC1_GATE",
"repository": "/Users/skotlov/d/projects/datafusion-comet",
"integration_branch": "branch-1.2",
"upstream_base_commit": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
- "observed_code_head_before_documentation_commit": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
- "implementation_started": false,
- "active_writer": null,
+ "observed_code_head_before_documentation_commit": "2b8f80b3f253559bc999f73769b05c88ebcd5fc2",
+ "implementation_started": true,
+ "active_writer": "Main coordinator; helpers prepare isolated patches and read-only reviews",
"worker_threads_created": [
{
"group": "G01",
"thread_id": "01a1e716-379a-7d92-b6bd-cbd9c8500afc",
"host_id": "local",
- "title_requested": "Comet G01: подготовка без кода",
+ "title_requested": "Comet G01: preparation without code",
"status": "IDLE_PREPARATION_COMPLETE_NO_IMPLEMENTATION_ASSIGNED",
"report": "G01-preparation.json",
"last_verified_head": "5e2f39b8a581d2bb055a4981cbf649c4bf059222",
@@ -39,15 +39,15 @@
"P03"
],
"depends_on": [],
- "status": "NOT_STARTED",
- "head_commit": null,
- "tests": "NOT_RUN",
- "handoff_report": null,
+ "status": "IMPLEMENTATION_COMPLETE_RC1_ACCEPTANCE_GATE",
+ "head_commit": "Resolve refs/tags/joom-1.2-1-rc1^{commit} after the acceptance gate.",
+ "tests": "Full runtime CI: 127 successful jobs, 17 conditional skips, zero failures. Current source/test/lockfile equality verified. Final exact-SHA RC receipt is delivered with artifacts and referenced by the annotated tag.",
+ "handoff_report": "G01-report.md",
"preparation": {
"group": "G01",
"thread_id": "01a1e716-379a-7d92-b6bd-cbd9c8500afc",
"host_id": "local",
- "title_requested": "Comet G01: подготовка без кода",
+ "title_requested": "Comet G01: preparation without code",
"status": "IDLE_PREPARATION_COMPLETE_NO_IMPLEMENTATION_ASSIGNED",
"report": "G01-preparation.json",
"last_verified_head": "5e2f39b8a581d2bb055a4981cbf649c4bf059222",
@@ -57,17 +57,8 @@
{
"id": "G02",
"branch": "joom/1.2-native-runtime",
- "cards": [
- "I05",
- "I06",
- "J14",
- "J15",
- "J16",
- "J22"
- ],
- "depends_on": [
- "G01"
- ],
+ "cards": ["I05", "I06", "J14", "J15", "J16", "J22"],
+ "depends_on": ["G01"],
"status": "NOT_STARTED",
"head_commit": null,
"tests": "NOT_RUN",
@@ -76,15 +67,8 @@
{
"id": "G03",
"branch": "joom/1.2-sort-spill-budget",
- "cards": [
- "D02",
- "D03",
- "D04",
- "D11"
- ],
- "depends_on": [
- "G01"
- ],
+ "cards": ["D02", "D03", "D04", "D11"],
+ "depends_on": ["G01"],
"status": "NOT_STARTED",
"head_commit": null,
"tests": "NOT_RUN",
@@ -106,9 +90,7 @@
"N08",
"N09"
],
- "depends_on": [
- "G01"
- ],
+ "depends_on": ["G01"],
"status": "NOT_STARTED",
"head_commit": null,
"tests": "NOT_RUN",
@@ -117,12 +99,8 @@
{
"id": "G05",
"branch": "joom/1.2-smj-filter",
- "cards": [
- "D05"
- ],
- "depends_on": [
- "G01"
- ],
+ "cards": ["D05"],
+ "depends_on": ["G01"],
"status": "NOT_STARTED",
"head_commit": null,
"tests": "NOT_RUN",
@@ -131,15 +109,8 @@
{
"id": "G06",
"branch": "joom/1.2-native-windows",
- "cards": [
- "N01",
- "N02",
- "N03"
- ],
- "depends_on": [
- "G01",
- "G02"
- ],
+ "cards": ["N01", "N02", "N03"],
+ "depends_on": ["G01", "G02"],
"status": "NOT_STARTED",
"head_commit": null,
"tests": "NOT_RUN",
@@ -162,10 +133,7 @@
"J26",
"J27"
],
- "depends_on": [
- "G01",
- "G02"
- ],
+ "depends_on": ["G01", "G02"],
"status": "NOT_STARTED",
"head_commit": null,
"tests": "NOT_RUN",
@@ -174,16 +142,8 @@
{
"id": "G08",
"branch": "joom/1.2-jvm-codegen",
- "cards": [
- "J05",
- "J06",
- "J07",
- "J17"
- ],
- "depends_on": [
- "G01",
- "G02"
- ],
+ "cards": ["J05", "J06", "J07", "J17"],
+ "depends_on": ["G01", "G02"],
"status": "NOT_STARTED",
"head_commit": null,
"tests": "NOT_RUN",
@@ -192,21 +152,8 @@
{
"id": "G09",
"branch": "joom/1.2-cost-based-engines",
- "cards": [
- "I07",
- "J01",
- "J02",
- "J03",
- "J04"
- ],
- "depends_on": [
- "G03",
- "G04",
- "G05",
- "G06",
- "G07",
- "G08"
- ],
+ "cards": ["I07", "J01", "J02", "J03", "J04"],
+ "depends_on": ["G03", "G04", "G05", "G06", "G07", "G08"],
"status": "NOT_STARTED",
"head_commit": null,
"tests": "NOT_RUN",
@@ -217,7 +164,7 @@
"id": "G10",
"branch": "joom/1.2-release-qualification",
"status": "APPROVED_RESERVE_ONLY_IF_NEEDED",
- "scope": "Межфункциональные тесты и необходимые ограниченные интеграционные исправления с отдельнымикоммитами. Есликонтрольдиффаобъективнотребуетразделитьодиниз9блоков, резервможетбытьпереназначеннаэтотsplitпослеобсужденияконкретнойграницы. НеобязательныйпустойPRнесоздаётся."
+ "scope": "Cross-feature tests and necessary limited integration fixes in separate commits. If review of one of the nine groups objectively requires a split, the reserve may be reassigned after discussing the concrete boundary. Do not create an unnecessary empty PR."
},
"docker_checks": {
"context": "desktop-linux",
@@ -231,13 +178,13 @@
"checkout_read_mount": "PASSED",
"output_write_mount": "PASSED",
"network": "none; pull=never; cached images only",
- "release_builder_compilation": "NOT_RUN",
+ "release_builder_compilation": "PASSED_BOTH_ARCHITECTURES",
"evidence": "/Users/skotlov/d/temp/20261116_comet12-execution-setup/docker-probes.json",
- "note": "Один неиспользуемый default builder указывает на недоступный /var/run/docker.sock. Рабочие desktop-linux/spark-builder доступны; переподключать или чинить default не требуется."
+ "note": "One unused default builder points to the unavailable /var/run/docker.sock. The working desktop-linux and spark-builder are available; no default-builder repair or reconnection is needed."
},
"build_tools": {
"standalone_mvn_cargo_rustc_in_current_PATH": false,
- "maven_wrapper": "PRESENT; pins3.9.6; not executed",
+ "maven_wrapper": "VERIFIED Maven3.9.6 Java17; registry baseline executed",
"host_rust_installation": "VERIFIED_USER_INSTALLED",
"java_sdkman_installations": "VERIFIED: Java/javac17.0.19-tem; global SDKMAN selection21.0.9-tem unchanged.",
"rustc_version": "1.100.0",
@@ -252,57 +199,69 @@
},
"permissions": {
"repository_operations_authorized": true,
- "implementation_authorized": false,
+ "implementation_authorized": true,
"git_mutation_authorized": true,
"repository_scope": "/Users/skotlov/d/projects/datafusion-comet",
"git_scope": "/Users/skotlov/d/projects/datafusion-comet/.git",
- "user_quote": "Отныне ты имеешь мое разрешение делать все необходимые манипуляции в /Users/skotlov/d/projects/datafusion-comet (в т.ч. в /Users/skotlov/d/projects/datafusion-comet/.git)",
- "interpretation": "Разрешение охватывает необходимые локальные изменения репозитория, ветки, коммиты, интеграцию и локальные теги. Подготовка передачи работы не означает, что реализация G01 уже начата.",
- "implementation_status": "NOT_STARTED_USER_CODE_HOLD",
+ "user_quote": "From now on you have my permission to perform all necessary operations in /Users/skotlov/d/projects/datafusion-comet, including /Users/skotlov/d/projects/datafusion-comet/.git.",
+ "interpretation": "Authorization covers necessary local repository changes, branches, commits, integration and local tags. Preparation of a handoff did not itself start G01 implementation.",
+ "implementation_status": "G01_AMD64_QUALIFICATION_PENDING_G02_NOT_AUTHORIZED",
"network_build_dependencies_authorized": true,
"network_scope_status": "APPROVED_WITH_USER_SAFETY_CONDITION",
"github_api_read_only_audit_authorized": true,
"automatic_worker_chat_creation_authorized": true,
"automatic_cross_chat_messaging_authorized": true,
"publishing_authorized": false,
- "remote_push_status": "USER_REPORTED_403; LOCAL_WORK_ONLY",
- "note": "Git/документация и создание/ведение рабочих сессий разрешены; сеть разрешена с условием безопасности. Последнее прямое указание запрещает писать код до нового поручения. Push, публикация и production-операции исключены.",
+ "remote_push_status": "AUTHORIZED_G01_PR_PREPARATION",
+ "note": "G01 implementation and PR28 feature-branch push/update authorized; branch-1.2 merge/update, RC1, G02+, external artifact publication and production operations remain held.",
"recorded_on": "2026-11-17",
"network_approval": {
"recorded_on": "2026-11-17",
"status": "APPROVED_WITH_USER_SAFETY_CONDITION",
- "source": "Прямой ответ пользователя на request_user_input_async в координаторском чате.",
- "user_quote": "Разрешаю на всю миграцию Comet1.2/Spark4.2.1 сетевые операции, перечисленные в docs/joom-migration/NETWORK.md: Maven/SBT, Rustup/Cargo, закреплённые git-зависимости и build tools с GitHub, Docker Hub и Ubuntu packages, включая дочерние процессы внутри контейнеров. При условии, что все скачиваемое на мой ноут из сети и запускаемое - является известными и широкоиспользуемыми артефактами, в безопасности которых для моей машины не приходится сомневаться. Без push, внешней публикации и production операций.",
+ "source": "Direct user reply to request_user_input_async in the coordinator chat; Russian quotations are translated into English.",
+ "user_quote": "I authorize, for the entire Comet 1.2 / Spark 4.2.1 migration, the network operations listed in docs/joom-migration/NETWORK.md: Maven/SBT, Rustup/Cargo, pinned Git dependencies and build tools from GitHub, Docker Hub and Ubuntu packages, including child processes inside containers. Everything downloaded to and executed on my laptop must be a known, widely used artifact whose safety for my machine is not reasonably in doubt. No push, external publication or production operations.",
"scope_document": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/NETWORK.md",
"approved_proposal_commit": "fbc250a9867515958c6a33920c1ac15b5d2d62b1",
"approved_proposal_sha256": "3bedf4a5dc5d14b1793d25c4ae9073b697a1691a614aa69494c37486bc8dac02",
- "scope": "На всю утверждённую миграцию: Maven/SBT, Rustup/Cargo, закреплённые git-зависимости и build tools с GitHub, Docker Hub и Ubuntu packages, включая дочерние процессы внутри контейнеров.",
- "user_safety_condition": "Все скачиваемые на ноутбук и запускаемые артефакты должны быть известными и широко используемыми, без обоснованных сомнений в их безопасности для машины пользователя.",
- "execution_rule": "Использовать известные зависимости выбранного upstream-проекта и официальные источники/реестры, закреплённые версии/commit и доступные publisher checksums. Перед запуском внешнего installer/build script проверить его происхождение и содержание. Наличие пакета в registry или lockfile само по себе не отменяет условие пользователя. Не добавлять и не запускать нестандартный, непроверенный или сомнительный инструмент/источник на основании одного этого разрешения; при такой необходимости согласовать отдельно.",
+ "scope": "For the entire approved migration: Maven/SBT, Rustup/Cargo, pinned Git dependencies and build tools from GitHub, Docker Hub and Ubuntu packages, including child processes inside containers.",
+ "user_safety_condition": "Downloaded and executed artifacts must be known and widely used, without reasonable doubts about their safety for the user's machine.",
+ "execution_rule": "Use known dependencies of the chosen upstream project and official sources/registries, pinned versions/commits and available publisher checksums. Check the provenance and content of an external installer/build script before running it. Registry or lockfile membership alone does not waive the user's safety condition. Do not introduce or run unusual, unverified or questionable tools/sources under this approval; obtain separate agreement if necessary.",
"excluded": [
"Git push",
- "Внешняя публикация артефактов/PR/images",
- "Production-операции",
- "Остальные исключения исходного NETWORK.md"
+ "External publication of artifacts/PRs/images",
+ "Production operations",
+ "Other exclusions in the original NETWORK.md"
],
"network_operations_performed_during_recording": false
},
"implementation_hold": {
"active": true,
- "user_quote": "Код не пиши до моего укзания.",
- "recorded_on": "2026-11-17",
- "allowed_now": "Документация и read-only подготовка; реализацию не начинать."
+ "applies_to_groups": [
+ "G02",
+ "G03",
+ "G04",
+ "G05",
+ "G06",
+ "G07",
+ "G08",
+ "G09",
+ "G10"
+ ],
+ "allowed_group": "G01",
+ "user_quote": "Start implementing G01 according to the approved plan. Report after completion and checks. Do not proceed to G02: I will explicitly authorize each of the ten groups after reviewing the preceding group."
},
"session_orchestration_authorization": {
"recorded_on": "2026-11-17",
- "user_quote": "Проект Comet добавлен (/Users/skotlov/d/projects/datafusion-comet). Создавай и веди рабочие сессии по утверждённому плану, передавай им задания и уточнения, проверяй результаты.",
+ "user_quote": "The Comet project has been added at /Users/skotlov/d/projects/datafusion-comet. Create and manage working sessions according to the approved plan, pass them assignments and clarifications, and review their results.",
"project_id": "79ec4996-572a-4070-96f7-2344b555eb2c",
"code_hold_preserved": true
- }
+ },
+ "implementation_authorized_groups": ["G01"],
+ "language_policy": "Commit messages and committed documentation must be English; user conversation may remain Russian. User quotations in this directory are English translations."
},
"delivery_scope": {
"status": "APPROVED_EXPLICITLY",
- "goal": "Реализовать согласованный перенос и получить локальные JAR-артефакты Comet и Delta integration для включения пользователем в Spark4.2 image.",
+ "goal": "Implement the agreed migration and obtain local Comet and Delta integration JAR artifacts for the user to include in the Spark 4.2 image.",
"targets": [
"Spark4.2.1",
"Scala2.13.18",
@@ -310,49 +269,1030 @@
"Delta4.4.1",
"GCP"
],
- "linux_release_architectures": [
- "amd64",
- "arm64"
- ],
- "checks": "Обязательные correctness/ANSI/regression/integration и packaging проверки сохраняются. Финальные JAR должны иметь SHA256, Git SHA и сведения о сборке. Полную проверку на production GCP jobs пользователь проведёт отдельно.",
- "user_quote": "Нам сейчас нужно сделать все задуманное, чтобы я получил jar артефакты для включения в наш spark 4.2 image. Проверять производительность я буду позже."
+ "linux_release_architectures": ["amd64", "arm64"],
+ "checks": "Retain mandatory correctness, ANSI, regression, integration and packaging checks. Final JARs must include SHA256, Git SHA and build information. The user will separately validate production GCP jobs.",
+ "user_quote": "We need to implement everything planned and obtain JAR artifacts for inclusion in our Spark 4.2 image. I will check performance later."
},
"performance_policy": {
"status": "APPROVED_EXPLICITLY",
- "user_quote": "ну по возможности ты ее конечно проверяй как сможешь во время реализации.",
- "during_implementation": "Существующие Scala microbenchmarks/Spark Benchmark, Rust Criterion и доступные TPC-H/DS runners в принятом стиле. При необходимости обычное расширение соответствующего benchmark; отдельную инфраструктуру ради миграции не создавать.",
- "production_benchmarks": "Полные замеры на реальных GCP jobs отложены на пользовательскую проверку и не являются текущим условием передачи локальных JAR.",
+ "user_quote": "Where possible, also check performance as you can during implementation.",
+ "during_implementation": "Use existing Scala microbenchmarks/Spark Benchmark, Rust Criterion and available TPC-H/DS runners in the established style. Extend the relevant existing benchmark normally if needed; do not create separate migration infrastructure.",
+ "production_benchmarks": "Full measurements on real GCP jobs are deferred to user validation and are not a current gate for handing over local JARs.",
"correctness_unchanged": true,
- "latest_user_quote": "Я хочу сказать, что с т.з. оценки производительности ты должен действовать точно так , как уже принято в этом проекте datafusion-comet. Не надо создавать специальных тестов, если там подобного никогда не было.",
+ "latest_user_quote": "For performance evaluation, follow exactly the practices already established in datafusion-comet. Do not create special tests if this project has never used that approach.",
"reference": "PERFORMANCE.md",
"correctness_tests_unchanged": true
},
- "pending_setup": [
- "Wait for explicit user instruction before writing implementation code.",
- "If actual tool/access gaps arise during authorized work, report them instead of long workarounds."
- ],
- "next_step": "Ожидать прямого разрешения писать код. Затем основной чат начинает G01, самостоятельно делегируя исследования/review помощникам; отдельный G01 prep-чат не является назначенным исполнителем реализации.",
- "qualification": "Comet/Spark/Delta correctness, ANSI, packaging and vendor tests not run. Local performance measurements to be attempted during implementation; production GCP benchmarks deferred to user.",
+ "pending_setup": [],
+ "next_step": "Run RC Build & Smoke for the final rewritten feature HEAD. After both native architectures pass and artifact download verification succeeds, fast-forward branch-1.2 and create the immutable annotated joom-1.2-1-rc1 tag. The tag annotation and delivered release-qualification.json record the final run/source/JAR identities without a recursive post-build source commit. Then stop; G02 requires explicit authorization.",
+ "qualification": "Runtime sources remain byte-identical to full Comet CI source 26590264b878fb8eae2ca439a8bcc0dfb513cb36. Prior B93 RC passed 7/7 jobs and 12/12 smoke cases per architecture. Rewritten final source must pass a fresh RC build; do not reuse B93 artifacts as RC1.",
"local_comet_project_id": "79ec4996-572a-4070-96f7-2344b555eb2c",
"cleanup_ledger": "cleanup-ledger.json",
"execution_model": {
"selected_on": "2026-11-17",
"status": "SELECTED_BY_COORDINATOR_WITHIN_USER_AUTHORIZED_ORCHESTRATION",
"primary": "ONE_MAIN_CHAT_EXECUTES_AND_COORDINATES",
- "main_chat_responsibility": "Последовательная реализация G01–G09 в утверждённых feature branches, интеграция, запуск необходимых проверок, состояние/решения/cleanup учёт; единственный пишущий исполнитель checkout.",
- "internal_helpers": "Ограниченные исследования, сверка совместимости, независимый review diff/тестового покрытия. По умолчанию read-only; не запускать параллельную запись в общий checkout или все тяжёлые builds одновременно.",
- "delegation_owner": "Координатор самостоятельно выбирает задачи, границы и время делегирования, передаёт контекст и проверяет результат. Пользователь не должен управлять помощниками вручную.",
+ "main_chat_responsibility": "Implement G01-G09 sequentially in the approved feature branches, integrate, run necessary checks and maintain state, decisions and cleanup records; own checkout integration writes.",
+ "internal_helpers": "Bounded research, compatibility checks and independent diff/test-coverage review. Read-only by default; do not allow overlapping writes to the shared checkout or run every heavy build concurrently.",
+ "delegation_owner": "The coordinator selects tasks, boundaries and timing, transfers context and verifies results. The user need not manage helpers manually.",
"delegation_checkpoints": {
- "G01": "Dependency graph/vendor provenance, Spark/Delta compatibility и Linux JAR packaging.",
- "G02_G07": "JNI/FFI, ownership/cancellation и buffer lifetime.",
- "G03_G06": "Sort/aggregate/SMJ/window memory/spill, outer-join/window correctness и ANSI semantics.",
+ "G01": "Dependency graph/vendor provenance, Spark/Delta compatibility and Linux JAR packaging.",
+ "G02_G07": "JNI/FFI, ownership/cancellation and buffer lifetime.",
+ "G03_G06": "Sort/aggregate/SMJ/window memory/spill, outer-join/window correctness and ANSI semantics.",
"G08": "Expression evaluation order, UDF/serialization.",
- "G09_FINAL": "Полнота 58 карточек, engine boundaries/AQE, совместные regressions и артефакты."
+ "G09_FINAL": "Coverage of all 58 cards, engine boundaries/AQE, combined regressions and artifacts."
},
- "new_user_visible_chat_policy": "Создавать по решению координатора при конкретной пользе от длительной изолированной задачи/отдельного ограниченного контекста или технической необходимости передачи продолжения. Новый PR/большой diff/compaction сами по себе не требуют нового чата. Сначала сообщить причину, сохранить checkpoint (actual HEAD, tree, tests, next step), передать контекст и проверить результат; повторное разрешение на каждую сессию не требуется в существующем scope.",
- "existing_g01_preparation_chat": "Оставить idle; подготовка завершена. Реализацию в нём автоматически не начинать; использовать при реальной необходимости отдельной проверки.",
- "user_involvement": "Явное начало кода, новые решения о поведении/объёме/разрешениях и реальные environment blockers. Не повседневное распределение задач.",
+ "new_user_visible_chat_policy": "The coordinator may create a session when a long isolated task, separate bounded context or technical handoff requirement gives a concrete benefit. A new PR, large diff or compaction alone does not require a new chat. First explain the reason, preserve a checkpoint (actual HEAD, tree, tests, next step), transfer context and verify the result; renewed permission per session is unnecessary within scope.",
+ "existing_g01_preparation_chat": "Leave idle; preparation is complete. Do not automatically start implementation there; use it only when a separate check is actually useful.",
+ "user_involvement": "Explicit authorization to start code, new decisions on behavior/scope/permissions, and real environment blockers; not day-to-day task assignment.",
"implementation_hold_preserved": true,
"approved_feature_groups_and_release_milestones_unchanged": true
+ },
+ "active_branch": "joom/1.2-dev",
+ "g01_work_directory": "/Users/skotlov/d/temp/20261117_comet12-g01",
+ "upstream_base_rechecked_on": "2026-11-17",
+ "upstream_base_current_sha": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
+ "active_sessions": {},
+ "validation_evidence": [
+ {
+ "check": "effective-pom",
+ "status": "PASSED",
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01/maven-effective-pom.log",
+ "versions": {
+ "spark": "4.2.1",
+ "scala": "2.13.18",
+ "delta": "delta-spark_4.2_2.13:4.4.1",
+ "java": "17"
+ }
+ },
+ {
+ "check": "registry-native-build",
+ "status": "PASSED",
+ "elapsed": "4m07s",
+ "command": "cargo build --locked -j4 -p datafusion-comet --manifest-path native/Cargo.toml",
+ "artifact": "native/target/debug/libcomet.dylib"
+ },
+ {
+ "check": "registry-jvm-initial",
+ "status": "FAILED_TEST_SELECTION",
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01/registry-baseline-jvm.log"
+ },
+ {
+ "check": "registry-jvm-discovery",
+ "status": "FAILED_LOCAL_RPC_ENVIRONMENT",
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01/registry-baseline-jvm-discovery.log"
+ },
+ {
+ "check": "loopback-smoke",
+ "status": "PASSED",
+ "tests": 10,
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01/registry-loopback-smoke.log",
+ "note": "SPARK_LOCAL_IP127.0.0.1; DPP+NativeBase suites andParquetEncryptionITCase passed."
+ },
+ {
+ "check": "delta-first-compile",
+ "status": "FAILED_FORMATTING_FIXED",
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01/registry-baseline-delta.log"
+ },
+ {
+ "check": "spotless",
+ "status": "PASSED",
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01/g01-spotless.log"
+ },
+ {
+ "check": "ci-config-static",
+ "status": "PASSED"
+ },
+ {
+ "check": "spark421-generated-diff-clean-apply",
+ "status": "PASSED",
+ "files": 120,
+ "diff": "dev/diffs/4.2.1.diff"
+ },
+ {
+ "check": "registry-full-delta",
+ "status": "283_PASS_1_FAIL_NEW_HADOOP_CONTEXT_GUARD_FIXED",
+ "tests": 284,
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01/registry-baseline-delta-formatted.log"
+ },
+ {
+ "check": "registry-context-guard-delta",
+ "status": "PASSED",
+ "tests": 164,
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01/registry-context-guard-tests.log",
+ "note": "Selected production/unit portions of upstream draftPR7593; local-only tests. Explicit gs control added afterward, pending next compilation."
+ },
+ {
+ "check": "registry-ansi-baseline",
+ "status": "PASSED_WITH_UPSTREAM_CANCELLATIONS",
+ "tests_passed": 65,
+ "tests_cancelled": 3,
+ "note": "Existing add/subtract/multiply generic tests cancel on Spark4.2 per issue4967; typed overflow fidelity and SQL ANSI fixtures pass. No claim of full ANSI compatibility.",
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01/registry-ansi-baseline.log"
+ },
+ {
+ "check": "vendor-graph",
+ "status": "PASSED",
+ "report": "/Users/skotlov/d/temp/20261117_comet12-g01/vendor-graph-verification.json",
+ "note": "One local physical-plan55.2.0; existing registry package versions/checksums unchanged; root native lockfile."
+ },
+ {
+ "check": "vendor-and-core-native-unit",
+ "status": "FAILED_UDF_LIBRARY_DISCOVERY_NOT_QUALIFIED",
+ "tests_passed": 961,
+ "tests_failed": 12,
+ "tests_ignored": 5,
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01/vendor-and-core-unit-direct-cargo.log",
+ "note": "Direct installed Cargo fixes libjvm loader environment. Twelve UDF tests fail missing test library path although comet-test-udfs build passed. Vendor tests not executed because core binary failed first. Investigate core/build.rs OUT_DIR ancestry under Cargo1.100; not yet established/fixed."
+ },
+ {
+ "check": "resumed-sandbox-access",
+ "status": "PASSED",
+ "report": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed/sandbox-probes.json"
+ },
+ {
+ "check": "native-workspace-unit",
+ "status": "PASSED",
+ "passed": 4326,
+ "ignored": 5,
+ "vendor_tests_passed": 1695,
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed/native-workspace-unit.log"
+ },
+ {
+ "check": "strict-workspace-clippy",
+ "status": "FAILED_PRISTINE_VENDOR_DEPRECATED_API",
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed/clippy-workspace-strict.log"
+ },
+ {
+ "check": "strict-comet-clippy",
+ "status": "PASSED",
+ "note": "All Comet workspace members selected, vendor excluded with --no-deps; explicit vendor tests mandatory.",
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed/clippy-comet-strict.log"
+ },
+ {
+ "check": "release-validator-publisher-negatives",
+ "status": "PASSED",
+ "cases": 19,
+ "report": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed/release-negative-results.json",
+ "note": "Synthetic validation fixtures; all HTTP/tag calls mocked; no external publication or native execution qualification."
+ },
+ {
+ "check": "vendor-statistics-test-utils",
+ "status": "PASSED",
+ "tests_passed": 42,
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed/vendor-statistics-stable-unit.log"
+ },
+ {
+ "check": "vendor-default-native-build",
+ "status": "PASSED",
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed/vendor-native-default-build.log",
+ "elapsed": "1m35s"
+ },
+ {
+ "check": "amd64-x86-64-v3-capability",
+ "status": "PASSED",
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed/amd64-cpu-probe-cpuid.log"
+ },
+ {
+ "check": "workspace-fmt",
+ "status": "PASSED",
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed/cargo-fmt-final.log"
+ },
+ {
+ "check": "vendor-delta-full-jvm",
+ "status": "PASSED",
+ "tests_passed": 291,
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed/vendor-delta-install.log",
+ "elapsed": "8m20s",
+ "note": "Root install also installs current fork; includes explicit gs control and all ADC/planning/serialization/native/DV/DML regressions."
+ },
+ {
+ "check": "vendor-ansi-runtime-jvm",
+ "status": "PASSED_WITH_UPSTREAM_CANCELLATIONS",
+ "tests_passed": 75,
+ "tests_cancelled": 3,
+ "log": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed/vendor-ansi-runtime-baseline.log",
+ "elapsed": "1m40s",
+ "note": "Typed ANSI/SQL fixtures plus DPP/NativeBase, and ParquetEncryptionITCase.3 old upstreamgeneric arithmetic tests cancel on Spark4.2."
+ },
+ {
+ "check": "final_g01_qualification",
+ "status": "PARTIAL_AMD64_RUNTIME_CHECK_INCOMPLETE",
+ "report": "G01-results.json",
+ "raw_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-final-qualification",
+ "source_commit": "2b8f80b3f253559bc999f73769b05c88ebcd5fc2"
+ }
+ ],
+ "pause": {
+ "status": "HISTORICAL_PAUSES_RESUMED_G01_FINALIZATION_AUTHORIZED",
+ "requested_on": "2026-11-17T05:06:02.577827+00:00",
+ "reason": "User requested restart checkpoint and interruption",
+ "all_task_builds_stopped": true,
+ "resume_requires_explicit_user_instruction": false,
+ "stopped_release_container": "de92fed928af",
+ "release_exit_code": 137,
+ "release_exit_explanation": "Deliberate docker stop for user pause; not an observed OOM or compiler failure.",
+ "resumed_on": "2026-11-17T13:34:17.545289+00:00"
+ },
+ "sandbox_access_verification": {
+ "checked_at": "2026-11-17T13:34:17.545289+00:00",
+ "protocbridge_cache_write": "PASSED_INSIDE_SANDBOX",
+ "loopback_bind_connect": "PASSED_INSIDE_SANDBOX",
+ "docker_server_version": "27.4.0",
+ "outside_sandbox_commands": "NONE_THIS_RESUME"
+ },
+ "toolchain_invocation_note": "Direct Cargo on macOS requires RUSTUP_TOOLCHAIN=stable to keep child rustc wrappers on installed1.100 rather than dependency-local nightly override; DYLD_LIBRARY_PATH points JAVA_HOME/lib/server. Docker pins RUSTUP_TOOLCHAIN1.100.0.",
+ "release_build_pending": {
+ "status": "PRIOR_B93_QUALIFIED_REPLACED_ONLY_AFTER_FINAL_EXACT_SHA_RC_SUCCESS",
+ "output_directory": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/gha-44388252264",
+ "source_commit": "b93d553856a0339af7468bcff2070e735a1a3c71",
+ "rc_tag": "joom-1.2-1-rc1; authorized only after final qualification"
+ },
+ "current_qualification_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-final-qualification",
+ "integration_gate": {
+ "status": "USER_AUTHORIZED_G01_INTEGRATION_AND_RC1_AFTER_SUCCESSFUL_FINAL_CHECKS",
+ "integration_head_before_g01": "9691385f35e4c5bd29514dca78d2044dd20e4bed",
+ "method": "Fast-forward only, retaining the qualified source SHA; no extra merge commit.",
+ "final_identity": "Annotated refs/tags/joom-1.2-1-rc1 and delivered release-qualification.json",
+ "not_authorized": [
+ "G02-G10",
+ "Java 21 qualification",
+ "JFrog/Nexus/Docker publication",
+ "production operations"
+ ]
+ },
+ "pr_preparation": {
+ "alignment_receipt": "G01-PR-preparation.json",
+ "body": "G01-PR.md",
+ "base_unchanged": "9691385f35e4c5bd29514dca78d2044dd20e4bed",
+ "test_change_commit": "235f3d306e3ec4be2d571b93a6c7dc99fb0af512",
+ "pull_request_created": true,
+ "created_manually_by_user": true,
+ "target_spark": "4.2.1",
+ "no_new_release_pipeline": true,
+ "url": "https://github.com/joomcode/datafusion-comet/pull/28",
+ "runner_fix": {
+ "reason": "Preflight/RequiredChecks on ubuntu-slim never acquired runner; zero executedsteps",
+ "replacement": "ubuntu-24.04",
+ "scope": "ci orchestrator, PR title, Iceberg coverage aggregator; commands/testcoverage unchanged",
+ "labeler_limit": "pull_request_target labeler still uses unchanged base workflow; no base change authorized"
+ },
+ "ci_preflight": {
+ "runner_fix_commit": "b406dccadc420b5896c0702083b60fe7da5c85db",
+ "observed_run": 44137489889,
+ "runner_acquired": "GitHub Actions 1000446763 / ubuntu-24.04",
+ "rat": "PASSED",
+ "pr_title_run": 44135662528,
+ "pr_title": "PASSED",
+ "observed_blocker": "Prettier formatting of four migration Markdown files; tests skipped before execution",
+ "local_formatting": "Prettier3.10.1 installed by user through Homebrew; four files formatted; whole-repository Markdown check passed",
+ "raw_evidence": "/Users/skotlov/d/temp/20261117_comet12-g01-runner-fix",
+ "native_amd64_qualification": "NOT_YET_EXECUTED_BY_GHA",
+ "labeler": "PASSED",
+ "formatting_fix_commit": "db2bd16584c1381853cc170abab7cd87f289c85c",
+ "preflight": "PASSED on rerun after temporary Maven Central HTTP429",
+ "detect_changes": "PASSED",
+ "full_test_verdict": "COMPLETED_FAILURE; see ci_monitoring.completed_result. G01notqualified."
+ }
+ },
+ "ci_wait_pause": {
+ "user_quote": "If you are only waiting for test results, do not keep this chat active because tests may take hours. I will tell you when they finish.",
+ "action": "Monitoring stopped. Resume only after user message. Local documentation checkpoint not pushed, to avoid restarting CI.",
+ "superseded_by": "User subsequently requested active monitoring every5minutes."
+ },
+ "ci_monitoring": {
+ "status": "COMPLETED_SUCCESS",
+ "interval_seconds": 300,
+ "pr_url": "https://github.com/joomcode/datafusion-comet/pull/28",
+ "source_commit": "30b6c4d7f531fad7b6550c5d3f1c92cefd096e58",
+ "head_sha": "26590264b878fb8eae2ca439a8bcc0dfb513cb36",
+ "run_id": 44205897075,
+ "raw_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha",
+ "run_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44205897075",
+ "last_checked_at": "2026-11-18T03:16:00.841276+00:00",
+ "counts": {
+ "success": 127,
+ "failure": 0,
+ "skipped": 17
+ },
+ "known_infrastructure_failure": {
+ "job_id": 134472513514,
+ "job": "Celeborn0.7.0 reflection compatibility",
+ "cause": "Maven Central429 resolving org.apache:apache:pom:23 before compile/checks",
+ "rerun_requested_now": "GitHub403: workflow still running",
+ "pending_action": null,
+ "receipt": "20261117_followup-gha/celeborn07-rerun-request.json",
+ "status": "RESOLVED_BY_SAME_SHA_FAILED_ONLY_RERUN"
+ },
+ "pending_infrastructure_rerun": null,
+ "fail_fast_policy": "decisions.json#ci_fail_fast_user_policy",
+ "run_attempt": 3,
+ "failure_classification_file": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/attempt-2-failure-classification.json",
+ "previous_attempt_final_counts": {
+ "success": 125,
+ "skipped": 17,
+ "failure": 2
+ },
+ "rerun_request": {
+ "requested_at": "2026-11-18T02:40:38.351472+00:00",
+ "run_id": 44205897075,
+ "head_sha": "26590264b878fb8eae2ca439a8bcc0dfb513cb36",
+ "from_attempt": 2,
+ "operation": "rerun-failed-jobs",
+ "http_status": 201,
+ "response": "{}",
+ "successful_jobs_preserved": true,
+ "full_rerun": false,
+ "remaining_direct_failure": "Maven parentPOM23 HTTP429 before compile/test"
+ },
+ "same_sha_attempts": [
+ {
+ "attempt": 1,
+ "success": 116,
+ "failure": 11,
+ "skipped": 17,
+ "causes": "8parentPOM429+2dependentcoverage+aggregate"
+ },
+ {
+ "attempt": 2,
+ "success": 125,
+ "failure": 2,
+ "skipped": 17,
+ "causes": "Spark34parentPOM429+aggregate"
+ },
+ {
+ "attempt": 3,
+ "success": 127,
+ "failure": 0,
+ "skipped": 17
+ }
+ ],
+ "final_qualification": {
+ "run_id": 44205897075,
+ "url": "https://github.com/joomcode/datafusion-comet/actions/runs/44205897075",
+ "attempt": 3,
+ "head_sha": "26590264b878fb8eae2ca439a8bcc0dfb513cb36",
+ "checked_at": "2026-11-18T03:16:00.841276+00:00",
+ "conclusion": "SUCCESS",
+ "job_counts": {
+ "success": 127,
+ "failure": 0,
+ "skipped": 17
+ },
+ "required_checks": "PASSED",
+ "retry_history": [
+ {
+ "attempt": 1,
+ "success": 116,
+ "failure": 11,
+ "skipped": 17,
+ "causes": "8parentPOM429+2dependentcoverage+aggregate"
+ },
+ {
+ "attempt": 2,
+ "success": 125,
+ "failure": 2,
+ "skipped": 17,
+ "causes": "Spark34parentPOM429+aggregate"
+ },
+ {
+ "attempt": 3,
+ "success": 127,
+ "failure": 0,
+ "skipped": 17
+ }
+ ],
+ "successful_jobs_preserved": true,
+ "source_unchanged_during_retries": true,
+ "target_spark42": {
+ "sql_shards": "9/9 PASSED",
+ "native": "2649passed/5existingignored",
+ "vendor": "1695passed/0ignored;statistics42passed",
+ "delta": "294passed/0failed/0canceled/0ignored",
+ "expressions": "2287passed/0failed/2existingcanceled/12ignored",
+ "ansi_three_arithmetic_cases": "ACTUALLY_EXECUTED_AND_PASSED",
+ "two_decimal_cast_cases": "ACTUALLY_EXECUTED_AND_PASSED",
+ "duplicate_reader_cases": "10ACTUALLY_EXECUTED_AND_PASSED",
+ "union_regressions": "Both conf-flip/native-grouping and live-copy/layout-binding tests passed on Spark4.1 and4.2"
+ },
+ "artifact_verification_receipts": [
+ "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/java-artifact-14386009386-verification.json",
+ "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/java-artifact-14386336295-verification.json",
+ "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/java-artifact-14386457154-verification.json",
+ "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/java-artifact-14386965743-verification.json"
+ ],
+ "raw_evidence_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha",
+ "qualification": "SOURCE_CI_PASSED; final exact-byte release JAR build/smoke remains RC gate; production/GCP validation not claimed",
+ "final_receipt": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/final-qualification-run-44205897075-head-26590264b.json",
+ "spark34_expressions": "2230passed/0failed/37canceled/12ignored"
+ },
+ "monitoring_stopped": true,
+ "pr_body_final_update_receipt": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha/pr28-final-success-body-update.json",
+ "local_progress_documents": "Saved uncommitted; no new source push, qualified HEAD unchanged"
+ },
+ "g01_ci_corrections_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes",
+ "g01_ci_corrections_protoc": {
+ "broken_system": "Homebrewprotobuf25.2 missingabseil2308dylib; nohostchanges",
+ "working_protoc": "/Users/skotlov/Library/Caches/com.thesamet.scalapb.protocbridge.protocbridge/v1/protoc-osx-aarch_64-4.33.5",
+ "verified_version": "libprotoc33.5",
+ "usage": "Explicitper-commandPROTOC; existingofficialSBTcache, nolang/globalconfigurationchange",
+ "cargo_invocation": "Directstabletoolchain/bin inPATH +RUSTUP_TOOLCHAIN=stable +DYLD_LIBRARY_PATHandDYLD_FALLBACK_LIBRARY_PATH=$JAVA_HOME/lib/server"
+ },
+ "g01_ci_corrections_checkpoint": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/implementation-checkpoint.json",
+ "previous_ci_monitoring": {
+ "status": "COMPLETED_FAILURE",
+ "interval_seconds": 300,
+ "run_id": 44137489889,
+ "head_sha": "db2bd16584c1381853cc170abab7cd87f289c85c",
+ "user_quote": "Can you monitor this GHA by checking its status every five minutes?",
+ "known_failures": {
+ "134232736818": "Delta build gate: Maven Central HTTP429 fetching org.apache:apache:pom:23; before compilation/tests",
+ "134233236967": "Strict Scala3.5 warnings: same Maven Central HTTP429 before compilation/tests",
+ "134247565203": "Spark4.0/JDK21 exec shard: Maven Central HTTP429 resolving org.apache:apache:pom:23 before compilation/tests",
+ "134232737645": {
+ "type": "test_assertion_failure",
+ "profile": "Spark4.0",
+ "succeeded": 292,
+ "failed": 1,
+ "test": "S3 deletion-vector context gates the real claim path after encryption algorithms",
+ "assertion": "DeltaScanContribSuite.scala2205 expected native scan in baseline, got Sparkfallback",
+ "result_correctness_claim": "No data-result comparison established by this planning-only failure",
+ "log": "job-134232737645-tail.log"
+ },
+ "134232736573": {
+ "type": "test_assertion_failure",
+ "profile": "Spark3.5",
+ "succeeded": 291,
+ "failed": 1,
+ "canceled": 1,
+ "test": "S3 deletion-vector context gates the real claim path after encryption algorithms",
+ "log": "job-134232736573-tail.log"
+ },
+ "134247565169": {
+ "type": "test_assertion_failure",
+ "profile": "Spark4.2/JDK17",
+ "succeeded": 2284,
+ "failed": 2,
+ "canceled": 2,
+ "ignored": 12,
+ "tests": [
+ "(ansi) failing cast of a literal raises Spark's own exception when reached",
+ "(ansi) cast string to negative-scale decimal fails when rescaling overflows an int"
+ ],
+ "assertions": [
+ "CometNativeCastSuite1646:Spark did not fail",
+ "CometNativeCastSuite1667:sparkError.isDefined was false"
+ ],
+ "baseline_behavior": "SparkwithoutComet did not raise expected exception; cause/parity diagnosis pending",
+ "log": "job-134247565169-tail.log"
+ },
+ "134254671669": {
+ "type": "dependency_resolution_failure_in_test",
+ "profile": "Spark4.2.1 SQL core-2",
+ "suite": "org.apache.spark.sql.SQLQuerySuite",
+ "succeeded": 1808,
+ "failed": 1,
+ "canceled": 1,
+ "ignored": 3440,
+ "exact_case_and_cause": "Ivy could not resolve org.springframework:spring-core:6.1.6 viarepo1.maven.org/spark-packages; HTTPstatusnotprovided",
+ "log": "job-134254671669-tail.log",
+ "complete_log_size_bytes": 10627700,
+ "exact_test": "SPARK-33084: Add jar support Ivy URI in SQL",
+ "evidence": "sql42-failure-diagnosis.json"
+ },
+ "134254671694": {
+ "type": "test_failure",
+ "profile": "Spark4.2.1 SQL core-4",
+ "suites": [
+ "org.apache.spark.sql.execution.datasources.parquet.ParquetIOSuite",
+ "org.apache.spark.sql.execution.datasources.parquet.ParquetFieldIdIOSuite"
+ ],
+ "succeeded": 7534,
+ "failed": 5,
+ "canceled": 0,
+ "ignored": 356,
+ "exact_case_and_cause": "3SPARK-59832 corruptedParquetdiagnosticmismatches;2SPARK-60262 missingmandatoryfieldID/namecollisionexception",
+ "log": "job-134254671694-tail.log",
+ "evidence": "parquet-core4-failures.json"
+ },
+ "134254671663": {
+ "type": "plan_fusion_partitioning_contract_assertions",
+ "profile": "Spark4.2.1 SQL core-1",
+ "succeeded": 10418,
+ "failed": 33,
+ "canceled": 64,
+ "ignored": 1155,
+ "groups": {
+ "UnionCodegen": 21,
+ "PushDownJoinThroughUnion": 2,
+ "SparkSessionExtension": 9,
+ "DataFrameSetOperations": 1
+ },
+ "limits": "Someearlierrowcheckspassed;12Unioncases and1DFcasefailbeforelaterresultchecks; noobservedvalue/errormismatch butfullcorrectnessnotqualified",
+ "evidence": [
+ "sql42-core1-diagnosis.json",
+ "sql42-core1-failure-cases.json",
+ "union-core1-failures.json"
+ ]
+ }
+ },
+ "raw_evidence": "/Users/skotlov/d/temp/20261117_comet12-g01-runner-fix",
+ "no_automatic_code_changes_or_push_for_monitoring": true,
+ "latest_summary": {
+ "snapshot_utc": "2026-11-17T19:29:03.032223+00:00",
+ "job_counts": {
+ "success": 117,
+ "failure": 10,
+ "skipped": 17
+ },
+ "target_spark42_sql": {
+ "passed": 6,
+ "failed": 3,
+ "failed_shards": ["core-1", "core-2", "core-4"],
+ "pending": 0
+ },
+ "target_comet": "scans/shuffle/execPASSED;expressions2284passed2failed2canceled12ignored",
+ "target_delta": "294PASSED0failed0canceled0ignored",
+ "rust_vendor": "Comet2639passed5skipped;vendor1695passed0skipped;statistics42passed",
+ "three_ansi_cases": "add/subtract/multiply ACTUALLY_PASSED onAMD64",
+ "highest_priority": "SPARK-60262 mandatoryPARQUET_FIELD_REQUESTED_MORE_THAN_ONCE missing inComet flat/nestedfieldID/name collisions",
+ "remaining_jobs": [],
+ "detailed_evidence": [
+ "first-test-job-failures.json",
+ "sql42-failure-diagnosis.json",
+ "parquet-core4-failures.json",
+ "sql42-core1-diagnosis.json",
+ "sql42-core1-failure-cases.json"
+ ],
+ "source_and_git_operations": "No source edits, commits, pushes or reruns duringmonitoring; onlylocalmigrationcheckpointupdates"
+ },
+ "completed_result": {
+ "run_id": 44137489889,
+ "url": "https://github.com/joomcode/datafusion-comet/actions/runs/44137489889",
+ "attempt": 2,
+ "head_sha": "db2bd16584c1381853cc170abab7cd87f289c85c",
+ "completed_at_utc": "2026-11-17T19:26:09Z",
+ "conclusion": "FAILURE",
+ "job_counts": {
+ "success": 117,
+ "failure": 10,
+ "skipped": 17
+ },
+ "independent_failed_jobs": 9,
+ "aggregate_failure": "Required Checks134305657089 follows the nine failedjobs",
+ "target_spark42": {
+ "native_vendor": "PASSED;2639Comet+1695vendor+42targetedstatistics;5existingnative skips",
+ "comet_suites": {
+ "scans": "PASSED",
+ "shuffle": "PASSED",
+ "exec": "PASSED",
+ "expressions": "2284passed/2failed/2canceled/12ignored"
+ },
+ "delta": "294passed/0failed/0canceled/0ignored",
+ "sql_shards": "6passed/3failed(core-1,core-2,core-4)",
+ "ansi_add_subtract_multiply": "ACTUALLY_PASSED_NATIVE_AMD64"
+ },
+ "macos": "Both scans andexpressionsPASSED",
+ "qualification": "G01_NOT_QUALIFIED; mandatoryerror/castcompatibility/testcontract issues unresolved",
+ "highest_priority": "MissingSPARK-60262 PARQUET_FIELD_REQUESTED_MORE_THAN_ONCE guard forflat/nested fieldID/namecollisions. Actualreturnedrows were notasserted; do not claimabsenceofsilentresult risk.",
+ "failure_groups": {
+ "dependency_resolution": "4jobs:3MavenHTTP429 +SQLcore2Ivydependency org.springframework:spring-core:6.1.6; IvyHTTPstatusnotreported",
+ "delta_fixture": "2oldprofilejobs Spark3.5/4.0, syntheticDVdescriptor serializerassertion; target4.2passes",
+ "ansi_casts": "2cases:obsolete literalexceptionexpectation andnative negative-scale errorwhereSpark4.2.1returnszero",
+ "sql_core1": "33operator/fusion/partitioningcontractassertions; some laterdata checksunreached; cannotblanketskip",
+ "sql_core4": "2missingmandatorymappingerrors +3corruptedParquetexceptiondiagnosticmismatches"
+ },
+ "raw_evidence_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-runner-fix",
+ "diagnosis_files": [
+ "first-test-job-failures.json",
+ "sql42-failure-diagnosis.json",
+ "parquet-core4-failures.json",
+ "sql42-core1-diagnosis.json",
+ "sql42-core1-failure-cases.json",
+ "union-core1-failures.json"
+ ],
+ "operations": "Monitoring/read-only diagnosis only; no sourceedits/commits/push/reruns duringmonitoring. Localmigrationdocumentsupdated; branch-1.2/RC1/G02unchanged."
+ }
+ },
+ "g01_ci_corrections": {
+ "status": "PUSHED_NATIVE_GHA_PENDING",
+ "source_commit": "6ae961dcbae949db09e349b05c91fc1cd6464d87",
+ "previous_pr_head": "db2bd16584c1381853cc170abab7cd87f289c85c",
+ "base_unchanged": "9691385f35e4c5bd29514dca78d2044dd20e4bed",
+ "fixes": [
+ "Spark SPARK-60119 tiny string-to-decimal rounding: version-contract flag, both Decimal128/256 paths, result/error and serde regressions",
+ "SPARK-60262: projected-field duplicate guard before data pages, exact physical names, nested error priority, structured factory cause and version-correct file-read envelope",
+ "Known-layout UNION: reject mismatched partition counts before unsafe concat; preserve declared partitions for all-empty inputs",
+ "Spark test fixtures retain original Spark contracts and add real native data/error/partition/observer checks; no new ignore/cancel",
+ "Delta synthetic DV fixture carries explicit bitmap offset compatible with older serializers",
+ "Maven model-only bounded transient parent-POM retry; actual compile/tests still run once; isolated Ivy local-repository fixture keeps transitive/exclude/class-loading checks"
+ ],
+ "checks": {
+ "rust": {
+ "parser": 5,
+ "tiny_decimal": 2,
+ "parquet_resolver": 10,
+ "structured_error": 1,
+ "failed": 0
+ },
+ "jvm_initial": {
+ "passed": 30,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "log": "jvm-regressions-concrete.log"
+ },
+ "jvm_converter_after_envelope_fix": {
+ "passed": 19,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "log": "jvm-error-envelope-final.log",
+ "note": "Rerun of converter tests after envelope correction; do not add to initial count as unique tests"
+ },
+ "spark_sql": {
+ "passed": 223,
+ "failed": 0,
+ "ignored": 3,
+ "canceled": 0,
+ "parts": {
+ "union": 189,
+ "extensions": 20,
+ "pushdown": 2,
+ "set_operations": 1,
+ "ivy": 2,
+ "field_selection": 3,
+ "corrupt_parquet": 6
+ },
+ "note": "Latest successful result per scenario, across targeted2/3/4/final logs. Three ignored Union tests are unchanged upstream ignores."
+ },
+ "delta_fixture": {
+ "passed": 1,
+ "failed": 0,
+ "log": "delta-fixture-final.log"
+ },
+ "ci_model_fault_injection": {
+ "passed": 4,
+ "failed": 0
+ },
+ "native_build": "PASSED",
+ "maven_install_spark42_delta": "PASSED",
+ "cargo_fmt": "PASSED",
+ "ci_config": "PASSED",
+ "spotless": "PASSED",
+ "spark_patch": "121 files; forward/reverse apply and exact patched-file equality passed",
+ "whitespace": "Actual Spark sources pass git diff --check v4.2.1; Comet sources excluding generated patch pass git diff --check. Generated patch preserves required single-space context lines."
+ },
+ "raw_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes",
+ "independent_review": "Runtime and CI reviews: no outstanding substantive finding after followups",
+ "push_strategy": "Single batched feature-branch push after local checks",
+ "gha_interval_seconds": 300,
+ "release_artifacts": "Previously built 2b8f80b3f JARs are historical; current corrections not release-packaged or RC tagged",
+ "scope": "G01 only; merge/base/RC1/G02 held",
+ "pr_head_commit": "ac3694504de3d5e3b93d19d8be87e48d6346b804"
+ },
+ "g01_ci_followup": {
+ "job_id": 134397478471,
+ "profile": "Spark4.2 scans",
+ "cases": 10,
+ "kind": "asserted legacy CometNativeException envelope after new footer prevalidation",
+ "data_contract": "Native scan still rejects file; no successful wrong data observed; ignoreCorruptFiles routes away from full native",
+ "fix": "Strict test-only assertion of actual structured Spark file-read envelope, exact path and native cause; old-profile assertion retained",
+ "patch": "20261117_parquet/native-reader-duplicate-envelope.patch",
+ "local_result": "PASSED; see native-reader-duplicate-envelope.log",
+ "source_changes": "Reader assertion adaptation committed in30b6c4d7f; final currenthead26590264b GHA PASSED",
+ "independent_review": "No weakening found"
+ },
+ "g01_union_followup": {
+ "status": "PASSED_LOCAL_AND_FINAL_GHA",
+ "current_code_location": "Committed and pushed on joom/1.2-dev; PR28 HEAD 26590264b878fb8eae2ca439a8bcc0dfb513cb36",
+ "production_changes": [
+ "Immutable legacy Union partitioning policy survives AQE/case-class copies; Spark4.2.1 delegates to authoritative snapshot",
+ "Preserve per-instance lazy layout, infer from live children of each copy; capture layout before executing children",
+ "Rebind snapshot-derived partitioning attributes to fixed Comet output after child reorder/replacement"
+ ],
+ "reader_test_change": "Strict typed file-read envelope/path/direct-cause; legacy CometNativeException assertion retained when guard absent",
+ "checks": {
+ "spark41_before_fix": "Reproduced wrong results:10rows(k,4) instead5rows(k,8); expected negative-control failure",
+ "spark41_after_fix": {
+ "passed": 12,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0
+ },
+ "spark42_union_after_fix": {
+ "passed": 12,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0
+ },
+ "spark42_reader_after_assertion_fix": {
+ "passed": 27,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0
+ },
+ "spark42_converter": {
+ "passed": 19,
+ "failed": 0
+ },
+ "spark42_native_spark_fixtures": {
+ "union": 21,
+ "extensions": 9,
+ "field_selection": 3,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "log": "spark-union-policy-followup.log"
+ },
+ "fresh_installed_test_jar_sha256": "4d448f7d253483dc92043d05564127544066290d21a72ad0d4fa704bce618f1b",
+ "maven42_install": "PASSED",
+ "format": "PASSED",
+ "spark35_scala212_legacy": {
+ "passed": 35,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "log": "union-reader-spark35-legacy.log",
+ "purpose": "Cross-compile changed shared API and preserve old raw duplicate-error contract"
+ }
+ },
+ "independent_review": "No outstanding finding; namespace followup matches recommended remap; tests not weakened",
+ "no_rust_or_vendor_change": true,
+ "raw_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes",
+ "source_patch_sha256": "403deeae1ebde3e20511e54c593beeae682f230c14677b034f471eaf98c1c8b1",
+ "pending": "Only user review/integration and final release-JAR acceptance; no unresolved GHA failures",
+ "active_local_builds": [],
+ "source_commit": "30b6c4d7f531fad7b6550c5d3f1c92cefd096e58",
+ "new_ci_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha",
+ "pr_head_commit": "26590264b878fb8eae2ca439a8bcc0dfb513cb36",
+ "run_id": 44205897075,
+ "run_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44205897075"
+ },
+ "ci_monitoring_history": [
+ {
+ "run_id": 44180095887,
+ "head_sha": "ac3694504de3d5e3b93d19d8be87e48d6346b804",
+ "html_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44180095887",
+ "status": "completed",
+ "conclusion": "failure",
+ "updated_at": "2026-11-17T23:26:56Z",
+ "job_counts": {
+ "success": 124,
+ "skipped": 17,
+ "failure": 3
+ },
+ "failed_jobs": [
+ {
+ "id": 134397478464,
+ "name": "PR Build (Linux) / Spark 4.1, JDK 17 [exec]",
+ "conclusion": "failure",
+ "html_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44180095887/job/134397478464"
+ },
+ {
+ "id": 134397478471,
+ "name": "PR Build (Linux) / Spark 4.2, JDK 17 [scans]",
+ "conclusion": "failure",
+ "html_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44180095887/job/134397478471"
+ },
+ {
+ "id": 134449352914,
+ "name": "Required Checks",
+ "conclusion": "failure",
+ "html_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44180095887/job/134449352914"
+ }
+ ],
+ "all_nine_previous_failed_jobs_passed": [
+ {
+ "id": 134382784825,
+ "name": "Delta Contrib Build Gate / Build-gate verification (default builds exclude Delta)",
+ "conclusion": "success"
+ },
+ {
+ "id": 134382785209,
+ "name": "Delta Contrib Tests (Spark 4.0) / Delta contrib (Spark 4.0)",
+ "conclusion": "success"
+ },
+ {
+ "id": 134382785727,
+ "name": "Delta Contrib Tests (Spark 3.5) / Delta contrib (Spark 3.5)",
+ "conclusion": "success"
+ },
+ {
+ "id": 134383206040,
+ "name": "PR Build (Linux) / Strict Scala warnings (Spark 3.5, JDK 17)",
+ "conclusion": "success"
+ },
+ {
+ "id": 134397478372,
+ "name": "PR Build (Linux) / Spark 4.0, JDK 21 [exec]",
+ "conclusion": "success"
+ },
+ {
+ "id": 134397478694,
+ "name": "PR Build (Linux) / Spark 4.2, JDK 17 [expressions]",
+ "conclusion": "success"
+ },
+ {
+ "id": 134403256068,
+ "name": "Spark SQL Tests (Spark 4.2) / spark-sql-sql_core-4/spark-4.2.1-jdk17",
+ "conclusion": "success"
+ },
+ {
+ "id": 134403256288,
+ "name": "Spark SQL Tests (Spark 4.2) / spark-sql-sql_core-1/spark-4.2.1-jdk17",
+ "conclusion": "success"
+ },
+ {
+ "id": 134403256457,
+ "name": "Spark SQL Tests (Spark 4.2) / spark-sql-sql_core-2/spark-4.2.1-jdk17",
+ "conclusion": "success"
+ }
+ ],
+ "tests_evidence": {
+ "native": {
+ "passed": 2649,
+ "skipped": 5,
+ "log": "job-134383205998-tail.log"
+ },
+ "vendor": {
+ "passed": 1695,
+ "skipped": 0,
+ "statistics_passed": 42,
+ "log": "job-134383205998-tail.log"
+ },
+ "delta42": {
+ "passed": 294,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "log": "job-134382785838-tail.log"
+ },
+ "delta40": {
+ "passed": 293,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 0,
+ "previous_case_verified": true,
+ "log": "job-134382785209-tail.log"
+ },
+ "delta35": {
+ "passed": 292,
+ "failed": 0,
+ "canceled": 1,
+ "ignored": 0,
+ "previous_case_verified": true,
+ "canceled_case_name_not_in_tail": true,
+ "log": "job-134382785727-tail.log"
+ },
+ "expressions42": {
+ "passed": 2287,
+ "failed": 0,
+ "canceled": 2,
+ "ignored": 12,
+ "three_ansi_and_two_cast_cases_executed_passed": true,
+ "receipt": "spark42-expressions-artifact-verification.json"
+ },
+ "sql42": {
+ "shards_passed": 9,
+ "shards_failed": 0,
+ "core1_counts": {
+ "passed": 10481,
+ "failed": 0,
+ "canceled": 64,
+ "ignored": 1155
+ },
+ "core2_counts": {
+ "passed": 1809,
+ "failed": 0,
+ "canceled": 1,
+ "ignored": 3440
+ },
+ "core4_counts": {
+ "passed": 7542,
+ "failed": 0,
+ "canceled": 0,
+ "ignored": 356
+ },
+ "previous_individual_cases_evidence": "Collected separately by coordinator helper under20261117_ci-reliability; see parent report"
+ }
+ },
+ "remaining_failures_classification": {
+ "134397478464": "Spark4.1 AQE union conf-flip query returns10rows(k,4) rather than5rows(k,8); actual result mismatch, CometExecSuite.scala4140",
+ "134397478471": "Ten duplicate-Parquet rejection tests reach expected read error, fail only cause-chain CometNativeException class assertion at CometNativeReaderSuite.scala123",
+ "134449352914": "Aggregate Required Checks reports the two failed jobs"
+ },
+ "monitor_actions": "Read-only API/logs/reports; no source/docs/Git changes, no rerun/push/cancel. Auxiliary JSON/log files only.",
+ "qualification_limit": "This verdict applies only to ac3694504. Later local uncommitted follow-up fixes are not covered by this run."
+ }
+ ],
+ "rc_workflow": {
+ "name": "RC Build & Smoke",
+ "file": ".github/workflows/rc_build_smoke.yml",
+ "auxiliary_directory": "/Users/skotlov/d/temp/20261118_comet12-rc-workflow",
+ "bootstrap_main_authorized": true,
+ "runtime_ci_baseline": "26590264b878fb8eae2ca439a8bcc0dfb513cb36",
+ "stage": "SUCCESS_DOWNLOADED_LOCALLY_VERIFIED",
+ "local_fault_tests": "22passed:9native-stage+7portable-smoke+6final-receipt",
+ "actionlint": "official1.7.12 downloadedin taskdir; checksumverified",
+ "source_runtime_changes": "NONE",
+ "bootstrap_commit": "fa7347e5ea549a01c77e325dbd3449d93c9d5700",
+ "workflow_validators": "actionlint1.7.12+CIconfig+Bashsyntax passed",
+ "head_sha": "b93d553856a0339af7468bcff2070e735a1a3c71",
+ "run_id": 44388252264,
+ "run_url": "https://github.com/joomcode/datafusion-comet/actions/runs/44388252264",
+ "first_run": {
+ "run_id": 44379092996,
+ "conclusion": "FAILURE_BEFORE_HEAVY_BUILDS",
+ "reason": "Negative duplicate-receipt fixture expected firstdiagnostic; Linuxfilesystemenumeration reachednativehashcheckfirst. Fixedfixturetoexclusivelyduplicateavalidreceipt, sameguardunchanged."
+ },
+ "cancelled_comet_ci": [44378830488, 44379067790, 44379346180],
+ "source_commit": "c6e9d375b",
+ "fixture_fix_commit": "966b8529b",
+ "previous_runs": [
+ {
+ "run_id": 44379355772,
+ "head_sha": "966b8529b4a0e32b272af9a21aebe29b721069ba",
+ "conclusion": "FAILURE",
+ "native_both": "PASSED",
+ "package_failure": "Separate dependency:build-classpath invocation lacked reactor artifacts in freshM2; fixedby sameMaveninvocation package+goal. No runtimechange."
+ },
+ {
+ "run_id": 44383036733,
+ "head_sha": "ee6bddb07e95af4bdb22ff801230871f913a6018",
+ "conclusion": "FAILURE",
+ "native_both": "PASSED",
+ "maven_package_release_validation": "PASSED",
+ "prepare_failure": "FocalGit2.25.1 ignoredcommand-scope safe.directory; UID1001checkoutrejected. Locallyreproducedwithexactbuilder; protectedsystemconfigurationpasses."
+ }
+ ],
+ "reactor_fix_commit": "ee6bddb07",
+ "late_auto_comet_ci": {
+ "run_id": 44383284941,
+ "status": "COMPLETED_FAILURE_BEFORE_HEAVY_JOBS",
+ "cause": "RATmissingASFheaders inREADME/G01-report; appearedafterinitialcancelpoll, alreadyterminalwhenobserved.",
+ "fix": "Bothheadersrestoredlocally; no runtime/toolchanges; do not restartRCfor documentation.",
+ "local_source_tarball_rat": "PASSED",
+ "rat_checked_document_tree": "ffbcbc1ac5e45926d0a98f4605f99b6ab2f9ee88",
+ "rat_log": "/Users/skotlov/d/temp/20261118_comet12-rc-workflow/rat-source-license-fix.log",
+ "not_yet_pushed": false,
+ "committed_fix": "b93d55385",
+ "exact_fixed_source_rat_log": "/Users/skotlov/d/temp/20261118_comet12-rc-workflow/rat-source-b93d55385.log"
+ },
+ "git_fix_commit": "d7f0df5bd",
+ "license_fix_commit": "b93d55385",
+ "latest_auto_ci": "44388222020 CANCELLED",
+ "exact_source_rat": "b93d55385 PASSED",
+ "git_fix_reproduction": "/Users/skotlov/d/temp/20261118_comet12-rc-workflow/docker-git-safe-directory-reproduction.json",
+ "conclusion": "SUCCESS",
+ "job_counts": {
+ "success": 7,
+ "failure": 0,
+ "skipped": 0
+ },
+ "download_verification": {
+ "run_id": 44388252264,
+ "source_commit": "b93d553856a0339af7468bcff2070e735a1a3c71",
+ "artifact_id": 14469302529,
+ "artifact_name": "rc-build-smoke-b93d553856a0339af7468bcff2070e735a1a3c71",
+ "artifact_api_digest": "sha256:1c633ecec930ce8a2e4f95a2fe859ddefcd89f3a2898fc77105775448ad6035b",
+ "downloaded_zip_sha256": "1c633ecec930ce8a2e4f95a2fe859ddefcd89f3a2898fc77105775448ad6035b",
+ "archive": "/Users/skotlov/d/temp/20261118_comet12-rc-workflow/qualified-artifact-44388252264.zip",
+ "output_directory": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/gha-44388252264",
+ "artifact_bytes": 189616637,
+ "unpacked_bytes": 429572457,
+ "jar_sha256": {
+ "comet-spark-spark4.2_2.13-1.2.0.jar": "9f7d57bfb88efceeecce15d96ca320a2d38f81aacc95f4bbf67d46915bfc0686",
+ "comet-contrib-delta-spark4.2_2.13-1.2.0.jar": "5581281ff5bf826f04a038318d802be13d9f66cb8084e3d8a40846b5175bd632"
+ },
+ "smoke_architectures": {
+ "amd64": {
+ "passed": 12,
+ "receipt_sha256": "43daf2cd5fa45b6823d9440ecf01d014f41d4b5c768ed3ad47f1da85bf592bde"
+ },
+ "aarch64": {
+ "passed": 12,
+ "receipt_sha256": "7d65b78eea3bd86777d4adc151a342611b5cb0a5d560e21b484fb44c9f68f3d1"
+ }
+ },
+ "archive_runtime_provenance_and_smoke_receipts_verified": true
+ },
+ "qualified_artifact_receipt": "/Users/skotlov/d/temp/20261118_comet12-rc-workflow/rc-run-44388252264-final-qualification.json",
+ "workflow_definition_in_main": "fa7347e5ea549a01c77e325dbd3449d93c9d5700 (onlyworkflowfile)",
+ "matching_bytes_both_architectures": true,
+ "monitoring_stopped": true,
+ "final_pr_body_update_receipt": "/Users/skotlov/d/temp/20261118_comet12-rc-workflow/pr28-rc-success-update.json"
+ },
+ "final_documentation_checkpoint": "refs/joom-migration/checkpoints/20261118-rc-complete-docs",
+ "finalization": {
+ "authorization_date": "2026-11-18",
+ "scope": "Finalize English documentation, consolidate service history, repeat exact-SHA RC, integrate G01 and tag RC1 after passing gates.",
+ "historical_fields": "Old CI, pause, preparation and artifact sections retain their original source-specific facts. They are evidence, not current authorization or final RC1 identity.",
+ "history_mapping": "G01-HISTORY.json",
+ "closeout": "G01-CLOSEOUT.md",
+ "original_documentation_archive": "/Users/skotlov/d/temp/20261118_comet12-g01-finalization/documentation-before.zip",
+ "external_final_receipt": "/Users/skotlov/d/temp/20261118_comet12-g01-finalization/completion.json",
+ "g02_authorized": false,
+ "java21_work_authorized": false
}
}
diff --git a/docs/joom-migration/approved-plan-ru.txt b/docs/joom-migration/approved-plan-ru.txt
deleted file mode 100644
index 4ad41457145..00000000000
--- a/docs/joom-migration/approved-plan-ru.txt
+++ /dev/null
@@ -1,137 +0,0 @@
-УКРУПНЁННЫЙ ПЛАН JOOM COMET1.2 — УТВЕРЖДЁН
-Дата утверждения:2026-11-16. 9 основных PR+резерв 10. Качество важнее дальнейшего укрупнения.
-Прежние 40 пакетов сохраняются как внутренние этапы/коммиты/проверки. Код пока не меняется.
-
-КАК ШАГ 2 ВХОДИТ В ПЛАН
-Шаг 2 целиком находится в G01 joom/1.2-dev. Прежняя организационная граница отдельных PR заменена отдельными этапами и коммитами. Порядок и требования к тестам остаются.
-
-G01.1 Spark/Delta на штатном DataFusion
-Настроить точные Spark4.2.1/Scala2.13.18 и Delta4.4.1, минимальные GCP ADC/planning адаптации; использовать штатные registry crates DataFusion55.2.0 с текущим target-lock Arrow/Parquet59.3.2. До vendoring получить результаты базовых Comet/Delta/ANSI fixtures.
-Проверка:EffectivePOM и actualtestSparkversion; native/Delta/schema/DV/rebase/DPP базовые результаты и ожидаемые ошибки. Никаких Joom-патчей DataFusion.
-
-G01.2 Чистый vendor отдельным коммитом
-Импортировать published standalone datafusion-physical-plan55.2.0 с provenance/checksum/license отдельным механическим коммитом. Не переносить старый Cargo55.1 и не наследовать Comet workspace.version1.2.0; имя/версия crate остаются datafusion-physical-plan55.2.0.
-Проверка:Исходники соответствуют pristine55.2; Joom алгоритмический diff отсутствует.
-
-G01.3 Подключение vendor и отдельный тестовый target
-Настроить root patch и включить vendored crate в native workspace с явным unit/regression test target. Использовать общий native/Cargo.lock/root patch; одного membership или cargo test --manifest-path недостаточно.
-Проверка:Проверить resolved graph: releaseComet и vendor-tests используют одну локальную physical-plan identity55.2.0, runtimeDF55.2/Arrow59.3.2. Unit-test dev-deps могут добавить lockentries, но не должны обновлять runtimegraph произвольно. CI явно выбирает package; Comet-only default-members не означают запуск vendor unit tests.
-
-G01.4 Повторная проверка и release tooling
-Повторить базовые проверки после подмены registry на чистый path crate. Подготовить build-release.sh/publish-jfrog.sh, multiarch packaging/provenance; отдельно проверить upstream native-library reload.
-Проверка:Сопоставить before/aftervendoring результаты и планы, прогнать именно vendorunit/regressions и Comet/Delta suites, проверить libcomet packaging/GLIBC/Scala. Итог G01 — внутренний RC1; полный Joom-набор ещё не перенесён.
-
-Joom алгоритмы в DataFusion vendor начинаются только в G03(sort),G04(aggregates),G05(SMJ). G01 сохраняет pristinevendor.
-
-ОСНОВНОЙ ПОРЯДОК
-
-G01 joom/1.2-dev
-База Spark/Scala, pristine DF vendor, Delta/GCP ADC/planning, release builder/publisher.
-Старые пакеты:B01,B03,B04,B05,B02,B06,B07
-Карточки:D01,I01,I02,I03,I04,I08,I09,I10,J18,P01,P02,P03
-До этого:
-Внутри одного PR сначала Spark/Delta на штатном DF55.2; затем отдельный pristinevendor commit, rootpatch/sharedlock/explicitvendor-tests и повторная проверка; затем release tooling. Все этапы — проверяемые коммиты без Joom-алгоритмов в vendor. Старые отдельные PR объединены, проверки не сокращены.
-
-G02 joom/1.2-native-runtime
-Offsets/FFI, TaskContext, block ownership, cancellation, timestamp compatibility.
-Старые пакеты:B08,B09,B10,B11,B27
-Карточки:I05,I06,J14,J15,J16,J22
-До этого:G01
-Сначала shared types/offsets, затем off-threadcontext, block lifecycle, cancellation, timestamp regressions/residuals. Каждая стадия проверяется до следующей; generic ANSI arithmetic/cast gaps здесь.
-
-G03 joom/1.2-sort-spill-budget
-Полный sort/wide-payload/spill/output-handoff контракт.
-Старые пакеты:B13,B14,B15
-Карточки:D02,D03,D04,D11
-До этого:G01
-Раздельные коммиты для reserve1/32, SpillWorkspace, wide sort, JVMhandoff1/4; один PR позволяет тестировать итоговый memorylifecycle безпромежуточныхнесогласованных API.
-
-G04 joom/1.2-native-aggregates
-Upstream aggregate parity, spill/offset/emit/drain, collect_list, FIRST/LAST/bool/string-value MIN_BY/MAX_BY, SortAggregate.
-Старые пакеты:B12,B18,B19,B20,B21,B22,B23
-Карточки:D06,D07,D08,D09,D10,N04,N05,N06,N07,N08,N09
-До этого:G01
-D08+N09 остаются одним согласованным подшагом. Остальные изменения отдельными коммитами; сохранить state/overflowguards. Остаточные SUM/AVG ANSI fixes иихтесты принадлежатэтому PR.
-
-G05 joom/1.2-smj-filter
-Filter-before-payload, outer-join correctness, one-side hoisting и ANSI eligibility.
-Старые пакеты:B16,B17
-Карточки:D05
-До этого:G01
-Сначала filter+всеобязательные outercorrectnessfixes, затем hoisting с can-throw/lazy проверками. Вобщуюветкупопадаетполныйбезопасныйблок, не временная некорректная outerjoin-версия.
-
-G06 joom/1.2-native-windows
-Spillable partition windows, tiny-partition batching и SortedWindow.
-Старые пакеты:B24,B25,B26
-Карточки:N01,N02,N03
-До этого:G01,G02
-Три самостоятельных review/test этапа внутриоднойветки. OffsetBudget уже есть. Сохранить FILTER fallback, RANGE/ANSI/ordering и required distribution.
-
-G07 joom/1.2-shuffle-and-c2r
-C2R borrowing/pool, BHJ/Kryo/offsets, writer optimisations/spill, direct read и coalescer.
-Старые пакеты:B28,B29,B30,B31,B32,B33
-Карточки:J08,J09,J10,J11,J12,J13,J21,J23,J24,J25,J26,J27
-До этого:G01,G02
-Не один монолитный commit: C2R+pool; broadcast; writers; spillcallback; initialdirectread; coalescerJVM/native/proto. Ownership и bufferlifetime проверятьпослекаждогошага. Самыйширокий interop блок;приобъективнойнеобозримости diff использоватьрезервразделения,нежертвовать review.
-
-G08 joom/1.2-jvm-codegen
-Scalar runtime arguments, JVM short-circuit, UDF pools, plan/task serialization; необходимые native expression evaluation-order fixes.
-Старые пакеты:B34,B35,B36,B37
-Карточки:J05,J06,J07,J17
-До этого:G01,G02
-JVMdispatch/пулы/serialization последовательнымикоммитами; shadedartifacttests. Native ANSI issue7246 шире J06:необходимые fixes явновыделитьотдельнымикоммитамивэтом PR,неподмешиватьисключённый PR5889.
-
-G09 joom/1.2-cost-based-engines
-BoundaryFormats, wide-row fallback и calibrated cost model вместе.
-Старые пакеты:B38,B39,B40
-Карточки:I07,J01,J02,J03,J04
-До этого:G03,G04,G05,G06,G07,G08
-Общийкод переноситсясразуцелостно:нетвременного standaloneConversionCount этапабез EngineCostModel. Сохранятьитоговуютаблицутега 13. Полная coston/off/AQE/mixedhash/nativecoverage матрицапослевсех native возможностей.
-
-РЕЗЕРВ
-joom/1.2-release-qualification
-Межфункциональные тесты и необходимые ограниченные интеграционные исправления с отдельнымикоммитами. Есликонтрольдиффаобъективнотребуетразделитьодиниз 9 блоков, резервможетбытьпереназначеннаэтот split послеобсужденияконкретнойграницы. Необязательныйпустой PR несоздаётся.
-
-ANSI/FIXES ВНУТРИ ЛИМИТА
-C01→G02:Arithmetic/cast/division ANSI correctness; нетотдельнойветкисверхлимита.
-C02→G08:Native evaluation order/JVMdispatch compatibility; J06 не объявлятьполнымзакрытием issue7246.
-C03→G04:Mixed SUM/AVG, runtimecodegenfallback, AVG+LIMIT.
-Q01→G10:Дополнительнаямежфункциональнаяквалификация, еслинуженотдельный diff.
-
-КАЧЕСТВО
-40 старыхпакетовсохраняютсякаквнутреннийчеклистсостава,зависимостейипроверок,анекак 40PR.
-Каждыйкрупный PR имеетатомарныекоммиты/смысловыеподшаги;проверкипослених,неединственныйтестовыйзапусквконце.
-Одиногромный squashcommit безпроверяемойструктурынепредлагается;сохранятьинформативнуюисторию.
-При review явноразделятьмеханический vendorimport иручныеизменения;происхождение/hash приведенывреестре.
-Общие CI/feature/runtime/ANSIgates неуменьшаются. Известный unsafe-result путьисправляетсяилиограничиваетсядо production.
-Еслипакетобъективнонельзякачественнопроверитьвэтомразмере,неукрупнятьдальшерадилимита;показатьконкретный diff ипредложитьразделениевпределахрезерва.
-Неоставлять C01/C02/C03/Q01 какскрытые 4 дополнительных PR:ихвладельцыявноназначенывтаблице.
-G09 квалифицируетсяпослевсех G03–G08. Группыидутпоосновномупоследовательномупорядку;параллельнаяработанетребуется.
-
-ПОЛЬЗА И ЦЕНА УКРУПНЕНИЯ
-Меньше CI/rebase/merge цикловбезпотерисостава.
-Убираютсяискусственныевременныеграницы BoundaryFormats↔costmodel.
-Вобщуюлиниюпопадаетполнаясвязаннаявозможностьсрегрессиями.
-Review крупнееиошибкувнутриблокаискатьсложнее;поэтомунужныатомарныекоммиты/подшаги.
-G01 содержитбольшоймеханический vendorimport;G04/G06/G07 могутдатьзначительный diff. Ихразмеркачественнооценитсяпофактическому patch,непообещанию.
-
-ТЕГИ
-joom-1.2-1-rc1 после G01:Внутренняябазоваясборка;неполный Joom перенос.
-joom-1.2-1-rc2 после G06:Общие runtime механизмы/native блокготовы;остальные JVM/shuffle/planner ещёнеполны.
-joom-1.2-1-rc3 после G09(+G10 еслипотребовался):Полныйкандидатсзакрытыми/ограниченнымиизвестными unsafe-result путями.
-joom-1.2-1 после Квалифицированпоследний RC:Тотже commit последнегоуспешного RC;никакихнепроверенныхизменений.
-
-НЕИЗМЕННЫЕ РЕШЕНИЯ
-ANSI=true;сравнивать Spark4.2.1 без/с Comet наоднихвходах,результатыиожидаемыеошибки.
-Максимальноиспользовать upstream,сохранятьвсенедостающие Joom части.
-PR5889 нетрогатьине включать.
-RC тегинеизменяемы;при дополнительныхпопыткахследующийномеррасходуется,поздниеплановыеномерасдвигаются.
-Тег/build/JFrogpublish/productionrollout—разныедействия.
-
-2026-11-16 пользователь разрешил все необходимые локальные операции в Joom Comet checkout и .git. Реализация G01–G09 ещё не начата; сетевой scope разрешён с условием безопасности; см.decisions.json. Публикация и production rollout не выполняются.
-
-ОБНОВЛЕНИЕ УСЛОВИЙ ИСПОЛНЕНИЯ 2026-11-16
-Локальные изменения Comet и .git разрешены. Сейчас готовится окружение; реализация групп не начата. Цель — проверенные локальные Linux JAR для Spark4.2 image. Локальные benchmarks выполнять по возможности; полные GCP benchmarks пользователь проведёт позже. Состав9+1, ANSI=true и correctness gates сохранены. Сетевой scope разрешён с условием безопасности; см.decisions.json.
-
-УТОЧНЕНИЯ2026-11-17
-Код не писать до нового прямого указания. Рабочие сессии и передача заданий разрешены; сейчас только read-only подготовка. Миграционная документация в репозитории только docs/joom-migration. Performance — штатные инструменты Comet; cleanup — ledger и итоговая инструкция; environment blockers сообщать пользователю без длительных обходов.
diff --git a/docs/joom-migration/approved-plan.json b/docs/joom-migration/approved-plan.json
index 63c5f653849..eb2be61527b 100644
--- a/docs/joom-migration/approved-plan.json
+++ b/docs/joom-migration/approved-plan.json
@@ -1,7 +1,7 @@
{
"date": "2026-11-16",
"status": "APPROVED_PLAN_NO_IMPLEMENTATION",
- "recommendation": "9основныхPRсрезервом10;неукрупнятьсортировку,агрегаты,SMJиокнавобщийбольшойnativePRрадиуменьшениячисла.",
+ "recommendation": "9 main PRs with a tenth in reserve; do not combine sort, aggregates, SMJ and windows into one large native PR merely to reduce the count.",
"supersedes_plan_for_review": "/Users/skotlov/d/temp/20261116_comet12-branch-release-plan/implementation-plan.json",
"refs": {
"source_tag": "joom-1.1-13",
@@ -33,15 +33,7 @@
{
"id": "G01",
"branch": "joom/1.2-dev",
- "old_work_packages": [
- "B01",
- "B03",
- "B04",
- "B05",
- "B02",
- "B06",
- "B07"
- ],
+ "old_work_packages": ["B01", "B03", "B04", "B05", "B02", "B06", "B07"],
"cards": [
"D01",
"I01",
@@ -57,94 +49,60 @@
"P03"
],
"depends_on": [],
- "scope": "База Spark/Scala, pristine DF vendor, Delta/GCP ADC/planning, release builder/publisher.",
- "internal_review_plan": "Внутри одного PR сначала Spark/Delta на штатном DF55.2; затем отдельный pristinevendor commit, rootpatch/sharedlock/explicitvendor-tests и повторная проверка; затем release tooling. Все этапы — проверяемые коммиты без Joom-алгоритмов в vendor. Старые отдельные PR объединены, проверки не сокращены.",
+ "scope": "Spark/Scala baseline, pristine DF vendor, Delta/GCP ADC/planning, release builder/publisher.",
+ "internal_review_plan": "Within one PR, first Spark/Delta on stock DF55.2; then a separate pristine vendor commit, root patch/shared lock/explicit vendor tests and repeat validation; then release tooling. All stages are reviewable commits without Joom algorithms in vendor. Earlier separate PRs are combined; checks are not reduced.",
"status": "PLAN_APPROVED_NOT_CREATED",
"internal_stages": [
{
"id": "G01.1",
- "name": "Spark/Delta на штатном DataFusion",
- "scope": "Настроить точные Spark4.2.1/Scala2.13.18 и Delta4.4.1, минимальные GCP ADC/planning адаптации; использовать штатные registry crates DataFusion55.2.0 с текущим target-lock Arrow/Parquet59.3.2. До vendoring получить результаты базовых Comet/Delta/ANSI fixtures.",
- "validation": "EffectivePOM и actualtestSparkversion; native/Delta/schema/DV/rebase/DPP базовые результаты и ожидаемые ошибки. Никаких Joom-патчей DataFusion."
+ "name": "Spark/Delta on stock DataFusion",
+ "scope": "Configure exact Spark4.2.1/Scala2.13.18 and Delta4.4.1, minimal GCP ADC/planning adaptations; use stock registry DataFusion55.2.0 crates with the current target lock's Arrow/Parquet59.3.2. Obtain baseline Comet/Delta/ANSI fixture results before vendoring.",
+ "validation": "Effective POM and actual test Spark version; baseline native/Delta/schema/DV/rebase/DPP results and expected errors. No Joom DataFusion patches."
},
{
"id": "G01.2",
- "name": "Чистый vendor отдельным коммитом",
- "scope": "Импортировать published standalone datafusion-physical-plan55.2.0 с provenance/checksum/license отдельным механическим коммитом. Не переносить старый Cargo55.1 и не наследовать Comet workspace.version1.2.0; имя/версия crate остаются datafusion-physical-plan55.2.0.",
- "validation": "Исходники соответствуют pristine55.2; Joom алгоритмический diff отсутствует."
+ "name": "Pristine vendor in a separate commit",
+ "scope": "Import the published standalone datafusion-physical-plan55.2.0 with provenance/checksum/license in a separate mechanical commit. Do not port the old Cargo55.1 or inherit Comet workspace.version1.2.0; the crate name/version remains datafusion-physical-plan55.2.0.",
+ "validation": "Sources match pristine55.2; no Joom algorithmic diff."
},
{
"id": "G01.3",
- "name": "Подключение vendor и отдельный тестовый target",
- "scope": "Настроить root patch и включить vendored crate в native workspace с явным unit/regression test target. Использовать общий native/Cargo.lock/root patch; одного membership или cargo test --manifest-path недостаточно.",
- "validation": "Проверить resolved graph: releaseComet и vendor-tests используют одну локальную physical-plan identity55.2.0, runtimeDF55.2/Arrow59.3.2. Unit-test dev-deps могут добавить lockentries, но не должны обновлять runtimegraph произвольно. CI явно выбирает package; Comet-only default-members не означают запуск vendor unit tests."
+ "name": "Connect vendor and a separate test target",
+ "scope": "Configure the root patch and include the vendored crate in the native workspace with an explicit unit/regression test target. Use shared native/Cargo.lock/root patch; membership alone or cargo test --manifest-path is insufficient.",
+ "validation": "Verify the resolved graph: release Comet and vendor tests use a single local physical-plan55.2.0 identity, runtime DF55.2/Arrow59.3.2. Unit-test dev dependencies may add lock entries, but must not arbitrarily update the runtime graph. CI explicitly selects the package; Comet-only default-members do not imply running vendor unit tests."
},
{
"id": "G01.4",
- "name": "Повторная проверка и release tooling",
- "scope": "Повторить базовые проверки после подмены registry на чистый path crate. Подготовить build-release.sh/publish-jfrog.sh, multiarch packaging/provenance; отдельно проверить upstream native-library reload.",
- "validation": "Сопоставить before/aftervendoring результаты и планы, прогнать именно vendorunit/regressions и Comet/Delta suites, проверить libcomet packaging/GLIBC/Scala. Итог G01 — внутренний RC1; полный Joom-набор ещё не перенесён."
+ "name": "Repeat validation and release tooling",
+ "scope": "Repeat baseline checks after replacing the registry crate with a pristine path crate. Prepare build-release.sh/publish-jfrog.sh, multiarch packaging/provenance; separately verify upstream native-library reload.",
+ "validation": "Compare before/after-vendoring results and plans, explicitly run vendor unit/regression tests and Comet/Delta suites, verify libcomet packaging/GLIBC/Scala. G01 produces an internal RC1; the full Joom capability set is not yet ported."
}
]
},
{
"id": "G02",
"branch": "joom/1.2-native-runtime",
- "old_work_packages": [
- "B08",
- "B09",
- "B10",
- "B11",
- "B27"
- ],
- "cards": [
- "I05",
- "I06",
- "J14",
- "J15",
- "J16",
- "J22"
- ],
- "depends_on": [
- "G01"
- ],
+ "old_work_packages": ["B08", "B09", "B10", "B11", "B27"],
+ "cards": ["I05", "I06", "J14", "J15", "J16", "J22"],
+ "depends_on": ["G01"],
"scope": "Offsets/FFI, TaskContext, block ownership, cancellation, timestamp compatibility.",
- "internal_review_plan": "Сначала shared types/offsets, затем off-threadcontext, block lifecycle, cancellation, timestamp regressions/residuals. Каждая стадия проверяется до следующей; generic ANSI arithmetic/cast gaps здесь.",
+ "internal_review_plan": "First shared types/offsets, then off-thread context, block lifecycle, cancellation, timestamp regressions/residuals. Validate each stage before the next; generic ANSI arithmetic/cast gaps belong here.",
"status": "PLAN_APPROVED_NOT_CREATED"
},
{
"id": "G03",
"branch": "joom/1.2-sort-spill-budget",
- "old_work_packages": [
- "B13",
- "B14",
- "B15"
- ],
- "cards": [
- "D02",
- "D03",
- "D04",
- "D11"
- ],
- "depends_on": [
- "G01"
- ],
- "scope": "Полный sort/wide-payload/spill/output-handoff контракт.",
- "internal_review_plan": "Раздельные коммиты дляreserve1/32, SpillWorkspace, wide sort, JVMhandoff1/4; один PR позволяет тестировать итоговыйmemorylifecycleбезпромежуточныхнесогласованныхAPI.",
+ "old_work_packages": ["B13", "B14", "B15"],
+ "cards": ["D02", "D03", "D04", "D11"],
+ "depends_on": ["G01"],
+ "scope": "Complete sort/wide-payload/spill/output-handoff contract.",
+ "internal_review_plan": "Separate commits for reserve1/32, SpillWorkspace, wide sort and JVM handoff1/4; one PR allows the final memory lifecycle to be tested without intermediate inconsistent APIs.",
"status": "PLAN_APPROVED_NOT_CREATED"
},
{
"id": "G04",
"branch": "joom/1.2-native-aggregates",
- "old_work_packages": [
- "B12",
- "B18",
- "B19",
- "B20",
- "B21",
- "B22",
- "B23"
- ],
+ "old_work_packages": ["B12", "B18", "B19", "B20", "B21", "B22", "B23"],
"cards": [
"D06",
"D07",
@@ -158,62 +116,35 @@
"N08",
"N09"
],
- "depends_on": [
- "G01"
- ],
+ "depends_on": ["G01"],
"scope": "Upstream aggregate parity, spill/offset/emit/drain, collect_list, FIRST/LAST/bool/string-value MIN_BY/MAX_BY, SortAggregate.",
- "internal_review_plan": "D08+N09остаются одним согласованным подшагом. Остальные изменения отдельными коммитами; сохранить state/overflowguards. Остаточные SUM/AVG ANSI fixes иихтесты принадлежатэтомуPR.",
+ "internal_review_plan": "D08+N09 remain a single coordinated substep. Other changes use separate commits; preserve state/overflow guards. Residual SUM/AVG ANSI fixes and their tests belong to this PR.",
"status": "PLAN_APPROVED_NOT_CREATED"
},
{
"id": "G05",
"branch": "joom/1.2-smj-filter",
- "old_work_packages": [
- "B16",
- "B17"
- ],
- "cards": [
- "D05"
- ],
- "depends_on": [
- "G01"
- ],
- "scope": "Filter-before-payload, outer-join correctness, one-side hoisting и ANSI eligibility.",
- "internal_review_plan": "Сначала filter+всеобязательныеoutercorrectnessfixes, затем hoisting сcan-throw/lazy проверками. Вобщуюветкупопадаетполныйбезопасныйблок, не временная некорректная outerjoin-версия.",
+ "old_work_packages": ["B16", "B17"],
+ "cards": ["D05"],
+ "depends_on": ["G01"],
+ "scope": "Filter-before-payload, outer-join correctness, one-side hoisting and ANSI eligibility.",
+ "internal_review_plan": "First filter plus all mandatory outer-join correctness fixes, then hoisting with can-throw/lazy checks. Integrate the complete safe block, not a temporary incorrect outer-join version.",
"status": "PLAN_APPROVED_NOT_CREATED"
},
{
"id": "G06",
"branch": "joom/1.2-native-windows",
- "old_work_packages": [
- "B24",
- "B25",
- "B26"
- ],
- "cards": [
- "N01",
- "N02",
- "N03"
- ],
- "depends_on": [
- "G01",
- "G02"
- ],
- "scope": "Spillable partition windows, tiny-partition batching и SortedWindow.",
- "internal_review_plan": "Три самостоятельных review/test этапа внутриоднойветки. OffsetBudget уже есть. Сохранить FILTER fallback, RANGE/ANSI/ordering иrequired distribution.",
+ "old_work_packages": ["B24", "B25", "B26"],
+ "cards": ["N01", "N02", "N03"],
+ "depends_on": ["G01", "G02"],
+ "scope": "Spillable partition windows, tiny-partition batching and SortedWindow.",
+ "internal_review_plan": "Three independent review/test stages within one branch. OffsetBudget already exists. Preserve FILTER fallback, RANGE/ANSI/ordering and required distribution.",
"status": "PLAN_APPROVED_NOT_CREATED"
},
{
"id": "G07",
"branch": "joom/1.2-shuffle-and-c2r",
- "old_work_packages": [
- "B28",
- "B29",
- "B30",
- "B31",
- "B32",
- "B33"
- ],
+ "old_work_packages": ["B28", "B29", "B30", "B31", "B32", "B33"],
"cards": [
"J08",
"J09",
@@ -228,62 +159,29 @@
"J26",
"J27"
],
- "depends_on": [
- "G01",
- "G02"
- ],
- "scope": "C2R borrowing/pool, BHJ/Kryo/offsets, writer optimisations/spill, direct read и coalescer.",
- "internal_review_plan": "Не один монолитный commit: C2R+pool; broadcast; writers; spillcallback; initialdirectread; coalescerJVM/native/proto. Ownershipиbufferlifetimeпроверятьпослекаждогошага. Самыйширокийinteropблок;приобъективнойнеобозримостиdiffиспользоватьрезервразделения,нежертвоватьreview.",
+ "depends_on": ["G01", "G02"],
+ "scope": "C2R borrowing/pool, BHJ/Kryo/offsets, writer optimizations/spill, direct read and coalescer.",
+ "internal_review_plan": "Not one monolithic commit: C2R+pool; broadcast; writers; spill callback; initial direct read; JVM/native/proto coalescer. Check ownership and buffer lifetime after every step. This is the broadest interop block; if the diff becomes objectively unreviewable, use the reserve split instead of sacrificing review.",
"status": "PLAN_APPROVED_NOT_CREATED"
},
{
"id": "G08",
"branch": "joom/1.2-jvm-codegen",
- "old_work_packages": [
- "B34",
- "B35",
- "B36",
- "B37"
- ],
- "cards": [
- "J05",
- "J06",
- "J07",
- "J17"
- ],
- "depends_on": [
- "G01",
- "G02"
- ],
- "scope": "Scalar runtime arguments, JVM short-circuit, UDF pools, plan/task serialization; необходимые native expression evaluation-order fixes.",
- "internal_review_plan": "JVMdispatch/пулы/serializationпоследовательнымикоммитами; shadedartifacttests. Native ANSI issue7246ширеJ06:необходимыеfixesявновыделитьотдельнымикоммитамивэтомPR,неподмешиватьисключённыйPR5889.",
+ "old_work_packages": ["B34", "B35", "B36", "B37"],
+ "cards": ["J05", "J06", "J07", "J17"],
+ "depends_on": ["G01", "G02"],
+ "scope": "Scalar runtime arguments, JVM short-circuit, UDF pools, plan/task serialization; necessary native expression evaluation-order fixes.",
+ "internal_review_plan": "Sequential commits for JVM dispatch/pools/serialization; shaded-artifact tests. Native ANSI issue7246 is broader than J06: identify necessary fixes in separate commits within this PR; do not mix in excluded PR5889.",
"status": "PLAN_APPROVED_NOT_CREATED"
},
{
"id": "G09",
"branch": "joom/1.2-cost-based-engines",
- "old_work_packages": [
- "B38",
- "B39",
- "B40"
- ],
- "cards": [
- "I07",
- "J01",
- "J02",
- "J03",
- "J04"
- ],
- "depends_on": [
- "G03",
- "G04",
- "G05",
- "G06",
- "G07",
- "G08"
- ],
- "scope": "BoundaryFormats, wide-row fallback и calibrated cost model вместе.",
- "internal_review_plan": "Общийкод переноситсясразуцелостно:нетвременного standaloneConversionCount этапабезEngineCostModel. Сохранятьитоговуютаблицутега13. Полнаяcoston/off/AQE/mixedhash/nativecoverageматрицапослевсехnativeвозможностей.",
+ "old_work_packages": ["B38", "B39", "B40"],
+ "cards": ["I07", "J01", "J02", "J03", "J04"],
+ "depends_on": ["G03", "G04", "G05", "G06", "G07", "G08"],
+ "scope": "BoundaryFormats, wide-row fallback and calibrated cost model together.",
+ "internal_review_plan": "Port the shared code as a coherent whole immediately: no temporary standaloneConversionCount stage without EngineCostModel. Preserve the final table from tag13. Run the complete cost on/off/AQE/mixed-hash/native-coverage matrix after all native capabilities.",
"status": "PLAN_APPROVED_NOT_CREATED"
}
],
@@ -291,17 +189,17 @@
"id": "G10",
"branch": "joom/1.2-release-qualification",
"status": "APPROVED_RESERVE_ONLY_IF_NEEDED",
- "scope": "Межфункциональные тесты и необходимые ограниченные интеграционные исправления с отдельнымикоммитами. Есликонтрольдиффаобъективнотребуетразделитьодиниз9блоков, резервможетбытьпереназначеннаэтотsplitпослеобсужденияконкретнойграницы. НеобязательныйпустойPRнесоздаётся."
+ "scope": "Cross-feature tests and necessary bounded integration fixes in separate commits. If controlling the diff objectively requires splitting one of the 9 blocks, the reserve may be reassigned to that split after discussing the concrete boundary. Do not create an unnecessary empty PR."
},
"maximum_planned_pr_count": 10,
"conditional_fix_ownership": {
"C01": {
"owner": "G02",
- "note": "Arithmetic/cast/division ANSI correctness; нетотдельнойветкисверхлимита."
+ "note": "Arithmetic/cast/division ANSI correctness; no extra branch beyond the limit."
},
"C02": {
"owner": "G08",
- "note": "Native evaluation order/JVMdispatch compatibility; J06 не объявлятьполнымзакрытиемissue7246."
+ "note": "Native evaluation order/JVM dispatch compatibility; do not claim J06 fully resolves issue7246."
},
"C03": {
"owner": "G04",
@@ -309,257 +207,141 @@
},
"Q01": {
"owner": "G10",
- "note": "Дополнительнаямежфункциональнаяквалификация, еслинуженотдельныйdiff."
+ "note": "Additional cross-feature qualification if a separate diff is needed."
}
},
"card_coverage": {
- "P01": [
- "G01"
- ],
- "I10": [
- "G01"
- ],
- "P03": [
- "G01"
- ],
- "D01": [
- "G01"
- ],
- "P02": [
- "G01"
- ],
- "I01": [
- "G01"
- ],
- "I02": [
- "G01"
- ],
- "I03": [
- "G01"
- ],
- "I04": [
- "G01"
- ],
- "I08": [
- "G01"
- ],
- "J18": [
- "G01"
- ],
- "I09": [
- "G01"
- ],
- "J14": [
- "G02"
- ],
- "I05": [
- "G02"
- ],
- "J16": [
- "G02"
- ],
- "J22": [
- "G02"
- ],
- "J15": [
- "G02"
- ],
- "D10": [
- "G04"
- ],
- "N06": [
- "G04"
- ],
- "D11": [
- "G03"
- ],
- "D02": [
- "G03"
- ],
- "D04": [
- "G03"
- ],
- "D03": [
- "G03"
- ],
- "D05": [
- "G05"
- ],
- "D06": [
- "G04"
- ],
- "D07": [
- "G04"
- ],
- "D08": [
- "G04"
- ],
- "N09": [
- "G04"
- ],
- "D09": [
- "G04"
- ],
- "N04": [
- "G04"
- ],
- "N05": [
- "G04"
- ],
- "N08": [
- "G04"
- ],
- "N07": [
- "G04"
- ],
- "N01": [
- "G06"
- ],
- "N02": [
- "G06"
- ],
- "N03": [
- "G06"
- ],
- "I06": [
- "G02"
- ],
- "J08": [
- "G07"
- ],
- "J09": [
- "G07"
- ],
- "J21": [
- "G07"
- ],
- "J25": [
- "G07"
- ],
- "J27": [
- "G07"
- ],
- "J10": [
- "G07"
- ],
- "J13": [
- "G07"
- ],
- "J11": [
- "G07"
- ],
- "J23": [
- "G07"
- ],
- "J12": [
- "G07"
- ],
- "J24": [
- "G07"
- ],
- "J26": [
- "G07"
- ],
- "J05": [
- "G08"
- ],
- "J06": [
- "G08"
- ],
- "J07": [
- "G08"
- ],
- "J17": [
- "G08"
- ],
- "J02": [
- "G09"
- ],
- "I07": [
- "G09"
- ],
- "J03": [
- "G09"
- ],
- "J04": [
- "G09"
- ],
- "J01": [
- "G09"
- ]
+ "P01": ["G01"],
+ "I10": ["G01"],
+ "P03": ["G01"],
+ "D01": ["G01"],
+ "P02": ["G01"],
+ "I01": ["G01"],
+ "I02": ["G01"],
+ "I03": ["G01"],
+ "I04": ["G01"],
+ "I08": ["G01"],
+ "J18": ["G01"],
+ "I09": ["G01"],
+ "J14": ["G02"],
+ "I05": ["G02"],
+ "J16": ["G02"],
+ "J22": ["G02"],
+ "J15": ["G02"],
+ "D10": ["G04"],
+ "N06": ["G04"],
+ "D11": ["G03"],
+ "D02": ["G03"],
+ "D04": ["G03"],
+ "D03": ["G03"],
+ "D05": ["G05"],
+ "D06": ["G04"],
+ "D07": ["G04"],
+ "D08": ["G04"],
+ "N09": ["G04"],
+ "D09": ["G04"],
+ "N04": ["G04"],
+ "N05": ["G04"],
+ "N08": ["G04"],
+ "N07": ["G04"],
+ "N01": ["G06"],
+ "N02": ["G06"],
+ "N03": ["G06"],
+ "I06": ["G02"],
+ "J08": ["G07"],
+ "J09": ["G07"],
+ "J21": ["G07"],
+ "J25": ["G07"],
+ "J27": ["G07"],
+ "J10": ["G07"],
+ "J13": ["G07"],
+ "J11": ["G07"],
+ "J23": ["G07"],
+ "J12": ["G07"],
+ "J24": ["G07"],
+ "J26": ["G07"],
+ "J05": ["G08"],
+ "J06": ["G08"],
+ "J07": ["G08"],
+ "J17": ["G08"],
+ "J02": ["G09"],
+ "I07": ["G09"],
+ "J03": ["G09"],
+ "J04": ["G09"],
+ "J01": ["G09"]
},
"historical_only": {
- "H01": "Отменённые доtag13правила не восстанавливаются; отдельная feature branch не нужна."
+ "H01": "Do not restore rules reverted before tag13; no separate feature branch is needed."
},
"quality_controls": [
- "40старыхпакетовсохраняютсякаквнутреннийчеклистсостава,зависимостейипроверок,анекак40PR.",
- "КаждыйкрупныйPRимеетатомарныекоммиты/смысловыеподшаги;проверкипослених,неединственныйтестовыйзапусквконце.",
- "Одиногромныйsquashcommitбезпроверяемойструктурынепредлагается;сохранятьинформативнуюисторию.",
- "Приreviewявноразделятьмеханическийvendorimportиручныеизменения;происхождение/hashприведенывреестре.",
- "ОбщиеCI/feature/runtime/ANSIgatesнеуменьшаются. Известныйunsafe-resultпутьисправляетсяилиограничиваетсядоproduction.",
- "Еслипакетобъективнонельзякачественнопроверитьвэтомразмере,неукрупнятьдальшерадилимита;показатьконкретныйdiffипредложитьразделениевпределахрезерва.",
- "НеоставлятьC01/C02/C03/Q01какскрытые4дополнительныхPR:ихвладельцыявноназначенывтаблице.",
- "G09квалифицируетсяпослевсехG03–G08. Группыидутпоосновномупоследовательномупорядку;параллельнаяработанетребуется."
+ "The 40 old packages remain an internal checklist of scope, dependencies and checks, rather than 40 PRs.",
+ "Each large PR has atomic commits/meaningful substeps, with checks after them rather than a single test run at the end.",
+ "A single enormous squash commit without reviewable structure is not proposed; retain informative history.",
+ "During review, clearly separate mechanical vendor import from manual changes; provenance/hashes are recorded in the register.",
+ "Overall CI/feature/runtime/ANSI gates are not reduced. Fix or restrict a known unsafe-result path before production.",
+ "If a package cannot objectively be reviewed well at this size, do not combine further merely to meet the limit; show the concrete diff and propose a split within the reserve.",
+ "Do not leave C01/C02/C03/Q01 as 4 hidden extra PRs: their owners are explicitly assigned in the table.",
+ "Qualify G09 after all of G03-G08. Groups follow the main sequential order; parallel implementation is not required."
],
"cost_benefit": {
"benefits": [
- "МеньшеCI/rebase/mergeцикловбезпотерисостава.",
- "УбираютсяискусственныевременныеграницыBoundaryFormats↔costmodel.",
- "Вобщуюлиниюпопадаетполнаясвязаннаявозможностьсрегрессиями."
+ "Fewer CI/rebase/merge cycles without losing scope.",
+ "Removes artificial temporary boundaries between BoundaryFormats and the cost model.",
+ "Integrates a complete related capability together with regressions."
],
"costs": [
- "Reviewкрупнееиошибкувнутриблокаискатьсложнее;поэтомунужныатомарныекоммиты/подшаги.",
- "G01содержитбольшоймеханическийvendorimport;G04/G06/G07могутдатьзначительныйdiff. Ихразмеркачественнооценитсяпофактическомуpatch,непообещанию."
+ "Review is larger and isolating a bug inside a block is harder; therefore atomic commits/substeps are needed.",
+ "G01 includes a large mechanical vendor import; G04/G06/G07 may produce substantial diffs. Assess their reviewability from the actual patch, not a promise."
]
},
"milestones": [
{
"planned_tag": "joom-1.2-1-rc1",
"after": "G01",
- "meaning": "Внутренняябазоваясборка;неполныйJoomперенос."
+ "meaning": "Internal baseline build; not the complete Joom migration."
},
{
"planned_tag": "joom-1.2-1-rc2",
"after": "G06",
- "meaning": "Общиеruntimeмеханизмы/nativeблокготовы;остальныеJVM/shuffle/plannerещёнеполны."
+ "meaning": "Shared runtime mechanisms/native block are ready; remaining JVM/shuffle/planner work is still incomplete."
},
{
"planned_tag": "joom-1.2-1-rc3",
- "after": "G09(+G10еслипотребовался)",
- "meaning": "Полныйкандидатсзакрытыми/ограниченнымиизвестнымиunsafe-resultпутями."
+ "after": "G09 (+G10 if needed)",
+ "meaning": "Complete candidate with known unsafe-result paths fixed/restricted."
},
{
"planned_tag": "joom-1.2-1",
- "after": "КвалифицированпоследнийRC",
- "meaning": "ТотжеcommitпоследнегоуспешногоRC;никакихнепроверенныхизменений."
+ "after": "Latest RC qualified",
+ "meaning": "The same commit as the latest successful RC; no unverified changes."
}
],
"unchanged_policies": [
- "ANSI=true;сравниватьSpark4.2.1без/сCometнаоднихвходах,результатыиожидаемыеошибки.",
- "Максимальноиспользоватьupstream,сохранятьвсенедостающиеJoomчасти.",
- "PR5889нетрогатьине включать.",
- "RCтегинеизменяемы;при дополнительныхпопыткахследующийномеррасходуется,поздниеплановыеномерасдвигаются.",
- "Тег/build/JFrogpublish/productionrollout—разныедействия."
+ "ANSI=true; compare Spark4.2.1 without/with Comet on identical inputs, checking results and expected errors.",
+ "Use upstream wherever possible; preserve all missing Joom parts.",
+ "Do not touch or include PR5889.",
+ "RC tags are immutable; additional attempts consume the next number, shifting later planned numbers.",
+ "Tag/build/JFrog publication/production rollout are separate actions."
],
- "authorization": "Рабочие сессии и локальные Git-операции разрешены, сеть — с условием безопасности. Последнее указание запрещает писать код до нового поручения; сейчас только документация и read-only подготовка. Push, публикация и production исключены.",
+ "authorization": "Work sessions and local Git operations are authorized; network access is subject to the safety condition. At this historical planning snapshot the latest instruction prohibited coding until a new task, so only documentation and read-only preparation were allowed. Push, publication and production were excluded. Later G01 authorizations are recorded in decisions.json.",
"approved_on": "2026-11-16",
- "approval_quote": "Согласен, зафиксируй план из 9 PR с резервом десятого.",
+ "approval_quote": "Agreed; record the plan of 9 PRs with a tenth in reserve.",
"supersedes_implementation_plan": "/Users/skotlov/d/temp/20261116_comet12-branch-release-plan/implementation-plan.json",
"step2_relationship": {
"status": "SAME_TECHNICAL_REQUIREMENTS_NEW_PR_GRANULARITY",
"owner": "G01 joom/1.2-dev",
- "superseded_requirement": "Spark/Delta/vendor/release scripts отдельными PR",
- "replacement": "Отдельные этапы и коммиты внутри G01. Технический порядок stockDF→pristinevendor→samegraphvendor-tests сохранён.",
+ "superseded_requirement": "Spark/Delta/vendor/release scripts in separate PRs",
+ "replacement": "Separate stages and commits within G01. The technical order stock DF -> pristine vendor -> same-graph vendor tests is preserved.",
"algorithm_ports": {
- "G03": "Sort/spill vendor patches вместе с Comet wiring.",
- "G04": "Aggregate vendor patches вместе с Comet functions/state guards.",
- "G05": "SMJ vendor patches вместе с eligibility/correctness tests."
+ "G03": "Sort/spill vendor patches together with Comet wiring.",
+ "G04": "Aggregate vendor patches together with Comet functions/state guards.",
+ "G05": "SMJ vendor patches together with eligibility/correctness tests."
}
},
"execution_decisions_ref": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/decisions.json",
"delivery_scope": {
"status": "APPROVED_EXPLICITLY",
- "goal": "Реализовать согласованный перенос и получить локальные JAR-артефакты Comet и Delta integration для включения пользователем в Spark4.2 image.",
+ "goal": "Implement the agreed migration and produce local Comet and Delta integration JAR artifacts for the user to include in the Spark4.2 image.",
"targets": [
"Spark4.2.1",
"Scala2.13.18",
@@ -567,23 +349,22 @@
"Delta4.4.1",
"GCP"
],
- "linux_release_architectures": [
- "amd64",
- "arm64"
- ],
- "checks": "Обязательные correctness/ANSI/regression/integration и packaging проверки сохраняются. Финальные JAR должны иметь SHA256, Git SHA и сведения о сборке. Полную проверку на production GCP jobs пользователь проведёт отдельно.",
- "user_quote": "Нам сейчас нужно сделать все задуманное, чтобы я получил jar артефакты для включения в наш spark 4.2 image. Проверять производительность я буду позже."
+ "linux_release_architectures": ["amd64", "arm64"],
+ "checks": "Mandatory correctness/ANSI/regression/integration and packaging checks remain. Final JARs must have SHA256, Git SHA and build details. The user will separately validate real production GCP jobs.",
+ "user_quote": "For now, we need to complete everything planned so that I receive JAR artifacts to include in our Spark 4.2 image. I will check performance later."
},
"performance_policy": {
"status": "APPROVED_EXPLICITLY",
- "user_quote": "ну по возможности ты ее конечно проверяй как сможешь во время реализации.",
- "during_implementation": "Существующие Scala microbenchmarks/Spark Benchmark, Rust Criterion и доступные TPC-H/DS runners в принятом стиле. При необходимости обычное расширение соответствующего benchmark; отдельную инфраструктуру ради миграции не создавать.",
- "production_benchmarks": "Полные замеры на реальных GCP jobs отложены на пользовательскую проверку и не являются текущим условием передачи локальных JAR.",
+ "user_quote": "Of course, check performance as far as you can during implementation where possible.",
+ "during_implementation": "Use existing Scala microbenchmarks/Spark Benchmark, Rust Criterion and available TPC-H/DS runners in the established style. Extend the relevant benchmark normally if necessary; do not create separate infrastructure for the migration.",
+ "production_benchmarks": "Full measurements on real GCP jobs are deferred to user validation and are not a current condition for handing over local JARs.",
"correctness_unchanged": true,
- "latest_user_quote": "Я хочу сказать, что с т.з. оценки производительности ты должен действовать точно так , как уже принято в этом проекте datafusion-comet. Не надо создавать специальных тестов, если там подобного никогда не было.",
+ "latest_user_quote": "For performance evaluation, follow exactly the established practice in the datafusion-comet project. Do not create special tests if nothing similar has existed there.",
"reference": "PERFORMANCE.md",
"correctness_tests_unchanged": true
},
"last_updated_on": "2026-11-17",
- "execution_model_ref": "decisions.json#execution_model"
+ "execution_model_ref": "decisions.json#execution_model",
+ "translation_note": "Translated from Russian on 2026-11-18 without changing the approved scope, card coverage, dependencies or milestones. User quotes are English translations. Historical execution holds describe the planning snapshot; decisions.json and STATE.json control current authorization/progress.",
+ "snapshot_note": "Approved scope snapshot; original planning statuses are historical. Track implementation in STATE.json and resolve final G01 acceptance through G01-CLOSEOUT.md, the annotated RC1 tag and delivered artifact receipts. No later group is automatically authorized."
}
diff --git a/docs/joom-migration/approved-plan.txt b/docs/joom-migration/approved-plan.txt
new file mode 100644
index 00000000000..be62408404a
--- /dev/null
+++ b/docs/joom-migration/approved-plan.txt
@@ -0,0 +1,158 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+CONSOLIDATED JOOM COMET1.2 PLAN — APPROVED
+Approval date: 2026-11-16. 9 main PRs plus a tenth in reserve. Quality takes priority over further consolidation.
+The previous 40 packages remain internal stages/commits/checks. No code was being changed at this historical planning snapshot.
+English translation finalized 2026-11-18; decisions.json and STATE.json govern later authorizations and actual progress.
+
+HOW STEP 2 FITS THE PLAN
+Step 2 is entirely inside G01 joom/1.2-dev. The previous organizational boundary of separate PRs is replaced by separate stages and commits. The order and test requirements remain.
+
+G01.1 Spark/Delta on stock DataFusion
+Configure exact Spark4.2.1/Scala2.13.18 and Delta4.4.1, minimal GCP ADC/planning adaptations; use stock registry DataFusion55.2.0 crates with the current target lock's Arrow/Parquet59.3.2. Obtain baseline Comet/Delta/ANSI fixture results before vendoring.
+Validation: Effective POM and actual test Spark version; baseline native/Delta/schema/DV/rebase/DPP results and expected errors. No Joom DataFusion patches.
+
+G01.2 Pristine vendor in a separate commit
+Import the published standalone datafusion-physical-plan55.2.0 with provenance/checksum/license in a separate mechanical commit. Do not port the old Cargo55.1 or inherit Comet workspace.version1.2.0; the crate name/version remains datafusion-physical-plan55.2.0.
+Validation: Sources match pristine55.2; no Joom algorithmic diff.
+
+G01.3 Connect vendor and a separate test target
+Configure the root patch and include the vendored crate in the native workspace with an explicit unit/regression test target. Use shared native/Cargo.lock/root patch; membership alone or cargo test --manifest-path is insufficient.
+Validation: Verify the resolved graph: release Comet and vendor tests use a single local physical-plan55.2.0 identity, runtime DF55.2/Arrow59.3.2. Unit-test dev dependencies may add lock entries, but must not arbitrarily update the runtime graph. CI explicitly selects the package; Comet-only default-members do not imply running vendor unit tests.
+
+G01.4 Repeat validation and release tooling
+Repeat baseline checks after replacing the registry crate with a pristine path crate. Prepare build-release.sh/publish-jfrog.sh, multiarch packaging/provenance; separately verify upstream native-library reload.
+Validation: Compare before/after-vendoring results and plans, explicitly run vendor unit/regression tests and Comet/Delta suites, verify libcomet packaging/GLIBC/Scala. G01 produces an internal RC1; the full Joom capability set is not yet ported.
+
+Joom algorithms in the DataFusion vendor begin only in G03 (sort), G04 (aggregates) and G05 (SMJ). G01 keeps the vendor pristine.
+
+MAIN ORDER
+
+G01 joom/1.2-dev
+Spark/Scala baseline, pristine DF vendor, Delta/GCP ADC/planning, release builder/publisher.
+Old packages: B01,B03,B04,B05,B02,B06,B07
+Cards: D01,I01,I02,I03,I04,I08,I09,I10,J18,P01,P02,P03
+Prerequisites:
+Within one PR, first Spark/Delta on stock DF55.2; then a separate pristine vendor commit, root patch/shared lock/explicit vendor tests and repeat validation; then release tooling. All stages are reviewable commits without Joom algorithms in vendor. Earlier separate PRs are combined; checks are not reduced.
+
+G02 joom/1.2-native-runtime
+Offsets/FFI, TaskContext, block ownership, cancellation, timestamp compatibility.
+Old packages: B08,B09,B10,B11,B27
+Cards: I05,I06,J14,J15,J16,J22
+Prerequisites: G01
+First shared types/offsets, then off-thread context, block lifecycle, cancellation, timestamp regressions/residuals. Validate each stage before the next; generic ANSI arithmetic/cast gaps belong here.
+
+G03 joom/1.2-sort-spill-budget
+Complete sort/wide-payload/spill/output-handoff contract.
+Old packages: B13,B14,B15
+Cards: D02,D03,D04,D11
+Prerequisites: G01
+Separate commits for reserve1/32, SpillWorkspace, wide sort and JVM handoff1/4; one PR allows the final memory lifecycle to be tested without intermediate inconsistent APIs.
+
+G04 joom/1.2-native-aggregates
+Upstream aggregate parity, spill/offset/emit/drain, collect_list, FIRST/LAST/bool/string-value MIN_BY/MAX_BY, SortAggregate.
+Old packages: B12,B18,B19,B20,B21,B22,B23
+Cards: D06,D07,D08,D09,D10,N04,N05,N06,N07,N08,N09
+Prerequisites: G01
+D08+N09 remain a single coordinated substep. Other changes use separate commits; preserve state/overflow guards. Residual SUM/AVG ANSI fixes and their tests belong to this PR.
+
+G05 joom/1.2-smj-filter
+Filter-before-payload, outer-join correctness, one-side hoisting and ANSI eligibility.
+Old packages: B16,B17
+Cards: D05
+Prerequisites: G01
+First filter plus all mandatory outer-join correctness fixes, then hoisting with can-throw/lazy checks. Integrate the complete safe block, not a temporary incorrect outer-join version.
+
+G06 joom/1.2-native-windows
+Spillable partition windows, tiny-partition batching and SortedWindow.
+Old packages: B24,B25,B26
+Cards: N01,N02,N03
+Prerequisites: G01,G02
+Three independent review/test stages within one branch. OffsetBudget already exists. Preserve FILTER fallback, RANGE/ANSI/ordering and required distribution.
+
+G07 joom/1.2-shuffle-and-c2r
+C2R borrowing/pool, BHJ/Kryo/offsets, writer optimizations/spill, direct read and coalescer.
+Old packages: B28,B29,B30,B31,B32,B33
+Cards: J08,J09,J10,J11,J12,J13,J21,J23,J24,J25,J26,J27
+Prerequisites: G01,G02
+Not one monolithic commit: C2R+pool; broadcast; writers; spill callback; initial direct read; JVM/native/proto coalescer. Check ownership and buffer lifetime after every step. This is the broadest interop block; if the diff becomes objectively unreviewable, use the reserve split instead of sacrificing review.
+
+G08 joom/1.2-jvm-codegen
+Scalar runtime arguments, JVM short-circuit, UDF pools, plan/task serialization; necessary native expression evaluation-order fixes.
+Old packages: B34,B35,B36,B37
+Cards: J05,J06,J07,J17
+Prerequisites: G01,G02
+Sequential commits for JVM dispatch/pools/serialization; shaded-artifact tests. Native ANSI issue7246 is broader than J06: identify necessary fixes in separate commits within this PR; do not mix in excluded PR5889.
+
+G09 joom/1.2-cost-based-engines
+BoundaryFormats, wide-row fallback and calibrated cost model together.
+Old packages: B38,B39,B40
+Cards: I07,J01,J02,J03,J04
+Prerequisites: G03,G04,G05,G06,G07,G08
+Port the shared code as a coherent whole immediately: no temporary standaloneConversionCount stage without EngineCostModel. Preserve the final table from tag13. Run the complete cost on/off/AQE/mixed-hash/native-coverage matrix after all native capabilities.
+
+RESERVE
+joom/1.2-release-qualification
+Cross-feature tests and necessary bounded integration fixes in separate commits. If controlling the diff objectively requires splitting one of the 9 blocks, the reserve may be reassigned to that split after discussing the concrete boundary. Do not create an unnecessary empty PR.
+
+ANSI/FIXES WITHIN THE LIMIT
+C01 -> G02: Arithmetic/cast/division ANSI correctness; no extra branch beyond the limit.
+C02 -> G08: Native evaluation order/JVM dispatch compatibility; do not claim J06 fully resolves issue7246.
+C03 -> G04: Mixed SUM/AVG, runtimecodegenfallback, AVG+LIMIT.
+Q01 -> G10: Additional cross-feature qualification if a separate diff is needed.
+
+QUALITY
+The 40 old packages remain an internal checklist of scope, dependencies and checks, rather than 40 PRs.
+Each large PR has atomic commits/meaningful substeps, with checks after them rather than a single test run at the end.
+A single enormous squash commit without reviewable structure is not proposed; retain informative history.
+During review, clearly separate mechanical vendor import from manual changes; provenance/hashes are recorded in the register.
+Overall CI/feature/runtime/ANSI gates are not reduced. Fix or restrict a known unsafe-result path before production.
+If a package cannot objectively be reviewed well at this size, do not combine further merely to meet the limit; show the concrete diff and propose a split within the reserve.
+Do not leave C01/C02/C03/Q01 as 4 hidden extra PRs: their owners are explicitly assigned in the table.
+Qualify G09 after all of G03-G08. Groups follow the main sequential order; parallel implementation is not required.
+
+BENEFITS AND COSTS OF CONSOLIDATION
+Fewer CI/rebase/merge cycles without losing scope.
+Removes artificial temporary boundaries between BoundaryFormats and the cost model.
+Integrates a complete related capability together with regressions.
+Review is larger and isolating a bug inside a block is harder; therefore atomic commits/substeps are needed.
+G01 includes a large mechanical vendor import; G04/G06/G07 may produce substantial diffs. Assess their reviewability from the actual patch, not a promise.
+
+TAGS
+joom-1.2-1-rc1; G01; Internal baseline build; not the complete Joom migration.
+joom-1.2-1-rc2; G06; Shared runtime mechanisms/native block are ready; remaining JVM/shuffle/planner work is still incomplete.
+joom-1.2-1-rc3; G09 (+G10 if needed); Complete candidate with known unsafe-result paths fixed/restricted.
+joom-1.2-1; Latest RC qualified; The same commit as the latest successful RC; no unverified changes.
+
+UNCHANGED DECISIONS
+ANSI=true; compare Spark4.2.1 without/with Comet on identical inputs, checking results and expected errors.
+Use upstream wherever possible; preserve all missing Joom parts.
+Do not touch or include PR5889.
+RC tags are immutable; additional attempts consume the next number, shifting later planned numbers.
+Tag/build/JFrog publication/production rollout are separate actions.
+
+On 2026-11-16 the user authorized all necessary local operations in the Joom Comet checkout and .git. G01-G09 implementation had not started at this historical snapshot; network scope was authorized subject to the safety condition; see decisions.json. No publication or production rollout.
+
+EXECUTION CONDITIONS UPDATE 2026-11-16
+Local changes in Comet and .git are authorized. At that snapshot the environment was being prepared; group implementation had not started. The goal is verified local Linux JARs for the Spark4.2 image. Run local benchmarks where possible; the user will run full GCP benchmarks later. The 9+1 scope, ANSI=true and correctness gates are preserved. Network scope is authorized subject to the safety condition; see decisions.json.
+
+CLARIFICATIONS 2026-11-17
+At that historical snapshot: do not write code until a new direct instruction. Work sessions and task handoff are authorized; read-only preparation only. Repository migration documentation belongs only in docs/joom-migration. Performance follows established Comet tools; cleanup uses the ledger and final guide; report environment blockers to the user without lengthy workarounds.
+
+CURRENT AUTHORIZATION NOTE 2026-11-18
+The historical code/merge/RC1 holds above are superseded for G01 only by the explicit finalization instruction in decisions.json#g01_finalization_authorization. G02-G10 remain gated. Java 21 work is on hold; artifact publication and production operations remain excluded.
diff --git a/docs/joom-migration/cleanup-ledger.json b/docs/joom-migration/cleanup-ledger.json
index 168207c2604..23b6edaeaf5 100644
--- a/docs/joom-migration/cleanup-ledger.json
+++ b/docs/joom-migration/cleanup-ledger.json
@@ -1,8 +1,8 @@
{
- "updated_on": "2026-11-17",
- "stage": "PRE_IMPLEMENTATION",
+ "updated_on": "2026-11-18T22:39:06.361313+00:00",
+ "stage": "G01_FINAL_HISTORY_AND_RC1_QUALIFICATION",
"baseline": "resource-baseline.json",
- "no_dependency_downloads_or_release_outputs_yet": true,
+ "no_dependency_downloads_or_release_outputs_yet": false,
"entries": [
{
"type": "migration_documentation",
@@ -24,9 +24,120 @@
"path": "/Users/skotlov/d/temp/20261117_comet12-session-setup",
"ownership": "TASK_CREATED",
"shared": false,
- "created_files": [
- "resource-baseline-command-results.json"
- ]
+ "created_files": ["resource-baseline-command-results.json"]
+ },
+ {
+ "type": "g01_auxiliary_outputs",
+ "path": "/Users/skotlov/d/temp/20261117_comet12-g01",
+ "ownership": "TASK_CREATED",
+ "stage": "G01",
+ "contains": "API verification, downloaded Maven POMs, build logs and selected download/fixture outputs; update after actual builds.",
+ "shared": false
+ },
+ {
+ "type": "local_spark_test_fixture",
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-spark-4.2.1",
+ "ownership": "TASK_CREATED",
+ "creation_command": "local git clone --shared from user Spark checkout; detach v4.2.1; apply generated Comet diff",
+ "shared": false,
+ "cleanup_conditions": "After Spark SQL qualification; preserve source diff in Git"
+ },
+ {
+ "type": "native_build_outputs",
+ "path": "/Users/skotlov/d/projects/datafusion-comet/native/target",
+ "ownership": "TASK_CREATED_BUILD_OUTPUT",
+ "shared": false,
+ "cleanup_conditions": "After preserving qualified release artifacts and evidence"
+ },
+ {
+ "type": "dependency_caches",
+ "paths": [
+ "/Users/skotlov/.m2",
+ "/Users/skotlov/.cargo/registry",
+ "/Users/skotlov/.cargo/git",
+ "/Users/skotlov/.sbt"
+ ],
+ "ownership": "SHARED_TOOL_CACHES",
+ "shared": true,
+ "cleanup_conditions": "Do not delete automatically; distinguish initial resource-baseline and new entries"
+ },
+ {
+ "type": "pause_checkpoint",
+ "path": "/Users/skotlov/d/temp/20261117_comet12-g01/20261117_pause-checkpoint",
+ "ownership": "TASK_CREATED",
+ "shared": false,
+ "preserve_until": "Resume verified; preserve patch/archive until code committed",
+ "contains": "Tracked binary diff, untracked source archive, Git status and file checksums"
+ },
+ {
+ "type": "official_build_tool_downloads",
+ "path": "/Users/skotlov/d/temp/20261117_comet12-g01",
+ "ownership": "TASK_CREATED",
+ "files": [
+ "protoc-30.2-linux-x86_64.zip",
+ "protoc-30.2-linux-aarch_64.zip",
+ "rustup-init.sh",
+ "protobuf-v30.2-release.json"
+ ],
+ "executed_on_host": false,
+ "note": "Official sources; archive/script hashes pinned in draft Dockerfile. Docker build not run."
+ },
+ {
+ "type": "g01_resumed_auxiliary_outputs",
+ "path": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed",
+ "ownership": "TASK_CREATED",
+ "shared": false,
+ "contains": "Sandbox probe report and resumed build/test/review evidence; update after qualification"
+ },
+ {
+ "type": "release_validator_test_models",
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release-validator-cases",
+ "ownership": "TASK_CREATED_BUILD_TEST_OUTPUT",
+ "shared": false,
+ "note": "Synthetic ELF/JAR/schema models and mock command scripts; not real release artifacts."
+ },
+ {
+ "type": "docker_builder_image",
+ "image": "comet-joom-builder-focal:1.2-arm64",
+ "id": "sha256:de016db98defac03c0715462aeddb86738cb89b44c34656d6be0ce511f082e7c",
+ "ownership": "TASK_CREATED",
+ "shared": false
+ },
+ {
+ "type": "docker_builder_image",
+ "image": "comet-joom-builder-focal:1.2-amd64",
+ "ownership": "TASK_CREATED",
+ "shared": false,
+ "id": "sha256:92564ee97ddb02f4b12cf71243936c313dd774878603221f59abaac281095a49"
+ },
+ {
+ "type": "planned_linux_release_outputs",
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1",
+ "ownership": "TASK_BUILD_OUTPUT",
+ "shared": false,
+ "preserve": "BothJARs,13-filechecksumset,commit/build manifest andqualificationevidence; cargo-target/.work separately disposable after qualification"
+ },
+ {
+ "type": "planned_docker_dependency_volumes",
+ "names": [
+ "comet-joom-1.2-cargo-arm64",
+ "comet-joom-1.2-cargo-amd64",
+ "comet-joom-1.2-cargo-git"
+ ],
+ "ownership": "TASK_CREATED_WHEN_RELEASE_STARTS",
+ "shared": false,
+ "note": "Do not confuse with preexistingSparkDocker volumes. Recordactualsize/IDs afterbuild."
+ },
+ {
+ "type": "g01_finalization",
+ "path": "/Users/skotlov/d/temp/20261118_comet12-g01-finalization",
+ "ownership": "TASK_CREATED",
+ "shared": false,
+ "contains": "Original documentation archive, history rewrite/translation verification, final RC API/logs and qualified ZIP, final integration receipt.",
+ "preserve_until": "User has retained RC1 JARs/provenance and verified final migration handoff.",
+ "cleanup_conditions": "No automatic deletion; archive and Git backup refs preserve pre-rewrite evidence.",
+ "expected_release_directory_pattern": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/gha-",
+ "expected_additional_disk": "Approximately 190 MiB downloaded ZIP plus 410 MiB unpacked qualified artifacts; builds execute on GitHub runners."
}
],
"preexisting_or_shared": {
@@ -39,10 +150,7 @@
"/Users/skotlov/d/temp/src/spark",
"/Users/skotlov/d/projects/spark"
],
- "user_installed_tools_preserve": [
- "Rust/Rustup via Homebrew",
- "SDKMAN JDKs"
- ],
+ "user_installed_tools_preserve": ["Rust/Rustup via Homebrew", "SDKMAN JDKs"],
"historical_auxiliary_directories_review_before_cleanup": [
"/Users/skotlov/d/temp/20261115_comet-upstream-audit",
"/Users/skotlov/d/temp/20261115_comet-upstream-branches-pr-audit",
@@ -73,5 +181,527 @@
"targeted_cleanup_command",
"cleanup_conditions"
],
- "final_guide": "CLEANUP.md"
+ "final_guide": "CLEANUP.md",
+ "resume_sizes_kib": {
+ "native_target": 15983300,
+ "spark_fixture": 252664,
+ "shared_m2": 6731496,
+ "shared_cargo": 820360,
+ "shared_sbt": 1020,
+ "prior_g01_aux": 88752
+ },
+ "second_pause_preserved_resources": {
+ "task_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-resumed",
+ "release_output": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1",
+ "task_container": "de92fed928af stopped deliberately; exit137",
+ "images": [
+ "comet-joom-builder-focal:1.2-arm64",
+ "comet-joom-builder-focal:1.2-amd64"
+ ],
+ "volumes": [
+ "comet-joom-1.2-cargo-arm64",
+ "comet-joom-1.2-cargo-amd64",
+ "comet-joom-1.2-cargo-git"
+ ],
+ "preserve": "All caches/partial targets for resuming. Shared Docker resources and Maven/Cargo caches are not disposable task outputs. No deletion/prune performed."
+ },
+ "final_qualification_resources": {
+ "task_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-final-qualification",
+ "release_output": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1",
+ "reusing_existing_cargo_caches_and_volumes": true,
+ "protocbridge_standard_cache": "/Users/skotlov/Library/Caches/com.thesamet.scalapb.protocbridge.protocbridge",
+ "cleanup": "Preserve shared caches; no prune/deletion."
+ },
+ "final_g01_inventory": {
+ "recorded_at_utc": "2026-11-17T14:12:24.384022+00:00",
+ "snapshot_while_builds_running": true,
+ "source_baseline": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/resource-baseline.json",
+ "source_ledger": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/cleanup-ledger.json",
+ "task_build_outputs": [
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/native/target",
+ "exists": true,
+ "allocated_kib": 27276308,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-spark-4.2.1",
+ "exists": true,
+ "allocated_kib": 841656,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1",
+ "exists": true,
+ "allocated_kib": 5760548,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1/cargo-target",
+ "exists": true,
+ "allocated_kib": 5181796,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1/.work",
+ "exists": true,
+ "allocated_kib": 0,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release-validator-cases",
+ "exists": true,
+ "allocated_kib": 1156,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/common/target",
+ "exists": true,
+ "allocated_kib": 136,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/contrib/delta-spark/target",
+ "exists": true,
+ "allocated_kib": 1592,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/spark/target",
+ "exists": true,
+ "allocated_kib": 409668,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/spark-integration/target",
+ "exists": true,
+ "allocated_kib": 36,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ }
+ ],
+ "shared_cache_usage_only": [
+ {
+ "path": "/Users/skotlov/.m2",
+ "exists": true,
+ "allocated_kib": 6922456,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/.cargo",
+ "exists": true,
+ "allocated_kib": 820828,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/.rustup",
+ "exists": true,
+ "allocated_kib": 519240,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/.sbt",
+ "exists": true,
+ "allocated_kib": 76212,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/.ivy2",
+ "exists": true,
+ "allocated_kib": 164,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/Library/Caches/Coursier",
+ "exists": true,
+ "allocated_kib": 300616,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "path": "/Users/skotlov/Library/Caches/com.thesamet.scalapb.protocbridge.protocbridge",
+ "exists": true,
+ "allocated_kib": 9236,
+ "exit_code": 0,
+ "note": null,
+ "measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ }
+ ],
+ "task_builder_images": [
+ {
+ "tag": "comet-joom-builder-focal:1.2-arm64",
+ "id": "sha256:de016db98defac03c0715462aeddb86738cb89b44c34656d6be0ce511f082e7c",
+ "architecture": "arm64",
+ "os": "linux",
+ "size_bytes": 1334036913,
+ "existed_at_baseline": false
+ },
+ {
+ "tag": "comet-joom-builder-focal:1.2-amd64",
+ "id": "sha256:92564ee97ddb02f4b12cf71243936c313dd774878603221f59abaac281095a49",
+ "architecture": "amd64",
+ "os": "linux",
+ "size_bytes": 1495106429,
+ "existed_at_baseline": false
+ }
+ ],
+ "task_dependency_volumes": [
+ {
+ "CreatedAt": "2026-11-17T04:39:33Z",
+ "Driver": "local",
+ "Labels": null,
+ "Mountpoint": "/var/lib/docker/volumes/comet-joom-1.2-cargo-arm64/_data",
+ "Name": "comet-joom-1.2-cargo-arm64",
+ "Options": null,
+ "Scope": "local",
+ "existed_at_baseline": false,
+ "usage_line": "comet-joom-1.2-cargo-arm64 0 725.4MB",
+ "usage_measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "CreatedAt": "2026-11-17T04:47:06Z",
+ "Driver": "local",
+ "Labels": null,
+ "Mountpoint": "/var/lib/docker/volumes/comet-joom-1.2-cargo-amd64/_data",
+ "Name": "comet-joom-1.2-cargo-amd64",
+ "Options": null,
+ "Scope": "local",
+ "existed_at_baseline": false,
+ "usage_line": "comet-joom-1.2-cargo-amd64 0 725.4MB",
+ "usage_measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "CreatedAt": "2026-11-17T04:39:33Z",
+ "Driver": "local",
+ "Labels": null,
+ "Mountpoint": "/var/lib/docker/volumes/comet-joom-1.2-cargo-git/_data",
+ "Name": "comet-joom-1.2-cargo-git",
+ "Options": null,
+ "Scope": "local",
+ "existed_at_baseline": false,
+ "usage_line": "comet-joom-1.2-cargo-git 0 17.98MB",
+ "usage_measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ }
+ ],
+ "docker_usage_selected_lines": [
+ "comet-joom-builder-focal 1.2-amd64 92564ee97ddb 10 hours ago 1.5GB 0B 1.495GB 1",
+ "comet-joom-builder-focal 1.2-arm64 de016db98def 10 hours ago 1.33GB 0B 1.334GB 0",
+ "comet-joom-1.2-cargo-amd64 0 725.4MB",
+ "comet-joom-1.2-cargo-git 0 17.98MB",
+ "comet-joom-1.2-cargo-arm64 0 725.4MB",
+ "Build cache usage: 3.199GB"
+ ],
+ "recommendations": {
+ "status": "INVENTORY_ONLY_NO_DELETION_PERFORMED",
+ "during_migration": "Retain native target, Spark fixture, both architecture Cargo targets, builder images and all three dependency volumes: G02-G09/RC2/RC3 can reuse them. Sizes may grow while current builds run.",
+ "preserve_release": "Retain final core and Delta JARs, both native libraries, commit.txt, build-manifest.json, effective-pom.xml, architecture metadata/toolchain/import reports, sha256.txt and smoke evidence. Back up before deleting any enclosing target directory.",
+ "after_entire_migration_only": [
+ {
+ "description": "Native host build outputs, not source/vendor",
+ "command": "rm -rf -- /Users/skotlov/d/projects/datafusion-comet/native/target"
+ },
+ {
+ "description": "Generated Spark qualification checkout; original user Spark checkout remains untouched",
+ "command": "rm -rf -- /Users/skotlov/d/projects/datafusion-comet/target/joom-spark-4.2.1"
+ },
+ {
+ "description": "Release compiler intermediates only; keeps final JARs/manifests/native files/smoke",
+ "command": "rm -rf -- /Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1/cargo-target /Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1/.work"
+ },
+ {
+ "description": "Synthetic negative-validator fixtures",
+ "command": "rm -rf -- /Users/skotlov/d/projects/datafusion-comet/target/joom-release-validator-cases"
+ },
+ {
+ "description": "Only these task-exclusive builder tags, after no task containers remain; no force and no prune",
+ "command": "docker --context desktop-linux image rm comet-joom-builder-focal:1.2-arm64 comet-joom-builder-focal:1.2-amd64"
+ },
+ {
+ "description": "Only these task-created dependency volumes, after no task containers remain; no force and no prune",
+ "command": "docker --context desktop-linux volume rm comet-joom-1.2-cargo-arm64 comet-joom-1.2-cargo-amd64 comet-joom-1.2-cargo-git"
+ }
+ ],
+ "shared_caches": "Measured usage only. No attribution of whole cache sizes to migration; do not delete or clean automatically. Preserve SDKMAN JDKs, Rustup toolchains and user-provided source checkouts.",
+ "avoid": "Do not run root mvnw clean, git clean -fdx, Docker system/builder/volume prune, or delete root target while release outputs and Spark fixture remain there.",
+ "accounting_limits": "du reports allocated KiB, may vary during active builds; nested release cargo-target/.work sizes are included in release total and must not be summed twice. Docker image Size is logical layer size and is not a guarantee of physical space reclaimed in Docker Desktop disk image. Docker Build cache is shared/unattributed.",
+ "branch_1_2_integration": "User now requires concrete preview and confirmation before ANY branch-1.2 update; no integration performed by this inventory task."
+ },
+ "measurement_history": [
+ {
+ "recorded_at_utc": "2026-11-17T13:36:32.575258+00:00",
+ "snapshot_while_builds_running": true,
+ "task_build_outputs": [
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/native/target",
+ "exists": true,
+ "allocated_kib": 27276308,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-spark-4.2.1",
+ "exists": true,
+ "allocated_kib": 689316,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1",
+ "exists": true,
+ "allocated_kib": 4612500,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1/cargo-target",
+ "exists": true,
+ "allocated_kib": 4478120,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1/.work",
+ "exists": true,
+ "allocated_kib": 0,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release-validator-cases",
+ "exists": true,
+ "allocated_kib": 1156,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/common/target",
+ "exists": true,
+ "allocated_kib": 140,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/contrib/delta-spark/target",
+ "exists": true,
+ "allocated_kib": 2276,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/spark/target",
+ "exists": true,
+ "allocated_kib": 869240,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/d/projects/datafusion-comet/spark-integration/target",
+ "exists": true,
+ "allocated_kib": 36,
+ "exit_code": 0,
+ "note": null
+ }
+ ],
+ "shared_cache_usage_only": [
+ {
+ "path": "/Users/skotlov/.m2",
+ "exists": true,
+ "allocated_kib": 6913140,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/.cargo",
+ "exists": true,
+ "allocated_kib": 820828,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/.rustup",
+ "exists": true,
+ "allocated_kib": 519240,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/.sbt",
+ "exists": true,
+ "allocated_kib": 76212,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/.ivy2",
+ "exists": true,
+ "allocated_kib": 164,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/Library/Caches/Coursier",
+ "exists": true,
+ "allocated_kib": 300616,
+ "exit_code": 0,
+ "note": null
+ },
+ {
+ "path": "/Users/skotlov/Library/Caches/com.thesamet.scalapb.protocbridge.protocbridge",
+ "exists": true,
+ "allocated_kib": 9236,
+ "exit_code": 0,
+ "note": null
+ }
+ ],
+ "task_dependency_volumes": [
+ {
+ "CreatedAt": "2026-11-17T04:39:33Z",
+ "Driver": "local",
+ "Labels": null,
+ "Mountpoint": "/var/lib/docker/volumes/comet-joom-1.2-cargo-arm64/_data",
+ "Name": "comet-joom-1.2-cargo-arm64",
+ "Options": null,
+ "Scope": "local",
+ "existed_at_baseline": false,
+ "usage_line": "comet-joom-1.2-cargo-arm64 0 725.4MB",
+ "usage_measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "CreatedAt": "2026-11-17T04:47:06Z",
+ "Driver": "local",
+ "Labels": null,
+ "Mountpoint": "/var/lib/docker/volumes/comet-joom-1.2-cargo-amd64/_data",
+ "Name": "comet-joom-1.2-cargo-amd64",
+ "Options": null,
+ "Scope": "local",
+ "existed_at_baseline": false,
+ "usage_line": "comet-joom-1.2-cargo-amd64 0 725.4MB",
+ "usage_measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ },
+ {
+ "CreatedAt": "2026-11-17T04:39:33Z",
+ "Driver": "local",
+ "Labels": null,
+ "Mountpoint": "/var/lib/docker/volumes/comet-joom-1.2-cargo-git/_data",
+ "Name": "comet-joom-1.2-cargo-git",
+ "Options": null,
+ "Scope": "local",
+ "existed_at_baseline": false,
+ "usage_line": "comet-joom-1.2-cargo-git 0 17.98MB",
+ "usage_measured_at_utc": "2026-11-17T14:12:24.384022+00:00"
+ }
+ ],
+ "docker_usage_selected_lines": [
+ "comet-joom-builder-focal 1.2-amd64 92564ee97ddb 9 hours ago 1.5GB 0B 1.495GB 0",
+ "comet-joom-builder-focal 1.2-arm64 de016db98def 10 hours ago 1.33GB 0B 1.334GB 1",
+ "comet-joom-1.2-cargo-arm64 1 725.4MB",
+ "comet-joom-1.2-cargo-amd64 0 444.7MB",
+ "comet-joom-1.2-cargo-git 1 17.98MB",
+ "Build cache usage: 3.199GB"
+ ]
+ }
+ ],
+ "snapshot_stage": "AFTER_BOTH_ARCH_RELEASE_BUILD_PASSED_DURING_AMD_PACKAGED_SMOKE",
+ "release_source_commit_reported_by_coordinator": "2b8f80b3f"
+ },
+ "final_artifacts_preserve": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/joom-1.2-1-rc1",
+ "raw_qualification_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-final-qualification",
+ "pr_preparation_auxiliary_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-pr-qualification",
+ "pr_ci_runner_fix_auxiliary_directory": {
+ "path": "/Users/skotlov/d/temp/20261117_comet12-g01-runner-fix",
+ "contents": "GitHub metadata/logs, local CI and RAT logs, formatting snapshots/receipt; prepared npm downloader was not executed and no npm package was downloaded",
+ "cleanup": "Review and remove task-created auxiliary files after migration evidence is no longer needed; preserve shared caches and user-installed Homebrew Prettier",
+ "allocated_kib_snapshot": 43888,
+ "snapshot_note": "FinalGHAmonitoring snapshot, approximately43MiB oflogs/JSON/scripts; no downloadednpm package; userinstalledHomebrewPrettierpreserved."
+ },
+ "g01_ci_corrections_resources": {
+ "directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes",
+ "ownership": "TASK_CREATED",
+ "contents": "Baseline/docsbackup, helperpatches, localregressionlogs, GitHubCIresults; standardexistingcachespreserved",
+ "cleanup": "Reviewtaskdirectoryafterqualification; donotdeleteusercheckouts/sharedcaches",
+ "planned_local_builds": [
+ "Existingnative/target incremental targetedRust tests/build",
+ "ExistingrootMaven outputs forSpark4.2.1/Delta targetedregressions",
+ "Existingtarget/joom-spark-4.2.1 targetedsbtfixtures; nofullsuite duplication"
+ ],
+ "local_validation_summary": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/local-validation.json",
+ "temporary_pristine_spark_subset": "Created for patch equality proof, then removed after validation; sources/user checkouts preserved",
+ "new_project_files": [
+ "/Users/skotlov/d/projects/datafusion-comet/dev/ci/prepare-maven-model.sh",
+ "/Users/skotlov/d/projects/datafusion-comet/dev/ci/test-maven-model.py"
+ ],
+ "status": "FINAL_GHA_PASSED_NO_NEW_RELEASE_ARTIFACTS",
+ "measured_kib": {
+ "auxiliary_directory": 21952,
+ "native_target_total_including_prior_builds": 41661508,
+ "spark_fixture_checkout_and_outputs_total": 848852
+ },
+ "followup_ci_directory": "/Users/skotlov/d/temp/20261117_comet12-g01-ci-fixes/20261117_followup-gha",
+ "additional_usage": "Existing Maven/SBT targets and standard caches for targeted Spark3.5/Scala2.12,4.1 and4.2 checks; no additional Rust/release build",
+ "large_console_streams": "Selected finished-job logs streamed with64MiB/55s cap and discarded; only small receipts/selected lines saved",
+ "measured_on": "2026-11-18T03:16:00.841276+00:00",
+ "final_gha_run": 44205897075,
+ "notes": "Small additional API/log receipts may slightly increase auxiliary size. Global tools/caches and Docker resources not modified during final monitoring."
+ },
+ "rc_workflow_resources": {
+ "directory": "/Users/skotlov/d/temp/20261118_comet12-rc-workflow",
+ "ownership": "TASK_CREATED",
+ "contents": "Beforepatch/docschecksums, helperpatches/validation, officialactionlintarchive+binary+checksums, GitHubcontrol/evidence andlaterartifactdownloads. No globaltoolsinstallation.",
+ "artifact_download_target": "target/joom-release/gha-",
+ "shared_caches": "No clearing or wholesale cleanup; m2RATplugin/additionalstandardcache mayhavegrown",
+ "status": "FINAL_ARTIFACTS_DOWNLOADED_VERIFIED",
+ "download_directory": "/Users/skotlov/d/projects/datafusion-comet/target/joom-release/gha-44388252264",
+ "unpacked_bytes": 429572457,
+ "downloaded_zip": "/Users/skotlov/d/temp/20261118_comet12-rc-workflow/qualified-artifact-44388252264.zip",
+ "downloaded_zip_bytes": 189616637,
+ "artifact_id": 14469302529,
+ "run_id": 44388252264,
+ "source_commit": "b93d553856a0339af7468bcff2070e735a1a3c71",
+ "local_docker_probes": "Existingcachedbuilderonly; --networknone; ephemeralcontainersremoved, hostGitconfigurationuntouched",
+ "tool_download": "Officialactionlint1.7.12archive/checksums/binary under task/20261118_tools; no globalinstallation",
+ "new_project_files": [
+ "/Users/skotlov/d/projects/datafusion-comet/.github/workflows/rc_build_smoke.yml",
+ "/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/native-build-provenance.py",
+ "/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/test-native-build-provenance.py",
+ "/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/verify-packaged-runtime.py",
+ "/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/test-packaged-runtime.py",
+ "/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/verify-rc-result.py",
+ "/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/test-rc-result.py",
+ "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/RC-BUILD-SMOKE.md"
+ ],
+ "allocated_kib": {
+ "auxiliary": 195040,
+ "downloaded_qualified_directory": 419620
+ }
+ }
}
diff --git a/docs/joom-migration/correctness-test-plan-ru.txt b/docs/joom-migration/correctness-test-plan-ru.txt
deleted file mode 100644
index 24651a36a1f..00000000000
--- a/docs/joom-migration/correctness-test-plan-ru.txt
+++ /dev/null
@@ -1,31 +0,0 @@
-УТВЕРЖДЁННЫЙ ПЛАН ПРОВЕРКИ КОРРЕКТНОСТИ — 2026-11-16
-
-Цель: Spark4.2.1 + JoomComet1.2 сохраняет ANSI=true и не публикует неверные результаты под видом успешно выполненной job. План принят пользователем; тестовый код и реализация сейчас не пишутся.
-
-Эталон и сравнение
-- Spark4.2.1 безComet и сComet; одинаковый snapshot входов, ANSI=true и значимые настройки.
-- Проверять значения и типы, NULL, duplicate rows, ключи и гарантированный SQL порядок. Count/checksum сами по себе недостаточны.
-- Отдельно сравнивать исходы: должен завершиться/должен упасть. Успех Comet вместо обязательной Spark ANSI-ошибки — нарушение корректности.
-- Не скрывать различия произвольными допусками/общей нормализацией. Для FIRST/LAST/MIN_BY/MAX_BY учитывать разрешённую Spark недетерминированность; фиксированные fixtures должны давать проверяемый результат.
-- Проверять initial/executed планы и фактический native execution: сравнение через случайный fallback не проверяет наш оператор.
-
-Приоритетные регрессии
-1. SUM/AVG decimal, mixed Spark-partial/native-final и обратные сочетания; сохранить overflow flags/guards и новые buffer codecs.
-2. Отдельно decimal SUM при runtime отказе Spark отcodegen: исходный NO_CODEGEN guard этот случай не закрывает.
-3. GroupeddecimalAVG сLIMIT; успешный Spark не должен превращаться в лишнюю Comet-ошибку.
-4. CAST/арифметика/деление на ноль; ленивые AND/OR/CASE/JVM-dispatch и SMJhoisting условных выражений.
-5. DATE/DECIMAL RANGE boundary арифметика custom окон и существующие fallback guards.
-6. ActiveJoom cost model/BoundaryFormats, mixed shuffle, AQE on/off, forced spill; корректность результатов и ожидаемых ошибок.
-7. Применимые известные silent-result cases независимо ANSI, включая invalidUTF8/BINARY→STRING.
-
-Условие выпуска
-- Подтверждённые несовместимости исправить или ограничить соответствующий native-путь ДО исполнения/публикации.
-- Не полагаться на post-write retry/fallback: неверные данные могут не вызвать исключения.
-- По возможности добавить детерминированные автоматические regression/differential tests в соответствующих feature PR; существенные непокрытые случаи явно описывать.
-- Финально проверить фактический shaded release artifact и целевые GCP paths. Ни зелёная job, ни зелёный CI сами по себе не доказывают эквивалентность.
-- Конечный набор тестов не даёт абсолютной гарантии отсутствия всех ошибок; цель — закрыть известные случаи и не выпускать подтверждённые небезопасные пути.
-
-Границы
-- ANSI=true не означает согласия на allowIncompatible=true.
-- PR#5889 пока не трогать и не включать; штатный fallback выбранной базы сохраняется.
-- Код, Git, кластеры, публикация и сами тесты сейчас не изменяются/не запускаются.
diff --git a/docs/joom-migration/correctness-test-plan.txt b/docs/joom-migration/correctness-test-plan.txt
new file mode 100644
index 00000000000..582c19f88ba
--- /dev/null
+++ b/docs/joom-migration/correctness-test-plan.txt
@@ -0,0 +1,58 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+APPROVED CORRECTNESS VALIDATION PLAN — 2026-11-16
+
+Goal: Spark4.2.1 + JoomComet1.2 keeps ANSI=true and does not publish incorrect results as a successful job. The user accepted the plan; no test or implementation code was being written at that historical snapshot.
+English translation finalized 2026-11-18; current execution authorization is in decisions.json and progress is in STATE.json.
+
+Reference and comparison
+- Spark4.2.1 without and with Comet; identical input snapshot, ANSI=true and significant settings.
+- Check values and types, NULLs, duplicate rows, keys and guaranteed SQL order. Count/checksum alone is insufficient.
+- Separately compare outcomes: must succeed/must fail. Comet success instead of a mandatory Spark ANSI error is a correctness violation.
+- Do not conceal differences through arbitrary tolerances/broad normalization. For FIRST/LAST/MIN_BY/MAX_BY respect Spark's permitted nondeterminism; fixed fixtures must produce a verifiable result.
+- Check initial/executed plans and actual native execution: a comparison that happens to fall back does not test our operator.
+
+Priority regressions
+1. Decimal SUM/AVG, mixed Spark-partial/native-final and reverse combinations; preserve overflow flags/guards and new buffer codecs.
+2. Separately cover decimal SUM when Spark abandons codegen at runtime: the original NO_CODEGEN guard does not cover that case.
+3. Grouped decimal AVG with LIMIT; Spark success must not turn into an additional Comet error.
+4. CAST/arithmetic/division by zero; lazy AND/OR/CASE/JVM dispatch and SMJ hoisting of conditional expressions.
+5. DATE/DECIMAL RANGE boundary arithmetic in custom windows and existing fallback guards.
+6. Active Joom cost model/BoundaryFormats, mixed shuffle, AQE on/off, forced spill; correct results and expected errors.
+7. Applicable known silent-result cases independently of ANSI, including invalid UTF8/BINARY-to-STRING.
+
+Release conditions
+- Fix confirmed incompatibilities or restrict the corresponding native path BEFORE execution/publication.
+- Do not rely on post-write retry/fallback: incorrect data may not raise an exception.
+- Where possible, add deterministic automated regression/differential tests in the corresponding feature PRs; explicitly describe significant uncovered cases.
+- Finally check the actual shaded release artifact and target GCP paths. Neither a green job nor green CI alone proves equivalence.
+- A finite test suite cannot absolutely guarantee the absence of all bugs; the aim is to cover known cases and not release confirmed unsafe paths.
+
+Boundaries
+- ANSI=true does not imply approval for allowIncompatible=true.
+- Do not touch or include PR#5889 for now; retain the selected base's standard fallback.
+- At the original historical snapshot, code, Git, clusters, publication and tests were not being changed/run.
+
+ADDENDUM APPROVED 2026-11-17
+- Local: fast checks of the changed area, focused regressions and diagnosis.
+- PR/GHA: full applicable native/vendor, Comet, Delta and Spark SQL suites on the required versions and SHA; inspect actual passed/skipped/canceled counts.
+- RC: a small smoke test of the actual finished release JARs. Overall correctness criteria/coverage are not reduced.
+- The three legacy ANSI add/subtract/multiply scenarios were enabled for execution on Spark4.2.1 native AMD64; the result was pending when this addendum was recorded. On failure, fix or narrowly fall back; do not mask it by cancellation.
+- An HDFS cluster is not part of Joom infrastructure: four HDFS-only ignored tests are not applicable; retain generic Hadoop/GCS paths.
+- At that historical point, no separate full release pipeline was to be created. The later explicitly authorized manual RC Build & Smoke workflow is recorded in decisions.json; it does not authorize artifact publication.
+Historical code-hold statements in the original snapshot do not override later G01 authorization. The 2026-11-18 instruction authorizes G01 finalization/integration/RC1 after verification. G02 and later groups still require explicit permission. Current results, including actual ANSI execution, are recorded separately in G01-results.json and STATE.json.
diff --git a/docs/joom-migration/created-files.txt b/docs/joom-migration/created-files.txt
index 780fb84a73c..d9075ca3e28 100644
--- a/docs/joom-migration/created-files.txt
+++ b/docs/joom-migration/created-files.txt
@@ -1,8 +1,214 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+/Users/skotlov/d/projects/datafusion-comet/dev/diffs/4.2.1.diff
+/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/Dockerfile
+/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/build-release.sh
+/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/publish-jfrog.sh
+/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/verify-release.py
/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/AGENTS.md
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/CHECKPOINT-VERIFICATION.json
/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/CLEANUP.md
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/G01-PR-preparation.json
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/G01-PR.md
/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/G01-preparation.json
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/G01-progress.md
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/G01-report.md
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/G01-results.json
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/G01-review-notes.json
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/HANDOFF.md
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/MANIFEST.json
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/NETWORK.md
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/PAUSED.md
/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/PERFORMANCE.md
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/README.md
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/STATE.json
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/approved-plan.txt
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/approved-plan.json
/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/cleanup-ledger.json
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/correctness-test-plan.txt
/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/created-files.txt
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/decisions.json
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/feature-register.json
/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/resource-baseline.json
-/Users/skotlov/d/temp/20261117_comet12-session-setup/resource-baseline-command-results.json
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/target-native-dependencies.json
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/vendor-provenance.json
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/Cargo.lock
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/Cargo.toml
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/Cargo.toml.orig
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/LICENSE.txt
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/NOTICE.txt
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/README.md
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/benches/aggregate_vectorized.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/benches/bounded_window.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/benches/compute_statistics.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/benches/dictionary_group_values.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/benches/hash_join_semi_anti.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/benches/multi_group_by.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/benches/partial_ordering.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/benches/sort_merge_join.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/benches/sort_preserving_merge.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/benches/spill_io.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/rustfmt.toml
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/final_table.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/ordered_final_table.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/ordered_partial_table.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/partial_reduce_table.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/partial_table.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/single_table.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/aggregate_stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/metrics.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/boolean.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/bytes.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/bytes_view.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/fixed_size_binary.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/primitive.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/multi_group_by/row_backed.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/null_builder.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/row.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/single_group_by/boolean.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/single_group_by/bytes.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/single_group_by/bytes_view.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/single_group_by/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/group_values/single_group_by/primitive.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/grouped_hash_stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/grouped_topk_stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/hash_stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/order/full.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/order/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/order/partial.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/ordered_final_stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/ordered_partial_stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/partial_reduce_stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/single_stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/skip_partial.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/topk/hash_table.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/topk/heap.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/topk/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/aggregates/topk/priority_map.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/analyze.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/async_func.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/buffer.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/coalesce/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/coalesce_batches.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/coalesce_partitions.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/column_rewriter.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/common.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/coop.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/display.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/distribution_requirements.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/empty.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/execution_plan.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/explain.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/filter.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/filter_pushdown.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/array_map.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/chain.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/cross_join.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/hash_join/exec.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/hash_join/inlist_builder.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/hash_join/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/hash_join/partitioned_hash_eval.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/hash_join/shared_bounds.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/hash_join/stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/join_filter.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/join_hash_map.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/nested_loop_join.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/piecewise_merge_join/classic_join.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/piecewise_merge_join/exec.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/piecewise_merge_join/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/piecewise_merge_join/utils.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/proto.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/sort_merge_join/bitwise_stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/sort_merge_join/exec.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/sort_merge_join/filter.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/sort_merge_join/materializing_stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/sort_merge_join/metrics.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/sort_merge_join/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/sort_merge_join/tests.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/stream_join_utils.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/symmetric_hash_join.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/test_utils.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/joins/utils.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/lib.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/limit.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/memory.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/metrics.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/operator_statistics/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/ordering.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/placeholder_row.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/projection.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/proto.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/recursive_query.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/render_tree.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/repartition/distributor_channels.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/repartition/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/scalar_subquery.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sort_pushdown.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sorts/builder.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sorts/cursor.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sorts/merge.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sorts/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sorts/multi_level_merge.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sorts/partial_sort.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sorts/partitioned_topk.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sorts/sort.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sorts/sort_preserving_merge.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sorts/stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/sorts/streaming_merge.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/spill/in_progress_spill_file.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/spill/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/spill/replayable_spill_input.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/spill/spill_manager.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/spill/spill_pool.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/statistics.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/stream.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/streaming.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/test.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/test/exec.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/topk/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/tree_node.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/union.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/unnest.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/visitor.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/windows/bounded_window_agg_exec.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/windows/mod.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/windows/proto.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/windows/utils.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/windows/window_agg_exec.rs
+/Users/skotlov/d/projects/datafusion-comet/native/vendor/datafusion-physical-plan/src/work_table.rs
+/Users/skotlov/d/projects/datafusion-comet/dev/ci/prepare-maven-model.sh
+/Users/skotlov/d/projects/datafusion-comet/dev/ci/test-maven-model.py
+
+RC Build & Smoke additions:
+/Users/skotlov/d/projects/datafusion-comet/.github/workflows/rc_build_smoke.yml
+/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/native-build-provenance.py
+/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/test-native-build-provenance.py
+/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/verify-packaged-runtime.py
+/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/test-packaged-runtime.py
+/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/verify-rc-result.py
+/Users/skotlov/d/projects/datafusion-comet/dev/release/joom/test-rc-result.py
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/RC-BUILD-SMOKE.md
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/G01-CLOSEOUT.md
+/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/G01-HISTORY.json
diff --git a/docs/joom-migration/decisions.json b/docs/joom-migration/decisions.json
index ebb17f6f701..91a5900ce7c 100644
--- a/docs/joom-migration/decisions.json
+++ b/docs/joom-migration/decisions.json
@@ -1,363 +1,269 @@
{
"date": "2026-11-16",
- "source": "Явные решения пользователя в текущем диалоге; пакет актуализирован2026-11-17. Историческое утверждение плана2026-11-16 сохранено.",
+ "source": "Explicit user decisions in the current conversation; package updated on 2026-11-17. Historical plan approval of 2026-11-16 is retained.",
"general_policy": {
"status": "APPROVED",
- "user_quote": "По максимому используем upstream реализацию и добавляем наши исправления , где имеет смысл.",
- "rule": "Максимально использовать upstream в качестве основы; сохранять все недостающие Joom-возможности, исправления и оптимизации. При потере поведения или оптимизации сохранять/адаптировать нашу часть.",
+ "user_quote": "Use upstream implementations as much as possible and add our fixes where appropriate.",
+ "rule": "Use upstream as the base wherever possible; preserve all missing Joom capabilities, fixes and optimizations. If behavior or an optimization is lost, preserve/adapt our part.",
"conditions": [
- "Функциональную эквивалентность подтвердить Joom regressions и проверкой native execution/планов.",
- "Для оптимизаций проверить memory/spill lifecycle и по возможности выполнить доступные локальные performance measurements; непроверенную производительность явно отметить. Полные GCP benchmarks отложены на пользовательскую проверку.",
- "Частичный аналог не означает полноценную замену Joom-возможности.",
- "Не исключать Joom-поведение только потому, что в upstream существует похожий механизм."
+ "Confirm functional equivalence with Joom regressions and verification of native execution/plans.",
+ "For optimizations, check memory/spill lifecycle and, where possible, run available local performance measurements; explicitly identify unverified performance. Full GCP benchmarks are deferred to user validation.",
+ "A partial analogue does not constitute a complete replacement for a Joom capability.",
+ "Do not exclude Joom behavior merely because a similar mechanism exists upstream."
]
},
"explicitly_approved": {
"I01": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать native Delta scanner Comet1.2; сохранить GCP ADC, сериализацию и защиту от planning deadlock, адаптируя недостающие Joom-особенности."
+ "decision": "Use the Comet 1.2 native Delta scanner; preserve GCP ADC, serialization and planning-deadlock protection, adapting missing Joom behavior."
},
"D10": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream Single/Final+MergeAsPartial и новые Spark aggregate-state codecs; сохранить устойчивость к spill и перенести Joom memory/offset guards, ordered drain и low-copy collect_list. Не переносить старую Joom-маршрутизацию PartialReduce без необходимости."
+ "decision": "Use upstream Single/Final+MergeAsPartial and the new Spark aggregate-state codecs; preserve spill resilience and port Joom memory/offset guards, ordered drain and low-copy collect_list. Do not port the old Joom PartialReduce routing unless needed."
},
"N08": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Сохранить upstream SortAggregate architecture и добавить недостающие Joom-оптимизации порядка/streaming и необходимые correctness guards."
+ "decision": "Keep the upstream SortAggregate architecture and add missing Joom ordering/streaming optimizations and necessary correctness guards."
},
"I06": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream timestamp producer/CASE fixes и Spark4.2 semantics; переносить нашу дополнительную нормализацию сравнений там, где она ещё необходима."
+ "decision": "Use upstream timestamp producer/CASE fixes and Spark 4.2 semantics; port our additional comparison normalization wherever it is still necessary."
},
"I07": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream native wide-decimal hashing вместо возвращения старого precision>18 запрета; проверить совместимость со Spark, включая mixed boundaries."
+ "decision": "Use upstream native wide-decimal hashing instead of restoring the old precision>18 prohibition; check compatibility with Spark, including mixed boundaries."
},
"N06": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream native string MIN/MAX; сохранить Joom cases, native coverage и недостающие guards."
+ "decision": "Use upstream native string MIN/MAX; preserve Joom cases, native coverage and missing guards."
},
"I04": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream DPP/empty-scan lifecycle; сохранить соответствующие Joom регрессии и недостающие исправления."
+ "decision": "Use the upstream DPP/empty-scan lifecycle; retain the corresponding Joom regressions and missing fixes."
},
"I05": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream recursive boolean FFI repair; сохранить дополнительные Joom nested/sliced regressions."
+ "decision": "Use the upstream recursive boolean FFI repair; retain additional Joom nested/sliced regressions."
},
"J18": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream однократную загрузку bundled native library; проверить исходный Joom reload-сценарий."
+ "decision": "Use upstream one-time loading of the bundled native library; verify the original Joom reload scenario."
},
"J21": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream нормализацию sliced broadcast offsets; сохранить Joom регрессии и добавить недостающие исправления при обнаружении потери поведения."
+ "decision": "Use upstream normalization of sliced broadcast offsets; retain Joom regressions and add missing fixes if behavior is lost."
},
"J24": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream small-batch/effective JVM shuffle batch cap; сохранить наши проверки."
+ "decision": "Use the upstream small-batch/effective JVM shuffle batch cap; retain our checks."
},
"P03": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Сохранить ANSI=true. Приоритет — отсутствие успешных jobs с некорректными результатами. Сравнить Spark4.2.1 Comet-off/on на одинаковых входах/ANSI; проверить результаты и обязательные ошибки; сохранить SUM/AVG overflow guards, исправлять несовместимости или заранее ограничивать native-путь. По возможности добавить автоматические regression/differential tests при реализации. PR#5889 в текущую миграцию не включать."
+ "decision": "Keep ANSI=true. The priority is to avoid successful jobs with incorrect results. Compare Spark 4.2.1 Comet-off/on with identical inputs/ANSI; verify results and mandatory errors; preserve SUM/AVG overflow guards, fix incompatibilities or restrict the native path before execution. Add automated regression/differential tests during implementation where possible. Do not include PR#5889 in this migration."
}
},
"covered_by_general_policy": {
"I03": {
"status": "RESOLVED_BY_APPROVED_POLICY",
- "decision": "Сохранить защиту от planning deadlock через актуальный upstream lifecycle с корректным execution partition count; старое безусловное UnknownPartitioning(0) не копировать. Проверить исходные Joom reproductions."
+ "decision": "Preserve planning-deadlock protection under the current upstream core contract: UnknownPartitioning(0) without reading perPartitionData during planning; execution obtains the actual count separately. This is a verified lifecycle adaptation, not a blind port of the old always-zero behavior.",
+ "technical_clarification": {
+ "previous_preinvestigation_wording": "Preserve planning-deadlock protection through the current upstream lifecycle with the correct execution partition count; do not copy the old unconditional UnknownPartitioning(0). Verify the original Joom reproductions.",
+ "checked_on": "2026-11-17",
+ "evidence": [
+ "upstream operators.scala:1382 forbids perPartitionData reads in outputPartitioning and prescribes Unknown0",
+ "operators.scala:1135 derives runtime count independently from perPartitionData.length",
+ "ordinary CometNativeScanExec.scala:131 and Spark4.2.1 DataSourceScanExec.scala:482 return Unknown0 for nonbucketed scans"
+ ],
+ "relative_plan_limit": "Compared with obsolete contrib Unknown1 path, a one-split Delta scan may get an additional SinglePartition exchange; not measured. Matches standard Spark and source Joom, so no agreed Joom feature is lost."
+ }
},
"J09": {
"status": "RESOLVED_BY_APPROVED_POLICY",
- "decision": "Сохранить Joom projection-instance pool поверх upstream generated-class cache, соблюдая lifetime/bounds guards. Частичный class cache сам по себе не заменяет pool; упрощение возможно только при подтверждённой эквивалентности по утверждённому правилу."
+ "decision": "Preserve the Joom projection-instance pool on top of the upstream generated-class cache, respecting lifetime/bounds guards. A partial class cache alone does not replace the pool; simplification is allowed only after equivalence is confirmed under the approved rule."
}
},
"not_decided": [],
"authorization_boundary": {
"repository_operations_authorized": true,
- "implementation_authorized": false,
+ "implementation_authorized": true,
"git_mutation_authorized": true,
"repository_scope": "/Users/skotlov/d/projects/datafusion-comet",
"git_scope": "/Users/skotlov/d/projects/datafusion-comet/.git",
- "user_quote": "Отныне ты имеешь мое разрешение делать все необходимые манипуляции в /Users/skotlov/d/projects/datafusion-comet (в т.ч. в /Users/skotlov/d/projects/datafusion-comet/.git)",
- "interpretation": "Разрешение охватывает необходимые локальные изменения репозитория, ветки, коммиты, интеграцию и локальные теги. Подготовка передачи работы не означает, что реализация G01 уже начата.",
- "implementation_status": "NOT_STARTED_USER_CODE_HOLD",
+ "user_quote": "From now on you have my permission to perform all necessary operations in /Users/skotlov/d/projects/datafusion-comet (including /Users/skotlov/d/projects/datafusion-comet/.git).",
+ "interpretation": "Permission covers necessary local repository changes, branches, commits, integration and local tags. Preparing a handoff does not mean that G01 implementation has already started.",
+ "implementation_status": "G01_IN_PROGRESS",
"network_build_dependencies_authorized": true,
"network_scope_status": "APPROVED_WITH_USER_SAFETY_CONDITION",
"github_api_read_only_audit_authorized": true,
"automatic_worker_chat_creation_authorized": true,
"automatic_cross_chat_messaging_authorized": true,
"publishing_authorized": false,
- "remote_push_status": "USER_REPORTED_403; LOCAL_WORK_ONLY",
- "note": "Git/документация и создание/ведение рабочих сессий разрешены; сеть разрешена с условием безопасности. Последнее прямое указание запрещает писать код до нового поручения. Push, публикация и production-операции исключены.",
+ "remote_push_status": "AUTHORIZED_FOR_G01_PR_PREPARATION; base publication only at unchanged SHA",
+ "note": "G01 PR preparation/push is now authorized by current request. User creates PR manually. No merge/base content change, RC1, G02, artifact publication or production operations.",
"recorded_on": "2026-11-17",
"network_approval": {
"recorded_on": "2026-11-17",
"status": "APPROVED_WITH_USER_SAFETY_CONDITION",
- "source": "Прямой ответ пользователя на request_user_input_async в координаторском чате.",
- "user_quote": "Разрешаю на всю миграцию Comet1.2/Spark4.2.1 сетевые операции, перечисленные в docs/joom-migration/NETWORK.md: Maven/SBT, Rustup/Cargo, закреплённые git-зависимости и build tools с GitHub, Docker Hub и Ubuntu packages, включая дочерние процессы внутри контейнеров. При условии, что все скачиваемое на мой ноут из сети и запускаемое - является известными и широкоиспользуемыми артефактами, в безопасности которых для моей машины не приходится сомневаться. Без push, внешней публикации и production операций.",
+ "source": "Direct user response to request_user_input_async in the coordinator chat.",
+ "user_quote": "I authorize, for the entire Comet1.2/Spark4.2.1 migration, the network operations listed in docs/joom-migration/NETWORK.md: Maven/SBT, Rustup/Cargo, pinned Git dependencies and build tools from GitHub, Docker Hub and Ubuntu packages, including child processes inside containers. This is subject to the condition that everything downloaded from the network to my laptop and executed is a known and widely used artifact whose safety for my machine is not reasonably in doubt. No push, external publication or production operations.",
"scope_document": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/NETWORK.md",
"approved_proposal_commit": "fbc250a9867515958c6a33920c1ac15b5d2d62b1",
"approved_proposal_sha256": "3bedf4a5dc5d14b1793d25c4ae9073b697a1691a614aa69494c37486bc8dac02",
- "scope": "На всю утверждённую миграцию: Maven/SBT, Rustup/Cargo, закреплённые git-зависимости и build tools с GitHub, Docker Hub и Ubuntu packages, включая дочерние процессы внутри контейнеров.",
- "user_safety_condition": "Все скачиваемые на ноутбук и запускаемые артефакты должны быть известными и широко используемыми, без обоснованных сомнений в их безопасности для машины пользователя.",
- "execution_rule": "Использовать известные зависимости выбранного upstream-проекта и официальные источники/реестры, закреплённые версии/commit и доступные publisher checksums. Перед запуском внешнего installer/build script проверить его происхождение и содержание. Наличие пакета в registry или lockfile само по себе не отменяет условие пользователя. Не добавлять и не запускать нестандартный, непроверенный или сомнительный инструмент/источник на основании одного этого разрешения; при такой необходимости согласовать отдельно.",
+ "scope": "For the entire approved migration: Maven/SBT, Rustup/Cargo, pinned Git dependencies and build tools from GitHub, Docker Hub and Ubuntu packages, including child processes inside containers.",
+ "user_safety_condition": "All artifacts downloaded to the laptop and executed must be known and widely used, with no reasonable doubts about their safety for the user's machine.",
+ "execution_rule": "Use known dependencies of the selected upstream project and official sources/registries, pinned versions/commits and available publisher checksums. Before running an external installer/build script, verify its origin and contents. Registry or lockfile inclusion alone does not override the user's condition. Do not add or run an unusual, unverified or questionable tool/source solely on the basis of this permission; obtain separate approval if needed.",
"excluded": [
"Git push",
- "Внешняя публикация артефактов/PR/images",
- "Production-операции",
- "Остальные исключения исходного NETWORK.md"
+ "External publication of artifacts/PRs/images",
+ "Production operations",
+ "Other exclusions in the original NETWORK.md"
],
"network_operations_performed_during_recording": false
},
"implementation_hold": {
"active": true,
- "user_quote": "Код не пиши до моего укзания.",
- "recorded_on": "2026-11-17",
- "allowed_now": "Документация и read-only подготовка; реализацию не начинать."
+ "applies_to_groups": [
+ "G02",
+ "G03",
+ "G04",
+ "G05",
+ "G06",
+ "G07",
+ "G08",
+ "G09",
+ "G10"
+ ],
+ "allowed_group": "G01",
+ "user_quote": "Start implementing G01 under the approved plan. Report after completion and checks. Do not proceed to G02 yet, because I will explicitly authorize each new one of the 10 groups after reviewing the previous one."
},
"session_orchestration_authorization": {
"recorded_on": "2026-11-17",
- "user_quote": "Проект Comet добавлен (/Users/skotlov/d/projects/datafusion-comet). Создавай и веди рабочие сессии по утверждённому плану, передавай им задания и уточнения, проверяй результаты.",
+ "user_quote": "The Comet project has been added (/Users/skotlov/d/projects/datafusion-comet). Create and manage work sessions under the approved plan, pass them tasks and clarifications, and check their results.",
"project_id": "79ec4996-572a-4070-96f7-2344b555eb2c",
"code_hold_preserved": true
- }
+ },
+ "implementation_authorized_groups": ["G01"],
+ "historical_scope_note": "Original stage-specific authorization/evidence retained. For current G01 finalization, g01_finalization_authorization supersedes earlier merge/tag holds; G02, Java 21, artifact publication and production remain unauthorized."
},
"ansi_policy": {
"status": "APPROVED_EXPLICITLY",
"spark_sql_ansi_enabled": true,
- "user_quote": "Давай сохраним `ansi=true`.",
- "rule": "Целевой режим Spark4.2.1 — ANSI=true. Подтверждённые несовместимости исправлять либо заранее ограничивать соответствующий native-путь; не отключать ANSI глобально для обхода расхождений."
+ "user_quote": "Let us keep `ansi=true`.",
+ "rule": "The target Spark 4.2.1 mode is ANSI=true. Fix confirmed incompatibilities or restrict the corresponding native path before execution; do not globally disable ANSI to work around differences."
},
"correctness_policy": {
"status": "APPROVED_EXPLICITLY",
- "priority": "Исключить ситуацию, когда job завершается без ошибки и публикует некорректный результат.",
- "user_quote": "Самое главное, мы тут должны сделать все, чтобы исключить ситуацию, когда джоба завершается без ошибки, но создает некорректный результат.",
+ "priority": "Prevent a job from completing without error and publishing an incorrect result.",
+ "user_quote": "Above all, we must do everything possible to prevent a job from completing without error while producing an incorrect result.",
"release_criteria": [
- "Известные небезопасные native-пути исправлены либо исключены плановым fallback до исполнения; зелёный статус job/CI сам по себе недостаточен.",
- "Сравнивать Spark4.2.1 безComet и сComet на одном snapshot входов, одинаковом ANSI=true и остальных значимых настройках.",
- "Сравнивать сами результаты, типы, NULL, duplicate rows и гарантированный SQL порядок; не ограничиваться count или отдельными суммами.",
- "Проверять обе стороны контракта: обязан завершиться/обязан упасть. Comet-success при обязательной Spark ANSI-ошибке считается нарушением корректности.",
- "Сохранять overflow detection и guards mixedSpark/native partial-final SUM/AVG, включая новый state codec.",
- "Для подтверждённой несовместимости fallback определяется до исполнения/публикации; не полагаться на retry вSpark после записи или ошибки.",
- "Проверять actual native execution/планы, чтобы fallback не маскировал отсутствие тестирования custom-пути.",
- "Не скрывать расхождения общей нормализацией, потерей duplicates или произвольными float tolerances; учитывать определённую Spark семантику и разрешённую недетерминированность."
+ "Known unsafe native paths are fixed or excluded through a planned fallback before execution; a green job/CI status alone is insufficient.",
+ "Compare Spark 4.2.1 without and with Comet on the same input snapshot, with identical ANSI=true and other significant settings.",
+ "Compare actual results, types, NULLs, duplicate rows and guaranteed SQL order; do not rely solely on counts or selected sums.",
+ "Check both sides of the contract: must succeed/must fail. Comet success instead of a mandatory Spark ANSI error is a correctness violation.",
+ "Preserve overflow detection and guards for mixed Spark/native partial-final SUM/AVG, including the new state codec.",
+ "For a confirmed incompatibility, select fallback before execution/publication; do not rely on retrying in Spark after a write or error.",
+ "Verify actual native execution/plans so that fallback does not conceal a lack of testing of the custom path.",
+ "Do not conceal differences through broad normalization, lost duplicates or arbitrary float tolerances; respect defined Spark semantics and permitted nondeterminism."
],
- "automated_regressions": "По возможности добавить автоматизированные regression/differential tests при реализации; сейчас фиксируется план без написания кода.",
+ "automated_regressions": "Where possible, add automated regression/differential tests during implementation; this historical entry records the plan without writing code.",
"priority_cases": [
- "Mixed Spark/native decimal SUM/AVG и промежуточное переполнение, forced spill и новый PartialMerge codec.",
- "Отдельно runtime codegen fallback decimal SUM; статических NO_CODEGEN guards недостаточно.",
- "GroupeddecimalAVG + LIMIT, malformedcasts, arithmeticoverflow, divisionbyzero и lazy AND/OR/CASE/JVM dispatch.",
- "SMJ hoisting: canthrow expressions/условные ветки/outer join, ANSI=true.",
- "Custom RANGE windows на DATE/DECIMAL границах и существующие fallback guards.",
- "Результаты с activeJoom cost/boundary flags, mixedshuffle, AQEon/off; проверки native-планов.",
- "Известные silent-result edgecases независимоANSI, включая BINARY→STRING/invalidUTF8, где применимо."
+ "Mixed Spark/native decimal SUM/AVG and intermediate overflow, forced spill and the new PartialMerge codec.",
+ "Separately cover decimal SUM runtime codegen fallback; static NO_CODEGEN guards are insufficient.",
+ "Grouped decimal AVG + LIMIT, malformed casts, arithmetic overflow, division by zero and lazy AND/OR/CASE/JVM dispatch.",
+ "SMJ hoisting: can-throw expressions/conditional branches/outer join, ANSI=true.",
+ "Custom RANGE windows at DATE/DECIMAL boundaries and existing fallback guards.",
+ "Results with active Joom cost/boundary flags, mixed shuffle, AQE on/off; verification of native plans.",
+ "Known silent-result edge cases independently of ANSI, including BINARY-to-STRING/invalid UTF8, where applicable."
],
- "limits": "Набор тестов не является математическим доказательством отсутствия всех ошибок. Непокрытые риски явно фиксировать; не объявлять совместимость доказанной без проверки."
+ "limits": "A test suite is not a mathematical proof of the absence of all bugs. Explicitly record uncovered risks; do not claim compatibility has been proved without verification."
},
"excluded_upstream_changes": {
"PR5889": {
"status": "EXCLUDED_FROM_CURRENT_MIGRATION",
- "user_quote": "Этот PR #5889 пока не трогаем и не включаем.",
- "rule": "Не переносить, не включать и не продолжать отдельную работу над PR#5889 без нового указания пользователя. Сохранять штатный безопасный fallback выбранной базы для неподдержанных struct scalar subqueries."
+ "user_quote": "Do not touch or include PR #5889 for now.",
+ "rule": "Do not port, include or continue separate work on PR#5889 without a new user instruction. Keep the selected base's standard safe fallback for unsupported struct scalar subqueries."
}
},
"implementation_plan": {
"status": "APPROVED_EXPLICITLY",
"date": "2026-11-16",
- "user_quote": "Согласен, зафиксируй план из 9 PR с резервом десятого.",
+ "user_quote": "Agreed; record the plan of 9 PRs with a tenth in reserve.",
"main_pr_count": 9,
"reserve_pr_count": 1,
"maximum_planned_pr_count": 10,
"path": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/approved-plan.json",
- "readable_path": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/approved-plan-ru.txt",
+ "readable_path": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/approved-plan.txt",
"step2_owner": "G01 joom/1.2-dev",
- "step2_order": "Spark4.2.1/Delta4.4.1 на registryDF55.2 → pristinevendor55.2 отдельнымкоммитом → rootpatch/sharedlock/explicitvendorunitCI → повторныепроверки и release tooling.",
+ "step2_order": "Spark4.2.1/Delta4.4.1 on registry DF55.2 -> pristine vendor55.2 in a separate commit -> root patch/shared lock/explicit vendor-unit CI -> repeat checks and release tooling.",
"step2_separate_prs_superseded": true,
"card_groups": {
- "P01": [
- "G01"
- ],
- "I10": [
- "G01"
- ],
- "P03": [
- "G01"
- ],
- "D01": [
- "G01"
- ],
- "P02": [
- "G01"
- ],
- "I01": [
- "G01"
- ],
- "I02": [
- "G01"
- ],
- "I03": [
- "G01"
- ],
- "I04": [
- "G01"
- ],
- "I08": [
- "G01"
- ],
- "J18": [
- "G01"
- ],
- "I09": [
- "G01"
- ],
- "J14": [
- "G02"
- ],
- "I05": [
- "G02"
- ],
- "J16": [
- "G02"
- ],
- "J22": [
- "G02"
- ],
- "J15": [
- "G02"
- ],
- "D10": [
- "G04"
- ],
- "N06": [
- "G04"
- ],
- "D11": [
- "G03"
- ],
- "D02": [
- "G03"
- ],
- "D04": [
- "G03"
- ],
- "D03": [
- "G03"
- ],
- "D05": [
- "G05"
- ],
- "D06": [
- "G04"
- ],
- "D07": [
- "G04"
- ],
- "D08": [
- "G04"
- ],
- "N09": [
- "G04"
- ],
- "D09": [
- "G04"
- ],
- "N04": [
- "G04"
- ],
- "N05": [
- "G04"
- ],
- "N08": [
- "G04"
- ],
- "N07": [
- "G04"
- ],
- "N01": [
- "G06"
- ],
- "N02": [
- "G06"
- ],
- "N03": [
- "G06"
- ],
- "I06": [
- "G02"
- ],
- "J08": [
- "G07"
- ],
- "J09": [
- "G07"
- ],
- "J21": [
- "G07"
- ],
- "J25": [
- "G07"
- ],
- "J27": [
- "G07"
- ],
- "J10": [
- "G07"
- ],
- "J13": [
- "G07"
- ],
- "J11": [
- "G07"
- ],
- "J23": [
- "G07"
- ],
- "J12": [
- "G07"
- ],
- "J24": [
- "G07"
- ],
- "J26": [
- "G07"
- ],
- "J05": [
- "G08"
- ],
- "J06": [
- "G08"
- ],
- "J07": [
- "G08"
- ],
- "J17": [
- "G08"
- ],
- "J02": [
- "G09"
- ],
- "I07": [
- "G09"
- ],
- "J03": [
- "G09"
- ],
- "J04": [
- "G09"
- ],
- "J01": [
- "G09"
- ]
+ "P01": ["G01"],
+ "I10": ["G01"],
+ "P03": ["G01"],
+ "D01": ["G01"],
+ "P02": ["G01"],
+ "I01": ["G01"],
+ "I02": ["G01"],
+ "I03": ["G01"],
+ "I04": ["G01"],
+ "I08": ["G01"],
+ "J18": ["G01"],
+ "I09": ["G01"],
+ "J14": ["G02"],
+ "I05": ["G02"],
+ "J16": ["G02"],
+ "J22": ["G02"],
+ "J15": ["G02"],
+ "D10": ["G04"],
+ "N06": ["G04"],
+ "D11": ["G03"],
+ "D02": ["G03"],
+ "D04": ["G03"],
+ "D03": ["G03"],
+ "D05": ["G05"],
+ "D06": ["G04"],
+ "D07": ["G04"],
+ "D08": ["G04"],
+ "N09": ["G04"],
+ "D09": ["G04"],
+ "N04": ["G04"],
+ "N05": ["G04"],
+ "N08": ["G04"],
+ "N07": ["G04"],
+ "N01": ["G06"],
+ "N02": ["G06"],
+ "N03": ["G06"],
+ "I06": ["G02"],
+ "J08": ["G07"],
+ "J09": ["G07"],
+ "J21": ["G07"],
+ "J25": ["G07"],
+ "J27": ["G07"],
+ "J10": ["G07"],
+ "J13": ["G07"],
+ "J11": ["G07"],
+ "J23": ["G07"],
+ "J12": ["G07"],
+ "J24": ["G07"],
+ "J26": ["G07"],
+ "J05": ["G08"],
+ "J06": ["G08"],
+ "J07": ["G08"],
+ "J17": ["G08"],
+ "J02": ["G09"],
+ "I07": ["G09"],
+ "J03": ["G09"],
+ "J04": ["G09"],
+ "J01": ["G09"]
},
"quality_requirements_unchanged": true,
- "no_implementation_authorization": true,
- "execution_not_started": true,
+ "no_implementation_authorization": false,
+ "execution_not_started": false,
"original_research_path": "/Users/skotlov/d/temp/20261116_comet12-consolidated-plan/consolidated-plan.json"
},
"delivery_scope": {
"status": "APPROVED_EXPLICITLY",
- "goal": "Реализовать согласованный перенос и получить локальные JAR-артефакты Comet и Delta integration для включения пользователем в Spark4.2 image.",
+ "goal": "Implement the agreed migration and produce local Comet and Delta integration JAR artifacts for the user to include in the Spark4.2 image.",
"targets": [
"Spark4.2.1",
"Scala2.13.18",
@@ -365,58 +271,214 @@
"Delta4.4.1",
"GCP"
],
- "linux_release_architectures": [
- "amd64",
- "arm64"
- ],
- "checks": "Обязательные correctness/ANSI/regression/integration и packaging проверки сохраняются. Финальные JAR должны иметь SHA256, Git SHA и сведения о сборке. Полную проверку на production GCP jobs пользователь проведёт отдельно.",
- "user_quote": "Нам сейчас нужно сделать все задуманное, чтобы я получил jar артефакты для включения в наш spark 4.2 image. Проверять производительность я буду позже."
+ "linux_release_architectures": ["amd64", "arm64"],
+ "checks": "Mandatory correctness/ANSI/regression/integration and packaging checks remain. Final JARs must have SHA256, Git SHA and build details. The user will separately validate real production GCP jobs.",
+ "user_quote": "For now, we need to complete everything planned so that I receive JAR artifacts to include in our Spark 4.2 image. I will check performance later."
},
"performance_policy": {
"status": "APPROVED_EXPLICITLY",
- "user_quote": "ну по возможности ты ее конечно проверяй как сможешь во время реализации.",
- "during_implementation": "Существующие Scala microbenchmarks/Spark Benchmark, Rust Criterion и доступные TPC-H/DS runners в принятом стиле. При необходимости обычное расширение соответствующего benchmark; отдельную инфраструктуру ради миграции не создавать.",
- "production_benchmarks": "Полные замеры на реальных GCP jobs отложены на пользовательскую проверку и не являются текущим условием передачи локальных JAR.",
+ "user_quote": "Of course, check performance as far as you can during implementation where possible.",
+ "during_implementation": "Use existing Scala microbenchmarks/Spark Benchmark, Rust Criterion and available TPC-H/DS runners in the established style. Extend the relevant benchmark normally if necessary; do not create separate infrastructure for the migration.",
+ "production_benchmarks": "Full measurements on real GCP jobs are deferred to user validation and are not a current condition for handing over local JARs.",
"correctness_unchanged": true,
- "latest_user_quote": "Я хочу сказать, что с т.з. оценки производительности ты должен действовать точно так , как уже принято в этом проекте datafusion-comet. Не надо создавать специальных тестов, если там подобного никогда не было.",
+ "latest_user_quote": "For performance evaluation, follow exactly the established practice in the datafusion-comet project. Do not create special tests if nothing similar has existed there.",
"reference": "PERFORMANCE.md",
"correctness_tests_unchanged": true
},
"last_updated_on": "2026-11-17",
"execution_documentation_policy": {
- "user_quote": "вся подобная документация для целей выполнения миграции, будет только в /Users/skotlov/d/projects/datafusion-comet/docs/joom-migration",
+ "user_quote": "All such documentation for carrying out the migration will be kept only in /Users/skotlov/d/projects/datafusion-comet/docs/joom-migration.",
"directory": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration",
- "rule": "В репозитории миграционные инструкции/решения/handoffs/прогресс/cleanup только здесь. Корневой AGENTS.md восстановлен. Raw logs/tool outputs — в датированных temp по общим правилам и в cleanup ledger."
+ "rule": "Keep repository migration instructions/decisions/handoffs/progress/cleanup only here. Root AGENTS.md has been restored. Keep raw logs/tool outputs in dated temp directories under the general rules and record them in the cleanup ledger."
},
"environment_blocker_policy": {
- "user_quote": "если ты видишь, что тебе что-то не хватает в окружении (тулов, прав,...), то ты не пытаешься это обойти трудными и долгими путями, а говоришь мне и мы решаем вопрос.",
- "rule": "Сообщать конкретный blocker; не строить длительные обходы. Обычная per-command PATH/JAVA_HOME настройка и штатный Maven wrapper допустимы."
+ "user_quote": "If you see that something is missing from the environment (tools, permissions, etc.), do not try difficult and lengthy workarounds; tell me and we will resolve it.",
+ "rule": "Report the specific blocker; do not build lengthy workarounds. Normal per-command PATH/JAVA_HOME settings and the standard Maven wrapper are allowed."
},
"cleanup_policy": {
- "user_quote": "Хотелось бы в конце получить перечень, что и куда было сохранено и создано, чтобы я потом мог почистить. Т.е. мне нужна инструкция как мне чистить более ненужное после окончания твоей работы.",
+ "user_quote": "At the end, I would like a list of what was saved and created, and where, so that I can clean it up later. I need instructions for removing what is no longer needed after your work is finished.",
"ledger": "cleanup-ledger.json",
"baseline": "resource-baseline.json",
"final_guide": "CLEANUP.md",
- "rule": "Записывать точные paths/IDs, размеры, происхождение и shared/task-owned статус; в конце точечные cleanup-команды с сохранением final JAR/checksums/build manifests."
+ "rule": "Record exact paths/IDs, sizes, origin and shared/task-owned status; finish with targeted cleanup commands while preserving final JARs/checksums/build manifests."
},
"execution_model": {
"selected_on": "2026-11-17",
"status": "SELECTED_BY_COORDINATOR_WITHIN_USER_AUTHORIZED_ORCHESTRATION",
"primary": "ONE_MAIN_CHAT_EXECUTES_AND_COORDINATES",
- "main_chat_responsibility": "Последовательная реализация G01–G09 в утверждённых feature branches, интеграция, запуск необходимых проверок, состояние/решения/cleanup учёт; единственный пишущий исполнитель checkout.",
- "internal_helpers": "Ограниченные исследования, сверка совместимости, независимый review diff/тестового покрытия. По умолчанию read-only; не запускать параллельную запись в общий checkout или все тяжёлые builds одновременно.",
- "delegation_owner": "Координатор самостоятельно выбирает задачи, границы и время делегирования, передаёт контекст и проверяет результат. Пользователь не должен управлять помощниками вручную.",
+ "main_chat_responsibility": "Sequential G01-G09 implementation in the approved feature branches, integration, necessary checks and state/decisions/cleanup records; sole writer in the checkout.",
+ "internal_helpers": "Bounded research, compatibility checks and independent review of diffs/test coverage. Read-only by default; do not write concurrently to the shared checkout or run all heavy builds simultaneously.",
+ "delegation_owner": "The coordinator independently chooses tasks, boundaries and delegation timing, passes context and checks results. The user should not have to manage helpers manually.",
"delegation_checkpoints": {
- "G01": "Dependency graph/vendor provenance, Spark/Delta compatibility и Linux JAR packaging.",
- "G02_G07": "JNI/FFI, ownership/cancellation и buffer lifetime.",
- "G03_G06": "Sort/aggregate/SMJ/window memory/spill, outer-join/window correctness и ANSI semantics.",
+ "G01": "Dependency graph/vendor provenance, Spark/Delta compatibility and Linux JAR packaging.",
+ "G02_G07": "JNI/FFI, ownership/cancellation and buffer lifetime.",
+ "G03_G06": "Sort/aggregate/SMJ/window memory/spill, outer-join/window correctness and ANSI semantics.",
"G08": "Expression evaluation order, UDF/serialization.",
- "G09_FINAL": "Полнота 58 карточек, engine boundaries/AQE, совместные regressions и артефакты."
+ "G09_FINAL": "Completeness of the 58 cards, engine boundaries/AQE, combined regressions and artifacts."
},
- "new_user_visible_chat_policy": "Создавать по решению координатора при конкретной пользе от длительной изолированной задачи/отдельного ограниченного контекста или технической необходимости передачи продолжения. Новый PR/большой diff/compaction сами по себе не требуют нового чата. Сначала сообщить причину, сохранить checkpoint (actual HEAD, tree, tests, next step), передать контекст и проверить результат; повторное разрешение на каждую сессию не требуется в существующем scope.",
- "existing_g01_preparation_chat": "Оставить idle; подготовка завершена. Реализацию в нём автоматически не начинать; использовать при реальной необходимости отдельной проверки.",
- "user_involvement": "Явное начало кода, новые решения о поведении/объёме/разрешениях и реальные environment blockers. Не повседневное распределение задач.",
+ "new_user_visible_chat_policy": "Create a new user-visible chat at the coordinator's discretion when a lengthy isolated task/separate bounded context or a technical handoff need offers concrete benefit. A new PR/large diff/compaction alone does not require a new chat. First explain the reason, save a checkpoint (actual HEAD, tree, tests, next step), transfer context and check the result; no repeated permission is needed for each session within the existing scope.",
+ "existing_g01_preparation_chat": "Leave idle; preparation is complete. Do not automatically start implementation there; use it if a genuinely separate check is needed.",
+ "user_involvement": "Explicit start of coding, new decisions about behavior/scope/permissions and actual environment blockers. Not everyday task allocation.",
"implementation_hold_preserved": true,
"approved_feature_groups_and_release_milestones_unchanged": true
+ },
+ "stage_start_policy": {
+ "explicit_user_approval_per_group": true,
+ "currently_authorized": ["G01"],
+ "groups_not_authorized": [
+ "G02",
+ "G03",
+ "G04",
+ "G05",
+ "G06",
+ "G07",
+ "G08",
+ "G09",
+ "G10"
+ ]
+ },
+ "execution_pause": {
+ "status": "G01_AMD64_QUALIFICATION_PENDING_USER_REVIEW",
+ "requested_on": "2026-11-17T05:06:02.577827+00:00",
+ "user_quote": "Let us pause again now. Save current progress so that I can restart you and resume from the stopping point.",
+ "resume_requires": "Explicit user resume; G01 only. No G02 authorization. Verify effective sandbox access.",
+ "unsandboxed_sbt_request": "Rejected by user; never executed. Do not retry without explicit consent.",
+ "previous_pause_and_resume": {
+ "status": "RESUMED_G01",
+ "requested_on": "2026-11-17",
+ "reason": "User wants scoped sandbox grants and restart instead of recurring unsandboxed approvals.",
+ "scope": "No implementation/builds until explicit resume. Documentation/checkpoint only during pause preparation.",
+ "resume": "Verify effective sandbox roots/network/socket access. Preserve existing network authorization and no-push/no-publish/no-production boundary.",
+ "resumed_on": "2026-11-17",
+ "resume_quote": "Continue G01 from /Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/PAUSED.md. First check the new permissions inside the sandbox."
+ },
+ "resumed_on": "2026-11-17T13:34:17.545289+00:00",
+ "resume_quote": "I added access to the ProtocBridge cache. Continue G01 from /Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/PAUSED.md. First check access inside the sandbox. Report after completion and checks; do not proceed to G02 yet.",
+ "historical_scope_note": "Original stage-specific authorization/evidence retained. For current G01 finalization, g01_finalization_authorization supersedes earlier merge/tag holds; G02, Java 21, artifact publication and production remain unauthorized."
+ },
+ "commit_history_cleanup": {
+ "status": "APPROVED_EXPLICITLY",
+ "user_quote": "Yes, let us at least combine the service/checkpoint commits later.",
+ "policy": "Combine migration service/checkpoint documentation commits before integration. Keep substantive implementation commits separate, including pristine vendor import. Rewrite before release provenance freeze; retain local pre-cleanup backup ref and mapping. Never move an existing release tag."
+ },
+ "integration_review_gate": {
+ "status": "APPROVED_EXPLICITLY",
+ "user_quote": "Before the final merge into branch-1.2, first show me what you are going to merge there.",
+ "rule": "Present concrete candidate commits/tree diff, tests and artifacts. Wait for user confirmation before any branch-1.2 ref update. G02 still requires separate explicit authorization.",
+ "historical_scope_note": "Original stage-specific authorization/evidence retained. For current G01 finalization, g01_finalization_authorization supersedes earlier merge/tag holds; G02, Java 21, artifact publication and production remain unauthorized."
+ },
+ "amd64_environment_limit": {
+ "user_answer": "Only the Mac is available for now; show the result with that limitation.",
+ "rule": "Show actual build/test results with AMD64 runtime gap. Do not claim botharch smoke passed or create qualified RC1. No lengthy additional emulation workarounds without new request.",
+ "historical_scope_note": "Original stage-specific authorization/evidence retained. For current G01 finalization, g01_finalization_authorization supersedes earlier merge/tag holds; G02, Java 21, artifact publication and production remain unauthorized."
+ },
+ "testing_distribution": {
+ "status": "APPROVED_EXPLICITLY",
+ "user_quote": "Agreed. Let us record this. Above all, the final correctness checks must not suffer.",
+ "local": "Fast changed-area checks, focused regressions and failure diagnosis. Do not repeat broad GHA suites locally without a concrete reason.",
+ "pr_gha": "Full applicable native/vendor, Comet, Delta and Spark SQL suites on exact target versions/SHA. Inspect actual jobs/test outcomes, including skipped/canceled.",
+ "rc": "Small smoke of actual final release JARs; retain artifact acceptance.",
+ "correctness": "No reduction in total correctness coverage; result/error parity and native execution remain required."
+ },
+ "release_infrastructure_policy": {
+ "status": "APPROVED_EXPLICITLY",
+ "user_quote": "Agreed. We definitely should not rebuild the release infrastructure.",
+ "previous_action": "Use existing native AMD64 GHA first. Do not add a standalone release producer/workflow now. Retain final JAR smoke at RC checkpoints.",
+ "action": "SUPERSEDED by explicit2026-11-18 authorization for separate manual RC Build & Smoke; artifact-only, no externaldeploy/publication."
+ },
+ "ansi_three_cases": {
+ "status": "USER_AUTHORIZED_EXECUTION_ON_AMD64_GHA",
+ "tests": [
+ "ANSI support for add",
+ "ANSI support for subtract",
+ "ANSI support for multiply"
+ ],
+ "change": "Exactly3 assume guards removed; inputs/native/error checks unchanged",
+ "result": "PASS on current HEAD26590264b, nativeAMD64 Spark4.2.1 expressions job134482241958; artifact14386009386 confirms all3 actually executed, not canceled",
+ "if_failure": "Investigate; fix or narrowly fall back rather than re-cancel silently",
+ "scope_limit": "Other upstream ANSI cancellations and full G02/G04/G08 ANSI scope remain."
+ },
+ "hdfs_scope": {
+ "user_quote": "We do not and will not have an HDFS cluster.",
+ "acceptance": "Four explicitly HDFS-cluster-only ignored tests are not applicable to Joom GCP; do not provision a cluster. Retain upstream tests and generic Hadoop/libhdfs code because it can also serve other schemes."
+ },
+ "vendor_lint_policy": {
+ "user_quote": "We will not change production vendor code merely to remove this Clippy warning.",
+ "action": "Keep production vendor pristine; strict Comet lint excludes only vendor/no-deps. Retain documented upstream deprecation warning."
+ },
+ "migration_docs_retention": {
+ "user_quote": "OK; we will decide before the stable release.",
+ "action": "Retain docs/joom-migration through migration; decide cleanup before stable, no automatic exclusion now."
+ },
+ "github_pr_preparation": {
+ "status": "AUTHORIZED_BY_CURRENT_G01_REQUEST",
+ "scope": "Prepare and push G01 feature branch; publish existing base ref at exactly unchanged SHA if absent, solely to enable PR. No merge or base content/ref update.",
+ "user_requested_actions": [
+ "Do it.",
+ "Do all of this and, above all, ensure the correctness and completeness of the resulting code and documentation."
+ ],
+ "user_followup": "I can create the PR manually if you provide the details.",
+ "pr_creation_actor": "USER_MANUAL; provide compare URL/title/body",
+ "repository": "joomcode/datafusion-comet",
+ "base": "branch-1.2",
+ "head": "joom/1.2-dev",
+ "integration_gate": "Still requires explicit user confirmation. RC1/G02 remain held.",
+ "excluded": "No artifact/JFrog/Docker publication or production operations.",
+ "historical_scope_note": "Original stage-specific authorization/evidence retained. For current G01 finalization, g01_finalization_authorization supersedes earlier merge/tag holds; G02, Java 21, artifact publication and production remain unauthorized."
+ },
+ "ci_runner_fix": {
+ "user_request": "GHA are not starting ... Can you fix this?",
+ "scope": "Replace unavailableubuntu-slim withubuntu24 in normalPRgatejobs; preservetests and base ref. Fix observedRATheaderfailuresin4migrationdocs. No releasepipeline/vendorproductionchanges."
+ },
+ "g01_ci_corrections": {
+ "status": "AUTHORIZED_EXPLICITLY",
+ "user_quote": "Fix the problems found within G01 without disabling or weakening correctness checks. Update the PR and check GHA. Do not change branch-1.2 yet or proceed to G02. Report after checks. Check GHA every 5 minutes.",
+ "scope": "CorrectobservedG01Spark4.2.1compatibility/correctness/testcontract/dependencyfailures; localtargetedchecks thenonepushandfullGHA; noG02featureports/basechanges/RCtag",
+ "success_criterion": "Correctexpectedresultsandmandatoryerrors; no suppressedorweakenedcorrectnesschecks",
+ "network": "Existingapprovedbuilddependencies+GitHubAPI/pushforPR28; no newprivatehosts orpublication"
+ },
+ "ci_fail_fast_user_policy": {
+ "recorded_on": "2026-11-18",
+ "user_quote": "Analyze all new errors. If you see a problem that will require changes and another GHA run, tell me and we will stop this build. Continuing the build is allowed only if, after it finishes, you can rerun only the failed jobs selectively.",
+ "rule": "Immediately inspect each new failure. Continue only for failures that can be rerun selectively on sameSHA. If source/CI changes and a new full run are needed, notify user immediately and stop current build instead of waiting for completion. Do not classify unknown test failure as infrastructure without evidence.",
+ "current_application": "Final run succeeded. Attempts1/2 contained only HTTP429 and dependent failures; only failed jobs rerun, no source push. Policy retained for future runs."
+ },
+ "rc_build_smoke_authorization": {
+ "status": "APPROVED_EXPLICITLY",
+ "user_quote": "Agreed. Implement a separate manual workflow called RC Build & Smoke (do not put Joom in the name), run builds and checks on both architectures, download the finished artifacts and report. Do not change branch-1.2 yet or proceed to G02.",
+ "comet_ci_policy": "Cancel any automatically started Comet CI; do not wait for a new full run. Prior runtime source qualification on26590264b retained separately.",
+ "bootstrap_authorization": "I authorize a bootstrap commit in main.",
+ "bootstrap_scope": "Only .github/workflows/rc_build_smoke.yml inorigin/main for firstworkflow_dispatch registration; never mergeG01sourceinto main orchangebranch-1.2/defaultbranch.",
+ "tag_or_publication_authorized": false,
+ "historical_scope_note": "Original stage-specific authorization/evidence retained. For current G01 finalization, g01_finalization_authorization supersedes earlier merge/tag holds; G02, Java 21, artifact publication and production remain unauthorized."
+ },
+ "translation_note": {
+ "language": "English",
+ "translated_on": "2026-11-18",
+ "user_quotes": "User quotations and answers originally written in Russian are faithful English translations, not verbatim English statements.",
+ "original_snapshot": "/Users/skotlov/d/temp/20261118_comet12-g01-finalization/documentation-before.zip",
+ "historical_entries": "Earlier authorization boundaries, pauses and holds remain as historical records. The latest g01_finalization_authorization controls G01 completion; G02-G10 still require separate explicit authorization."
+ },
+ "g01_finalization_authorization": {
+ "recorded_on": "2026-11-18",
+ "status": "APPROVED_EXPLICITLY",
+ "user_request_translation": "Do nothing further on Java 21 compatibility for now. Finish G01: finalize documentation; combine service/checkpoint commits; communicate in Russian but commit only English text, including docs/joom-migration, translating carefully without losing important meaning. Squash history before RC1 (all service/checkpoint commits; other commits at your discretion). Repeat RC Build & Smoke if needed. If there are no remaining questions or unfinished work, integrate into branch-1.2 and create joom-1.2-1-rc1. Afterwards carefully check that translation and squashing lost or damaged nothing important and that final branch-1.2 contains all required changes and documentation. Do not start G02 without explicit permission.",
+ "scope": [
+ "English-only committed documentation and commit messages",
+ "Consolidate service/checkpoint history before RC1; preserve meaningful reviewable implementation history",
+ "Repeat exact-source RC qualification after history/provenance changes where required",
+ "Integrate qualified G01 into branch-1.2 and create immutable joom-1.2-1-rc1",
+ "Verify final source/documentation completeness and correctness after translation, history rewrite and integration"
+ ],
+ "supersedes_for_G01": [
+ "Historical no-merge/no-RC1 holds in integration_review_gate, github_pr_preparation and rc_build_smoke_authorization",
+ "Historical execution pause once G01 was explicitly resumed"
+ ],
+ "remaining_boundaries": [
+ "No Java 21 compatibility work",
+ "No G02-G10 implementation without new explicit authorization",
+ "No JFrog/Maven/Docker artifact publication or production operations"
+ ],
+ "verification_required_before_integration": true
}
}
diff --git a/docs/joom-migration/feature-register.json b/docs/joom-migration/feature-register.json
index d12ede35ed9..bc4f4ed0e5c 100644
--- a/docs/joom-migration/feature-register.json
+++ b/docs/joom-migration/feature-register.json
@@ -50,16 +50,10 @@
"N08"
],
"phases": [
- [
- "0",
- "Утверждение границ и upstream-аналогов",
- [
- "H01"
- ]
- ],
+ ["0", "Approve scope boundaries and upstream equivalents", ["H01"]],
[
"1",
- "Базовая сборка, совместимость и инфраструктура проверок",
+ "Baseline build, compatibility and validation infrastructure",
[
"P01",
"I10",
@@ -79,32 +73,17 @@
],
[
"2",
- "Общие offset/lifecycle guards и ранняя проверка аналогов",
- [
- "J14",
- "J16",
- "J22",
- "J15",
- "J18",
- "J24",
- "J21",
- "J26"
- ]
+ "Shared offset/lifecycle guards and early verification of equivalents",
+ ["J14", "J16", "J22", "J15", "J18", "J24", "J21", "J26"]
],
[
"3",
- "Сортировка, spill и независимый блок SMJ",
- [
- "D11",
- "D02",
- "D04",
- "D03",
- "D05"
- ]
+ "Sorting, spill and the independent SMJ block",
+ ["D11", "D02", "D04", "D03", "D05"]
],
[
"4",
- "Агрегаты: upstream architecture и недостающие улучшения",
+ "Aggregates: upstream architecture and missing improvements",
[
"D10",
"D06",
@@ -119,81 +98,40 @@
"N07"
]
],
- [
- "5",
- "Окна",
- [
- "N01",
- "N02",
- "N03"
- ]
- ],
- [
- "6",
- "Shuffle",
- [
- "J10",
- "J11",
- "J13",
- "J23",
- "J12"
- ]
- ],
+ ["5", "Windows", ["N01", "N02", "N03"]],
+ ["6", "Shuffle", ["J10", "J11", "J13", "J23", "J12"]],
[
"7",
- "JVM/codegen, сериализация и преобразование строк",
- [
- "J05",
- "J06",
- "J07",
- "J17",
- "J08",
- "J09",
- "J25",
- "J27"
- ]
+ "JVM/codegen, serialization and row conversion",
+ ["J05", "J06", "J07", "J17", "J08", "J09", "J25", "J27"]
],
- [
- "8",
- "Выбор движков и форматов",
- [
- "J02",
- "J03",
- "J04",
- "J01"
- ]
- ]
+ ["8", "Engine and format selection", ["J02", "J03", "J04", "J01"]]
],
"entries": [
{
"id": "H01",
- "title": "Отменённые stage-wide эвристики: учёт истории без восстановления",
- "contract": "RevertIsolatedNativeOperators и UnifyStageEngines были добавлены, затем удалены до joom-1.1-13. Их отсутствие является состоянием исходного тега; действующие BoundaryFormats/cost model и wide-row fallback учтены отдельно.",
- "source_commits": [
- "2eda5358d",
- "edaf6cae3",
- "0a92f716c",
- "4f92985f5"
- ],
+ "title": "Reverted stage-wide heuristics: retain history without restoring them",
+ "contract": "RevertIsolatedNativeOperators and UnifyStageEngines were introduced and then removed before joom-1.1-13. Their absence is the state of the source tag; active BoundaryFormats/cost-model and wide-row fallback features are tracked separately.",
+ "source_commits": ["2eda5358d", "edaf6cae3", "0a92f716c", "4f92985f5"],
"source_paths": [],
- "proposed_action": "Зафиксировать как исторические изменения, отсутствующие в конечном теге. Не воскрешать их при переносе истории; это не исключение действующей возможности.",
+ "proposed_action": "Record these as historical changes absent from the final tag. Do not resurrect them when porting history; this does not exclude an active feature.",
"upstream": {
- "base": "В upstream есть собственные transition-heavy-stage правила; они не превращают удалённые Joom правила в часть тега13."
+ "base": "Upstream has its own transition-heavy-stage rules; these do not make the removed Joom rules part of tag13."
},
"risks": [
- "Механический cherry-pick всей истории способен временно вернуть отменённые правила и смешать их с новым cost model."
+ "Mechanically cherry-picking the entire history can temporarily restore reverted rules and mix them with the new cost model."
],
"dependencies": [],
"acceptance": [
- "В новой линии отсутствуют случайно восстановленные Joom UnifyStageEngines/RevertIsolatedNativeOperators."
+ "The new line contains no accidentally restored Joom UnifyStageEngines/RevertIsolatedNativeOperators."
],
- "open_decisions": "Утвердить границу состава по конечному тегу; дополнительных технических вопросов нет.",
+ "open_decisions": "Approve scope based on the final tag; there are no additional technical questions.",
"approval_status": "AWAITING_USER_DECISION",
"scope_kind": "historical_removed",
"prerequisites": [],
"phase": 0,
"requires_upstream_discussion": false,
- "decision_category": "история",
+ "decision_category": "history",
"same_pr_or_co_review": [],
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -201,7 +139,7 @@
},
{
"id": "P01",
- "title": "Точный профиль Spark 4.2.1 / Scala 2.13",
+ "title": "Exact Spark 4.2.1 / Scala 2.13 profile",
"scope": "new_target_prerequisite",
"source_commits": [],
"source_files": [
@@ -210,25 +148,25 @@
"dev/local-ci.sh",
"dev/diffs/4.2.1.diff"
],
- "contract": "Сборка и тесты используют Spark 4.2.1 вместо исходного upstream 4.2.0; фактическая версия Scala 2.13 согласована в effective POM.",
+ "contract": "Builds and tests use Spark 4.2.1 instead of the original upstream 4.2.0; the actual Scala 2.13 version is consistent in the effective POM.",
"upstream_analogue": {
- "base_1_2": "Spark4.2profileготов, ноpin4.2.0; Scala2.13.18/Java17.",
- "outside_base": "НеявляетсяотсутствующейJoomфичей; этоусловиемиграции."
+ "base_1_2": "The Spark4.2 profile is ready but pins4.2.0; Scala2.13.18/Java17.",
+ "outside_base": "This is a migration prerequisite, not a missing Joom feature."
},
- "proposed_action": "Обновить Spark pin и все inputs CI и Spark SQL suite. Создать dev/diffs/4.2.1.diff через предусмотренный процесс изменения Spark source и генерации diff.",
+ "proposed_action": "Update the Spark pin and all CI and Spark SQL suite inputs. Create dev/diffs/4.2.1.diff through the prescribed Spark-source modification and diff-generation process.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"dependencies": [],
"risks": [
- "Замены версии только в POM недостаточно: CI scripts и Spark SQL checkout могут остаться на 4.2.0.",
- "Совместимость всей сборки подтверждается компиляцией и исполнением, а не наличием профиля."
+ "Changing only the POM version is insufficient: CI scripts and the Spark SQL checkout can remain on 4.2.0.",
+ "Whole-build compatibility is demonstrated by compilation and execution, not the existence of a profile."
],
"acceptance_tests": [
"Effective POM: Spark 4.2.1, Scala 2.13.18, Java 17.",
- "Comet tests и Spark SQL matrix на точном Spark 4.2.1.",
- "Обновлённые snapshots планов получены из реального запуска."
+ "Comet tests and the Spark SQL matrix on exact Spark 4.2.1.",
+ "Updated plan snapshots obtained from actual execution."
],
- "user_decision": "Цель Spark 4.2.1 уже согласована; дополнительных продуктовых вопросов нет.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "The Spark 4.2.1 target is already agreed; no additional product questions remain.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 10,
"source_paths": [
"pom.xml",
@@ -237,37 +175,35 @@
"dev/diffs/4.2.1.diff"
],
"upstream": {
- "base_1_2": "Spark4.2profileготов, ноpin4.2.0; Scala2.13.18/Java17.",
- "outside_base": "НеявляетсяотсутствующейJoomфичей; этоусловиемиграции."
+ "base_1_2": "The Spark4.2 profile is ready but pins4.2.0; Scala2.13.18/Java17.",
+ "outside_base": "This is a migration prerequisite, not a missing Joom feature."
},
"acceptance": [
"Effective POM: Spark 4.2.1, Scala 2.13.18, Java 17.",
- "Comet tests и Spark SQL matrix на точном Spark 4.2.1.",
- "Обновлённые snapshots планов получены из реального запуска."
+ "Comet tests and the Spark SQL matrix on exact Spark 4.2.1.",
+ "Updated plan snapshots obtained from actual execution."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"original_dependency_notes": [],
"scope_kind": "new_target_work",
"prerequisites": [],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G01"
- ],
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "approved_decision": "Обновить Spark pin и все inputs CI и Spark SQL suite. Создать dev/diffs/4.2.1.diff через предусмотренный процесс изменения Spark source и генерации diff.",
+ "approved_decision": "Update the Spark pin and all CI and Spark SQL suite inputs. Create dev/diffs/4.2.1.diff through the prescribed Spark-source modification and diff-generation process.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Цель Spark 4.2.1 уже согласована; дополнительных продуктовых вопросов нет.",
+ "prior_open_decisions": "The Spark 4.2.1 target is already agreed; no additional product questions remain.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "I10",
- "title": "Совместимость Scala/Rust и включение Joom-тестов в CI",
+ "title": "Scala/Rust compatibility and inclusion of Joom tests in CI",
"scope": "existing_joom_supporting_changes",
"source_commits": [
"1a52c8ca6",
@@ -287,29 +223,26 @@
"native/core/src/execution/memory_pools/unified_pool.rs",
"native/spark-expr/src/conversion_funcs/numeric.rs"
],
- "contract": "Все активные Joom-возможности компилируются на целевых Scala 2.13 и Rust; все их регрессии действительно запускаются в CI. Изменения fakeSpark/cfg для тестов и style fixes не являются отдельными SQL-возможностями.",
+ "contract": "All active Joom features compile on the target Scala 2.13 and Rust; all their regressions actually run in CI. Test-related fakeSpark/cfg changes and style fixes are not separate SQL features.",
"upstream_analogue": {
- "base_1_2": "Частьupstreamтеперьизмененаиначе; targetужеScala2.13forSpark4.2, нонашиновыесuitesещёнадоподключить.",
- "outside_base": "Historyreverts/stylecommitsнепереносятсяслепымcherrypick; сохранитьфинальныйсмысл."
+ "base_1_2": "Some upstream changes now take a different approach; the target already uses Scala2.13 for Spark4.2, but our new suites still need to be wired in.",
+ "outside_base": "Do not blindly cherry-pick historical reverts/style commits; preserve their final intent."
},
- "proposed_action": "Адаптировать совместимость по фактическим compile/lint ошибкам в соответствующих feature PR. Явно обеспечить тесты Spark 4.2.1, native Delta и vendor; списки CI suites должны включать все новые регрессии.",
+ "proposed_action": "Adapt compatibility to actual compile/lint errors in the corresponding feature PRs. Explicitly provide Spark 4.2.1, native Delta and vendor tests; CI suite lists must include all new regressions.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "dependencies": [
- "P01",
- "allfeaturePRs"
- ],
+ "dependencies": ["P01", "allfeaturePRs"],
"risks": [
- "Зелёный upstream PR CI по умолчанию не доказывает прохождение Spark 4.2 lane.",
- "В старом Joom workspace vendor исключён; обычный cargo test не запускает его внутренние unit-тесты.",
- "Нельзя отключать тесты ради устранения ошибок компиляции Scala 2.13."
+ "A green default upstream PR CI does not demonstrate that the Spark 4.2 lane passes.",
+ "Vendor is excluded in the old Joom workspace; ordinary cargo test does not run its internal unit tests.",
+ "Do not disable tests to resolve Scala 2.13 compilation errors."
],
"acceptance_tests": [
- "Проверить полный состав Joom suites и правила их выбора CI.",
- "Comet Rust/JVM tests, native Delta и отдельный vendor target на точном Spark 4.2.1.",
- "Spark SQL target 4.2.1; изменения dev/diffs генерировать через Spark checkout по установленной процедуре."
+ "Check the complete Joom suite inventory and CI selection rules.",
+ "Comet Rust/JVM tests, native Delta and a separate vendor target on exact Spark 4.2.1.",
+ "Spark SQL target 4.2.1; generate dev/diffs changes through a Spark checkout using the established procedure."
],
- "user_decision": "Утвердить как обязательную проверочную часть каждого PR, без исключения доработок из-за сложности тестирования.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "Approve this as a mandatory validation part of every PR, without excluding improvements because they are difficult to test.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 12,
"source_paths": [
".github/workflows/ci.yml",
@@ -321,42 +254,35 @@
"native/spark-expr/src/conversion_funcs/numeric.rs"
],
"upstream": {
- "base_1_2": "Частьupstreamтеперьизмененаиначе; targetужеScala2.13forSpark4.2, нонашиновыесuitesещёнадоподключить.",
- "outside_base": "Historyreverts/stylecommitsнепереносятсяслепымcherrypick; сохранитьфинальныйсмысл."
+ "base_1_2": "Some upstream changes now take a different approach; the target already uses Scala2.13 for Spark4.2, but our new suites still need to be wired in.",
+ "outside_base": "Do not blindly cherry-pick historical reverts/style commits; preserve their final intent."
},
"acceptance": [
- "Проверить полный состав Joom suites и правила их выбора CI.",
- "Comet Rust/JVM tests, native Delta и отдельный vendor target на точном Spark 4.2.1.",
- "Spark SQL target 4.2.1; изменения dev/diffs генерировать через Spark checkout по установленной процедуре."
- ],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
- "original_dependency_notes": [
- "P01",
- "allfeaturePRs"
+ "Check the complete Joom suite inventory and CI selection rules.",
+ "Comet Rust/JVM tests, native Delta and a separate vendor target on exact Spark 4.2.1.",
+ "Spark SQL target 4.2.1; generate dev/diffs changes through a Spark checkout using the established procedure."
],
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
+ "original_dependency_notes": ["P01", "allfeaturePRs"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G01"
- ],
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "approved_decision": "Адаптировать совместимость по фактическим compile/lint ошибкам в соответствующих feature PR. Явно обеспечить тесты Spark 4.2.1, native Delta и vendor; списки CI suites должны включать все новые регрессии.",
+ "approved_decision": "Adapt compatibility to actual compile/lint errors in the corresponding feature PRs. Explicitly provide Spark 4.2.1, native Delta and vendor tests; CI suite lists must include all new regressions.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Утвердить как обязательную проверочную часть каждого PR, без исключения доработок из-за сложности тестирования.",
+ "prior_open_decisions": "Approve this as a mandatory validation part of every PR, without excluding improvements because they are difficult to test.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "P03",
- "title": "Проверка семантики и fallback на Spark 4.2.1",
+ "title": "Verify semantics and fallback on Spark 4.2.1",
"scope": "new_target_validation_gate",
"source_commits": [],
"source_files": [
@@ -366,39 +292,34 @@
"native/spark-expr/src/datetime_funcs/timestamp_trunc.rs",
"spark/src/test/scala/org/apache/comet/CometAnsiExpressionSuite.scala"
],
- "contract": "Целевой Spark4.2.1 работает с ANSI=true. Сохраняются upstream/Joom correctness guards и безопасные fallback; successful job не должна маскировать неверный результат или отсутствие обязательной ANSI-ошибки.",
+ "contract": "The target Spark4.2.1 runs with ANSI=true. Preserve upstream/Joom correctness guards and safe fallbacks; a successful job must not conceal an incorrect result or a missing mandatory ANSI error.",
"upstream_analogue": {
- "base_1_2": "Уже есть SPARK-58821 sequence и SPARK-60118 string→decimal (#7263), SPARK-59602 signed-zero arrays (#6989), date_trunc overflow (#6740), FILTER window fallback (#4950). Struct scalar subquery остаётся ограничен; ряд ANSI overflow тестов пропускается для 4.2. Обновлённая branch-1.2 содержит #7489 (версионная семантика float collect_set), #7510 (required child distribution), #7512 (arrays_overlap scalar-column).",
- "outside_base": "PR#5889 пользователем явно исключён из текущей миграции: не трогать и не включать. Для unsupported struct scalar subqueries сохраняется fallback выбранной базы."
+ "base_1_2": "Already includes SPARK-58821 sequence and SPARK-60118 string→decimal (#7263), SPARK-59602 signed-zero arrays (#6989), date_trunc overflow (#6740), FILTER window fallback (#4950). Struct scalar subqueries remain limited; some ANSI overflow tests are skipped for4.2. The refreshed branch-1.2 contains #7489 (version-specific float collect_set semantics), #7510 (required child distribution), #7512 (arrays_overlap scalar-column).",
+ "outside_base": "The user explicitly excluded PR#5889 from this migration: do not touch or include it. Retain the selected base fallback for unsupported struct scalar subqueries."
},
- "proposed_action": "Сохранить ANSI=true. Приоритет — отсутствие успешных jobs с некорректными результатами. Сравнить Spark4.2.1 Comet-off/on на одинаковых входах/ANSI; проверить результаты и обязательные ошибки; сохранить SUM/AVG overflow guards, исправлять несовместимости или заранее ограничивать native-путь. По возможности добавить автоматические regression/differential tests при реализации. PR#5889 в текущую миграцию не включать.",
+ "proposed_action": "Keep ANSI=true. Prioritize preventing successful jobs with incorrect results. Compare Spark4.2.1 Comet-off/on with identical inputs/ANSI; verify results and mandatory errors; preserve SUM/AVG overflow guards, fix incompatibilities or restrict the native path before execution. Add automated regression/differential tests during implementation where possible. Do not include PR#5889 in this migration.",
"approval_status": "APPROVED_EXPLICITLY",
- "dependencies": [
- "P01",
- "customwindows",
- "cost/boundaryrules",
- "I06"
- ],
+ "dependencies": ["P01", "customwindows", "cost/boundaryrules", "I06"],
"risks": [
- "FILTER в window aggregate должен оставаться fallback, пока отдельная поддержка не согласована и не реализована.",
- "Merged struct scalar subqueries могут снижать native coverage в обычном SQL, включая UNION.",
- "Расхождения ANSI overflow нельзя считать только различием текста ошибки. У Spark 4 ANSI включён по умолчанию, у 3.5 выключен.",
- "Custom aggregate/window/join wrappers должны сохранять requiredChildDistribution из нового #7510.",
- "Float collect_set PartialMerge fallback #7489 действует для Spark ДО 4.2; на нашем 4.2.1 используется нормализованный путь. Сохранить эту версионную семантику."
+ "FILTER in a window aggregate must remain a fallback until separate support is agreed and implemented.",
+ "Merged struct scalar subqueries can reduce native coverage in ordinary SQL, including UNION.",
+ "ANSI overflow differences cannot be treated as merely differences in error text. Spark4 enables ANSI by default;3.5 disables it.",
+ "Custom aggregate/window/join wrappers must preserve requiredChildDistribution from the new #7510.",
+ "The float collect_set PartialMerge fallback #7489 applies to Spark BEFORE4.2; our4.2.1 uses the normalized path. Preserve this version-specific semantics."
],
"acceptance_tests": [
- "Sequence с Long extremes; decimal zero/exponent overflow; NaN и signed zero в массивах; переполнение date_trunc SECOND/MILLISECOND.",
- "FILTER window и q77a/struct scalar subquery: результат и ожидаемый безопасный план.",
- "ANSI арифметика на границах: результат, error class и SQLSTATE при явно заданных true/false.",
- "Новые регрессии #7489/#7510/#7512: float collect_set DISTINCT chains, distribution/AQE, arrays_overlap scalar-column и длинная константа без повторения списка.",
- "Spark4.2.1 Comet-off/on: одинаковые snapshots/configs/ANSI=true; сравнить rows/types/nulls/duplicates и ожидаемые успехи/ошибки, не только count.",
- "Mixed Spark/native SUM/AVG overflow guards и state codecs; отдельно воспроизвести runtimecodegenfallback SUM и groupedAVG+LIMIT.",
- "Проверить lazy evaluation casts/UDF/арифметики в SMJhoisting, AND/OR/CASE; при несовместимости исправление или плановый fallback.",
- "Custom DATE/DECIMAL RANGE boundary cases и relevant fallback guards; результаты с active cost/boundary flags.",
- "По возможности автоматизировать regression/differential cases; фактические native-планы подтверждают исполнение проверяемого пути."
- ],
- "user_decision": "Сохранить ANSI=true. Приоритет — отсутствие успешных jobs с некорректными результатами. Сравнить Spark4.2.1 Comet-off/on на одинаковых входах/ANSI; проверить результаты и обязательные ошибки; сохранить SUM/AVG overflow guards, исправлять несовместимости или заранее ограничивать native-путь. По возможности добавить автоматические regression/differential tests при реализации. PR#5889 в текущую миграцию не включать.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "Sequence with Long extremes; decimal zero/exponent overflow; NaN and signed zero in arrays; date_trunc SECOND/MILLISECOND overflow.",
+ "FILTER window and q77a/struct scalar subquery: result and expected safe plan.",
+ "ANSI boundary arithmetic: result, error class and SQLSTATE with true/false explicitly set.",
+ "New #7489/#7510/#7512 regressions: float collect_set DISTINCT chains, distribution/AQE, arrays_overlap scalar-column and a long constant without repeating the list.",
+ "Spark4.2.1 Comet-off/on: identical snapshots/configs/ANSI=true; compare rows/types/nulls/duplicates and expected successes/errors, not only count.",
+ "Mixed Spark/native SUM/AVG overflow guards and state codecs; separately reproduce runtime-codegen-fallback SUM and groupedAVG+LIMIT.",
+ "Verify lazy evaluation of casts/UDF/arithmetic in SMJ hoisting, AND/OR/CASE; fix incompatibility or use planning-time fallback.",
+ "Custom DATE/DECIMAL RANGE boundary cases and relevant fallback guards; results with active cost/boundary flags.",
+ "Automate regression/differential cases where possible; actual native plans confirm execution of the path under test."
+ ],
+ "user_decision": "Keep ANSI=true. Prioritize preventing successful jobs with incorrect results. Compare Spark4.2.1 Comet-off/on with identical inputs/ANSI; verify results and mandatory errors; preserve SUM/AVG overflow guards, fix incompatibilities or restrict the native path before execution. Add automated regression/differential tests during implementation where possible. Do not include PR#5889 in this migration.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 95,
"source_paths": [
"spark/src/main/scala/org/apache/comet/serde/arrays.scala",
@@ -408,21 +329,21 @@
"spark/src/test/scala/org/apache/comet/CometAnsiExpressionSuite.scala"
],
"upstream": {
- "base_1_2": "Уже есть SPARK-58821 sequence и SPARK-60118 string→decimal (#7263), SPARK-59602 signed-zero arrays (#6989), date_trunc overflow (#6740), FILTER window fallback (#4950). Struct scalar subquery остаётся ограничен; ряд ANSI overflow тестов пропускается для 4.2. Обновлённая branch-1.2 содержит #7489 (версионная семантика float collect_set), #7510 (required child distribution), #7512 (arrays_overlap scalar-column).",
- "outside_base": "PR#5889 пользователем явно исключён из текущей миграции: не трогать и не включать. Для unsupported struct scalar subqueries сохраняется fallback выбранной базы."
+ "base_1_2": "Already includes SPARK-58821 sequence and SPARK-60118 string→decimal (#7263), SPARK-59602 signed-zero arrays (#6989), date_trunc overflow (#6740), FILTER window fallback (#4950). Struct scalar subqueries remain limited; some ANSI overflow tests are skipped for4.2. The refreshed branch-1.2 contains #7489 (version-specific float collect_set semantics), #7510 (required child distribution), #7512 (arrays_overlap scalar-column).",
+ "outside_base": "The user explicitly excluded PR#5889 from this migration: do not touch or include it. Retain the selected base fallback for unsupported struct scalar subqueries."
},
"acceptance": [
- "Sequence с Long extremes; decimal zero/exponent overflow; NaN и signed zero в массивах; переполнение date_trunc SECOND/MILLISECOND.",
- "FILTER window и q77a/struct scalar subquery: результат и ожидаемый безопасный план.",
- "ANSI арифметика на границах: результат, error class и SQLSTATE при явно заданных true/false.",
- "Новые регрессии #7489/#7510/#7512: float collect_set DISTINCT chains, distribution/AQE, arrays_overlap scalar-column и длинная константа без повторения списка.",
- "Spark4.2.1 Comet-off/on: одинаковые snapshots/configs/ANSI=true; сравнить rows/types/nulls/duplicates и ожидаемые успехи/ошибки, не только count.",
- "Mixed Spark/native SUM/AVG overflow guards и state codecs; отдельно воспроизвести runtimecodegenfallback SUM и groupedAVG+LIMIT.",
- "Проверить lazy evaluation casts/UDF/арифметики в SMJhoisting, AND/OR/CASE; при несовместимости исправление или плановый fallback.",
- "Custom DATE/DECIMAL RANGE boundary cases и relevant fallback guards; результаты с active cost/boundary flags.",
- "По возможности автоматизировать regression/differential cases; фактические native-планы подтверждают исполнение проверяемого пути."
- ],
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
+ "Sequence with Long extremes; decimal zero/exponent overflow; NaN and signed zero in arrays; date_trunc SECOND/MILLISECOND overflow.",
+ "FILTER window and q77a/struct scalar subquery: result and expected safe plan.",
+ "ANSI boundary arithmetic: result, error class and SQLSTATE with true/false explicitly set.",
+ "New #7489/#7510/#7512 regressions: float collect_set DISTINCT chains, distribution/AQE, arrays_overlap scalar-column and a long constant without repeating the list.",
+ "Spark4.2.1 Comet-off/on: identical snapshots/configs/ANSI=true; compare rows/types/nulls/duplicates and expected successes/errors, not only count.",
+ "Mixed Spark/native SUM/AVG overflow guards and state codecs; separately reproduce runtime-codegen-fallback SUM and groupedAVG+LIMIT.",
+ "Verify lazy evaluation of casts/UDF/arithmetic in SMJ hoisting, AND/OR/CASE; fix incompatibility or use planning-time fallback.",
+ "Custom DATE/DECIMAL RANGE boundary cases and relevant fallback guards; results with active cost/boundary flags.",
+ "Automate regression/differential cases where possible; actual native plans confirm execution of the path under test."
+ ],
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
"original_dependency_notes": [
"P01",
"customwindows",
@@ -430,22 +351,18 @@
"I06"
],
"scope_kind": "new_target_work",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "ANSI и критерии корректности утверждены",
+ "decision_category": "ANSI and correctness criteria approved",
"same_pr_or_co_review": [],
"approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Сохранить ANSI=true. Приоритет — отсутствие успешных jobs с некорректными результатами. Сравнить Spark4.2.1 Comet-off/on на одинаковых входах/ANSI; проверить результаты и обязательные ошибки; сохранить SUM/AVG overflow guards, исправлять несовместимости или заранее ограничивать native-путь. По возможности добавить автоматические regression/differential tests при реализации. PR#5889 в текущую миграцию не включать.",
+ "approved_decision": "Keep ANSI=true. Prioritize preventing successful jobs with incorrect results. Compare Spark4.2.1 Comet-off/on with identical inputs/ANSI; verify results and mandatory errors; preserve SUM/AVG overflow guards, fix incompatibilities or restrict the native path before execution. Add automated regression/differential tests during implementation where possible. Do not include PR#5889 in this migration.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Сохранить guards и fixes upstream; добавить differential edge fixtures на 4.2.1. Не обходить ограничения молчаливым изменением ANSI или SQL-семантики.",
- "prior_open_decisions": "Сохранить ANSI=true. Приоритет — отсутствие успешных jobs с некорректными результатами. Сравнить Spark4.2.1 Comet-off/on на одинаковых входах/ANSI; проверить результаты и обязательные ошибки; сохранить SUM/AVG overflow guards, исправлять несовместимости или заранее ограничивать native-путь. По возможности добавить автоматические regression/differential tests при реализации. PR#5889 в текущую миграцию не включать.",
- "implementation_plan_groups": [
- "G01"
- ],
+ "original_proposal": "Preserve upstream guards and fixes; add differential edge fixtures on4.2.1. Do not circumvent limitations by silently changing ANSI or SQL semantics.",
+ "prior_open_decisions": "Keep ANSI=true. Prioritize preventing successful jobs with incorrect results. Compare Spark4.2.1 Comet-off/on with identical inputs/ANSI; verify results and mandatory errors; preserve SUM/AVG overflow guards, fix incompatibilities or restrict the native path before execution. Add automated regression/differential tests during implementation where possible. Do not include PR#5889 in this migration.",
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -453,35 +370,33 @@
},
{
"id": "D01",
- "title": "Standalone vendor DataFusion 55.2.0 и согласованный dependency graph",
- "contract": "Сохранить локально изменяемый physical-plan внутри Comet; остальные DataFusion crates55.2.0 и Arrow/Parquet59.3.2 остаются target-lock. Отдельный форк всего DataFusion не нужен.",
- "source_commits": [
- "6eaf12e80"
- ],
+ "title": "Standalone DataFusion55.2.0 vendor and a consistent dependency graph",
+ "contract": "Keep a locally modifiable physical-plan inside Comet; other DataFusion55.2.0 and Arrow/Parquet59.3.2 crates remain at the target lock. No separate fork of the entire DataFusion project is required.",
+ "source_commits": ["6eaf12e80"],
"source_paths": [
"native/Cargo.toml",
"native/Cargo.lock",
"native/vendor/datafusion-physical-plan/Cargo.toml"
],
"upstream": {
- "base": "Штатный physical-plan55.2.0 registry dependency; vendor отсутствует.",
- "outside_base": "Не выбирать DF56 автоматически ради отдельных аналогов."
+ "base": "Standard physical-plan55.2.0 registry dependency; no vendor.",
+ "outside_base": "Do not automatically select DF56 for individual equivalents."
},
- "proposed_action": "В G01 joom/1.2-dev сначала проверить Spark4.2.1/Delta4.4.1 на registryDF55.2, затем добавить pristine published crate55.2.0 отдельным механическим коммитом; подключить rootpatch/sharedlock/nativeworkspace и явный vendorunitCI-target, повторить baseline. Не отдельный PR: границы укрупнены утверждённым планом9+1. Joom алгоритмы — позднееG03/G04/G05.",
+ "proposed_action": "In G01 joom/1.2-dev, first validate Spark4.2.1/Delta4.4.1 on registryDF55.2, then add the pristine published crate55.2.0 as a separate mechanical commit; configure root patch/shared lock/native workspace and explicit vendor-unit CI target, and repeat the baseline. This is not a separate PR: boundaries were consolidated by the approved9+1 plan. Joom algorithms follow later in G03/G04/G05.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Каталог из Git55.2 наследует DF workspace.*; буквально скопированный Cargo.toml под Comet неверен.",
- "Не копировать старый lock55.1/Arrow59.3.0; сохранить serde/display55.2 и одну package identity physical-plan.",
- "Текущий Jo om excludes vendor; обычные Comet tests не запускают собственные unit tests vendor."
+ "The Git55.2 directory inherits DF workspace.*; copying its Cargo.toml literally under Comet is incorrect.",
+ "Do not copy the old lock55.1/Arrow59.3.0; preserve serde/display55.2 and one physical-plan package identity.",
+ "Current Joom excludes vendor; ordinary Comet tests do not run vendor internal unit tests."
],
"dependencies": [
- "Подтверждённая upstream-база Comet и базовая сборка Spark4.2.1"
+ "Confirmed upstream Comet base and baseline Spark4.2.1 build"
],
"acceptance": [
- "Проверить resolved graph/path-source и версии; cargo build с locked graph; отдельные unit/regression vendor tests на согласованных dependencies; baseline Comet tests до патчей.",
- "Workspace membership недостаточно: отдельныйCI-target явно запускает vendorunit/regression suite; runtimegraph согласован сrelease, test-onlydevdeps могутдобавитьсявобщийlockfile."
+ "Verify the resolved graph/path-source and versions; cargo build with a locked graph; separate vendor unit/regression tests with consistent dependencies; baseline Comet tests before patches.",
+ "Workspace membership is insufficient: a separate CI target explicitly runs the vendor unit/regression suite; the runtime graph is consistent with release, while test-only dev-deps may add entries to the shared lockfile."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 2,
"source_commit_details": [
{
@@ -644,30 +559,26 @@
],
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [
- "Подтверждённая upstream-база Comet и базовая сборка Spark4.2.1"
+ "Confirmed upstream Comet base and baseline Spark4.2.1 build"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G01"
- ],
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "В G01 joom/1.2-dev сначала проверить Spark4.2.1/Delta4.4.1 на registryDF55.2, затем добавить pristine published crate55.2.0 отдельным механическим коммитом; подключить rootpatch/sharedlock/nativeworkspace и явный vendorunitCI-target, повторить baseline. Не отдельный PR: границы укрупнены утверждённым планом9+1. Joom алгоритмы — позднееG03/G04/G05.",
+ "approved_decision": "In G01 joom/1.2-dev, first validate Spark4.2.1/Delta4.4.1 on registryDF55.2, then add the pristine published crate55.2.0 as a separate mechanical commit; configure root patch/shared lock/native workspace and explicit vendor-unit CI target, and repeat the baseline. This is not a separate PR: boundaries were consolidated by the approved9+1 plan. Joom algorithms follow later in G03/G04/G05.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "P02",
- "title": "Согласование Delta 4.4.1 со Spark 4.2",
+ "title": "Align Delta4.4.1 with Spark4.2",
"scope": "new_target_prerequisite",
"source_commits": [],
"source_files": [
@@ -675,27 +586,25 @@
"contrib/delta-spark/pom.xml",
".github/workflows/ci.yml"
],
- "contract": "Native contrib и runtime используют совместимый со Spark 4.2 артефакт io.delta:delta-spark_4.2_2.13:4.4.1. Не смешивать его со старым kernel-backed профилем contrib-delta.",
+ "contract": "Native contrib and runtime use the Spark4.2-compatible io.delta:delta-spark_4.2_2.13:4.4.1 artifact. Do not mix it with the old kernel-backed contrib-delta profile.",
"upstream_analogue": {
- "base_1_2": "POMspark-4.2ещёуказываетDelta4.3.1; этонеподходящееготовоеpairing.",
- "outside_base": "PR#7593на15ноябряOPEN DRAFT,head60deafe942261abba4577645d0478a0ca3f10007:pairing4.4.1,CI/regressions; источникдляreview/backport,неproductionqualification."
+ "base_1_2": "The spark-4.2 POM still specifies Delta4.3.1; this is not a suitable ready-made pairing.",
+ "outside_base": "PR#7593 was OPEN DRAFT on15November, head60deafe942261abba4577645d0478a0ca3f10007: pairing4.4.1, CI/regressions; a review/backport source, not production qualification."
},
- "proposed_action": "Обсудить использование выбранных изменений PR #7593 как заготовки. Адаптировать их к проверенной branch-1.2, дополнить I02 для GCP и квалифицировать Spark 4.2.1.",
+ "proposed_action": "Discuss using selected PR#7593 changes as a starting point. Adapt them to verified branch-1.2, add I02 for GCP and qualify Spark4.2.1.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "dependencies": [
- "P01"
- ],
+ "dependencies": ["P01"],
"risks": [
- "PR #7593 был draft в проверенном снимке 15 ноября; это не доказательство успешной квалификации и не утверждение об актуальном статусе PR.",
- "Не использовать Delta артефакт _4.1 в runtime Spark 4.2. Внутренние Delta API warehouse требуют отдельной проверки общей миграции."
+ "PR#7593 was draft in the inspected15November snapshot; this does not prove successful qualification and is not a statement about its current status.",
+ "Do not use the Delta _4.1 artifact in a Spark4.2 runtime. Warehouse usage of internal Delta APIs requires a separate check in the overall migration."
],
"acceptance_tests": [
- "Native Delta Scala/Rust tests на Spark 4.2.1 и Delta 4.4.1.",
- "Deletion vectors, column mapping, совместимость чтений после DML, GCP ADC.",
- "CI запускает профиль -Pdelta, а не только старый -Pcontrib-delta."
+ "Native Delta Scala/Rust tests on Spark4.2.1 and Delta4.4.1.",
+ "Deletion vectors, column mapping, read compatibility after DML, GCP ADC.",
+ "CI runs the -Pdelta profile, not only the old -Pcontrib-delta."
],
- "user_decision": "Обсудить использование PR #7593 как источника реализации. Требуемое совместимое pairing определено.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "Discuss using PR#7593 as an implementation source. The required compatible pairing is defined.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 11,
"source_paths": [
"pom.xml",
@@ -703,34 +612,28 @@
".github/workflows/ci.yml"
],
"upstream": {
- "base_1_2": "POMspark-4.2ещёуказываетDelta4.3.1; этонеподходящееготовоеpairing.",
- "outside_base": "PR#7593на15ноябряOPEN DRAFT,head60deafe942261abba4577645d0478a0ca3f10007:pairing4.4.1,CI/regressions; источникдляreview/backport,неproductionqualification."
+ "base_1_2": "The spark-4.2 POM still specifies Delta4.3.1; this is not a suitable ready-made pairing.",
+ "outside_base": "PR#7593 was OPEN DRAFT on15November, head60deafe942261abba4577645d0478a0ca3f10007: pairing4.4.1, CI/regressions; a review/backport source, not production qualification."
},
"acceptance": [
- "Native Delta Scala/Rust tests на Spark 4.2.1 и Delta 4.4.1.",
- "Deletion vectors, column mapping, совместимость чтений после DML, GCP ADC.",
- "CI запускает профиль -Pdelta, а не только старый -Pcontrib-delta."
- ],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
- "original_dependency_notes": [
- "P01"
+ "Native Delta Scala/Rust tests on Spark4.2.1 and Delta4.4.1.",
+ "Deletion vectors, column mapping, read compatibility after DML, GCP ADC.",
+ "CI runs the -Pdelta profile, not only the old -Pcontrib-delta."
],
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
+ "original_dependency_notes": ["P01"],
"scope_kind": "new_target_work",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G01"
- ],
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "approved_decision": "Обсудить использование выбранных изменений PR #7593 как заготовки. Адаптировать их к проверенной branch-1.2, дополнить I02 для GCP и квалифицировать Spark 4.2.1.",
+ "approved_decision": "Discuss using selected PR#7593 changes as a starting point. Adapt them to verified branch-1.2, add I02 for GCP and qualify Spark4.2.1.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Обсудить использование PR #7593 как источника реализации. Требуемое совместимое pairing определено.",
+ "prior_open_decisions": "Discuss using PR#7593 as an implementation source. The required compatible pairing is defined.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
@@ -739,9 +642,7 @@
"id": "I01",
"title": "Native Delta scan: Spark planning + native Parquet/DV/rebase execution",
"scope": "existing_joom_feature",
- "source_commits": [
- "c69bd9c39"
- ],
+ "source_commits": ["c69bd9c39"],
"source_files": [
"contrib/delta-spark/**",
"native/core/src/execution/delta_dv.rs",
@@ -752,28 +653,25 @@
"spark/src/main/scala/org/apache/comet/ContribServices.scala",
"spark/src/main/scala/org/apache/comet/rules/CometScanContrib.scala"
],
- "contract": "Сохранить native чтение Delta через планирование файлов в Spark: deletion vectors, column mapping/schema adaptation, datetime rebase, partition/data/runtime filters и поддерживаемые scan shapes; unsupported shapes безопасно fallback. Не подменять этот scanner старым kernel-backed contrib-delta.",
+ "contract": "Preserve native Delta reads through Spark file planning: deletion vectors, column mapping/schema adaptation, datetime rebase, partition/data/runtime filters and supported scan shapes; safely fall back for unsupported shapes. Do not replace this scanner with the old kernel-backed contrib-delta.",
"upstream_analogue": {
- "base_1_2": "Аналог уже есть: native contrib/delta-spark — upstream реализация исходного PR#5365. Содержит дальнейшие исправления и более строгие gates. Не переносить старые54файла поверх новой базы.",
- "outside_base": "PR#7593 — отдельная заготовка pairing Spark4.2/Delta4.4.1, не замена GCP адаптации."
+ "base_1_2": "An equivalent already exists: native contrib/delta-spark is the upstream implementation of original PR#5365. It includes subsequent fixes and stricter gates. Do not port the old54files over the new base.",
+ "outside_base": "PR#7593 is a separate Spark4.2/Delta4.4.1 pairing starting point, not a substitute for GCP adaptation."
},
- "proposed_action": "Использовать native Delta scanner Comet1.2; сохранить GCP ADC, сериализацию и защиту от planning deadlock, адаптируя недостающие Joom-особенности.",
+ "proposed_action": "Use the Comet1.2 native Delta scanner; preserve GCP ADC, serialization and planning-deadlock protection, adapting missing Joom specifics.",
"approval_status": "APPROVED_EXPLICITLY",
- "dependencies": [
- "P01 exact Spark4.2.1",
- "P02 Delta4.4.1 pairing"
- ],
+ "dependencies": ["P01 exact Spark4.2.1", "P02 Delta4.4.1 pairing"],
"risks": [
- "Delta/Spark internal APIs и binary pairing4.2 требуют квалификации.",
- "Старый scanner имеет отличающиеся interfaces/partition-count assumptions; blind cherry-pick способен откатить upstream fixes."
+ "Delta/Spark internal APIs and binary pairing4.2 require qualification.",
+ "The old scanner has different interfaces/partition-count assumptions; blind cherry-picking can revert upstream fixes."
],
"acceptance_tests": [
- "CometDeltaNativeScanSuite, DeltaScanContribSuite, DeltaPlanDataInjectorSuite, CometDeltaDmlReproSuite наSpark4.2.1/Delta4.4.1",
- "Результаты и ошибки Comet-on/off, планы с native Delta, DV/column-mapping/rebase/runtime-filter fixtures",
- "GCP интеграционные чтения в реальном release image; AWS не является целью квалификации"
+ "CometDeltaNativeScanSuite, DeltaScanContribSuite, DeltaPlanDataInjectorSuite, CometDeltaDmlReproSuite on Spark4.2.1/Delta4.4.1",
+ "Comet-on/off results and errors, native Delta plans, DV/column-mapping/rebase/runtime-filter fixtures",
+ "GCP integration reads in the real release image; AWS is not a qualification target"
],
- "user_decision": "Использовать native Delta scanner Comet1.2; сохранить GCP ADC, сериализацию и защиту от planning deadlock, адаптируя недостающие Joom-особенности.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "Use the Comet1.2 native Delta scanner; preserve GCP ADC, serialization and planning-deadlock protection, adapting missing Joom specifics.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 20,
"source_paths": [
"contrib/delta-spark/**",
@@ -786,36 +684,32 @@
"spark/src/main/scala/org/apache/comet/rules/CometScanContrib.scala"
],
"upstream": {
- "base_1_2": "Аналог уже есть: native contrib/delta-spark — upstream реализация исходного PR#5365. Содержит дальнейшие исправления и более строгие gates. Не переносить старые54файла поверх новой базы.",
- "outside_base": "PR#7593 — отдельная заготовка pairing Spark4.2/Delta4.4.1, не замена GCP адаптации."
+ "base_1_2": "An equivalent already exists: native contrib/delta-spark is the upstream implementation of original PR#5365. It includes subsequent fixes and stricter gates. Do not port the old54files over the new base.",
+ "outside_base": "PR#7593 is a separate Spark4.2/Delta4.4.1 pairing starting point, not a substitute for GCP adaptation."
},
"acceptance": [
- "CometDeltaNativeScanSuite, DeltaScanContribSuite, DeltaPlanDataInjectorSuite, CometDeltaDmlReproSuite наSpark4.2.1/Delta4.4.1",
- "Результаты и ошибки Comet-on/off, планы с native Delta, DV/column-mapping/rebase/runtime-filter fixtures",
- "GCP интеграционные чтения в реальном release image; AWS не является целью квалификации"
+ "CometDeltaNativeScanSuite, DeltaScanContribSuite, DeltaPlanDataInjectorSuite, CometDeltaDmlReproSuite on Spark4.2.1/Delta4.4.1",
+ "Comet-on/off results and errors, native Delta plans, DV/column-mapping/rebase/runtime-filter fixtures",
+ "GCP integration reads in the real release image; AWS is not a qualification target"
],
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
"original_dependency_notes": [
"P01 exact Spark4.2.1",
"P02 Delta4.4.1 pairing"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P02"
- ],
+ "prerequisites": ["P02"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "upstream-основа утверждена",
+ "decision_category": "upstream base approved",
"same_pr_or_co_review": [],
"approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Использовать native Delta scanner Comet1.2; сохранить GCP ADC, сериализацию и защиту от planning deadlock, адаптируя недостающие Joom-особенности.",
+ "approved_decision": "Use the Comet1.2 native Delta scanner; preserve GCP ADC, serialization and planning-deadlock protection, adapting missing Joom specifics.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Обсудить: принять native Delta из1.2 как основу, перенести/добавить Joom regression contracts и только остаточные адаптации I02/I03. Состав DV/rebase/filter гарантий подтвердить тестами.",
- "prior_open_decisions": "Утвердить использование upstream native Delta1.2 вместо повторного переноса старой реализации; остаточные особенности остаются отдельными пунктами.",
- "implementation_plan_groups": [
- "G01"
- ],
+ "original_proposal": "Discuss accepting native Delta from1.2 as the base, porting/adding Joom regression contracts and only residual I02/I03 adaptations. Verify DV/rebase/filter guarantees with tests.",
+ "prior_open_decisions": "Approve use of upstream native Delta1.2 instead of porting the old implementation again; residual specifics remain separate items.",
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -825,143 +719,119 @@
"id": "I02",
"title": "GCS/GKE ADC-equivalent Hadoop auth flags",
"scope": "existing_joom_feature",
- "source_commits": [
- "a43523692"
- ],
+ "source_commits": ["a43523692"],
"source_files": [
"contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala",
"contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala"
],
- "contract": "При gs:// не отклонять native Delta только из-за fs.gs.auth.service.account.enable=true / google.cloud.auth.service.account.enable=true или fs.gs.auth.type=COMPUTE_ENGINE/APPLICATION_DEFAULT. Keyfiles, emails/private keys/client ids, false/non-ADC auth types по-прежнему должны decline.",
+ "contract": "For gs://, do not reject native Delta solely because fs.gs.auth.service.account.enable=true / google.cloud.auth.service.account.enable=true or fs.gs.auth.type=COMPUTE_ENGINE/APPLICATION_DEFAULT. Keyfiles, emails/private keys/client ids, false/non-ADC auth types must still decline.",
"upstream_analogue": {
- "base_1_2": "Отсутствует whitelist ADC-equivalent flags: gcsHadoopOnlyAuthReason всё ещё отклоняет любой непустой auth key.",
- "outside_base": "Просмотренный PR#7593 эту GCP адаптацию не добавляет."
+ "base_1_2": "The ADC-equivalent flag whitelist is absent: gcsHadoopOnlyAuthReason still rejects any nonempty auth key.",
+ "outside_base": "Inspected PR#7593 does not add this GCP adaptation."
},
- "proposed_action": "Адаптировать whitelist в актуальный DeltaScanSupport, сохранить отрицательные credential tests и выполнить GCP ADC integration.",
+ "proposed_action": "Adapt the whitelist in current DeltaScanSupport, retain negative credential tests and run GCP ADC integration.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "dependencies": [
- "I01",
- "P02"
- ],
+ "dependencies": ["I01", "P02"],
"risks": [
- "Без адаптации GCP Delta может целиком уйти в Spark fallback.",
- "Нельзя расширять whitelist на реальные Hadoop-only credentials: native object_store их не получает."
+ "Without adaptation, GCP Delta can fall back entirely to Spark.",
+ "Do not broaden the whitelist to real Hadoop-only credentials: native object_store does not receive them."
],
"acceptance_tests": [
- "Два теста a435: ADC flags допускаются; keyfile рядом с enable=true, enable=false и SERVICE_ACCOUNT_JSON_KEYFILE отклоняются",
- "Scheme-scoped tests: другие URI не используютGCS gate",
- "Native Delta plan + чтение через GKE Workload Identity/metadata ADC"
+ "Two a435 tests: ADC flags are accepted; keyfile together with enable=true, enable=false and SERVICE_ACCOUNT_JSON_KEYFILE are rejected",
+ "Scheme-scoped tests: other URIs do not use the GCS gate",
+ "Native Delta plan and reads through GKE Workload Identity/metadata ADC"
],
- "user_decision": "Утвердить перенос поведения whitelist без расширения списка разрешённых механизмов аутентификации.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "Approve porting whitelist behavior without broadening the permitted authentication mechanisms.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 21,
"source_paths": [
"contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala",
"contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala"
],
"upstream": {
- "base_1_2": "Отсутствует whitelist ADC-equivalent flags: gcsHadoopOnlyAuthReason всё ещё отклоняет любой непустой auth key.",
- "outside_base": "Просмотренный PR#7593 эту GCP адаптацию не добавляет."
+ "base_1_2": "The ADC-equivalent flag whitelist is absent: gcsHadoopOnlyAuthReason still rejects any nonempty auth key.",
+ "outside_base": "Inspected PR#7593 does not add this GCP adaptation."
},
"acceptance": [
- "Два теста a435: ADC flags допускаются; keyfile рядом с enable=true, enable=false и SERVICE_ACCOUNT_JSON_KEYFILE отклоняются",
- "Scheme-scoped tests: другие URI не используютGCS gate",
- "Native Delta plan + чтение через GKE Workload Identity/metadata ADC"
- ],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
- "original_dependency_notes": [
- "I01",
- "P02"
+ "Two a435 tests: ADC flags are accepted; keyfile together with enable=true, enable=false and SERVICE_ACCOUNT_JSON_KEYFILE are rejected",
+ "Scheme-scoped tests: other URIs do not use the GCS gate",
+ "Native Delta plan and reads through GKE Workload Identity/metadata ADC"
],
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
+ "original_dependency_notes": ["I01", "P02"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "I01"
- ],
+ "prerequisites": ["I01"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G01"
- ],
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "approved_decision": "Адаптировать whitelist в актуальный DeltaScanSupport, сохранить отрицательные credential tests и выполнить GCP ADC integration.",
+ "approved_decision": "Adapt the whitelist in current DeltaScanSupport, retain negative credential tests and run GCP ADC integration.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Утвердить перенос поведения whitelist без расширения списка разрешённых механизмов аутентификации.",
+ "prior_open_decisions": "Approve porting whitelist behavior without broadening the permitted authentication mechanisms.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "I03",
- "title": "Delta outputPartitioning: отсутствие планировочного deadlock",
+ "title": "Delta outputPartitioning: prevent planning deadlock",
"scope": "existing_joom_feature",
- "source_commits": [
- "0e17ce3e5"
- ],
+ "source_commits": ["0e17ce3e5"],
"source_files": [
"contrib/delta-spark/src/main/scala/org/apache/spark/sql/comet/CometDeltaNativeScanExec.scala",
"contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala"
],
- "contract": "Чтение outputPartitioning во время планирования/cost-boundary анализа не должно запускать file listing/подзапросы под QueryExecution monitor и deadlock; исполнение fused native parent должно получить правильное количество partitions.",
+ "contract": "Reading outputPartitioning during planning/cost-boundary analysis must not start file listing/subqueries under the QueryExecution monitor and deadlock; execution of a fused native parent must obtain the correct partition count.",
"upstream_analogue": {
- "base_1_2": "Частичный аналог: guard hasUnevaluableSubqueryFilter возвращаетUnknownPartitioning(0) для unresolved runtime filters; иначе perPartitionData.length. Upstream комментарий требует реальное количество partitions для fused parent.",
- "outside_base": "Нельзя считать старое безусловноеUnknownPartitioning(0) готовым совместимым исправлением."
+ "base_1_2": "Partial equivalent: hasUnevaluableSubqueryFilter returns UnknownPartitioning(0) for unresolved runtime filters; otherwise perPartitionData.length. The upstream comment requires the real partition count for a fused parent.",
+ "outside_base": "Do not treat the old unconditional UnknownPartitioning(0) as a ready compatible fix."
},
- "proposed_action": "Сохранить защиту от planning deadlock через актуальный upstream lifecycle с корректным execution partition count; старое безусловное UnknownPartitioning(0) не копировать. Проверить исходные Joom reproductions.",
+ "proposed_action": "Preserve planning-deadlock protection through the current upstream lifecycle with the correct execution partition count; do not copy the old unconditional UnknownPartitioning(0). Verify the original Joom reproductions.",
"approval_status": "RESOLVED_BY_APPROVED_POLICY",
- "dependencies": [
- "I01",
- "native boundary/cost integration"
- ],
+ "dependencies": ["I01", "native boundary/cost integration"],
"risks": [
- "Реальный конфликт контрактов: blind port always0 может ломать native block sizing.",
- "Статический аудит не доказывает ни отсутствие, ни воспроизводимость остаточного deadlock в новой версии."
+ "A real contract conflict: blindly porting always0 can break native block sizing.",
+ "Static auditing proves neither absence nor reproducibility of residual deadlock in the new version."
],
"acceptance_tests": [
- "Старые regression cases0e17 с scalar subquery+DPP и planning-time partitioning access под timeout",
- "AQE on/off; native fused parent; ноль/одна/несколько execution partitions",
- "Проверка отсутствия file listing/subquery execution при безопасных planning metadata reads"
+ "Old 0e17 regression cases with scalar subquery+DPP and planning-time partitioning access under timeout",
+ "AQE on/off; native fused parent; zero/one/multiple execution partitions",
+ "Check that safe planning metadata reads do not trigger file listing/subquery execution"
],
- "user_decision": "Сохранить защиту от planning deadlock через актуальный upstream lifecycle с корректным execution partition count; старое безусловное UnknownPartitioning(0) не копировать. Проверить исходные Joom reproductions.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "Preserve planning-deadlock protection through the current upstream lifecycle with the correct execution partition count; do not copy the old unconditional UnknownPartitioning(0). Verify the original Joom reproductions.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 23,
"source_paths": [
"contrib/delta-spark/src/main/scala/org/apache/spark/sql/comet/CometDeltaNativeScanExec.scala",
"contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/CometDeltaNativeScanSuite.scala"
],
"upstream": {
- "base_1_2": "Частичный аналог: guard hasUnevaluableSubqueryFilter возвращаетUnknownPartitioning(0) для unresolved runtime filters; иначе perPartitionData.length. Upstream комментарий требует реальное количество partitions для fused parent.",
- "outside_base": "Нельзя считать старое безусловноеUnknownPartitioning(0) готовым совместимым исправлением."
+ "base_1_2": "Partial equivalent: hasUnevaluableSubqueryFilter returns UnknownPartitioning(0) for unresolved runtime filters; otherwise perPartitionData.length. The upstream comment requires the real partition count for a fused parent.",
+ "outside_base": "Do not treat the old unconditional UnknownPartitioning(0) as a ready compatible fix."
},
"acceptance": [
- "Старые regression cases0e17 с scalar subquery+DPP и planning-time partitioning access под timeout",
- "AQE on/off; native fused parent; ноль/одна/несколько execution partitions",
- "Проверка отсутствия file listing/subquery execution при безопасных planning metadata reads"
- ],
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
- "original_dependency_notes": [
- "I01",
- "native boundary/cost integration"
+ "Old 0e17 regression cases with scalar subquery+DPP and planning-time partitioning access under timeout",
+ "AQE on/off; native fused parent; zero/one/multiple execution partitions",
+ "Check that safe planning metadata reads do not trigger file listing/subquery execution"
],
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
+ "original_dependency_notes": ["I01", "native boundary/cost integration"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "I01"
- ],
+ "prerequisites": ["I01"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "определено утверждённым общим правилом",
+ "decision_category": "defined by the approved general rule",
"same_pr_or_co_review": [],
"approval": "RESOLVED_BY_APPROVED_POLICY",
- "approved_decision": "Сохранить защиту от planning deadlock через актуальный upstream lifecycle с корректным execution partition count; старое безусловное UnknownPartitioning(0) не копировать. Проверить исходные Joom reproductions.",
+ "approved_decision": "Preserve planning-deadlock protection through the current upstream lifecycle with the correct execution partition count; do not copy the old unconditional UnknownPartitioning(0). Verify the original Joom reproductions.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Обсудить/адаптировать контракт: начать с upstream lifecycle, запустить старую deadlock regression и добавить сочетание с cost/boundary rules. При воспроизведении исправить принудительное разрешение подзапросов без нарушения execution partition count.",
- "prior_open_decisions": "Утвердить сохранение поведения без deadlock через адаптацию upstream контракта, а не копированиеalways0.",
- "implementation_plan_groups": [
- "G01"
- ],
+ "original_proposal": "Discuss/adapt the contract: start from the upstream lifecycle, run the old deadlock regression and add combinations with cost/boundary rules. If reproduced, fix forced subquery resolution without violating the execution partition count.",
+ "prior_open_decisions": "Approve preservation of deadlock-free behavior by adapting the upstream contract rather than copying always0.",
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -969,39 +839,33 @@
},
{
"id": "I04",
- "title": "Native scan: DPP lifecycle и zero partitions после pruning",
+ "title": "Native scan: DPP lifecycle and zero partitions after pruning",
"scope": "existing_joom_feature",
- "source_commits": [
- "65be8b751",
- "45fad9eea"
- ],
+ "source_commits": ["65be8b751", "45fad9eea"],
"source_files": [
"spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala",
"spark/src/main/scala/org/apache/spark/sql/comet/operators.scala",
"spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala"
],
- "contract": "Native scan сохраняется при допустимом DPP; planning metadata не запускает преждевременно unresolved subqueries. Полностью отсечённый scan создаётноль partitions, не пытаясь читать отсутствующий source-key и не изобретая partition.",
+ "contract": "Retain native scan for eligible DPP; planning metadata must not prematurely run unresolved subqueries. A fully pruned scan creates zero partitions without attempting to read an absent source-key or inventing a partition.",
"upstream_analogue": {
- "base_1_2": "Функциональные аналоги есть: #6268/695ca3ab7 дляDPP lifecycle; operators.scala1120–1124 определяет count черезscan.perPartitionData.length, включая0.",
- "outside_base": "Это части составного warehouse commit65be8, не весь commit целиком.",
- "version_history": "Обычный zero-partition scan уже обрабатывался в official1.1.0; Joom45 исправлял собственный новый lookup path после65. Отдельный planning DPP guard#6268 отсутствует в1.1.0 и включён в1.2.0."
+ "base_1_2": "Functional equivalents exist: #6268/695ca3ab7 for DPP lifecycle; operators.scala1120–1124 derives count from scan.perPartitionData.length, including0.",
+ "outside_base": "These are parts of composite warehouse commit65be8, not the whole commit.",
+ "version_history": "Ordinary zero-partition scans were already handled in official1.1.0; Joom45 fixed its own new lookup path introduced after65. The separate planning DPP guard#6268 is absent in1.1.0 and included in1.2.0."
},
- "proposed_action": "Использовать upstream DPP/empty-scan lifecycle; сохранить соответствующие Joom регрессии и недостающие исправления.",
+ "proposed_action": "Use the upstream DPP/empty-scan lifecycle; preserve corresponding Joom regressions and missing fixes.",
"approval_status": "APPROVED_EXPLICITLY",
- "dependencies": [
- "P01",
- "I01 forDelta combinations"
- ],
+ "dependencies": ["P01", "I01 forDelta combinations"],
"risks": [
- "Spark4.2 AQE/subquery shapes меняются; regression должен утверждать native coverage, а не только правильный Spark fallback.",
- "Старый scan нельзя переносить целиком: новый plan-data lifecycle распространяется также на Iceberg/contrib scans."
+ "Spark4.2 AQE/subquery shapes change; the regression must assert native coverage, not only correct Spark fallback.",
+ "Do not port the old scan wholesale: the new plan-data lifecycle also applies to Iceberg/contrib scans."
],
"acceptance_tests": [
- "CometDppFallbackRepro3949Suite: pruned zero-partition case и warehouse reproducer",
+ "CometDppFallbackRepro3949Suite: pruned zero-partition case and warehouse reproducer",
"AQE on/off, native/Spark boundary, empty scan under projection/join/aggregate"
],
- "user_decision": "Использовать upstream DPP/empty-scan lifecycle; сохранить соответствующие Joom регрессии и недостающие исправления.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "Use the upstream DPP/empty-scan lifecycle; preserve corresponding Joom regressions and missing fixes.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 24,
"source_paths": [
"spark/src/main/scala/org/apache/spark/sql/comet/CometNativeScanExec.scala",
@@ -1009,36 +873,29 @@
"spark/src/test/scala/org/apache/spark/sql/comet/CometDppFallbackRepro3949Suite.scala"
],
"upstream": {
- "base_1_2": "Функциональные аналоги есть: #6268/695ca3ab7 дляDPP lifecycle; operators.scala1120–1124 определяет count черезscan.perPartitionData.length, включая0.",
- "outside_base": "Это части составного warehouse commit65be8, не весь commit целиком.",
- "version_history": "Обычный zero-partition scan уже обрабатывался в official1.1.0; Joom45 исправлял собственный новый lookup path после65. Отдельный planning DPP guard#6268 отсутствует в1.1.0 и включён в1.2.0."
+ "base_1_2": "Functional equivalents exist: #6268/695ca3ab7 for DPP lifecycle; operators.scala1120–1124 derives count from scan.perPartitionData.length, including0.",
+ "outside_base": "These are parts of composite warehouse commit65be8, not the whole commit.",
+ "version_history": "Ordinary zero-partition scans were already handled in official1.1.0; Joom45 fixed its own new lookup path introduced after65. The separate planning DPP guard#6268 is absent in1.1.0 and included in1.2.0."
},
"acceptance": [
- "CometDppFallbackRepro3949Suite: pruned zero-partition case и warehouse reproducer",
+ "CometDppFallbackRepro3949Suite: pruned zero-partition case and warehouse reproducer",
"AQE on/off, native/Spark boundary, empty scan under projection/join/aggregate"
],
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
- "original_dependency_notes": [
- "P01",
- "I01 forDelta combinations"
- ],
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
+ "original_dependency_notes": ["P01", "I01 forDelta combinations"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "upstream-основа утверждена",
+ "decision_category": "upstream base approved",
"same_pr_or_co_review": [],
"approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Использовать upstream DPP/empty-scan lifecycle; сохранить соответствующие Joom регрессии и недостающие исправления.",
+ "approved_decision": "Use the upstream DPP/empty-scan lifecycle; preserve corresponding Joom regressions and missing fixes.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Обсудить: сохранить upstreamDPP/partition-data реализацию, перенести смысл старых regression fixtures и новые cost/boundary combinations.",
- "prior_open_decisions": "Утвердить upstream-аналоги при сохранении исходных регрессий; старые тела методов не копировать автоматически.",
- "implementation_plan_groups": [
- "G01"
- ],
+ "original_proposal": "Discuss retaining upstream DPP/partition-data implementation and porting the intent of old regression fixtures plus new cost/boundary combinations.",
+ "prior_open_decisions": "Approve upstream equivalents while preserving original regressions; do not automatically copy old method bodies.",
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -1048,10 +905,7 @@
"id": "I05",
"title": "Nested sliced boolean FFI offsets",
"scope": "existing_joom_backport",
- "source_commits": [
- "5618dec71",
- "64e096c91"
- ],
+ "source_commits": ["5618dec71", "64e096c91"],
"source_files": [
"native/common/src/ffi_offsets.rs",
"native/common/src/lib.rs",
@@ -1062,27 +916,25 @@
"spark/src/test/scala/org/apache/comet/exec/CometExecSuite.scala",
"docs/source/contributor-guide/ffi.md"
],
- "contract": "Boolean bitmap offsets нормализуются рекурсивно на всех уровнях list/struct перед JNI export; значения/nulls sliced arrays не меняются.",
+ "contract": "Boolean bitmap offsets are normalized recursively at every list/struct level before JNI export; values/nulls in sliced arrays remain unchanged.",
"upstream_analogue": {
- "base_1_2": "Upstream#6339 уже включён; changelog1.2 явно содержит тот же repair.",
- "outside_base": "Joom5618 — backport upstream(#6339/#6449); дополнительный64e nested two-level fixture необходимо сохранить.",
- "version_history": "Recursive FFI repair уже включён в official1.1.0 через branch1.1 backport66800830e(#6339/#6449). ffi_offsets.rs побайтово совпадает в1.1.0,1.2.0 иJo omtag13; дополнительный two-struct regression64e — Joom."
+ "base_1_2": "Upstream#6339 is already included; the1.2 changelog explicitly lists the same repair.",
+ "outside_base": "Joom5618 is an upstream backport (#6339/#6449); preserve the additional64e nested two-level fixture.",
+ "version_history": "Recursive FFI repair is already included in official1.1.0 through branch1.1 backport66800830e (#6339/#6449). ffi_offsets.rs is byte-identical in1.1.0,1.2.0 and Joom tag13; the additional two-struct regression64e is Joom-specific."
},
- "proposed_action": "Использовать upstream recursive boolean FFI repair; сохранить дополнительные Joom nested/sliced regressions.",
+ "proposed_action": "Use upstream recursive boolean FFI repair; preserve additional Joom nested/sliced regressions.",
"approval_status": "APPROVED_EXPLICITLY",
- "dependencies": [
- "P01"
- ],
+ "dependencies": ["P01"],
"risks": [
- "При переносе nested-offset budget/slicing и custom windows/shuffle можно повторно нарушить FFI invariant."
+ "Porting nested-offset budgeting/slicing and custom windows/shuffle can break the FFI invariant again."
],
"acceptance_tests": [
"Nested two-struct-level boolean OFFSET regression64e",
- "JNI/UDF/explode/scan paths послеslice; nulls и nonzerooffsets",
- "Планы должны исполняться нативно в проверяемых местах"
+ "JNI/UDF/explode/scan paths after slicing; nulls and nonzero offsets",
+ "Plans must execute natively at the points under test"
],
- "user_decision": "Использовать upstream recursive boolean FFI repair; сохранить дополнительные Joom nested/sliced regressions.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "Use upstream recursive boolean FFI repair; preserve additional Joom nested/sliced regressions.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 25,
"source_paths": [
"native/common/src/ffi_offsets.rs",
@@ -1095,36 +947,30 @@
"docs/source/contributor-guide/ffi.md"
],
"upstream": {
- "base_1_2": "Upstream#6339 уже включён; changelog1.2 явно содержит тот же repair.",
- "outside_base": "Joom5618 — backport upstream(#6339/#6449); дополнительный64e nested two-level fixture необходимо сохранить.",
- "version_history": "Recursive FFI repair уже включён в official1.1.0 через branch1.1 backport66800830e(#6339/#6449). ffi_offsets.rs побайтово совпадает в1.1.0,1.2.0 иJo omtag13; дополнительный two-struct regression64e — Joom."
+ "base_1_2": "Upstream#6339 is already included; the1.2 changelog explicitly lists the same repair.",
+ "outside_base": "Joom5618 is an upstream backport (#6339/#6449); preserve the additional64e nested two-level fixture.",
+ "version_history": "Recursive FFI repair is already included in official1.1.0 through branch1.1 backport66800830e (#6339/#6449). ffi_offsets.rs is byte-identical in1.1.0,1.2.0 and Joom tag13; the additional two-struct regression64e is Joom-specific."
},
"acceptance": [
"Nested two-struct-level boolean OFFSET regression64e",
- "JNI/UDF/explode/scan paths послеslice; nulls и nonzerooffsets",
- "Планы должны исполняться нативно в проверяемых местах"
- ],
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
- "original_dependency_notes": [
- "P01"
+ "JNI/UDF/explode/scan paths after slicing; nulls and nonzero offsets",
+ "Plans must execute natively at the points under test"
],
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
+ "original_dependency_notes": ["P01"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "upstream-основа утверждена",
+ "decision_category": "upstream base approved",
"same_pr_or_co_review": [],
"approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Использовать upstream recursive boolean FFI repair; сохранить дополнительные Joom nested/sliced regressions.",
+ "approved_decision": "Use upstream recursive boolean FFI repair; preserve additional Joom nested/sliced regressions.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Обсудить: оставить upstreamrepair и перенести только недостающие Joom regressions/FFI contracts.",
- "prior_open_decisions": "Утвердить upstream repair вместо дублирующегоbackport, сохранив дополнительныеJoomtests.",
- "implementation_plan_groups": [
- "G02"
- ],
+ "original_proposal": "Discuss retaining upstream repair and porting only missing Joom regressions/FFI contracts.",
+ "prior_open_decisions": "Approve upstream repair instead of the duplicate backport, retaining additional Joom tests.",
+ "implementation_plan_groups": ["G02"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -1132,13 +978,9 @@
},
{
"id": "I06",
- "title": "Timestamp timezone labels: producer repairs и сравнения разных producers",
+ "title": "Timestamp timezone labels: producer repairs and comparisons across producers",
"scope": "existing_joom_feature_and_backports",
- "source_commits": [
- "65be8b751",
- "c9574d243",
- "d78393122"
- ],
+ "source_commits": ["65be8b751", "c9574d243", "d78393122"],
"source_files": [
"native/core/src/execution/planner.rs",
"native/spark-expr/src/datetime_funcs/timestamp_trunc.rs",
@@ -1148,28 +990,25 @@
"spark/src/test/scala/org/apache/comet/CometTimestampComparisonSuite.scala",
"spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala"
],
- "contract": "Native timestamp producers сохраняют корректные Arrow timezone labels; сравнения корректно работают междуtimestamp_seconds, date_trunc, Spark/native scan и прочими producers безпаники/неверногоimplicitcast, при сохранении Spark instant semantics.",
+ "contract": "Native timestamp producers preserve correct Arrow timezone labels; comparisons work correctly between timestamp_seconds, date_trunc, Spark/native scans and other producers without panics/incorrect implicit casts, preserving Spark instant semantics.",
"upstream_analogue": {
- "base_1_2": "#6341 timestamp_seconds UTC; #6345 date_trunc возвращает label входа; #6347 CASE/COALESCE branches relabel; более сильные test assertions #6868/#6930. Общий Joom comparison-side relabel отсутствует.",
- "outside_base": "Producer fixes покрывают известные части, но полная эквивалентность произвольных producer combinations статически не доказана."
+ "base_1_2": "#6341 timestamp_seconds UTC; #6345 date_trunc returns the input label; #6347 CASE/COALESCE branch relabeling; stronger test assertions #6868/#6930. General Joom comparison-side relabeling is absent.",
+ "outside_base": "Producer fixes cover known parts, but full equivalence across arbitrary producer combinations is not statically proven."
},
- "proposed_action": "Использовать upstream timestamp producer/CASE fixes и Spark4.2 semantics; переносить нашу дополнительную нормализацию сравнений там, где она ещё необходима.",
+ "proposed_action": "Use upstream timestamp producer/CASE fixes and Spark4.2 semantics; port our additional comparison normalization where still necessary.",
"approval_status": "APPROVED_EXPLICITLY",
- "dependencies": [
- "P01",
- "native expression/planner integration"
- ],
+ "dependencies": ["P01", "native expression/planner integration"],
"risks": [
- "Нельзя заменить весь новый plannerстарым: потеряются новые typedcoercions/semantics.",
- "Spark4.2 date_trunc overflow flag из#6740 нужно сохранить; сравнитьUTC aliases/sessionzones, DST и отрицательные timestamps."
+ "Do not replace the entire new planner with the old one: new typed coercions/semantics would be lost.",
+ "Preserve the Spark4.2 date_trunc overflow flag from#6740; compare UTC aliases/session zones, DST and negative timestamps."
],
"acceptance_tests": [
- "CometTimestampComparisonSuite иtimestamp_seconds_timezone.sql",
+ "CometTimestampComparisonSuite and timestamp_seconds_timezone.sql",
"CometDateTimeUtilsSuite UTC aliases/sessionzones",
- "Comet-on/off наUTC/nonUTC, mixedproducers, nulls, casts, ANSIon/off, timestamp/date_truncboundaries"
+ "Comet-on/off on UTC/nonUTC, mixed producers, nulls, casts, ANSI on/off, timestamp/date_trunc boundaries"
],
- "user_decision": "Использовать upstream timestamp producer/CASE fixes и Spark4.2 semantics; переносить нашу дополнительную нормализацию сравнений там, где она ещё необходима.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "Use upstream timestamp producer/CASE fixes and Spark4.2 semantics; port our additional comparison normalization where still necessary.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 26,
"source_paths": [
"native/core/src/execution/planner.rs",
@@ -1181,36 +1020,32 @@
"spark/src/test/scala/org/apache/comet/CometDateTimeUtilsSuite.scala"
],
"upstream": {
- "base_1_2": "#6341 timestamp_seconds UTC; #6345 date_trunc возвращает label входа; #6347 CASE/COALESCE branches relabel; более сильные test assertions #6868/#6930. Общий Joom comparison-side relabel отсутствует.",
- "outside_base": "Producer fixes покрывают известные части, но полная эквивалентность произвольных producer combinations статически не доказана."
+ "base_1_2": "#6341 timestamp_seconds UTC; #6345 date_trunc returns the input label; #6347 CASE/COALESCE branch relabeling; stronger test assertions #6868/#6930. General Joom comparison-side relabeling is absent.",
+ "outside_base": "Producer fixes cover known parts, but full equivalence across arbitrary producer combinations is not statically proven."
},
"acceptance": [
- "CometTimestampComparisonSuite иtimestamp_seconds_timezone.sql",
+ "CometTimestampComparisonSuite and timestamp_seconds_timezone.sql",
"CometDateTimeUtilsSuite UTC aliases/sessionzones",
- "Comet-on/off наUTC/nonUTC, mixedproducers, nulls, casts, ANSIon/off, timestamp/date_truncboundaries"
+ "Comet-on/off on UTC/nonUTC, mixed producers, nulls, casts, ANSI on/off, timestamp/date_trunc boundaries"
],
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
"original_dependency_notes": [
"P01",
"native expression/planner integration"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "upstream-основа утверждена",
+ "decision_category": "upstream base approved",
"same_pr_or_co_review": [],
"approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Использовать upstream timestamp producer/CASE fixes и Spark4.2 semantics; переносить нашу дополнительную нормализацию сравнений там, где она ещё необходима.",
+ "approved_decision": "Use upstream timestamp producer/CASE fixes and Spark4.2 semantics; port our additional comparison normalization where still necessary.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Обсудить частичный аналог. Сначала upstream producer guarantees + все Joom comparison regressions; если остаётся несовпадение, адаптировать только residual comparison normalization в новый split planner.",
- "prior_open_decisions": "Утвердить проверку upstreamрешения с переносом только недостающего поведения, а не обязательное сохранение старогоcomparisoncast.",
- "implementation_plan_groups": [
- "G02"
- ],
+ "original_proposal": "Discuss the partial equivalent. First use upstream producer guarantees plus all Joom comparison regressions; if a mismatch remains, adapt only residual comparison normalization to the new split planner.",
+ "prior_open_decisions": "Approve verification of the upstream solution with only missing behavior ported, rather than mandatory retention of the old comparison cast.",
+ "implementation_plan_groups": ["G02"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -1220,9 +1055,7 @@
"id": "I07",
"title": "Wide-decimal hash shuffle correctness",
"scope": "existing_joom_backport",
- "source_commits": [
- "88cfb2929"
- ],
+ "source_commits": ["88cfb2929"],
"source_files": [
"spark/src/main/scala/org/apache/comet/serde/hash.scala",
"spark/src/main/scala/org/apache/comet/rules/BoundaryFormats.scala",
@@ -1233,28 +1066,25 @@
"spark/src/test/scala/org/apache/comet/CometFuzzTestSuite.scala",
"spark/src/test/resources/tpcds-plan-stability/**"
],
- "contract": "Partition hashing Decimal precision>18 должен совпадать со Spark во всехnative/Spark mixedproducer-consumer combinations. СтарыйJoombackport#6005 обеспечивалэто fallbackrestriction.",
+ "contract": "Partition hashing for Decimal precision>18 must match Spark in all native/Spark mixed producer-consumer combinations. The old Joom backport#6005 ensured this with a fallback restriction.",
"upstream_analogue": {
- "base_1_2": "#6005 входит в1.2; дополнительно#6548 реализует native wide-decimal hashing. Старый запрет уже не является единственным корректным способом.",
- "outside_base": "Не возвращать автоматически obsolete precision>18 запрет в нашBoundaryFormats."
+ "base_1_2": "#6005 is included in1.2; additionally#6548 implements native wide-decimal hashing. The old restriction is no longer the only correct approach.",
+ "outside_base": "Do not automatically restore the obsolete precision>18 restriction in our BoundaryFormats."
},
- "proposed_action": "Использовать upstream native wide-decimal hashing вместо возвращения старого precision>18 запрета; проверить совместимость со Spark, включая mixed boundaries.",
+ "proposed_action": "Use upstream native wide-decimal hashing instead of restoring the old precision>18 restriction; verify Spark compatibility, including mixed boundaries.",
"approval_status": "APPROVED_EXPLICITLY",
- "dependencies": [
- "P01",
- "boundary formats integration"
- ],
+ "dependencies": ["P01", "boundary formats integration"],
"risks": [
- "Ошибкиmixedhashдают неверныеjoin/aggregate результаты, не просто медленнее.",
- "Тестирование только native-native не выявляетSpark/native partition mismatch."
+ "Mixed-hash errors produce incorrect join/aggregate results, not merely slower execution.",
+ "Testing native-native only does not expose Spark/native partition mismatches."
],
"acceptance_tests": [
- "CometNativeShuffleSuite precision18/19/38 positive/negative/null и hash repartition",
+ "CometNativeShuffleSuite precision18/19/38 positive/negative/null and hash repartition",
"Spark/native/columnar/row boundary mixedjoin+aggregate; AQE on/off",
- "Regenerate target4.2.1 approvedplans изреальныхпланов, не копироватьстарыеsnapshots"
+ "Regenerate approved target4.2.1 plans from actual plans; do not copy old snapshots"
],
- "user_decision": "Использовать upstream native wide-decimal hashing вместо возвращения старого precision>18 запрета; проверить совместимость со Spark, включая mixed boundaries.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "Use upstream native wide-decimal hashing instead of restoring the old precision>18 restriction; verify Spark compatibility, including mixed boundaries.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 27,
"source_paths": [
"spark/src/main/scala/org/apache/comet/serde/hash.scala",
@@ -1267,36 +1097,29 @@
"spark/src/test/resources/tpcds-plan-stability/**"
],
"upstream": {
- "base_1_2": "#6005 входит в1.2; дополнительно#6548 реализует native wide-decimal hashing. Старый запрет уже не является единственным корректным способом.",
- "outside_base": "Не возвращать автоматически obsolete precision>18 запрет в нашBoundaryFormats."
+ "base_1_2": "#6005 is included in1.2; additionally#6548 implements native wide-decimal hashing. The old restriction is no longer the only correct approach.",
+ "outside_base": "Do not automatically restore the obsolete precision>18 restriction in our BoundaryFormats."
},
"acceptance": [
- "CometNativeShuffleSuite precision18/19/38 positive/negative/null и hash repartition",
+ "CometNativeShuffleSuite precision18/19/38 positive/negative/null and hash repartition",
"Spark/native/columnar/row boundary mixedjoin+aggregate; AQE on/off",
- "Regenerate target4.2.1 approvedplans изреальныхпланов, не копироватьстарыеsnapshots"
- ],
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
- "original_dependency_notes": [
- "P01",
- "boundary formats integration"
+ "Regenerate approved target4.2.1 plans from actual plans; do not copy old snapshots"
],
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
+ "original_dependency_notes": ["P01", "boundary formats integration"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "upstream-основа утверждена",
+ "decision_category": "upstream base approved",
"same_pr_or_co_review": [],
"approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Использовать upstream native wide-decimal hashing вместо возвращения старого precision>18 запрета; проверить совместимость со Spark, включая mixed boundaries.",
+ "approved_decision": "Use upstream native wide-decimal hashing instead of restoring the old precision>18 restriction; verify Spark compatibility, including mixed boundaries.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Обсудить: использовать upstreamtypedhash/nativewide-decimal support; адаптироватьBoundaryFormats к актуальномуsupport predicate и перенести Joom mixed-boundary regressions.",
- "prior_open_decisions": "Утвердить upstreamwide-decimal capability вместо сохранения старого ограничения при сохраненииhash-correctnesscontract.",
- "implementation_plan_groups": [
- "G09"
- ],
+ "original_proposal": "Discuss using upstream typed hashing/native wide-decimal support; adapt BoundaryFormats to the current support predicate and port Joom mixed-boundary regressions.",
+ "prior_open_decisions": "Approve upstream wide-decimal capability instead of retaining the old restriction, preserving the hash-correctness contract.",
+ "implementation_plan_groups": ["G09"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -1304,21 +1127,19 @@
},
{
"id": "I08",
- "title": "Joom release builder для двух архитектур и упаковка libcomet",
+ "title": "Joom release builder for two architectures and libcomet packaging",
"scope": "existing_joom_release_infrastructure",
- "source_commits": [
- "e6b7d5eb4"
- ],
+ "source_commits": ["e6b7d5eb4"],
"source_files": [
"dev/release/joom/Dockerfile",
"dev/release/joom/build-release.sh"
],
- "contract": "Из чистого проверенного HEAD получить Linux amd64 и arm64 библиотеки, core и Delta JAR. Сохранить commit и контрольные суммы; проверить ELF, GLIBC и байты обеих библиотек в shaded JAR.",
+ "contract": "From a clean verified HEAD, produce Linux amd64 and arm64 libraries, core and Delta JARs. Preserve commit and checksums; verify ELF, GLIBC and the bytes of both libraries in the shaded JAR.",
"upstream_analogue": {
- "base_1_2": "АналогаJoomreleasepipelineнет; общийupstreamreleaseпроцессдругой.",
- "outside_base": "СтарыйscriptжёсткоSpark3.5/Scala2.12/Comet1.1.0; targetменяетсянаSpark4.2/Scala2.13/Comet1.2.0."
+ "base_1_2": "No equivalent Joom release pipeline exists; the general upstream release process differs.",
+ "outside_base": "The old script hardcodes Spark3.5/Scala2.12/Comet1.1.0; the target changes to Spark4.2/Scala2.13/Comet1.2.0."
},
- "proposed_action": "Адаптировать builder и pipeline под точные Spark 4.2.1, Delta 4.4.1, актуальный Maven reactor и target Cargo.lock; зафиксировать Rust toolchain. Сохранить проверки обеих архитектур и provenance.",
+ "proposed_action": "Adapt the builder and pipeline to exact Spark4.2.1, Delta4.4.1, the current Maven reactor and target Cargo.lock; pin the Rust toolchain. Preserve both-architecture checks and provenance.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"dependencies": [
"P01",
@@ -1327,36 +1148,36 @@
"all runtimefeaturePRs beforefinalrelease"
],
"risks": [
- "Старый Maven -pl нельзя копировать: текущая инструкция требует сборку из корня reactor.",
- "Spark-профиль 4.2 задаёт Scala 2.13.18, а отдельный профиль scala-2.13 позднее в POM задаёт 2.13.16. Не заменять профили механически; проверить effective POM.",
- "Сохранить GLIBC <=2.31 и проверить сборку на целевых GCP CPU, включая amd64 x86-64-v3. Фактический CPU узлов этим аудитом не проверен.",
- "Старый release build пропускает тесты: требуется отдельный пройденный test gate того же коммита.",
- "При нашем локальном запуске явно задавать рабочие и выходные каталоги внутри датированного каталога /Users/skotlov/d/temp. Старый publisher использует bare mktemp -d; новая реализация должна позволять задать рабочий каталог, сохраняя переносимость CI без hardcoded личного пути."
+ "Do not copy old Maven -pl: current instructions require building from the reactor root.",
+ "The Spark4.2 profile sets Scala2.13.18, while the separate scala-2.13 profile later in the POM sets2.13.16. Do not mechanically substitute profiles; check the effective POM.",
+ "Preserve GLIBC<=2.31 and verify the build on target GCP CPUs, including amd64 x86-64-v3. Actual node CPUs were not checked in this audit.",
+ "The old release build skips tests: a separate passing test gate for the same commit is required.",
+ "For our local runs, explicitly set working/output directories inside a dated /Users/skotlov/d/temp directory. The old publisher uses bare mktemp -d; the new implementation must accept a working directory while remaining portable in CI without a hardcoded personal path."
],
"acceptance_tests": [
- "Artifact IDs comet-spark-spark4.2_2.13 и comet-contrib-delta-spark4.2_2.13; правильный Scala/Spark/Delta classpath.",
- "ELF machine, GLIBC symbol versions, ровно две Linux libcomet.so без dylib, побайтовое сравнение packaged libraries.",
- "Smoke-тест Spark 4.2.1, native operators, Delta/GCS и UDF в Linux release image каждой используемой архитектуры.",
- "commit.txt, sha256.txt и согласованный locked dependency graph."
+ "Artifact IDs comet-spark-spark4.2_2.13 and comet-contrib-delta-spark4.2_2.13; correct Scala/Spark/Delta classpath.",
+ "ELF machine, GLIBC symbol versions, exactly two Linux libcomet.so files with no dylib, byte-identical packaged libraries.",
+ "Smoke-test Spark4.2.1, native operators, Delta/GCS and UDF in a Linux release image for each used architecture.",
+ "commit.txt, sha256.txt and a consistent locked dependency graph."
],
- "user_decision": "Утвердить сохранение контракта двух архитектур, GLIBC и provenance с новыми профилями. Изменение аппаратного baseline требует отдельного решения.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "Approve retention of the two-architecture, GLIBC and provenance contract with new profiles. Changing the hardware baseline requires a separate decision.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 90,
"source_paths": [
"dev/release/joom/Dockerfile",
"dev/release/joom/build-release.sh"
],
"upstream": {
- "base_1_2": "АналогаJoomreleasepipelineнет; общийupstreamreleaseпроцессдругой.",
- "outside_base": "СтарыйscriptжёсткоSpark3.5/Scala2.12/Comet1.1.0; targetменяетсянаSpark4.2/Scala2.13/Comet1.2.0."
+ "base_1_2": "No equivalent Joom release pipeline exists; the general upstream release process differs.",
+ "outside_base": "The old script hardcodes Spark3.5/Scala2.12/Comet1.1.0; the target changes to Spark4.2/Scala2.13/Comet1.2.0."
},
"acceptance": [
- "Artifact IDs comet-spark-spark4.2_2.13 и comet-contrib-delta-spark4.2_2.13; правильный Scala/Spark/Delta classpath.",
- "ELF machine, GLIBC symbol versions, ровно две Linux libcomet.so без dylib, побайтовое сравнение packaged libraries.",
- "Smoke-тест Spark 4.2.1, native operators, Delta/GCS и UDF в Linux release image каждой используемой архитектуры.",
- "commit.txt, sha256.txt и согласованный locked dependency graph."
+ "Artifact IDs comet-spark-spark4.2_2.13 and comet-contrib-delta-spark4.2_2.13; correct Scala/Spark/Delta classpath.",
+ "ELF machine, GLIBC symbol versions, exactly two Linux libcomet.so files with no dylib, byte-identical packaged libraries.",
+ "Smoke-test Spark4.2.1, native operators, Delta/GCS and UDF in a Linux release image for each used architecture.",
+ "commit.txt, sha256.txt and a consistent locked dependency graph."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"original_dependency_notes": [
"P01",
"P02",
@@ -1364,103 +1185,81 @@
"all runtimefeaturePRs beforefinalrelease"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01",
- "P02",
- "D01"
- ],
+ "prerequisites": ["P01", "P02", "D01"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G01"
- ],
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "approved_decision": "Адаптировать builder и pipeline под точные Spark 4.2.1, Delta 4.4.1, актуальный Maven reactor и target Cargo.lock; зафиксировать Rust toolchain. Сохранить проверки обеих архитектур и provenance.",
+ "approved_decision": "Adapt the builder and pipeline to exact Spark4.2.1, Delta4.4.1, the current Maven reactor and target Cargo.lock; pin the Rust toolchain. Preserve both-architecture checks and provenance.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Утвердить сохранение контракта двух архитектур, GLIBC и provenance с новыми профилями. Изменение аппаратного baseline требует отдельного решения.",
+ "prior_open_decisions": "Approve retention of the two-architecture, GLIBC and provenance contract with new profiles. Changing the hardware baseline requires a separate decision.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "I09",
- "title": "Неизменяемые версии при публикации в JFrog",
+ "title": "Immutable versions when publishing to JFrog",
"scope": "existing_joom_release_infrastructure",
- "source_commits": [
- "e6b7d5eb4"
- ],
- "source_files": [
- "dev/release/joom/publish-jfrog.sh"
- ],
- "contract": "Maven-версия соответствует Git-тегу; commit.txt совпадает с коммитом тега; контрольные суммы валидны. Существующая версия не перезаписывается. Скачанный после upload JAR побайтово совпадает с собранным.",
+ "source_commits": ["e6b7d5eb4"],
+ "source_files": ["dev/release/joom/publish-jfrog.sh"],
+ "contract": "The Maven version matches the Git tag; commit.txt matches the tagged commit; checksums are valid. An existing version is never overwritten. A JAR downloaded after upload is byte-identical to the built JAR.",
"upstream_analogue": {
- "base_1_2": "Joom-specificJFrogpublisherотсутствует.",
- "outside_base": "СтарыйpublisherжёсткоartifactSpark3.5_2.12/Comet1.1.0 filenames; нужноадаптироватьSpark4.2_2.13/1.2.0."
+ "base_1_2": "No Joom-specific JFrog publisher exists.",
+ "outside_base": "The old publisher hardcodes Spark3.5_2.12/Comet1.1.0 artifact filenames; adapt to Spark4.2_2.13/1.2.0."
},
- "proposed_action": "Адаптировать имена артефактов и metadata для Spark 4.2 / Scala 2.13 / Comet 1.2. Сохранить проверки тега, коммита и хешей, отказ от перезаписи и проверку скачивания. Поддержать согласованные RC и стабильные теги.",
+ "proposed_action": "Adapt artifact names and metadata for Spark4.2 / Scala2.13 / Comet1.2. Preserve tag, commit and hash checks, refusal to overwrite, and download verification. Support agreed RC and stable tags.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "dependencies": [
- "I08"
- ],
+ "dependencies": ["I08"],
"risks": [
- "Нельзя опубликовать старый JAR под новым именем или артефакт другого коммита.",
- "Публикация сама не обновляет datasci Docker images; rollout — последующий отдельный этап.",
- "При нашем локальном запуске явно задавать рабочие и выходные каталоги внутри датированного каталога /Users/skotlov/d/temp. Старый publisher использует bare mktemp -d; новая реализация должна позволять задать рабочий каталог, сохраняя переносимость CI без hardcoded личного пути."
+ "Do not publish an old JAR under a new name or an artifact from a different commit.",
+ "Publishing does not update datasci Docker images itself; rollout is a subsequent separate stage.",
+ "For our local runs, explicitly set working/output directories inside a dated /Users/skotlov/d/temp directory. The old publisher uses bare mktemp -d; the new implementation must accept a working directory while remaining portable in CI without a hardcoded personal path."
],
"acceptance_tests": [
- "Проверка параметров и артефактов конкретного RC.",
- "При публикации: tag commit = commit.txt, валидные хеши, отказ от overwrite, скачанный JAR идентичен исходному.",
- "Версии joom-1.2-1-rcN и joom-1.2-1 без перемещения опубликованных тегов."
+ "Check parameters and artifacts for the specific RC.",
+ "On publication: tag commit=commit.txt, valid hashes, refusal to overwrite, downloaded JAR identical to the original.",
+ "Versions joom-1.2-1-rcN and joom-1.2-1 without moving published tags."
],
- "user_decision": "Утвердить сохранение контракта неизменяемых версий. Реальную публикацию выполнять после отдельного указания пользователя.",
- "implementation_clarity": "Целевое поведение определено. Детали API/проверки указаны в рисках; незаданных продуктовых требований нет.",
+ "user_decision": "Approve preservation of immutable-version guarantees. Perform actual publication only after a separate user instruction.",
+ "implementation_clarity": "Target behavior is defined. API/validation details are identified in the risks; no unspecified product requirements remain.",
"order": 91,
- "source_paths": [
- "dev/release/joom/publish-jfrog.sh"
- ],
+ "source_paths": ["dev/release/joom/publish-jfrog.sh"],
"upstream": {
- "base_1_2": "Joom-specificJFrogpublisherотсутствует.",
- "outside_base": "СтарыйpublisherжёсткоartifactSpark3.5_2.12/Comet1.1.0 filenames; нужноадаптироватьSpark4.2_2.13/1.2.0."
+ "base_1_2": "No Joom-specific JFrog publisher exists.",
+ "outside_base": "The old publisher hardcodes Spark3.5_2.12/Comet1.1.0 artifact filenames; adapt to Spark4.2_2.13/1.2.0."
},
"acceptance": [
- "Проверка параметров и артефактов конкретного RC.",
- "При публикации: tag commit = commit.txt, валидные хеши, отказ от overwrite, скачанный JAR идентичен исходному.",
- "Версии joom-1.2-1-rcN и joom-1.2-1 без перемещения опубликованных тегов."
- ],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
- "original_dependency_notes": [
- "I08"
+ "Check parameters and artifacts for the specific RC.",
+ "On publication: tag commit=commit.txt, valid hashes, refusal to overwrite, downloaded JAR identical to the original.",
+ "Versions joom-1.2-1-rcN and joom-1.2-1 without moving published tags."
],
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
+ "original_dependency_notes": ["I08"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "I08"
- ],
+ "prerequisites": ["I08"],
"phase": 1,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G01"
- ],
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "approved_decision": "Адаптировать имена артефактов и metadata для Spark 4.2 / Scala 2.13 / Comet 1.2. Сохранить проверки тега, коммита и хешей, отказ от перезаписи и проверку скачивания. Поддержать согласованные RC и стабильные теги.",
+ "approved_decision": "Adapt artifact names and metadata for Spark4.2 / Scala2.13 / Comet1.2. Preserve tag, commit and hash checks, refusal to overwrite, and download verification. Support agreed RC and stable tags.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Утвердить сохранение контракта неизменяемых версий. Реальную публикацию выполнять после отдельного указания пользователя.",
+ "prior_open_decisions": "Approve preservation of immutable-version guarantees. Perform actual publication only after a separate user instruction.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J14",
- "title": "Общий бюджет 32-bit offsets и разбиение explode/concat до overflow",
- "contract": "OffsetBudget/offset_extents учитываютповторение nestedvariablewidthdata; explode умеетразделитьрезультатоднойinputrowмеждуbatches; coalescer/windowconcat не переполняют offsets.",
- "source_commits": [
- "dfa6439e5"
- ],
+ "title": "Shared32-bit offset budget and splitting explode/concat before overflow",
+ "contract": "OffsetBudget/offset_extents account for repeated nested variable-width data; explode can split the result of one input row across batches; coalescer/window concat do not overflow offsets.",
+ "source_commits": ["dfa6439e5"],
"source_files": [
"native/common/src/offset_extents.rs",
"native/common/src/lib.rs",
@@ -1468,11 +1267,11 @@
"native/core/src/execution/operators/partition_aggregate_window.rs",
"native/shuffle/src/read_coalescer.rs"
],
- "upstream_base": "Base1.2 explode movedto native/operators/src/explode.rs иявнонеразделяетsingleinputrow; нашегоoffsetbudget нет.",
- "proposal": "Перенести commonutility и адаптироватьconsumerpaths; сэтапомwindow/coalescer согласоватьиспользование.",
+ "upstream_base": "Base1.2 moved explode to native/operators/src/explode.rs and explicitly does not split a single input row; our offset budget is absent.",
+ "proposal": "Port the common utility and adapt consumer paths; coordinate usage with the window/coalescer stage.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Существеннаяадаптация newoperatorscrate; dictionary/nested/map/list/nulls иoffsetarithmetics; syntheticbudgettests вместообязательныхmultiGBallocations."
+ "Substantial adaptation to the new operators crate; dictionary/nested/map/list/nulls and offset arithmetic; synthetic-budget tests instead of mandatory multi-GB allocations."
],
"dependencies": [],
"acceptance": [
@@ -1480,7 +1279,7 @@
],
"implementation_questions": [],
"other_upstream_candidates": [
- "PR#4791 LargeUtf8/LargeBinary — alternateoffsetwidth, неполныйэквивалент; PR#7537 top-leveldictionarysplit такжечастичный."
+ "PR#4791 LargeUtf8/LargeBinary uses an alternate offset width, not a full equivalent; PR#7537 top-level dictionary splitting is also partial."
],
"config": [],
"source_commit_details": [
@@ -1504,7 +1303,7 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/native/core/src/execution/operators/partition_aggregate_window.rs",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/native/shuffle/src/read_coalescer.rs"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"native/common/src/offset_extents.rs",
"native/common/src/lib.rs",
@@ -1512,50 +1311,44 @@
"native/core/src/execution/operators/partition_aggregate_window.rs",
"native/shuffle/src/read_coalescer.rs"
],
- "proposed_action": "Перенести commonutility и адаптироватьconsumerpaths; сэтапомwindow/coalescer согласоватьиспользование.",
+ "proposed_action": "Port the common utility and adapt consumer paths; coordinate usage with the window/coalescer stage.",
"upstream": {
- "base": "Base1.2 explode movedto native/operators/src/explode.rs иявнонеразделяетsingleinputrow; нашегоoffsetbudget нет.",
+ "base": "Base1.2 moved explode to native/operators/src/explode.rs and explicitly does not split a single input row; our offset budget is absent.",
"outside_base": [
- "PR#4791 LargeUtf8/LargeBinary — alternateoffsetwidth, неполныйэквивалент; PR#7537 top-leveldictionarysplit такжечастичный."
+ "PR#4791 LargeUtf8/LargeBinary uses an alternate offset width, not a full equivalent; PR#7537 top-level dictionary splitting is also partial."
]
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 2,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G02"
- ],
+ "implementation_plan_groups": ["G02"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести commonutility и адаптироватьconsumerpaths; сэтапомwindow/coalescer согласоватьиспользование.",
+ "approved_decision": "Port the common utility and adapt consumer paths; coordinate usage with the window/coalescer stage.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J16",
- "title": "TaskContext захватывается до чтения результата в другом потоке",
- "contract": "CometExecIterator сохраняетtaskcontext приcreation ииспользуетего когдаoutputiteratorчитаетсявнеtaskthread; taskmetrics/cancellation/lifecycle сохраняются.",
- "source_commits": [
- "042ab7d3f"
- ],
+ "title": "Capture TaskContext before results are read on another thread",
+ "contract": "CometExecIterator captures task context at creation and uses it when the output iterator is read outside the task thread; preserve task metrics/cancellation/lifecycle.",
+ "source_commits": ["042ab7d3f"],
"source_files": [
"spark/src/main/scala/org/apache/comet/CometExecIterator.scala"
],
- "upstream_base": "Base1.2 nextbatch path at257 stillTaskContext.get(), targetcapture отсутствует.",
- "proposal": "Небольшойцелевойперенос поверхновогоiteratorlifecycle.",
+ "upstream_base": "The base1.2 next-batch path at257 still uses TaskContext.get(); the target capture is absent.",
+ "proposal": "A small targeted port on top of the new iterator lifecycle.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Нельзяпереноситьiteratorцелымфайлом:1.2NativeExecContext/UDF/resourcecleanup отличаются; offthreadcleanup ordering."
+ "Do not port the whole iterator file:1.2 NativeExecContext/UDF/resource cleanup differ; consider off-thread cleanup ordering."
],
"dependencies": [],
"acceptance": [
@@ -1580,52 +1373,46 @@
"source_urls": [
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/CometExecIterator.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/comet/CometExecIterator.scala"
],
- "proposed_action": "Небольшойцелевойперенос поверхновогоiteratorlifecycle.",
+ "proposed_action": "A small targeted port on top of the new iterator lifecycle.",
"upstream": {
- "base": "Base1.2 nextbatch path at257 stillTaskContext.get(), targetcapture отсутствует.",
+ "base": "The base1.2 next-batch path at257 still uses TaskContext.get(); the target capture is absent.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 2,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G02"
- ],
+ "implementation_plan_groups": ["G02"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Небольшойцелевойперенос поверхновогоiteratorlifecycle.",
+ "approved_decision": "A small targeted port on top of the new iterator lifecycle.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J22",
- "title": "Безопасное владение fetched shuffle blocks и early-close",
- "contract": "FetchedBlocksInputStreamзакрываеткаждыйполученныйblockодинраздоnext; closeнеfetchesremainingblocks; killcheckмеждублоками; taskcompletionнеdouble-releaseNettybuffers.",
- "source_commits": [
- "705fc113f"
- ],
+ "title": "Safe ownership of fetched shuffle blocks and early close",
+ "contract": "FetchedBlocksInputStream closes every received block exactly once before next; close does not fetch remaining blocks; kill checks occur between blocks; task completion must not double-release Netty buffers.",
+ "source_commits": ["705fc113f"],
"source_files": [
"spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala"
],
- "upstream_base": "Base1.2 rawpathSequenceInputStream+FilterInputStreamимеетkillchecks, ноcloseвсёещёпроходитоставшиесяfetches; полногоаналогаownershipfixнет.",
- "proposal": "Перенестицелевойstreamlifecyclefixдоcoalescer; оставитьновые1.2interruptchecks.",
+ "upstream_base": "The base1.2 raw path SequenceInputStream+FilterInputStream has kill checks, but close still traverses remaining fetches; no full equivalent of the ownership fix exists.",
+ "proposal": "Port the targeted stream-lifecycle fix before the coalescer; retain new1.2 interrupt checks.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Spark4ShuffleBlockFetcherIterator completionlisteners/Nettyrefcount/cancelI/O; нельзязакрыватьqueuebuffersповторно."
+ "Spark4 ShuffleBlockFetcherIterator completion listeners/Netty refcounts/cancel I/O; do not close queued buffers twice."
],
"dependencies": [],
"acceptance": [
@@ -1648,44 +1435,38 @@
"source_urls": [
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala"
],
- "proposed_action": "Перенестицелевойstreamlifecyclefixдоcoalescer; оставитьновые1.2interruptchecks.",
+ "proposed_action": "Port the targeted stream-lifecycle fix before the coalescer; retain new1.2 interrupt checks.",
"upstream": {
- "base": "Base1.2 rawpathSequenceInputStream+FilterInputStreamимеетkillchecks, ноcloseвсёещёпроходитоставшиесяfetches; полногоаналогаownershipfixнет.",
+ "base": "The base1.2 raw path SequenceInputStream+FilterInputStream has kill checks, but close still traverses remaining fetches; no full equivalent of the ownership fix exists.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "J16"
- ],
+ "prerequisites": ["J16"],
"phase": 2,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G02"
- ],
+ "implementation_plan_groups": ["G02"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенестицелевойstreamlifecyclefixдоcoalescer; оставитьновые1.2interruptchecks.",
+ "approved_decision": "Port the targeted stream-lifecycle fix before the coalescer; retain new1.2 interrupt checks.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J15",
- "title": "Остановка killed task внутри native plan",
- "contract": "Taskkillwatcher вызываетcancelPlan; cancellable native streams/rowloops/shufflewriter/JVMcodegen checks прекращаютработуибезопасноосвобождаютresources.",
- "source_commits": [
- "55c5b6fe8"
- ],
+ "title": "Stop killed tasks inside the native plan",
+ "contract": "The task-kill watcher calls cancelPlan; cancellable native streams/row loops/shuffle writer/JVM codegen checks stop work and safely release resources.",
+ "source_commits": ["55c5b6fe8"],
"source_files": [
"native/common/src/cancellation.rs",
"native/core/src/execution/operators/cancellable.rs",
@@ -1696,22 +1477,19 @@
"spark/src/main/scala/org/apache/comet/Native.scala",
"spark/src/main/scala/org/apache/spark/comet/CometTaskContextShim.scala"
],
- "upstream_base": "Base1.2 releasePlan/dropcleanup есть, ноJoomtaskkillwatcher/cancelPlan нет.",
- "proposal": "Адаптировать кновымexecutioncontexts/asyncstreams/resourcecleanup, включаяJNI/JVMloop checks.",
+ "upstream_base": "Base1.2 has releasePlan/drop cleanup, but not the Joom task-kill watcher/cancelPlan.",
+ "proposal": "Adapt to new execution contexts/async streams/resource cleanup, including JNI/JVM loop checks.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "RacecancelPlan vsreleasePlan, cancellationwhileblocked/runningnative, handlelifetime/useafterfree; cleanupпослеearlytermination."
- ],
- "dependencies": [
- "J16",
- "J22"
+ "cancelPlan versus releasePlan races, cancellation while blocked/running native code, handle lifetime/use-after-free; cleanup after early termination."
],
+ "dependencies": ["J16", "J22"],
"acceptance": [
"CometNativeTaskKillSuite: cancellongrunningnative/shuffle/JVMdispatch; cleanupmetrics/reservations; repeatedcancel/completionraces; noexecutor-taskreaperkill."
],
"implementation_questions": [],
"other_upstream_candidates": [
- "PR#7181 PythonArrowUDF killmonitor — другойscope, необщийnativeplancancel."
+ "PR#7181 PythonArrowUDF kill monitor has a different scope; it is not general native-plan cancellation."
],
"config": [],
"source_commit_details": [
@@ -1749,7 +1527,7 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/Native.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/spark/comet/CometTaskContextShim.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"native/common/src/cancellation.rs",
"native/core/src/execution/operators/cancellable.rs",
@@ -1760,61 +1538,49 @@
"spark/src/main/scala/org/apache/comet/Native.scala",
"spark/src/main/scala/org/apache/spark/comet/CometTaskContextShim.scala"
],
- "proposed_action": "Адаптировать кновымexecutioncontexts/asyncstreams/resourcecleanup, включаяJNI/JVMloop checks.",
+ "proposed_action": "Adapt to new execution contexts/async streams/resource cleanup, including JNI/JVM loop checks.",
"upstream": {
- "base": "Base1.2 releasePlan/dropcleanup есть, ноJoomtaskkillwatcher/cancelPlan нет.",
+ "base": "Base1.2 has releasePlan/drop cleanup, but not the Joom task-kill watcher/cancelPlan.",
"outside_base": [
- "PR#7181 PythonArrowUDF killmonitor — другойscope, необщийnativeplancancel."
+ "PR#7181 PythonArrowUDF kill monitor has a different scope; it is not general native-plan cancellation."
]
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "J16",
- "J22"
- ],
+ "original_dependency_notes": ["J16", "J22"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "J16",
- "J22"
- ],
+ "prerequisites": ["J16", "J22"],
"phase": 2,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G02"
- ],
+ "implementation_plan_groups": ["G02"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать кновымexecutioncontexts/asyncstreams/resourcecleanup, включаяJNI/JVMloop checks.",
+ "approved_decision": "Adapt to new execution contexts/async streams/resource cleanup, including JNI/JVM loop checks.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J18",
- "title": "Native library загружается один раз при повторной инициализации NativeBase",
- "contract": "Неизвлекать/незагружать вторуюlibcomet приresetloaded ссохранениемbundledlibrary.",
- "source_commits": [
- "1b63d8be6"
- ],
- "source_files": [
- "spark/src/main/java/org/apache/comet/NativeBase.java"
- ],
- "upstream_base": "Функциональныйаналогupstream#6493 ужеестьвbase1.2(bundledLibrary/perclassloaderstate).",
- "proposal": "Обсудитьпринятиеupstreamimplementation + regression; старыйpatchавтоматическинеприменять.",
+ "title": "Load the native library once when NativeBase is reinitialized",
+ "contract": "Do not extract/load a second libcomet when loaded is reset while bundledLibrary is preserved.",
+ "source_commits": ["1b63d8be6"],
+ "source_files": ["spark/src/main/java/org/apache/comet/NativeBase.java"],
+ "upstream_base": "Functional upstream equivalent#6493 is already in base1.2 (bundledLibrary/per-classloader state).",
+ "proposal": "Discuss accepting the upstream implementation plus regression; do not automatically apply the old patch.",
"approval": "APPROVED_EXPLICITLY",
"risks": [
- "Qualificationexecutorclassloader/reload/shadedartifact; sourcepresenceнеruntimeproof."
+ "Qualify executor classloader/reload/shaded artifact; source presence is not runtime proof."
],
"dependencies": [],
"acceptance": [
- "ПовторнаяинициализацияNativeBase втомжеclassloader; load/extractcount; Linuxamd64/arm64nativeJAR."
+ "Reinitialize NativeBase in the same classloader; load/extract count; Linux amd64/arm64 native JAR."
],
"implementation_questions": [
- "Подтвердитьupstreamреализациюпослепроверкистарогосценария; отдельныйJoomduplicate предлагаетсянесоздавать."
+ "Confirm the upstream implementation after checking the old scenario; a separate Joom duplicate is not proposed."
],
"other_upstream_candidates": [],
"config": [],
@@ -1822,45 +1588,37 @@
{
"sha": "1b63d8be654dc0bd60c6e745fad5453b98ca4027",
"subject": "fix: load the native library once per JVM when NativeBase is reloaded",
- "files": [
- "spark/src/main/java/org/apache/comet/NativeBase.java"
- ],
+ "files": ["spark/src/main/java/org/apache/comet/NativeBase.java"],
"url": "https://github.com/joomcode/datafusion-comet/commit/1b63d8be654dc0bd60c6e745fad5453b98ca4027"
}
],
"source_urls": [
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/java/org/apache/comet/NativeBase.java"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
- "source_paths": [
- "spark/src/main/java/org/apache/comet/NativeBase.java"
- ],
- "proposed_action": "Использовать upstream однократную загрузку bundled native library; проверить исходный Joom reload-сценарий.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
+ "source_paths": ["spark/src/main/java/org/apache/comet/NativeBase.java"],
+ "proposed_action": "Use upstream one-time bundled native-library loading; verify the original Joom reload scenario.",
"upstream": {
- "base": "Функциональныйаналогupstream#6493 ужеестьвbase1.2(bundledLibrary/perclassloaderstate).",
+ "base": "Functional upstream equivalent#6493 is already in base1.2 (bundledLibrary/per-classloader state).",
"outside_base": []
},
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
"approval_status": "APPROVED_EXPLICITLY",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 2,
"requires_upstream_discussion": false,
- "decision_category": "upstream-основа утверждена",
+ "decision_category": "upstream base approved",
"same_pr_or_co_review": [],
- "approved_decision": "Использовать upstream однократную загрузку bundled native library; проверить исходный Joom reload-сценарий.",
+ "approved_decision": "Use upstream one-time bundled native-library loading; verify the original Joom reload scenario.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Обсудитьпринятиеupstreamimplementation + regression; старыйpatchавтоматическинеприменять.",
+ "original_proposal": "Discuss accepting the upstream implementation plus regression; do not automatically apply the old patch.",
"prior_open_decisions": [
- "Подтвердитьupstreamреализациюпослепроверкистарогосценария; отдельныйJoomduplicate предлагаетсянесоздавать."
- ],
- "implementation_plan_groups": [
- "G01"
+ "Confirm the upstream implementation after checking the old scenario; a separate Joom duplicate is not proposed."
],
+ "implementation_plan_groups": ["G01"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -1868,28 +1626,26 @@
},
{
"id": "J24",
- "title": "Малый batchSize и cap JVM shuffle batch size в месте использования",
- "contract": "ПоложительныймалыйbatchSizeнепадаетнаинициализацииCometConf; effectiveJVMshuffleBatch=min(configuredJVM,batchSize).",
- "source_commits": [
- "ea38aad51"
- ],
+ "title": "Small batchSize and capping JVM shuffle batch size at its use site",
+ "contract": "A positive small batchSize does not fail during CometConf initialization; effectiveJVMshuffleBatch=min(configuredJVM,batchSize).",
+ "source_commits": ["ea38aad51"],
"source_files": [
"spark/src/main/scala/org/apache/comet/CometConf.scala",
"spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java",
"spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java"
],
- "upstream_base": "Полныйфункциональныйаналогвbase1.2 CometConf.scala1006–1028,jvmShuffleBatchSize(conf).",
- "proposal": "Обсудитьиспользованиеupstreamimplementation+нашиvalidationcases.",
+ "upstream_base": "A full functional equivalent exists in base1.2 CometConf.scala1006–1028,jvmShuffleBatchSize(conf).",
+ "proposal": "Discuss use of the upstream implementation plus our validation cases.",
"approval": "APPROVED_EXPLICITLY",
"risks": [
- "Небольшойриск: заменитьимявызоваhelper, неоткатитьSQLConfcapture1.2."
+ "Small risk: change the helper call name without reverting1.2 SQLConf capture."
],
"dependencies": [],
"acceptance": [
"CometConfSuite batchSize512/smallpositive/default8192,bothwriters effectivecap; clusterexecutorsessionconf."
],
"implementation_questions": [
- "Подтвердитьupstreamаналог; копироватьстарыйhelperнепредлагается."
+ "Confirm the upstream equivalent; copying the old helper is not proposed."
],
"other_upstream_candidates": [],
"config": [],
@@ -1911,38 +1667,34 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/comet/CometConf.scala",
"spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/CometDiskBlockWriter.java",
"spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java"
],
- "proposed_action": "Использовать upstream small-batch/effective JVM shuffle batch cap; сохранить наши проверки.",
+ "proposed_action": "Use the upstream small-batch/effective JVM shuffle batch cap; preserve our checks.",
"upstream": {
- "base": "Полныйфункциональныйаналогвbase1.2 CometConf.scala1006–1028,jvmShuffleBatchSize(conf).",
+ "base": "A full functional equivalent exists in base1.2 CometConf.scala1006–1028,jvmShuffleBatchSize(conf).",
"outside_base": []
},
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
"approval_status": "APPROVED_EXPLICITLY",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 2,
"requires_upstream_discussion": false,
- "decision_category": "upstream-основа утверждена",
+ "decision_category": "upstream base approved",
"same_pr_or_co_review": [],
- "approved_decision": "Использовать upstream small-batch/effective JVM shuffle batch cap; сохранить наши проверки.",
+ "approved_decision": "Use the upstream small-batch/effective JVM shuffle batch cap; preserve our checks.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Обсудитьиспользованиеupstreamimplementation+нашиvalidationcases.",
+ "original_proposal": "Discuss use of the upstream implementation plus our validation cases.",
"prior_open_decisions": [
- "Подтвердитьupstreamаналог; копироватьстарыйhelperнепредлагается."
- ],
- "implementation_plan_groups": [
- "G07"
+ "Confirm the upstream equivalent; copying the old helper is not proposed."
],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -1950,29 +1702,27 @@
},
{
"id": "J21",
- "title": "Нормализация offsets sliced broadcast batches",
- "contract": "При coalescing broadcast sliced string/binary/list values сохраняют точные границы и результаты; проверить обычный и reused/AQE broadcast.",
- "source_commits": [
- "d913126f8"
- ],
+ "title": "Normalize offsets in sliced broadcast batches",
+ "contract": "When coalescing broadcast sliced string/binary/list values, preserve exact boundaries and results; verify ordinary and reused/AQE broadcast.",
+ "source_commits": ["d913126f8"],
"source_files": [
"spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala",
"native/common/src/ffi_offsets.rs",
"native/core/src/execution/utils.rs",
"native/spark-expr/src/jvm_udf/mod.rs"
],
- "upstream_base": "Аналогиесть: Utils.scala273–277normalizesrootbeforeIPCserialize(другойэтапчемJoomappend); ffi_offsets.rs zero_offsets рекурсивныйupstream#6339.",
- "proposal": "Обсудитьиспользованиеupstreamimplementations; переноситьрегрессиииостаткиесливыявятся.",
+ "upstream_base": "Equivalents exist: Utils.scala273–277 normalizes the root before IPC serialization (a different stage from Joom append); ffi_offsets.rs zero_offsets is recursive upstream#6339.",
+ "proposal": "Discuss using upstream implementations; port regressions and residual changes if identified.",
"approval": "APPROVED_EXPLICITLY",
"risks": [
- "Проверитьвсепутиобычныйbroadcast/FFI/JVMdispatch; upstreamnormalizationраньшепоlifecycleдолжнапокрытьJoomlatefix."
+ "Verify all ordinary broadcast/FFI/JVM-dispatch paths; upstream normalization earlier in the lifecycle must cover the Joom late fix."
],
"dependencies": [],
"acceptance": [
- "Исходные sliced broadcast regressions из CometJoinSuite и UtilsSuite на Spark4.2.1, включая reused/AQE exchange."
+ "Original sliced-broadcast regressions from CometJoinSuite and UtilsSuite on Spark4.2.1, including reused/AQE exchange."
],
"implementation_questions": [
- "Подтвердитьupstreamаналоги после выполнениястарыхтестов; продублироватьнормализациювдвухместахнепредлагается."
+ "Confirm upstream equivalents after running old tests; duplicating normalization in two places is not proposed."
],
"other_upstream_candidates": [],
"config": [],
@@ -1993,38 +1743,34 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/spark/sql/comet/util/Utils.scala",
"spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala",
"spark/src/test/scala/org/apache/spark/sql/comet/util/UtilsSuite.scala"
],
- "proposed_action": "Использовать upstream нормализацию sliced broadcast offsets; сохранить Joom регрессии и добавить недостающие исправления при обнаружении потери поведения.",
+ "proposed_action": "Use upstream sliced-broadcast offset normalization; retain Joom regressions and add missing fixes if behavior is lost.",
"upstream": {
- "base": "Upstream нормализует source VectorSchemaRoot через slice ДО IPC serialization; Joom исправлял offsets позже, при append. Обсудить применение upstream-аналога и сохранить регрессии.",
+ "base": "Upstream normalizes the source VectorSchemaRoot through slicing BEFORE IPC serialization; Joom repaired offsets later, during append. Discuss using the upstream equivalent and preserve regressions.",
"outside_base": []
},
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
"approval_status": "APPROVED_EXPLICITLY",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 2,
"requires_upstream_discussion": false,
- "decision_category": "upstream-основа утверждена",
+ "decision_category": "upstream base approved",
"same_pr_or_co_review": [],
- "approved_decision": "Использовать upstream нормализацию sliced broadcast offsets; сохранить Joom регрессии и добавить недостающие исправления при обнаружении потери поведения.",
+ "approved_decision": "Use upstream sliced-broadcast offset normalization; retain Joom regressions and add missing fixes if behavior is lost.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Обсудитьиспользованиеupstreamimplementations; переноситьрегрессиииостаткиесливыявятся.",
+ "original_proposal": "Discuss using upstream implementations; port regressions and residual changes if identified.",
"prior_open_decisions": [
- "Подтвердитьupstreamаналоги после выполнениястарыхтестов; продублироватьнормализациювдвухместахнепредлагается."
- ],
- "implementation_plan_groups": [
- "G07"
+ "Confirm upstream equivalents after running old tests; duplicating normalization in two places is not proposed."
],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -2032,30 +1778,26 @@
},
{
"id": "J26",
- "title": "Диагностика памяти крупных batches на decode/FFI границах",
- "contract": "ОпциональныйCOMET_DEBUG_BATCH_MEMORY=1 пишетrows/sharedbufferbytes/nativeallocated/reservedприbatch>=16MiB; вобычномрежименеактивен.",
- "source_commits": [
- "ca6b963f0"
- ],
+ "title": "Large-batch memory diagnostics at decode/FFI boundaries",
+ "contract": "Optional COMET_DEBUG_BATCH_MEMORY=1 logs rows/shared-buffer bytes/native allocated/reserved for batch>=16MiB; inactive in normal mode.",
+ "source_commits": ["ca6b963f0"],
"source_files": [
"native/core/src/execution/jni_api.rs",
"native/core/src/execution/operators/shuffle_scan.rs"
],
- "upstream_base": "ТочнотакойJoomdiagnosticpathвbase1.2 не найден.",
- "proposal": "Перенести как существующийopt-indiagnosticконтракт; можнообъединитьсcoalescerPR.",
+ "upstream_base": "No identical Joom diagnostic path was found in base1.2.",
+ "proposal": "Port as an existing opt-in diagnostic contract; may combine with the coalescer PR.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "MemorycounterAPI59.3.2/sharedIPCbuffers; логированиепотенциальнобольшое, сохраняемopt-in."
+ "Memory-counter API59.3.2/shared IPC buffers; logging can be extensive, so retain opt-in behavior."
],
"dependencies": [],
"acceptance": [
- "Unit/integrationlogtestдляflagoff/on,sharedbufferaccounting; noalways-onlogging."
+ "Unit/integration log test for flag off/on and shared-buffer accounting; no always-on logging."
],
"implementation_questions": [],
"other_upstream_candidates": [],
- "config": [
- "COMET_DEBUG_BATCH_MEMORY=1"
- ],
+ "config": ["COMET_DEBUG_BATCH_MEMORY=1"],
"source_commit_details": [
{
"sha": "ca6b963f0d00c48262644e9fbe12944ae4186673",
@@ -2071,94 +1813,76 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/native/core/src/execution/jni_api.rs",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/native/core/src/execution/operators/shuffle_scan.rs"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"native/core/src/execution/jni_api.rs",
"native/core/src/execution/operators/shuffle_scan.rs"
],
- "proposed_action": "Перенести как существующийopt-indiagnosticконтракт; можнообъединитьсcoalescerPR.",
+ "proposed_action": "Port as an existing opt-in diagnostic contract; may combine with the coalescer PR.",
"upstream": {
- "base": "ТочнотакойJoomdiagnosticpathвbase1.2 не найден.",
+ "base": "No identical Joom diagnostic path was found in base1.2.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 2,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G07"
- ],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести как существующийopt-indiagnosticконтракт; можнообъединитьсcoalescerPR.",
+ "approved_decision": "Port as an existing opt-in diagnostic contract; may combine with the coalescer PR.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "D11",
- "title": "Ограничение начального sort merge reserve до 1/32 task budget",
- "contract": "При off-heap execution начальный datafusion.execution.sort_spill_reservation_bytes ограничивается min(upstream_default, max(off_heap_limit / max(executor_cores / max(task_cpus,1),1) /32,1)). Большие tasks сохраняют upstream default. Явные spark.comet.datafusion.* overrides применяются ПОСЛЕ cap и имеют приоритет. Это отдельная защита от D03 spillBeforeOutputThreshold.",
- "source_commits": [
- "65be8b751",
- "d153b3a85",
- "4ff37081a"
- ],
- "source_paths": [
- "native/core/src/execution/jni_api.rs"
- ],
+ "title": "Cap initial sort merge reserve at1/32 of task budget",
+ "contract": "With off-heap execution, initial datafusion.execution.sort_spill_reservation_bytes is capped at min(upstream_default, max(off_heap_limit / max(executor_cores / max(task_cpus,1),1) /32,1)). Large tasks retain the upstream default. Explicit spark.comet.datafusion.* overrides apply AFTER the cap and take precedence. This protection is separate from D03 spillBeforeOutputThreshold.",
+ "source_commits": ["65be8b751", "d153b3a85", "4ff37081a"],
+ "source_paths": ["native/core/src/execution/jni_api.rs"],
"upstream": {
- "base": "В выбранной Comet1.2-базе нет Joom configure_sort_spill_reservation. Базовый DataFusion sort reserve сам по себе не заменяет cap для небольшого Spark task budget."
+ "base": "The selected Comet1.2 base lacks Joom configure_sort_spill_reservation. The baseline DataFusion sort reserve alone does not replace the cap for a small Spark task budget."
},
- "proposed_action": "Адаптировать настройку SessionConfig в актуальный native context до прохода explicit overrides; сохранить итоговое восстановленное поведение тега13.",
+ "proposed_action": "Adapt the SessionConfig setup in the current native context before applying explicit overrides; preserve the final restored behavior of tag13.",
"risks": [
- "Не перенести промежуточный revert d153b3a85 вместо финального восстановления4ff37081a.",
- "Не перепутать1/32 eager merge reserve с1/4 threshold выдачи JVM и не перезаписать пользовательские overrides."
- ],
- "dependencies": [
- "D01"
- ],
- "original_dependency_notes": [
- "D01"
+ "Do not port intermediate revert d153b3a85 instead of final restoration4ff37081a.",
+ "Do not confuse the1/32 eager merge reserve with the1/4 JVM-output threshold or overwrite user overrides."
],
+ "dependencies": ["D01"],
+ "original_dependency_notes": ["D01"],
"acceptance": [
- "sort_merge_reserve_scales_with_the_task_budget:64MiB/2tasks даёт1MiB,8GiB/4tasks сохраняет default; дополнить explicit override и on-heap/no-offheap кейсы.",
- "Малый task budget допускает первый sort batch и последующий spill; общий native/JVM memory regression D03."
+ "sort_merge_reserve_scales_with_the_task_budget:64MiB/2tasks gives1MiB;8GiB/4tasks preserves the default; add explicit-override and on-heap/no-offheap cases.",
+ "A small task budget permits the first sort batch and subsequent spill; combined native/JVM memory regression D03."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D01"
- ],
+ "prerequisites": ["D01"],
"phase": 3,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G03"
- ],
+ "implementation_plan_groups": ["G03"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "approved_decision": "Адаптировать настройку SessionConfig в актуальный native context до прохода explicit overrides; сохранить итоговое восстановленное поведение тега13.",
+ "approved_decision": "Adapt the SessionConfig setup in the current native context before applying explicit overrides; preserve the final restored behavior of tag13.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить перенос именно конечного контракта.",
+ "prior_open_decisions": "Required behavior is defined; porting the exact final contract needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "D02",
- "title": "Память сортировки, SpillWorkspace и multipass merge",
- "contract": "Сохранить резервы при уменьшении Spark grant, через все merge passes; учитывать encoded Rows и shared buffers один раз; оставлять headroom для downstream; уменьшать read-ahead/разбивать слишком широкий spill batch.",
+ "title": "Sort memory, SpillWorkspace and multipass merge",
+ "contract": "Preserve reservations when Spark shrinks the grant, across all merge passes; count encoded Rows and shared buffers once; leave downstream headroom; reduce read-ahead/split overly wide spill batches.",
"source_commits": [
"943936fed",
"880575b8d",
@@ -2184,23 +1908,21 @@
"native/vendor/datafusion-physical-plan/src/stream.rs"
],
"upstream": {
- "base": "DF55.2 уже передаёт final spill merge reservation.take(); это baseline, не полная замена Joom расширений. Comet PlanMemoryPool и SpillReplayPool решают другой lifecycle.",
- "outside_base": "DF56 #24740 MergeMemoryPool/WorkspaceLoan — существенный альтернативный механизм; отсутствует в выбранной55.2, не повод автоматически обновлять весь DF."
+ "base": "DF55.2 already transfers the final spill merge reservation.take(); this is baseline, not a full replacement of Joom extensions. Comet PlanMemoryPool and SpillReplayPool address a different lifecycle.",
+ "outside_base": "DF56 #24740 MergeMemoryPool/WorkspaceLoan is a significant alternative mechanism; absent in selected55.2, it is not a reason to automatically upgrade all of DF."
},
- "proposed_action": "Перенести residual Joom memory lifecycle и связанные regressions поверх pristine55.2.",
+ "proposed_action": "Port residual Joom memory lifecycle changes and associated regressions on top of pristine55.2.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Новые Comet1.2 memory contexts/cleanup и aggregate replay требуют проверить ownership/reservation lifetimes.",
- "Это блок зависимых исправлений; выборочные cherry-pick могут сохранить leaks/двойной учёт.",
- "Из чтения нельзя доказать bounded RSS под реальным Spark share."
- ],
- "dependencies": [
- "D01"
+ "New Comet1.2 memory contexts/cleanup and aggregate replay require verification of ownership/reservation lifetimes.",
+ "This is a block of dependent fixes; selective cherry-picks can retain leaks/double accounting.",
+ "Reading source cannot prove bounded RSS under a real Spark share."
],
+ "dependencies": ["D01"],
"acceptance": [
- "Vendor comet_memory_tests + регрессии shrunk grant/multipass/final merge/wide batches/shared slices; проверить output equality, spill metrics, ограничение памяти и отсутствие зависания."
+ "Vendor comet_memory_tests plus shrunk-grant/multipass/final-merge/wide-batch/shared-slice regressions; verify output equality, spill metrics, memory bounds and absence of hangs."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -2327,33 +2049,26 @@
}
],
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "D01"
- ],
+ "original_dependency_notes": ["D01"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D01",
- "D11"
- ],
+ "prerequisites": ["D01", "D11"],
"phase": 3,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G03"
- ],
+ "implementation_plan_groups": ["G03"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести residual Joom memory lifecycle и связанные regressions поверх pristine55.2.",
+ "approved_decision": "Port residual Joom memory lifecycle changes and associated regressions on top of pristine55.2.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "D04",
- "title": "Wide-payload sort: keys-first, BinaryView, prefix и gather",
- "contract": "Сортировать narrow keys, payload собирать один раз из действительно referenced batches; использовать BinaryView для широких binary columns, eight-byte key prefix с полным сравнением collisions; coalesce small batches; учитывать concat pool budget и sliced nested offsets.",
+ "title": "Wide-payload sort: keys first, BinaryView, prefix and gather",
+ "contract": "Sort narrow keys, gather payload once from genuinely referenced batches; use BinaryView for wide binary columns, an eight-byte key prefix with full comparison on collisions; coalesce small batches; account for concat pool budget and sliced nested offsets.",
"source_commits": [
"39f49a033",
"be99cc1bd",
@@ -2374,24 +2089,21 @@
"native/vendor/datafusion-physical-plan/benches/sort_wide_payload.rs"
],
"upstream": {
- "base": "Указанные Joom modules отсутствуют; изменяемые base sort files совпадают с pristine55.1.",
- "outside_base": "Полный аналог не найден в ранее проверенных main/DF56/PR."
+ "base": "The specified Joom modules are absent; modified base sort files match pristine55.1.",
+ "outside_base": "No full equivalent was found in previously checked main/DF56/PRs."
},
- "proposed_action": "Перенести coherent wide-sort блок поверх D02; адаптировать Arrow59.3.2 APIs при необходимости.",
+ "proposed_action": "Port the coherent wide-sort block on top of D02; adapt Arrow59.3.2 APIs if necessary.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
"Preserve null ordering, ascending/descending, prefix collisions, schema/Binary vs BinaryView, slices/dictionaries/nested offsets.",
- "Не подменять новым алгоритмом ограниченные Spark collation/floating semantics без guards.",
- "Performance сохраняется только после benchmarks; source equality снижает текстовые конфликты, не заменяет измерения."
- ],
- "dependencies": [
- "D01",
- "D02"
+ "Do not replace guarded Spark collation/floating semantics with the new algorithm without guards.",
+ "Performance preservation requires benchmarks; source equality reduces textual conflicts but does not replace measurement."
],
+ "dependencies": ["D01", "D02"],
"acceptance": [
"Vendor late_materialize/comet_memory tests; wide binary, sparse referenced batches, repeated/sliced dictionary and nested data, prefix collision; compare Spark/stock sort results and measured allocations/copies."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -2415,9 +2127,7 @@
{
"sha": "5da77919c7ae7ff50afea8d863ace0d4e475054a",
"subject": "test: measure dictionary expansion before wide binary sort",
- "paths": [
- "native/core/benches/sort_payload.rs"
- ]
+ "paths": ["native/core/benches/sort_payload.rs"]
},
{
"sha": "3b37b4a1254e8584d0305a8f4cbb39a4f75b8344",
@@ -2485,33 +2195,26 @@
}
],
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "D01",
- "D02"
- ],
+ "original_dependency_notes": ["D01", "D02"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D02"
- ],
+ "prerequisites": ["D02"],
"phase": 3,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G03"
- ],
+ "implementation_plan_groups": ["G03"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести coherent wide-sort блок поверх D02; адаптировать Arrow59.3.2 APIs при необходимости.",
+ "approved_decision": "Port the coherent wide-sort block on top of D02; adapt Arrow59.3.2 APIs if necessary.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "D03",
- "title": "Output merge и передача native sort результата JVM",
- "contract": "Если JVM-потребитель мог бы голодать из-за удерживаемого native input, разрешить spill-before-output; не навязывать его только-native цепочке. При нехватке output-merge бюджета spill группами, сохранять precheck grant и ограниченный overcommit. Не смешивать с отдельным начальным резервом D11: spillBeforeOutputThreshold по умолчанию равен 1/4 task off-heap share, в on-heap режиме отключён; явный0 отключает механизм.",
+ "title": "Output merge and handing native sort results to the JVM",
+ "contract": "Allow spill-before-output when retained native input could starve a JVM consumer; do not impose it on a native-only chain. When output-merge budget is insufficient, spill in groups, preserve the precheck grant and bounded overcommit. Keep separate from D11 initial reserve: spillBeforeOutputThreshold defaults to1/4 of the task off-heap share, is disabled on-heap, and explicit0 disables the mechanism.",
"source_commits": [
"65be8b751",
"d153b3a85",
@@ -2532,25 +2235,21 @@
"spark/src/main/scala/org/apache/comet/CometConf.scala"
],
"upstream": {
- "base": "Нет with_spill_before_output/SpillWorkspace. Обычный native spill не учитывает именно JVM-output границу и retain-grant precheck.",
- "outside_base": "DF56 workspace partial overlap не включает все Comet JVM-consumer guards."
+ "base": "No with_spill_before_output/SpillWorkspace. Ordinary native spill does not account for this specific JVM-output boundary and retain-grant precheck.",
+ "outside_base": "Partial DF56 workspace overlap does not include all Comet JVM-consumer guards."
},
- "proposed_action": "Адаптировать vendor API и Comet planner/session-config wiring вместе. Сохранить финальное состояние tag13, а не промежуточный revert reservation cap.",
+ "proposed_action": "Adapt the vendor API and Comet planner/session-config wiring together. Preserve final tag13 state rather than the intermediate reservation-cap revert.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Нужно правильно определять границу после нового planner/операторных wrappers Comet1.2.",
- "Старый d153b3a85 отменял cap, 4ff37081a восстановил: нельзя включить только один исторический commit.",
- "Порог public config и bounded-overcommit условия сохранять, а не менять ради прохода тестов."
- ],
- "dependencies": [
- "D01",
- "D02",
- "D04"
+ "Identify the boundary correctly after the new Comet1.2 planner/operator wrappers.",
+ "Old d153b3a85 reverted the cap;4ff37081a restored it: including only one historical commit is insufficient.",
+ "Preserve the public configuration threshold and bounded-overcommit conditions rather than changing them to pass tests."
],
+ "dependencies": ["D01", "D02", "D04"],
"acceptance": [
- "Child-sort→JVM handoff regression, native-only no-unneeded-spill, grouped output merge при budget refusal, удержание already granted reservation, output results и spill/memory metrics."
+ "Child-sort→JVM handoff regression, no unnecessary spill for native-only execution, grouped output merge on budget refusal, retention of already granted reservation, output results and spill/memory metrics."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -2593,9 +2292,7 @@
{
"sha": "5f6cbb8715cadb47cff2c0d34566061f064987e8",
"subject": "test: bound unreserved sort output at the JVM handoff",
- "paths": [
- "native/core/src/execution/jni_api.rs"
- ]
+ "paths": ["native/core/src/execution/jni_api.rs"]
},
{
"sha": "568955d21dd82ce83142a27068b0e47a42f808d8",
@@ -2612,9 +2309,7 @@
{
"sha": "6b11f2e6963089d3029b96035b9e95f0bbba4a97",
"subject": "fix: clone the shared sort source with Arc::clone in the handoff test",
- "paths": [
- "native/core/src/execution/jni_api.rs"
- ]
+ "paths": ["native/core/src/execution/jni_api.rs"]
},
{
"sha": "2b28fed53a3d97b172ebed6103983f93263cfd13",
@@ -2654,35 +2349,26 @@
}
],
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "D01",
- "D02",
- "D04"
- ],
+ "original_dependency_notes": ["D01", "D02", "D04"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D02",
- "D04"
- ],
+ "prerequisites": ["D02", "D04"],
"phase": 3,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G03"
- ],
+ "implementation_plan_groups": ["G03"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать vendor API и Comet planner/session-config wiring вместе. Сохранить финальное состояние tag13, а не промежуточный revert reservation cap.",
+ "approved_decision": "Adapt the vendor API and Comet planner/session-config wiring together. Preserve final tag13 state rather than the intermediate reservation-cap revert.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "D05",
- "title": "Sort-merge join: filter до payload и hoisting",
- "contract": "Не материализовать payload для отвергнутых пар; buffered-only expressions считать один раз на buffered row, streamed-only — один раз на run. Сохранить streamed ordering, FULL/LEFT/RIGHT null/duplicate correctness и volatile-expression ограничения.",
+ "title": "Sort-merge join: filter before payload and hoisting",
+ "contract": "Do not materialize payload for rejected pairs; evaluate buffered-only expressions once per buffered row and streamed-only expressions once per run. Preserve streamed ordering, FULL/LEFT/RIGHT null/duplicate correctness and volatile-expression restrictions.",
"source_commits": [
"f761b31c3",
"eedede1a6",
@@ -2704,23 +2390,21 @@
"native/vendor/datafusion-physical-plan/benches/sort_merge_join.rs"
],
"upstream": {
- "base": "DF55.2 materializing_stream.rs:1542–1568 по-прежнему сначала materializes output, затем filter; Joom меняет именно порядок и reuse.",
- "outside_base": "В проверенной DF56 реализации также не найден полной замены."
+ "base": "DF55.2 materializing_stream.rs:1542–1568 still materializes output before filtering; Joom changes precisely the order and reuse.",
+ "outside_base": "No full replacement was found in the inspected DF56 implementation either."
},
- "proposed_action": "Перенести оптимизацию и correctness fixes одним согласованным блоком; cost model интегрировать/калибровать после него.",
+ "proposed_action": "Port the optimization and correctness fixes as one coherent block; integrate/calibrate the cost model afterwards.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Спорные SQL edge cases: outer join null staging, repeated keys, condition over both sides, CASE/IF, volatile expr, ANSI casts.",
- "Joom fuzz suites исторически ANSI=false; дополнить проверкой Spark4.2.1 ANSI=true/false, не объявлять ANSI=false полным покрытием.",
- "Сохранить 89b6ad738 requiredChildDistribution delegation в Spark Comet aggregate/window/join operators; при AQE/skew rebalance она предотвращает неправильное разбиение clustered input."
- ],
- "dependencies": [
- "D01"
+ "Difficult SQL edge cases: outer-join null staging, repeated keys, conditions over both sides, CASE/IF, volatile expressions, ANSI casts.",
+ "Joom fuzz suites historically use ANSI=false; add Spark4.2.1 ANSI=true/false coverage and do not claim ANSI=false provides complete coverage.",
+ "Preserve89b6ad738 requiredChildDistribution delegation in Spark Comet aggregate/window/join operators; under AQE/skew rebalance it prevents incorrect partitioning of clustered input."
],
+ "dependencies": ["D01"],
"acceptance": [
- "CometSmjJoinFilterFuzzSuite и vendor SMJ tests; nullable duplicate keys, full/left/right, both-side CASE/IF, hoisted expressions; native-plan+result assertions; AQE required-distribution regression."
+ "CometSmjJoinFilterFuzzSuite and vendor SMJ tests; nullable duplicate keys, full/left/right, both-side CASE/IF, hoisted expressions; native-plan+result assertions; AQE required-distribution regression."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -2826,58 +2510,48 @@
}
],
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "D01"
- ],
+ "original_dependency_notes": ["D01"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D01"
- ],
+ "prerequisites": ["D01"],
"phase": 3,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G05"
- ],
+ "implementation_plan_groups": ["G05"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести оптимизацию и correctness fixes одним согласованным блоком; cost model интегрировать/калибровать после него.",
+ "approved_decision": "Port the optimization and correctness fixes as one coherent block; integrate/calibrate the cost model afterwards.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "D10",
- "title": "PartialMerge/COUNT(DISTINCT) spill: решение об upstream-замене",
- "contract": "Промежуточный merge должен выдавать каждый ключ ровно один раз даже при spill, сохранять native state schema и multi-stage aggregate correctness. PartialReduce уже существовал в DataFusion; Joom изменил Comet mapping и маршрутизацию существующего режима через spilling FinalHash/OrderedFinal streams с выдачей states (emit_states/next_state_output_batch).",
- "source_commits": [
- "6c8193e76"
- ],
+ "title": "PartialMerge/COUNT(DISTINCT) spill: upstream replacement decision",
+ "contract": "Intermediate merging must emit every key exactly once even with spill, preserving native state schema and multistage aggregate correctness. PartialReduce already existed in DataFusion; Joom changed Comet mapping and routing of the existing mode through spilling FinalHash/OrderedFinal streams that emit states (emit_states/next_state_output_batch).",
+ "source_commits": ["6c8193e76"],
"source_paths": [
"native/vendor/datafusion-physical-plan/src/aggregates/mod.rs",
"native/vendor/datafusion-physical-plan/src/aggregates/partial_reduce_stream.rs",
"native/core/src/execution/planner.rs"
],
"upstream": {
- "base": "Comet1.2 planner/aggregate.rs:159–164 использует Single+MergeAsPartial; :250–275 prefix-sorted ветка Final+MergeAsPartial; state structs unpacked. Это готовый функциональный аналог, а не отсутствие возможности.",
- "outside_base": "DF56 clustered streams меняют ещё раз; не брать в этой миграции."
+ "base": "Comet1.2 planner/aggregate.rs:159–164 uses Single+MergeAsPartial;:250–275 uses Final+MergeAsPartial for the prefix-sorted branch; state structs are unpacked. This is an existing functional equivalent, not a missing feature.",
+ "outside_base": "DF56 changes clustered streams again; do not adopt it in this migration."
},
- "proposed_action": "Использовать upstream Single/Final+MergeAsPartial и новые Spark aggregate-state codecs; сохранить устойчивость к spill и перенести Joom memory/offset guards, ordered drain и low-copy collect_list. Не переносить старую Joom-маршрутизацию PartialReduce без необходимости.",
+ "proposed_action": "Use upstream Single/Final+MergeAsPartial and new Spark aggregate-state codecs; preserve spill robustness and port Joom memory/offset guards, ordered drain and low-copy collect_list. Do not port old Joom PartialReduce routing unless necessary.",
"approval": "APPROVED_EXPLICITLY",
"risks": [
- "Нужны binary collect buffers, prefix-sorted/fully-sorted/unordered, DISTINCT и mixed aggregate modes под pressure; результаты/планы должны соответствовать contract.",
- "D07–D09 надо адаптировать к новому wrapper, а не вернуть старый planner целиком.",
- "Сохранить новый1a10bc99e collect_set float PartialMerge guard/нормализацию; для4.2.1 isSpark42Plus=true, староверсионный fallback не навязывать всем профилям."
- ],
- "dependencies": [
- "D01"
+ "Need binary collect buffers, prefix-sorted/fully-sorted/unordered inputs, DISTINCT and mixed aggregate modes under pressure; results/plans must satisfy the contract.",
+ "Adapt D07–D09 to the new wrapper rather than restoring the entire old planner.",
+ "Preserve the new1a10bc99e float collect_set PartialMerge guard/normalization; for4.2.1 isSpark42Plus=true, do not impose the older-version fallback on all profiles."
],
+ "dependencies": ["D01"],
"acceptance": [
- "Partial→PartialMerge→Final под forced spill; COUNT(DISTINCT), multiple distinct/grouped, collect states, first/last, min/max, ordered inputs, no duplicate groups."
+ "Partial→PartialMerge→Final under forced spill; COUNT(DISTINCT), multiple distinct/grouped, collect states, first/last, min/max, ordered inputs, no duplicate groups."
],
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
"plan_step": 3,
"source_commit_details": [
{
@@ -2897,25 +2571,19 @@
}
],
"approval_status": "APPROVED_EXPLICITLY",
- "original_dependency_notes": [
- "D01"
- ],
+ "original_dependency_notes": ["D01"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D01"
- ],
+ "prerequisites": ["D01"],
"phase": 4,
"requires_upstream_discussion": false,
- "decision_category": "upstream-основа утверждена",
+ "decision_category": "upstream base approved",
"same_pr_or_co_review": [],
- "approved_decision": "Использовать upstream Single/Final+MergeAsPartial и новые Spark aggregate-state codecs; сохранить устойчивость к spill и перенести Joom memory/offset guards, ordered drain и low-copy collect_list. Не переносить старую Joom-маршрутизацию PartialReduce без необходимости.",
+ "approved_decision": "Use upstream Single/Final+MergeAsPartial and new Spark aggregate-state codecs; preserve spill robustness and port Joom memory/offset guards, ordered drain and low-copy collect_list. Do not port old Joom PartialReduce routing unless necessary.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Предлагаю сохранить upstream Single/Final+MergeAsPartial и его Spark state decoder; перенести наши regression scenarios. При подтверждённой parity не переносить Joom-модификации PartialReduce stream routing/emit_states. Сам upstream enum/режим PartialReduce не удалять. Решение ожидает утверждения пользователя.",
- "prior_open_decisions": "Нужно утвердить upstream Single/Final+MergeAsPartial как замену старого PartialReduce при прохождении наших regressions. Никакая часть возможности пока не исключена.",
- "implementation_plan_groups": [
- "G04"
- ],
+ "original_proposal": "Recommend preserving upstream Single/Final+MergeAsPartial and its Spark state decoder, and porting our regression scenarios. With confirmed parity, do not port Joom PartialReduce stream-routing/emit_states modifications. Do not remove the upstream PartialReduce enum/mode itself. The decision awaits user approval.",
+ "prior_open_decisions": "Need approval of upstream Single/Final+MergeAsPartial as a replacement for old PartialReduce if our regressions pass. No part of the feature has been excluded yet.",
+ "implementation_plan_groups": ["G04"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -2923,35 +2591,29 @@
},
{
"id": "D06",
- "title": "Aggregate initial reservation и fully ordered accounting",
- "contract": "Не выдавать ложный ResourcesExhausted при отказе первой reservation агрегата, способного spill; уже выделенную память fully ordered final учитывать без ошибочного повторного fallible отказа.",
- "source_commits": [
- "85bef53f6",
- "c6178ee56"
- ],
+ "title": "Aggregate initial reservation and fully ordered accounting",
+ "contract": "Do not raise a false ResourcesExhausted when the first reservation is denied for a spill-capable aggregate; account for already allocated fully ordered final memory without a mistaken repeated fallible refusal.",
+ "source_commits": ["85bef53f6", "c6178ee56"],
"source_paths": [
"native/vendor/datafusion-physical-plan/src/aggregates/hash_stream.rs",
"native/vendor/datafusion-physical-plan/src/aggregates/ordered_final_stream.rs",
"native/vendor/datafusion-physical-plan/src/aggregates/starved_spill_tests.rs"
],
"upstream": {
- "base": "Joom call-site исправлений в55.2 нет; Comet SpillReplayPool покрывает часть replay, но не всю initial/main-input ветку.",
- "outside_base": "DF56 #25900/#26734 и clustered stream refactor — alternative/partial, вне выбранной базы."
+ "base": "Joom call-site fixes are absent in55.2; Comet SpillReplayPool covers part of replay, but not the entire initial/main-input branch.",
+ "outside_base": "DF56 #25900/#26734 and the clustered-stream refactor are alternative/partial approaches outside the selected base."
},
- "proposed_action": "Перенести residual call-site guards, сохранив upstream1.2 replay wrapper.",
+ "proposed_action": "Port residual call-site guards while preserving the upstream1.2 replay wrapper.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Нельзя трактовать infallible учёт уже allocation как разрешение неограниченной новой allocation.",
- "Проверить взаимодействие с новым Single/Final+MergeAsPartial и SpillReplayPool."
- ],
- "dependencies": [
- "D01",
- "D10:решение upstream-аналога"
+ "Infallible accounting of existing allocations must not be treated as permission for unbounded new allocations.",
+ "Verify interaction with new Single/Final+MergeAsPartial and SpillReplayPool."
],
+ "dependencies": ["D01", "D10: upstream-equivalent decision"],
"acceptance": [
- "starved_spill_tests при empty initial hash table и ordered output; Partial/Final/Single, initial reservation refusal, memory cleanup, результаты Spark."
+ "starved_spill_tests with an empty initial hash table and ordered output; Partial/Final/Single, initial reservation refusal, memory cleanup, Spark results."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -2977,26 +2639,18 @@
}
],
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "D01",
- "D10:решение upstream-аналога"
- ],
+ "original_dependency_notes": ["D01", "D10: upstream-equivalent decision"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D01",
- "D10"
- ],
+ "prerequisites": ["D01", "D10"],
"phase": 4,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G04"
- ],
+ "implementation_plan_groups": ["G04"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести residual call-site guards, сохранив upstream1.2 replay wrapper.",
+ "approved_decision": "Port residual call-site guards while preserving the upstream1.2 replay wrapper.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
@@ -3004,10 +2658,8 @@
{
"id": "D07",
"title": "Aggregate group-key 32-bit offset budget",
- "contract": "До overflow variable/nested group-key offsets выдать готовое состояние (Partial) либо spill (Final/Single); покрыть все group-value layouts, а не только memory-pool refusal.",
- "source_commits": [
- "6450a99f5"
- ],
+ "contract": "Before variable/nested group-key offsets overflow, emit ready state (Partial) or spill (Final/Single); cover all group-value layouts, not only memory-pool refusal.",
+ "source_commits": ["6450a99f5"],
"source_paths": [
"native/vendor/datafusion-physical-plan/src/aggregates/group_values/mod.rs",
"native/vendor/datafusion-physical-plan/src/aggregates/group_values/single_group_by/bytes.rs",
@@ -3016,25 +2668,21 @@
"native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common.rs"
],
"upstream": {
- "base": "DF55.2 не имеет Joom has_offset_room/max_offset_extent/early emit branch.",
- "outside_base": "PR4791 draft LargeUtf8/Binary offsets — альтернативный частичный scope; не считается принятым аналогом."
+ "base": "DF55.2 lacks Joom has_offset_room/max_offset_extent/early-emit branches.",
+ "outside_base": "Draft PR4791 LargeUtf8/Binary offsets is an alternative partial scope; it is not an accepted equivalent."
},
- "proposed_action": "Перенести guard и все callers в streams; не переносить автоматически old PartialReduce planner.",
+ "proposed_action": "Port the guard and all stream callers; do not automatically port the old PartialReduce planner.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Новый PartialMerge wrapper state schema и режимы требуют учитывать раннюю выдачу без duplicate groups.",
- "Высокий memory budget не предотвращает overflow: тестировать искусственными малыми offset budgets, а не полагаться на OOM.",
- "Сохранить новый1a10bc99e collect_set float PartialMerge guard/нормализацию; для4.2.1 isSpark42Plus=true, староверсионный fallback не навязывать всем профилям."
- ],
- "dependencies": [
- "D01",
- "D06",
- "D10:решение upstream-аналога"
+ "The new PartialMerge wrapper state schema and modes require early emission without duplicate groups.",
+ "A high memory budget does not prevent overflow: test with artificially small offset budgets rather than relying on OOM.",
+ "Preserve the new1a10bc99e float collect_set PartialMerge guard/normalization; for4.2.1 isSpark42Plus=true, do not impose the older-version fallback on all profiles."
],
+ "dependencies": ["D01", "D06", "D10: upstream-equivalent decision"],
"acceptance": [
"Nested/string/binary grouped-key offset thresholds, early emit/spill, no duplicate/missing groups, partial→merge→final correctness."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3063,34 +2711,28 @@
"original_dependency_notes": [
"D01",
"D06",
- "D10:решение upstream-аналога"
+ "D10: upstream-equivalent decision"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D06"
- ],
+ "prerequisites": ["D06"],
"phase": 4,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G04"
- ],
+ "implementation_plan_groups": ["G04"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести guard и все callers в streams; не переносить автоматически old PartialReduce planner.",
+ "approved_decision": "Port the guard and all stream callers; do not automatically port the old PartialReduce planner.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "D08",
- "title": "Aggregate emit pre-reservation и lifetime output",
- "contract": "До materialization оценивать emit_size variable states/keys/indices; резервировать дополнительную память; учитывать уже материализованный output до освобождения.",
- "source_commits": [
- "056ff7097"
- ],
+ "title": "Aggregate emit pre-reservation and output lifetime",
+ "contract": "Estimate emit_size for variable states/keys/indices before materialization; reserve additional memory; account for already materialized output until it is released.",
+ "source_commits": ["056ff7097"],
"source_paths": [
"native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common.rs",
"native/vendor/datafusion-physical-plan/src/aggregates/hash_stream.rs",
@@ -3098,27 +2740,27 @@
"native/core/tests/aggregate_emit_memory.rs"
],
"upstream": {
- "base": "emit_size и Joom pre-reservation нет. SpillReplayPool не заменяет этот mechanism.",
- "outside_base": "DF56 output-spill/reservation изменения partial overlap, вне базы."
+ "base": "emit_size and Joom pre-reservation are absent. SpillReplayPool does not replace this mechanism.",
+ "outside_base": "DF56 output-spill/reservation changes partially overlap and are outside the base."
},
- "proposed_action": "Перенести вместе с N09 collect_list и test fixture из того же исходного commit.",
+ "proposed_action": "Port together with N09 collect_list and the test fixture from the same original commit.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Оценка должна учитывать новые packed/struct states MergeAsPartial.",
- "Нельзя добавлять fallible accounting после уже произошедшей allocation: сохранить корректный порядок и cleanup.",
- "Сохранить новый1a10bc99e collect_set float PartialMerge guard/нормализацию; для4.2.1 isSpark42Plus=true, староверсионный fallback не навязывать всем профилям."
+ "Estimation must account for the new packed/struct MergeAsPartial states.",
+ "Do not add fallible accounting after allocation has already occurred: preserve correct ordering and cleanup.",
+ "Preserve the new1a10bc99e float collect_set PartialMerge guard/normalization; for4.2.1 isSpark42Plus=true, do not impose the older-version fallback on all profiles."
],
"dependencies": [
"D01",
"D06",
"D07",
- "D10:решение upstream-аналога",
- "N09:единый PR"
+ "D10: upstream-equivalent decision",
+ "N09: same PR"
],
"acceptance": [
- "aggregate_emit_memory.rs + vendor emit tests; large collect_list/string states/row keys, refused reservation, retained output slices и pool cleanup."
+ "aggregate_emit_memory.rs plus vendor emit tests; large collect_list/string states/row keys, refused reservation, retained output slices and pool cleanup."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3137,27 +2779,20 @@
"D01",
"D06",
"D07",
- "D10:решение upstream-аналога",
- "N09:единый PR"
+ "D10: upstream-equivalent decision",
+ "N09: same PR"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D07"
- ],
+ "prerequisites": ["D07"],
"phase": 4,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
- "same_pr_or_co_review": [
- "D08",
- "N09"
- ],
- "implementation_plan_groups": [
- "G04"
- ],
+ "decision_category": "scope approved in the implementation plan",
+ "same_pr_or_co_review": ["D08", "N09"],
+ "implementation_plan_groups": ["G04"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести вместе с N09 collect_list и test fixture из того же исходного commit.",
+ "approved_decision": "Port together with N09 collect_list and the test fixture from the same original commit.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
@@ -3165,33 +2800,28 @@
{
"id": "N09",
"title": "collect_list low-copy materialization",
- "contract": "При сборке runs использовать copy_runs/MutableArrayData вместо slice object на каждый run; поддержать variable-size capacities/struct recursion и сохранить result array order/schema.",
- "source_commits": [
- "056ff7097"
- ],
+ "contract": "When gathering runs, use copy_runs/MutableArrayData instead of a slice object per run; support variable-size capacities/struct recursion and preserve result-array order/schema.",
+ "source_commits": ["056ff7097"],
"source_paths": [
"native/spark-expr/src/agg_funcs/collect.rs",
"native/core/tests/aggregate_emit_memory.rs"
],
"upstream": {
- "base": "Comet1.2 collect.rs:384–408 по-прежнему slices+concat; Joom copy_runs отсутствует.",
- "outside_base": "Полной замены не найдено."
+ "base": "Comet1.2 collect.rs:384–408 still uses slices+concat; Joom copy_runs is absent.",
+ "outside_base": "No full replacement was found."
},
- "proposed_action": "Перенести low-copy path одновременно с D08 и original memory regression.",
+ "proposed_action": "Port the low-copy path together with D08 and the original memory regression.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
"Arrow59.3.2 nested struct/list/null arrays; dtype/schema must match.",
- "Без D08 reduced temporary objects не решают reservation перед большой materialization.",
- "Сохранить новый1a10bc99e collect_set float PartialMerge guard/нормализацию; для4.2.1 isSpark42Plus=true, староверсионный fallback не навязывать всем профилям."
- ],
- "dependencies": [
- "D01",
- "D08:единый PR"
+ "Without D08, fewer temporary objects do not solve reservation before large materialization.",
+ "Preserve the new1a10bc99e float collect_set PartialMerge guard/normalization; for4.2.1 isSpark42Plus=true, do not impose the older-version fallback on all profiles."
],
+ "dependencies": ["D01", "D08: same PR"],
"acceptance": [
"collect.rs unit tests and aggregate_emit_memory; multi-run nested/string states, nulls, output order/schema, memory/allocation benchmark."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3206,28 +2836,18 @@
}
],
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "D01",
- "D08:единый PR"
- ],
+ "original_dependency_notes": ["D01", "D08: same PR"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D07"
- ],
+ "prerequisites": ["D07"],
"phase": 4,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
- "same_pr_or_co_review": [
- "D08",
- "N09"
- ],
- "implementation_plan_groups": [
- "G04"
- ],
+ "decision_category": "scope approved in the implementation plan",
+ "same_pr_or_co_review": ["D08", "N09"],
+ "implementation_plan_groups": ["G04"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести low-copy path одновременно с D08 и original memory regression.",
+ "approved_decision": "Port the low-copy path together with D08 and the original memory regression.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
@@ -3235,36 +2855,33 @@
{
"id": "D09",
"title": "Ordered aggregate drain once",
- "contract": "Один раз materialize большой завершённый ordered range, далее отдавать slices; memory и pending state корректно учитываются при pressure.",
- "source_commits": [
- "ee5d84128",
- "3ef370e25"
- ],
+ "contract": "Materialize a large completed ordered range once, then return slices; correctly account for memory and pending state under pressure.",
+ "source_commits": ["ee5d84128", "3ef370e25"],
"source_paths": [
"native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common.rs",
"native/vendor/datafusion-physical-plan/src/aggregates/aggregate_hash_table/common_ordered.rs",
"native/vendor/datafusion-physical-plan/src/aggregates/ordered_drain_tests.rs"
],
"upstream": {
- "base": "DF55.2 clamps emit к batch_size; Joom pending materialized ranges отсутствует.",
- "outside_base": "DF56 #25312/#25639/#25984 уже реализуют основную цель иначе; не входит в55.2."
+ "base": "DF55.2 clamps emit to batch_size; Joom pending materialized ranges are absent.",
+ "outside_base": "DF56 #25312/#25639/#25984 already implement the main goal differently; not included in55.2."
},
- "proposed_action": "Адаптировать drain-once к upstream PartialMerge wrappers; простой !is_final перенос недостаточен.",
+ "proposed_action": "Adapt drain-once to upstream PartialMerge wrappers; simply porting !is_final is insufficient.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Prefix-sorted PartialMerge теперь Final+MergeAsPartial, evaluate которого возвращает state. Старое условие исключит этот fast path.",
- "Pending slices должны удерживать memory accounting; output order и partial states не менять."
+ "Prefix-sorted PartialMerge is now Final+MergeAsPartial whose evaluate returns state. The old condition would exclude this fast path.",
+ "Pending slices must retain memory accounting; do not change output order or partial states."
],
"dependencies": [
"D01",
"D07",
"D08",
- "D10:решение upstream-аналога"
+ "D10: upstream-equivalent decision"
],
"acceptance": [
- "ordered_drain_tests; huge completed prefix groups, batches sizes and pressure, Partial + prefix-sorted PartialMerge/Final-wrapper; verify bounded allocations/materialization count и results."
+ "ordered_drain_tests; huge completed prefix groups, batch sizes and pressure, Partial plus prefix-sorted PartialMerge/Final-wrapper; verify bounded allocations/materialization count and results."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3291,24 +2908,19 @@
"D01",
"D07",
"D08",
- "D10:решение upstream-аналога"
+ "D10: upstream-equivalent decision"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D08",
- "N09"
- ],
+ "prerequisites": ["D08", "N09"],
"phase": 4,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G04"
- ],
+ "implementation_plan_groups": ["G04"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать drain-once к upstream PartialMerge wrappers; простой !is_final перенос недостаточен.",
+ "approved_decision": "Adapt drain-once to upstream PartialMerge wrappers; simply porting !is_final is insufficient.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
@@ -3316,11 +2928,8 @@
{
"id": "N04",
"title": "Unordered FIRST/LAST GroupsAccumulator",
- "contract": "Сохранить быстрый grouped accumulator Spark FIRST/LAST без ordering, включая ignore-null behavior и supported scalar/string/binary/boolean types.",
- "source_commits": [
- "2fa930bc6",
- "25f52bfce"
- ],
+ "contract": "Preserve the fast grouped accumulator for Spark FIRST/LAST without ordering, including ignore-null behavior and supported scalar/string/binary/boolean types.",
+ "source_commits": ["2fa930bc6", "25f52bfce"],
"source_paths": [
"native/spark-expr/src/agg_funcs/first_last.rs",
"native/spark-expr/benches/first_last.rs",
@@ -3328,24 +2937,21 @@
"spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala"
],
"upstream": {
- "base": "SQL FIRST/LAST есть, но DF55.2 groups accumulator требует nonempty order_bys; Spark unordered path попадает на adapter.",
- "outside_base": "В проверенной DF56 first_last тоже не полная замена Joom empty-order groups path."
+ "base": "SQL FIRST/LAST exists, but the DF55.2 groups accumulator requires nonempty order_bys; the Spark unordered path uses an adapter.",
+ "outside_base": "The inspected DF56 first_last is also not a full replacement for the Joom empty-order groups path."
},
- "proposed_action": "Перенести custom wrapper в spark-expr и подключить в новом aggregate planner.",
+ "proposed_action": "Port the custom wrapper into spark-expr and wire it into the new aggregate planner.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Нельзя требовать одинаковый недетерминированный FIRST/LAST результат на произвольно различном порядке input; сравнивать controlled input/allowed semantics.",
- "PartialMerge schema/state order, IGNORE NULLS, nullable boolean/string/binary проверить."
- ],
- "dependencies": [
- "D01",
- "D10:решение upstream-аналога"
+ "Do not require identical nondeterministic FIRST/LAST results for arbitrarily different input orders; compare controlled input/permitted semantics.",
+ "Verify PartialMerge schema/state order, IGNORE NULLS, nullable boolean/string/binary."
],
+ "dependencies": ["D01", "D10: upstream-equivalent decision"],
"acceptance": [
"first_last.rs unit tests/bench, CometAggregateSuite, differential fuzz first/last with controlled ordering and nulls; verify GroupsAccumulator selected.",
- "Точное имя source-suite: CometFirstLastBoolAggFuzzSuite."
+ "Exact source-suite name: CometFirstLastBoolAggFuzzSuite."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3371,60 +2977,47 @@
}
],
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "D01",
- "D10:решение upstream-аналога"
- ],
+ "original_dependency_notes": ["D01", "D10: upstream-equivalent decision"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D10"
- ],
+ "prerequisites": ["D10"],
"phase": 4,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G04"
- ],
+ "implementation_plan_groups": ["G04"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести custom wrapper в spark-expr и подключить в новом aggregate planner.",
+ "approved_decision": "Port the custom wrapper into spark-expr and wire it into the new aggregate planner.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "N05",
- "title": "Boolean MIN/MAX через native groups path",
- "contract": "Для Boolean MIN/MAX использовать эквивалентные bool_and/bool_or groups accumulators вместо scalar adapter, сохраняя null semantics.",
- "source_commits": [
- "2fa930bc6",
- "25f52bfce"
- ],
+ "title": "Boolean MIN/MAX through the native groups path",
+ "contract": "Use equivalent bool_and/bool_or groups accumulators for Boolean MIN/MAX instead of the scalar adapter, preserving null semantics.",
+ "source_commits": ["2fa930bc6", "25f52bfce"],
"source_paths": [
"native/core/src/execution/planner.rs",
"spark/src/test/scala/org/apache/comet/exec/CometAggregateSuite.scala"
],
"upstream": {
- "base": "SQL bool min/max уже есть; optimised Joom wiring нет. DF bool_and/or groups есть, bool min/max groups нет.",
- "outside_base": "Полного принятого аналогичного planner wiring не найдено."
+ "base": "SQL boolean min/max already exists; optimized Joom wiring does not. DF has bool_and/or groups, but not boolean min/max groups.",
+ "outside_base": "No fully accepted equivalent planner wiring was found."
},
- "proposed_action": "Добавить Boolean branch в новый min_max_udaf; сохранить SparkMinMax branch upstream для Float32/Float64.",
+ "proposed_action": "Add a Boolean branch to the new min_max_udaf; preserve the upstream SparkMinMax branch for Float32/Float64.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Замена нового helper целиком старым Joom кодом потеряет upstream float NaN/signed-zero correctness.",
- "Null-only/empty/all-group and partial states должны совпасть."
- ],
- "dependencies": [
- "D01",
- "D10:решение upstream-аналога"
+ "Replacing the entire new helper with old Joom code would lose upstream float NaN/signed-zero correctness.",
+ "Null-only/empty/all-group and partial states must match."
],
+ "dependencies": ["D01", "D10: upstream-equivalent decision"],
"acceptance": [
"CometAggregateSuite/fuzz boolean min/max, null-only/empty/partialmerge; assert native groups fast path and preserve float regressions.",
- "Точное имя source-suite: CometFirstLastBoolAggFuzzSuite."
+ "Exact source-suite name: CometFirstLastBoolAggFuzzSuite."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3450,58 +3043,46 @@
}
],
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "D01",
- "D10:решение upstream-аналога"
- ],
+ "original_dependency_notes": ["D01", "D10: upstream-equivalent decision"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D10"
- ],
+ "prerequisites": ["D10"],
"phase": 4,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G04"
- ],
+ "implementation_plan_groups": ["G04"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Добавить Boolean branch в новый min_max_udaf; сохранить SparkMinMax branch upstream для Float32/Float64.",
+ "approved_decision": "Add a Boolean branch to the new min_max_udaf; preserve the upstream SparkMinMax branch for Float32/Float64.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "N06",
- "title": "Native string MIN/MAX: upstream-аналог",
- "contract": "Сохранить native MIN/MAX по строковым значениям с default collation и корректный fallback для unsupported collations.",
- "source_commits": [
- "6f7a970eb"
- ],
+ "title": "Native string MIN/MAX: upstream equivalent",
+ "contract": "Preserve native MIN/MAX on string values with default collation and correct fallback for unsupported collations.",
+ "source_commits": ["6f7a970eb"],
"source_paths": [
"spark/src/main/scala/org/apache/comet/serde/aggregates.scala",
"spark/src/test/resources/sql-tests/expressions/aggregate/min_max.sql"
],
"upstream": {
- "base": "Есть функциональный аналог: minMaxSupportLevel допускает StringType/BinaryType default collation; upstream #6927. Это уже покрывает string capability.",
- "outside_base": "Не требуется брать main ради этого."
+ "base": "A functional equivalent exists: minMaxSupportLevel permits StringType/BinaryType with default collation; upstream#6927. This already covers string capability.",
+ "outside_base": "No need to use main for this."
},
- "proposed_action": "Использовать upstream native string MIN/MAX; сохранить Joom cases, native coverage и недостающие guards.",
+ "proposed_action": "Use upstream native string MIN/MAX; preserve Joom cases, native coverage and missing guards.",
"approval": "APPROVED_EXPLICITLY",
"risks": [
- "Spark4 collations; не переносить старый strict-float guard поверх улучшенного SparkMinMax upstream.",
- "Memory guards для variable states — отдельный D08, наличие SQL string support не заменяет его."
- ],
- "dependencies": [
- "D01",
- "D08:memory qualification"
+ "Spark4 collations; do not port the old strict-float guard over improved upstream SparkMinMax.",
+ "Variable-state memory guards are separate D08; SQL string support does not replace them."
],
+ "dependencies": ["D01", "D08:memory qualification"],
"acceptance": [
"Joom min/max string cases, empty/nulls/grouped/partialmerge/default and unsupported collations; result+native plan assertions."
],
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3520,26 +3101,19 @@
}
],
"approval_status": "APPROVED_EXPLICITLY",
- "original_dependency_notes": [
- "D01",
- "D08:memory qualification"
- ],
+ "original_dependency_notes": ["D01", "D08:memory qualification"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D10"
- ],
+ "prerequisites": ["D10"],
"phase": 4,
"requires_upstream_discussion": false,
- "decision_category": "upstream-основа утверждена",
+ "decision_category": "upstream base approved",
"same_pr_or_co_review": [],
- "approved_decision": "Использовать upstream native string MIN/MAX; сохранить Joom cases, native coverage и недостающие guards.",
+ "approved_decision": "Use upstream native string MIN/MAX; preserve Joom cases, native coverage and missing guards.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Предлагаю оставить upstream implementation и перенести Joom cases/plan assertions; отдельно подтвердить parity.",
- "prior_open_decisions": "Нужно утвердить использование upstream string MIN/MAX вместо копирования старой Joom реализации, при подтверждении parity.",
- "implementation_plan_groups": [
- "G04"
- ],
+ "original_proposal": "Recommend retaining the upstream implementation and porting Joom cases/plan assertions; separately verify parity.",
+ "prior_open_decisions": "Need approval to use upstream string MIN/MAX instead of copying the old Joom implementation, subject to parity confirmation.",
+ "implementation_plan_groups": ["G04"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -3547,14 +3121,9 @@
},
{
"id": "N08",
- "title": "SortAggregate native conversion, streaming и order-sensitive guard",
- "contract": "Сохранять корректный ordering contract и order-sensitive FIRST/LAST/MIN_BY/MAX_BY behavior; native conversion не должен провоцировать unnecessary re-sort, если ordered consumer может использовать streaming aggregation.",
- "source_commits": [
- "6f7a970eb",
- "4d7eec2d2",
- "e930327f4",
- "9153d794f"
- ],
+ "title": "Native SortAggregate conversion, streaming and order-sensitive guard",
+ "contract": "Preserve the correct ordering contract and order-sensitive FIRST/LAST/MIN_BY/MAX_BY behavior; native conversion must not cause unnecessary re-sorting when an ordered consumer can use streaming aggregation.",
+ "source_commits": ["6f7a970eb", "4d7eec2d2", "e930327f4", "9153d794f"],
"source_paths": [
"spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala",
"spark/src/main/scala/org/apache/spark/sql/comet/operators.scala",
@@ -3562,25 +3131,22 @@
"native/proto/src/proto/operator.proto"
],
"upstream": {
- "base": "Comet1.2 имеет CometSortAggregateExec, ordered_by_grouping_keys и restore-sort при утраченном native ordering. Реализация другая; общая возможность есть.",
- "outside_base": "Не следует считать entire Joom block отсутствующим."
+ "base": "Comet1.2 has CometSortAggregateExec, ordered_by_grouping_keys and restore-sort when native ordering is lost. The implementation differs, but the general capability exists.",
+ "outside_base": "Do not treat the entire Joom block as absent."
},
- "proposed_action": "Сохранить upstream SortAggregate architecture и добавить недостающие Joom-оптимизации порядка/streaming и необходимые correctness guards.",
+ "proposed_action": "Preserve upstream SortAggregate architecture and add missing Joom ordering/streaming optimizations and required correctness guards.",
"approval": "APPROVED_EXPLICITLY",
"risks": [
- "Upstream restore-sort может быть необходим для cached sorted relation→unordered native Scan; слепое удаление нарушит correctness.",
- "Joom stream consumer optimization / conditional output ordering не доказана полностью заменённой; нужен exact plan comparison.",
- "Сохранить 89b6ad738 requiredChildDistribution delegation в Spark Comet aggregate/window/join operators; при AQE/skew rebalance она предотвращает неправильное разбиение clustered input.",
- "Сохранить новый1a10bc99e collect_set float PartialMerge guard/нормализацию; для4.2.1 isSpark42Plus=true, староверсионный fallback не навязывать всем профилям."
- ],
- "dependencies": [
- "D01",
- "D10:решение upstream-аналога"
+ "Upstream restore-sort may be necessary for a cached sorted relation→unordered native Scan; blindly removing it violates correctness.",
+ "Joom stream-consumer optimization / conditional output ordering is not proven fully replaced; exact plan comparison is needed.",
+ "Preserve89b6ad738 requiredChildDistribution delegation in Spark Comet aggregate/window/join operators; under AQE/skew rebalance it prevents incorrect partitioning of clustered input.",
+ "Preserve the new1a10bc99e float collect_set PartialMerge guard/normalization; for4.2.1 isSpark42Plus=true, do not impose the older-version fallback on all profiles."
],
+ "dependencies": ["D01", "D10: upstream-equivalent decision"],
"acceptance": [
"Aggregate suite sortaggregate+firstlast/minby cases, cached sorted input, ordered downstream, nullable group keys, no unnecessary sort where guaranteed and required restore-sort where not."
],
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3631,26 +3197,19 @@
"N07: shared order-sensitive eligibility cases; implementation order upstream architecture decision first, then N07 support."
],
"approval_status": "APPROVED_EXPLICITLY",
- "original_dependency_notes": [
- "D01",
- "D10:решение upstream-аналога"
- ],
+ "original_dependency_notes": ["D01", "D10: upstream-equivalent decision"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D10"
- ],
+ "prerequisites": ["D10"],
"phase": 4,
"requires_upstream_discussion": false,
- "decision_category": "upstream-основа утверждена",
+ "decision_category": "upstream base approved",
"same_pr_or_co_review": [],
- "approved_decision": "Сохранить upstream SortAggregate architecture и добавить недостающие Joom-оптимизации порядка/streaming и необходимые correctness guards.",
+ "approved_decision": "Preserve upstream SortAggregate architecture and add missing Joom ordering/streaming optimizations and required correctness guards.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Сначала согласовать upstream conversion как базу; сверить Joom eligibility/order-sensitive/streaming случаи и переносить только реально недостающие residual behavior.",
- "prior_open_decisions": "Нужно утвердить сохранение upstream SortAggregate architecture с переносом доказанных residual Joom optimisations; полную эквивалентность до plan/regression comparison не заявляем.",
- "implementation_plan_groups": [
- "G04"
- ],
+ "original_proposal": "First agree on upstream conversion as the base; compare Joom eligibility/order-sensitive/streaming cases and port only genuinely missing residual behavior.",
+ "prior_open_decisions": "Need approval to retain upstream SortAggregate architecture while porting proven residual Joom optimizations; do not claim full equivalence before plan/regression comparison.",
+ "implementation_plan_groups": ["G04"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -3658,12 +3217,9 @@
},
{
"id": "N07",
- "title": "Native MIN_BY/MAX_BY со строковым value",
- "contract": "Сохранить native MIN_BY/MAX_BY, где возвращаемое value — string; ordering и null/tie/PartialMerge contracts исходного Joom кода не потерять.",
- "source_commits": [
- "92f4b7297",
- "9153d794f"
- ],
+ "title": "Native MIN_BY/MAX_BY with a string value",
+ "contract": "Preserve native MIN_BY/MAX_BY when the returned value is string; do not lose ordering and null/tie/PartialMerge contracts from original Joom code.",
+ "source_commits": ["92f4b7297", "9153d794f"],
"source_paths": [
"spark/src/main/scala/org/apache/comet/serde/aggregates.scala",
"spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala",
@@ -3672,24 +3228,24 @@
"spark/src/test/resources/sql-tests/expressions/aggregate/min_by.sql"
],
"upstream": {
- "base": "CometMaxMinBy value guard по minMaxDataTypeSupported исключает string, хотя обычный string MIN/MAX разрешён отдельно. Полного аналога возможности нет.",
- "outside_base": "Полной принятой замены не найдено."
+ "base": "The CometMaxMinBy value guard through minMaxDataTypeSupported excludes string, although ordinary string MIN/MAX is permitted separately. No full equivalent exists.",
+ "outside_base": "No fully accepted replacement was found."
},
- "proposed_action": "Адаптировать string value support и order-sensitive SortAggregate eligibility поверх1.2 serde/native accumulator.",
+ "proposed_action": "Adapt string-value support and order-sensitive SortAggregate eligibility on top of1.2 serde/native accumulator.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "SortAggregate conversion может поменять выбор при ties: следовать Spark permitted nondeterminism, не обещать единственный tie result.",
- "String collation/value vs ordering constraints разделять; preserve null/merge state schema."
+ "SortAggregate conversion can change tie selection: follow Spark-permitted nondeterminism rather than promising a unique tie result.",
+ "Separate string collation/value constraints from ordering constraints; preserve null/merge state schema."
],
"dependencies": [
"D01",
- "N08:решение conversion/ordering",
- "D10:решение upstream-аналога"
+ "N08: conversion/ordering decision",
+ "D10: upstream-equivalent decision"
],
"acceptance": [
"max_by.sql/min_by.sql Joom cases + aggregate suite; string/null values, ties, merge states, ordered/unordered inputs, native coverage and unsupported collation fallback."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3720,24 +3276,20 @@
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [
"D01",
- "N08:решение conversion/ordering",
- "D10:решение upstream-аналога"
+ "N08: conversion/ordering decision",
+ "D10: upstream-equivalent decision"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "N08"
- ],
+ "prerequisites": ["N08"],
"phase": 4,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G04"
- ],
+ "implementation_plan_groups": ["G04"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать string value support и order-sensitive SortAggregate eligibility поверх1.2 serde/native accumulator.",
+ "approved_decision": "Adapt string-value support and order-sensitive SortAggregate eligibility on top of1.2 serde/native accumulator.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
@@ -3745,13 +3297,8 @@
{
"id": "N01",
"title": "Spillable non-streaming partition windows",
- "contract": "Whole-partition aggregates/first-last-nth, NTILE/PERCENT_RANK/CUME_DIST и поддержанные suffix ROWS/RANGE frames сохраняют Spark semantics; partition rows и reverse-pass buffers могут spill; неподдержанные expressions сохраняют fallback.",
- "source_commits": [
- "65be8b751",
- "e90fcec24",
- "e43546613",
- "bc64dfceb"
- ],
+ "contract": "Whole-partition aggregates/first-last-nth, NTILE/PERCENT_RANK/CUME_DIST and supported suffix ROWS/RANGE frames preserve Spark semantics; partition rows and reverse-pass buffers can spill; unsupported expressions retain fallback.",
+ "source_commits": ["65be8b751", "e90fcec24", "e43546613", "bc64dfceb"],
"source_paths": [
"native/core/src/execution/operators/partition_aggregate_window.rs",
"native/core/src/execution/planner.rs",
@@ -3759,26 +3306,26 @@
"spark/src/main/scala/org/apache/comet/CometConf.scala"
],
"upstream": {
- "base": "Comet1.2 строит BoundedWindowAggExec/WindowAggExec; DF55.2 unbounded buffers input, Joom spilling executor отсутствует.",
- "outside_base": "PR6279 partitions+accounting не имеет Joom spilling; не полная замена."
+ "base": "Comet1.2 builds BoundedWindowAggExec/WindowAggExec; DF55.2 unbounded execution buffers input, and the Joom spilling executor is absent.",
+ "outside_base": "PR6279 partitions+accounting lacks Joom spilling; it is not a full replacement."
},
- "proposed_action": "Адаптировать custom executor к новому operators crate/window planner/FFI config; сохранить library flag default=false и deploy-enabled поведение datasci.",
+ "proposed_action": "Adapt the custom executor to the new operators crate/window planner/FFI configuration; preserve the library flag default=false and datasci deployment-enabled behavior.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Spark4.2 FILTER window aggregates нужно по-прежнему отвергать upstream guard, иначе можно потерять filter.",
- "IGNORE NULLS, suffix frames, range peer/null/descending ordering, Decimal/ANSI semantics должны сохраниться.",
- "Импортирует OffsetBudget из Comet common: обязательная зависимость от общего nested-offset guard.",
- "Сохранить 89b6ad738 requiredChildDistribution delegation в Spark Comet aggregate/window/join operators; при AQE/skew rebalance она предотвращает неправильное разбиение clustered input."
+ "Continue rejecting Spark4.2 FILTER window aggregates through the upstream guard; otherwise the filter can be lost.",
+ "Preserve IGNORE NULLS, suffix frames, RANGE peer/null/descending ordering and Decimal/ANSI semantics.",
+ "Imports OffsetBudget from Comet common: mandatory dependency on the shared nested-offset guard.",
+ "Preserve89b6ad738 requiredChildDistribution delegation in Spark Comet aggregate/window/join operators; under AQE/skew rebalance it prevents incorrect partitioning of clustered input."
],
"dependencies": [
"D01",
- "Общий Comet OffsetBudget/nested-offset guard",
- "Базовый window planner1.2"
+ "Shared Comet OffsetBudget/nested-offset guard",
+ "Baseline1.2 window planner"
],
"acceptance": [
"whole_partition_aggregates_spill_and_preserve_rows, whole_partition_values_spill, partition_size_functions_spill, suffix_frames_spill, mixed_node_spills, unsupported_expressions_keep_window_agg_exec, spark_semantics; Spark4.2.1 differential+plan assertions."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3840,59 +3387,48 @@
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [
"D01",
- "Общий Comet OffsetBudget/nested-offset guard",
- "Базовый window planner1.2"
+ "Shared Comet OffsetBudget/nested-offset guard",
+ "Baseline1.2 window planner"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D01",
- "J14"
- ],
+ "prerequisites": ["D01", "J14"],
"phase": 5,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G06"
- ],
+ "implementation_plan_groups": ["G06"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать custom executor к новому operators crate/window planner/FFI config; сохранить library flag default=false и deploy-enabled поведение datasci.",
+ "approved_decision": "Adapt the custom executor to the new operators crate/window planner/FFI configuration; preserve the library flag default=false and datasci deployment-enabled behavior.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "N02",
- "title": "Batching tiny partition windows и coalescing output",
- "contract": "Много малых partitions внутри input batch вычислять вместе; partitions across batch boundaries и spill сохраняют rows/order; output batches coalesce с учётом limits.",
- "source_commits": [
- "95dc9ae9b",
- "d60929066"
- ],
+ "title": "Batch tiny partition windows and coalesce output",
+ "contract": "Compute many small partitions within one input batch together; partitions spanning batch boundaries and spill preserve rows/order; coalesce output batches subject to limits.",
+ "source_commits": ["95dc9ae9b", "d60929066"],
"source_paths": [
"native/core/src/execution/operators/partition_aggregate_window.rs"
],
"upstream": {
- "base": "Joom batching paths отсутствуют.",
- "outside_base": "PR6279 partition-at-time не является tiny-partition vectorized replacement."
+ "base": "Joom batching paths are absent.",
+ "outside_base": "PR6279 partition-at-a-time is not a vectorized tiny-partition replacement."
},
- "proposed_action": "Перенести поверх N01, можно отдельным зависимым PR.",
+ "proposed_action": "Port on top of N01, possibly as a separate dependent PR.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Нельзя объединять разные partition peer/state, особенно suffix RANGE offsets; сохранить unsupported-case path.",
- "Output coalescing использует общий nested-offset budget.",
- "Сохранить 89b6ad738 requiredChildDistribution delegation в Spark Comet aggregate/window/join operators; при AQE/skew rebalance она предотвращает неправильное разбиение clustered input."
- ],
- "dependencies": [
- "N01",
- "Общий Comet OffsetBudget/nested-offset guard"
+ "Do not combine peer/state information from different partitions, especially suffix RANGE offsets; preserve the unsupported-case path.",
+ "Output coalescing uses the shared nested-offset budget.",
+ "Preserve89b6ad738 requiredChildDistribution delegation in Spark Comet aggregate/window/join operators; under AQE/skew rebalance it prevents incorrect partitioning of clustered input."
],
+ "dependencies": ["N01", "Shared Comet OffsetBudget/nested-offset guard"],
"acceptance": [
"small_partitions_within_and_across_batches, mixed spill tests, tiny partitions benchmark; output count/schema/order + plan stays native."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3913,34 +3449,28 @@
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [
"N01",
- "Общий Comet OffsetBudget/nested-offset guard"
+ "Shared Comet OffsetBudget/nested-offset guard"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "N01"
- ],
+ "prerequisites": ["N01"],
"phase": 5,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G06"
- ],
+ "implementation_plan_groups": ["G06"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести поверх N01, можно отдельным зависимым PR.",
+ "approved_decision": "Port on top of N01, possibly as a separate dependent PR.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "N03",
- "title": "SortedWindowExec для простых streaming окон",
- "contract": "ROW_NUMBER/RANK/DENSE_RANK и LEAD/LAG с постоянным offset/default без IGNORE NULLS обрабатывать целыми sorted batches, удерживая минимальный cross-batch state; unsupported cases идут другим window path.",
- "source_commits": [
- "47ce539e4"
- ],
+ "title": "SortedWindowExec for simple streaming windows",
+ "contract": "Process ROW_NUMBER/RANK/DENSE_RANK and LEAD/LAG with constant offset/default without IGNORE NULLS across whole sorted batches, retaining minimal cross-batch state; unsupported cases use another window path.",
+ "source_commits": ["47ce539e4"],
"source_paths": [
"native/core/src/execution/operators/sorted_window.rs",
"native/core/benches/sorted_window.rs",
@@ -3950,24 +3480,21 @@
"spark/src/main/scala/org/apache/spark/sql/comet/CometWindowExec.scala"
],
"upstream": {
- "base": "Нет SortedWindowExec; WindowGroupLimit/PartitionedRankLimitExec — другая оптимизация.",
- "outside_base": "Полного аналога не найдено в проверенных refs."
+ "base": "No SortedWindowExec; WindowGroupLimit/PartitionedRankLimitExec is a different optimization.",
+ "outside_base": "No full equivalent was found in the inspected refs."
},
- "proposed_action": "Адаптировать executor и selection rule; сохранить flag window.sorted.enabled default=true, исходные eligibility ограничения.",
+ "proposed_action": "Adapt the executor and selection rule; preserve window.sorted.enabled default=true and original eligibility restrictions.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Spark4.2 expression serialization/physical ordering и Arrow cross-batch peers проверить.",
- "LEAD/LAG offset cap1024 и null/default eligibility сохранить; не расширять scope автоматически.",
- "Сохранить 89b6ad738 requiredChildDistribution delegation в Spark Comet aggregate/window/join operators; при AQE/skew rebalance она предотвращает неправильное разбиение clustered input."
- ],
- "dependencies": [
- "D01",
- "Базовый window planner1.2"
+ "Check Spark4.2 expression serialization/physical ordering and Arrow cross-batch peers.",
+ "Preserve the LEAD/LAG offset cap1024 and null/default eligibility; do not automatically broaden scope.",
+ "Preserve89b6ad738 requiredChildDistribution delegation in Spark Comet aggregate/window/join operators; under AQE/skew rebalance it prevents incorrect partitioning of clustered input."
],
+ "dependencies": ["D01", "Baseline1.2 window planner"],
"acceptance": [
"CometWindowExecSuite Joom cases + native sorted_window tests/benchmark; cross-batch partitions/peers, rank ties, null ordering, lead/lag boundary/default, unsupported IGNORE NULLS and offsets fallback."
],
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"plan_step": 3,
"source_commit_details": [
{
@@ -3989,52 +3516,42 @@
}
],
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "D01",
- "Базовый window planner1.2"
- ],
+ "original_dependency_notes": ["D01", "Baseline1.2 window planner"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "D01",
- "J14"
- ],
+ "prerequisites": ["D01", "J14"],
"phase": 5,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G06"
- ],
+ "implementation_plan_groups": ["G06"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать executor и selection rule; сохранить flag window.sorted.enabled default=true, исходные eligibility ограничения.",
+ "approved_decision": "Adapt the executor and selection rule; preserve window.sorted.enabled default=true and original eligibility restrictions.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Дополнительного вопроса о требуемом поведении нет: сохранить описанный контракт; рекомендация ещё требует утверждения пользователя.",
+ "prior_open_decisions": "No additional question about required behavior remains: preserve the described contract; the recommendation still requires user approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J10",
- "title": "Без копирования unique dictionary values при JVM shuffle row conversion",
- "contract": "Если nondictionaryfallback нужен, аnonnullkeysссылаются науникальныеvalues по insertionorder, reusevaluesbuffer вместоcast; повторяющиесяkeys/nullvalues имеютsafeordinaryfallback.",
- "source_commits": [
- "d9ea39240"
- ],
+ "title": "Avoid copying unique dictionary values during JVM-shuffle row conversion",
+ "contract": "When nondictionary fallback is needed and nonnull keys reference unique values in insertion order, reuse the values buffer instead of casting; repeated keys/null values use safe ordinary fallback.",
+ "source_commits": ["d9ea39240"],
"source_files": [
"native/shuffle/src/spark_unsafe/row.rs",
"native/shuffle/benches/row_columnar.rs"
],
- "upstream_base": "Точная цепочка target93: SpillWriter.writeSortedFileNative / SpillSorter.writeSortedPartitionsNative → JNI → process_sorted_row_partition/process_sorted_partitions → SortedRowEncoder.encode_batch → builder_to_array. Для Utf8/Binary при prefer_dictionary_ratio>1 неэффективный dictionary в обоих случаях безусловно идёт в cast(dict,plain), строки1777/1796. Joom unique_dictionary_to_plain guard/reuse в этом общем месте отсутствует.",
- "proposal": "Перенести unique_dictionary_to_plain в общий builder_to_array новой row.rs. Сохранить guards: non-null dictionary values, ключи уникальны и идут в insertion order; иначе обычный cast. Проверить оба новых вызывающих writer пути. Это сохраняет Joom специализацию; выигрыш поверх Arrow59.3.2 подтверждается benchmark, не объявляется доказанным чтением Comet.",
+ "upstream_base": "Exact target93 chain: SpillWriter.writeSortedFileNative / SpillSorter.writeSortedPartitionsNative → JNI → process_sorted_row_partition/process_sorted_partitions → SortedRowEncoder.encode_batch → builder_to_array. For Utf8/Binary when prefer_dictionary_ratio>1, an inefficient dictionary unconditionally takes cast(dict,plain) in both cases, lines1777/1796. Joom unique_dictionary_to_plain guard/reuse is absent from this shared location.",
+ "proposal": "Port unique_dictionary_to_plain into the shared builder_to_array in new row.rs. Preserve guards: non-null dictionary values and keys unique in insertion order; otherwise use ordinary cast. Verify both new writer caller paths. This preserves the Joom specialization; any benefit over Arrow59.3.2 requires benchmarks and is not proven by reading Comet.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Новый shared SortedRowEncoder и buffered IPC output в1.2: не возвращать старую row.rs целиком. Локальная вставка остаётся в builder_to_array.",
- "Нужно сохранить null/sliced-offset/insertion-order guards и неизменяемый buffer lifetime. Внутренняя реализация Arrow59.3.2 cast отдельно не исследована; performance benefit подтверждается сравнением allocations/benchmark."
+ "New shared SortedRowEncoder and buffered IPC output in1.2: do not restore the entire old row.rs. Keep the local insertion in builder_to_array.",
+ "Preserve null/sliced-offset/insertion-order guards and immutable buffer lifetime. The internal Arrow59.3.2 cast implementation was not separately investigated; confirm performance benefit through allocation comparison/benchmarks."
],
"dependencies": [],
"acceptance": [
- "Исходные unique_binary_dictionary_matches_cast, unique_string_dictionary_matches_cast, repeated_dictionary_values_are_not_reused; дополнительно output data buffer shares dictionary values allocation в reuse path.",
- "Для legacy single-partition SpillWriter и нового multi-partition SpillSorter: одинаковые данные/checksums, nulls/empty/wide binary и repeated keys; row_columnar benchmark на59.3.2."
+ "Original unique_binary_dictionary_matches_cast, unique_string_dictionary_matches_cast, repeated_dictionary_values_are_not_reused; additionally assert that the output data buffer shares the dictionary-values allocation in the reuse path.",
+ "For legacy single-partition SpillWriter and new multipartition SpillSorter: identical data/checksums, nulls/empty/wide binary and repeated keys; row_columnar benchmark on59.3.2."
],
"implementation_questions": [],
"other_upstream_candidates": [],
@@ -4054,85 +3571,63 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/native/shuffle/src/spark_unsafe/row.rs",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/native/shuffle/benches/row_columnar.rs"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"upstream_source_evidence": [
{
"ref": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
"path": "native/shuffle/src/spark_unsafe/row.rs",
- "lines": [
- 1173,
- 1202,
- 1452,
- 1479,
- 1541,
- 1587,
- 1616,
- 1657,
- 1753,
- 1800
- ]
+ "lines": [1173, 1202, 1452, 1479, 1541, 1587, 1616, 1657, 1753, 1800]
},
{
"ref": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
"path": "spark/src/main/java/org/apache/spark/shuffle/sort/SpillSorter.java",
- "lines": [
- 246,
- 263
- ]
+ "lines": [246, 263]
},
{
"ref": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
"path": "spark/src/main/java/org/apache/spark/sql/comet/execution/shuffle/SpillWriter.java",
- "lines": [
- 223
- ]
+ "lines": [223]
}
],
"source_paths": [
"native/shuffle/src/spark_unsafe/row.rs",
"native/shuffle/benches/row_columnar.rs"
],
- "proposed_action": "Перенести unique_dictionary_to_plain в общий builder_to_array новой row.rs. Сохранить guards: non-null dictionary values, ключи уникальны и идут в insertion order; иначе обычный cast. Проверить оба новых вызывающих writer пути. Это сохраняет Joom специализацию; выигрыш поверх Arrow59.3.2 подтверждается benchmark, не объявляется доказанным чтением Comet.",
+ "proposed_action": "Port unique_dictionary_to_plain into the shared builder_to_array in new row.rs. Preserve guards: non-null dictionary values and keys unique in insertion order; otherwise use ordinary cast. Verify both new writer caller paths. This preserves the Joom specialization; any benefit over Arrow59.3.2 requires benchmarks and is not proven by reading Comet.",
"upstream": {
- "base": "Точная цепочка target93: SpillWriter.writeSortedFileNative / SpillSorter.writeSortedPartitionsNative → JNI → process_sorted_row_partition/process_sorted_partitions → SortedRowEncoder.encode_batch → builder_to_array. Для Utf8/Binary при prefer_dictionary_ratio>1 неэффективный dictionary в обоих случаях безусловно идёт в cast(dict,plain), строки1777/1796. Joom unique_dictionary_to_plain guard/reuse в этом общем месте отсутствует.",
+ "base": "Exact target93 chain: SpillWriter.writeSortedFileNative / SpillSorter.writeSortedPartitionsNative → JNI → process_sorted_row_partition/process_sorted_partitions → SortedRowEncoder.encode_batch → builder_to_array. For Utf8/Binary when prefer_dictionary_ratio>1, an inefficient dictionary unconditionally takes cast(dict,plain) in both cases, lines1777/1796. Joom unique_dictionary_to_plain guard/reuse is absent from this shared location.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 6,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G07"
- ],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести unique_dictionary_to_plain в общий builder_to_array новой row.rs. Сохранить guards: non-null dictionary values, ключи уникальны и идут в insertion order; иначе обычный cast. Проверить оба новых вызывающих writer пути. Это сохраняет Joom специализацию; выигрыш поверх Arrow59.3.2 подтверждается benchmark, не объявляется доказанным чтением Comet.",
+ "approved_decision": "Port unique_dictionary_to_plain into the shared builder_to_array in new row.rs. Preserve guards: non-null dictionary values and keys unique in insertion order; otherwise use ordinary cast. Verify both new writer caller paths. This preserves the Joom specialization; any benefit over Arrow59.3.2 requires benchmarks and is not proven by reading Comet.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J11",
- "title": "Другие Spark memory consumers могут заставить JVM shuffle writer spill",
- "contract": "OwnerSpill callback allocator→CometShuffleExternalSorter; выполнениетолько ownerthread сbusy/closed guards, безreentrant/concurrent unsafe spill.",
- "source_commits": [
- "b65fdbe01"
- ],
+ "title": "Other Spark memory consumers can trigger JVM shuffle-writer spill",
+ "contract": "OwnerSpill callback allocator→CometShuffleExternalSorter; execute only on the owner thread with busy/closed guards, without reentrant/concurrent unsafe spill.",
+ "source_commits": ["b65fdbe01"],
"source_files": [
"spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java",
"spark/src/main/java/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocator.java",
"spark/src/main/java/org/apache/spark/shuffle/sort/CometShuffleExternalSorter.java"
],
- "upstream_base": "Base1.2 allocator spill() возвращает0; нашcallbackотсутствует.",
- "proposal": "Адаптировать кSpark4.2.1 MemoryConsumer иновому allocator retry lifecycle.",
+ "upstream_base": "Base1.2 allocator spill() returns0; our callback is absent.",
+ "proposal": "Adapt to Spark4.2.1 MemoryConsumer and the new allocator retry lifecycle.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
"Reentrantallocation/spilldeadlocks, otherthread callbacks, taskcompletion races; zero/failedspill retries."
@@ -4143,7 +3638,7 @@
],
"implementation_questions": [],
"other_upstream_candidates": [
- "PR#6336/#6440 memory accounting partiallyrelated, неOwnerSpillreplacement."
+ "PR#6336/#6440 memory accounting is partly related, not an OwnerSpill replacement."
],
"config": [],
"source_commit_details": [
@@ -4164,58 +3659,52 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/java/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocator.java",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/java/org/apache/spark/shuffle/sort/CometShuffleExternalSorter.java"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/java/org/apache/spark/shuffle/comet/CometShuffleMemoryAllocatorTrait.java",
"spark/src/main/java/org/apache/spark/shuffle/comet/CometUnifiedShuffleMemoryAllocator.java",
"spark/src/main/java/org/apache/spark/shuffle/sort/CometShuffleExternalSorter.java"
],
- "proposed_action": "Адаптировать кSpark4.2.1 MemoryConsumer иновому allocator retry lifecycle.",
+ "proposed_action": "Adapt to Spark4.2.1 MemoryConsumer and the new allocator retry lifecycle.",
"upstream": {
- "base": "Base1.2 allocator spill() возвращает0; нашcallbackотсутствует.",
+ "base": "Base1.2 allocator spill() returns0; our callback is absent.",
"outside_base": [
- "PR#6336/#6440 memory accounting partiallyrelated, неOwnerSpillreplacement."
+ "PR#6336/#6440 memory accounting is partly related, not an OwnerSpill replacement."
]
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 6,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G07"
- ],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать кSpark4.2.1 MemoryConsumer иновому allocator retry lifecycle.",
+ "approved_decision": "Adapt to Spark4.2.1 MemoryConsumer and the new allocator retry lifecycle.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J13",
- "title": "Gather широких shuffle rows только из используемых batches + shared type alignment",
- "contract": "Partition chunkсобираетpayloadизreferencedbatchsubset; alignnestedtypeinstances кwriterschema безкопированияvalues; сохранитьroworder/null/metadata guards.",
- "source_commits": [
- "94abcc0fc"
- ],
+ "title": "Gather wide shuffle rows only from used batches plus shared type alignment",
+ "contract": "A partition chunk gathers payload from the referenced batch subset; align nested type instances to the writer schema without copying values; preserve row-order/null/metadata guards.",
+ "source_commits": ["94abcc0fc"],
"source_files": [
"native/shuffle/src/partitioners/partitioned_batch_iterator.rs",
"native/shuffle/src/shuffle_writer.rs",
"native/shuffle/src/type_align.rs"
],
- "upstream_base": "Base1.2 RowIteratorinterleave используетвсеrecord_batches; Joomchunk_batches/batch_slots/type_align отсутствуют.",
- "proposal": "Адаптировать локальныйwriterpath, сохранив новыерemote/local/round-robincontracts.",
+ "upstream_base": "Base1.2 RowIterator interleave uses all record_batches; Joom chunk_batches/batch_slots/type_align are absent.",
+ "proposal": "Adapt the local writer path while preserving new remote/local/round-robin contracts.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Схемаnestednullability/metadata можетбытьравнапоdatatype, нонеидентичнапопойнтеру; важныslicedarrays иincompatiblefallback."
+ "Nested nullability/metadata schemas can be equal by datatype but not pointer-identical; sliced arrays and incompatible fallback matter."
],
"dependencies": [],
"acceptance": [
@@ -4242,56 +3731,50 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/native/shuffle/src/shuffle_writer.rs",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/native/shuffle/src/type_align.rs"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"native/shuffle/src/partitioners/partitioned_batch_iterator.rs",
"native/shuffle/src/shuffle_writer.rs",
"native/shuffle/src/type_align.rs"
],
- "proposed_action": "Адаптировать локальныйwriterpath, сохранив новыерemote/local/round-robincontracts.",
+ "proposed_action": "Adapt the local writer path while preserving new remote/local/round-robin contracts.",
"upstream": {
- "base": "Base1.2 RowIteratorinterleave используетвсеrecord_batches; Joomchunk_batches/batch_slots/type_align отсутствуют.",
+ "base": "Base1.2 RowIterator interleave uses all record_batches; Joom chunk_batches/batch_slots/type_align are absent.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "J14"
- ],
+ "prerequisites": ["J14"],
"phase": 6,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G07"
- ],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать локальныйwriterpath, сохранив новыерemote/local/round-robincontracts.",
+ "approved_decision": "Adapt the local writer path while preserving new remote/local/round-robin contracts.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J23",
- "title": "Direct shuffle read выбирается уже при первоначальном planning",
- "contract": "ShuffleScanprotoвыбираетсяinitialconversion, не толькопозднемAQE; nativeparentне сохраняетобычныйScanсдорогимJVMroundtrip; JVMdispatch соответствуетserializedproto.",
- "source_commits": [
- "4956d831b"
- ],
+ "title": "Select direct shuffle reads during initial planning",
+ "contract": "Select ShuffleScan proto during initial conversion, not only late AQE; a native parent must not retain ordinary Scan with an expensive JVM round trip; JVM dispatch must match serialized proto.",
+ "source_commits": ["4956d831b"],
"source_files": [
"spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala",
"spark/src/main/scala/org/apache/spark/sql/comet/operators.scala"
],
- "upstream_base": "Initial ShuffleExchangeExec конвертируется CometExecRule.scala664–666 через handler CometShuffleExchangeExec. Этот handler наследует CometSink.convert, который всегда создаёт обычный Scan (CometSink.scala53–84), затем createExec (CometShuffleExchangeExec.scala455–468) создаёт sink placeholder с тем же proto. Поэтому уже-native parent получает обычный Scan на initial/non-AQE path даже при directRead=true. Upstream CometExchangeSink умеет ShuffleScan для existing Comet exchange/AQE wrappers, а refreshStaleShuffleScans (CometExecRule754–819) меняет stale Scan только когда child feeder уже имеет ShuffleScan: это частичный поздний AQE analogue, не Joom initial conversion.",
- "proposal": "Адаптировать Joom4956d831b: handler первоначального ShuffleExchangeExec conversion должен сразу выдавать ShuffleScan при directRead=true и сохранять обычный Scan при false. Сохранить новый upstream refreshStaleShuffleScans и его AQE/DPP ограничения. Проверять выбор input stream по serialized proto, а не отдельно по текущей conf.",
+ "upstream_base": "Initial ShuffleExchangeExec is converted at CometExecRule.scala664–666 through handler CometShuffleExchangeExec. This handler inherits CometSink.convert, which always creates ordinary Scan (CometSink.scala53–84); createExec (CometShuffleExchangeExec.scala455–468) then creates a sink placeholder with the same proto. Thus an already-native parent receives ordinary Scan on the initial/non-AQE path even with directRead=true. Upstream CometExchangeSink can create ShuffleScan for existing Comet exchange/AQE wrappers, while refreshStaleShuffleScans (CometExecRule754–819) changes stale Scan only when the child feeder already has ShuffleScan: a partial late-AQE equivalent, not Joom initial conversion.",
+ "proposal": "Adapt Joom4956d831b: the initial ShuffleExchangeExec conversion handler must immediately emit ShuffleScan for directRead=true and retain ordinary Scan for false. Preserve the new upstream refreshStaleShuffleScans and its AQE/DPP restrictions. Select the input stream by serialized proto, not independently from current configuration.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Новый upstream AQE stale-scan refresh и DPP stage-link lifecycle нельзя заменить старой логикой: исправление initial conversion дополняет их.",
- "Новые CapturedSQLConf/local/remote paths; proto input indices должны совпадать с JNI stream indices. Source trace определяет непокрытую initial ветку; runtime-тестами ещё предстоит подтвердить end-to-end."
+ "Do not replace the new upstream AQE stale-scan refresh and DPP stage-link lifecycle with old logic: the initial-conversion fix complements them.",
+ "New CapturedSQLConf/local/remote paths; proto input indices must match JNI stream indices. Source tracing identifies the uncovered initial branch; end-to-end behavior still requires runtime tests."
],
"dependencies": [],
"acceptance": [
@@ -4316,87 +3799,61 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"upstream_source_evidence": [
{
"ref": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
"path": "spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala",
- "lines": [
- 664,
- 666,
- 754,
- 819
- ]
+ "lines": [664, 666, 754, 819]
},
{
"ref": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
"path": "spark/src/main/scala/org/apache/comet/serde/operator/CometSink.scala",
- "lines": [
- 53,
- 84,
- 95,
- 149
- ]
+ "lines": [53, 84, 95, 149]
},
{
"ref": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
"path": "spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala",
- "lines": [
- 343,
- 349,
- 455,
- 468
- ]
+ "lines": [343, 349, 455, 468]
},
{
"ref": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
"path": "spark/src/main/scala/org/apache/spark/sql/comet/operators.scala",
- "lines": [
- 1061,
- 1066,
- 1105
- ]
+ "lines": [1061, 1066, 1105]
}
],
"source_paths": [
"spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala",
"spark/src/main/scala/org/apache/spark/sql/comet/operators.scala"
],
- "proposed_action": "Адаптировать Joom4956d831b: handler первоначального ShuffleExchangeExec conversion должен сразу выдавать ShuffleScan при directRead=true и сохранять обычный Scan при false. Сохранить новый upstream refreshStaleShuffleScans и его AQE/DPP ограничения. Проверять выбор input stream по serialized proto, а не отдельно по текущей conf.",
+ "proposed_action": "Adapt Joom4956d831b: the initial ShuffleExchangeExec conversion handler must immediately emit ShuffleScan for directRead=true and retain ordinary Scan for false. Preserve the new upstream refreshStaleShuffleScans and its AQE/DPP restrictions. Select the input stream by serialized proto, not independently from current configuration.",
"upstream": {
- "base": "Initial ShuffleExchangeExec конвертируется CometExecRule.scala664–666 через handler CometShuffleExchangeExec. Этот handler наследует CometSink.convert, который всегда создаёт обычный Scan (CometSink.scala53–84), затем createExec (CometShuffleExchangeExec.scala455–468) создаёт sink placeholder с тем же proto. Поэтому уже-native parent получает обычный Scan на initial/non-AQE path даже при directRead=true. Upstream CometExchangeSink умеет ShuffleScan для existing Comet exchange/AQE wrappers, а refreshStaleShuffleScans (CometExecRule754–819) меняет stale Scan только когда child feeder уже имеет ShuffleScan: это частичный поздний AQE analogue, не Joom initial conversion.",
+ "base": "Initial ShuffleExchangeExec is converted at CometExecRule.scala664–666 through handler CometShuffleExchangeExec. This handler inherits CometSink.convert, which always creates ordinary Scan (CometSink.scala53–84); createExec (CometShuffleExchangeExec.scala455–468) then creates a sink placeholder with the same proto. Thus an already-native parent receives ordinary Scan on the initial/non-AQE path even with directRead=true. Upstream CometExchangeSink can create ShuffleScan for existing Comet exchange/AQE wrappers, while refreshStaleShuffleScans (CometExecRule754–819) changes stale Scan only when the child feeder already has ShuffleScan: a partial late-AQE equivalent, not Joom initial conversion.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 6,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G07"
- ],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать Joom4956d831b: handler первоначального ShuffleExchangeExec conversion должен сразу выдавать ShuffleScan при directRead=true и сохранять обычный Scan при false. Сохранить новый upstream refreshStaleShuffleScans и его AQE/DPP ограничения. Проверять выбор input stream по serialized proto, а не отдельно по текущей conf.",
+ "approved_decision": "Adapt Joom4956d831b: the initial ShuffleExchangeExec conversion handler must immediately emit ShuffleScan for directRead=true and retain ordinary Scan for false. Preserve the new upstream refreshStaleShuffleScans and its AQE/DPP restrictions. Select the input stream by serialized proto, not independently from current configuration.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J12",
- "title": "Shuffle read coalescing с ограничениями rows и bytes",
- "contract": "ОбеJVM/native directread стороны coalesce smallblocks доbatchSizeили64MiB; nestedoffsetbudget safe; блокбольшеbyteslimitпередаётсякакесть.64MiBэтоtarget дляjoinedbatches, не жёсткийlimit любойoutputbatch.",
- "source_commits": [
- "2d30872c8",
- "192bad884"
- ],
+ "title": "Shuffle-read coalescing with row and byte limits",
+ "contract": "Both JVM/native direct-read sides coalesce small blocks up to batchSize or64MiB; nested-offset budget remains safe; a block larger than the byte limit passes through unchanged.64MiB is a target for joined batches, not a strict limit on every output batch.",
+ "source_commits": ["2d30872c8", "192bad884"],
"source_files": [
"native/shuffle/src/read_coalescer.rs",
"native/core/src/execution/operators/shuffle_scan.rs",
@@ -4406,25 +3863,21 @@
"spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala",
"spark/src/main/scala/org/apache/comet/Native.scala"
],
- "upstream_base": "Base1.2 не имеетJoomreadcoalescer/flags.",
- "proposal": "Перенести native/JVM/proto/JNI sides в одномсогласованномPR.",
+ "upstream_base": "Base1.2 lacks Joom read-coalescer/flags.",
+ "proposal": "Port native/JVM/proto/JNI sides in one coherent PR.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Новыйshuffle/readdecoder lifecycle, compression/framing,memoryownership; joinedbatchesнеучитываютсяpool, bytesbound важен.",
- "Припереносенеможнооткатитьlatestzstdcachedbatchcapacityfix/base."
- ],
- "dependencies": [
- "J14",
- "J22",
- "J23"
+ "New shuffle/read-decoder lifecycle, compression/framing, memory ownership; joined batches are not pool-accounted, so the byte bound matters.",
+ "Do not revert the latest zstd cached-batch-capacity fix/base during porting."
],
+ "dependencies": ["J14", "J22", "J23"],
"acceptance": [
"CometShuffleReadCoalesceSuite/native read_coalescer tests; JVM/native/direct paths; rows/bytes/oversizedblock/null/nested/compression; releasebuffers once."
],
"implementation_questions": [],
"other_upstream_candidates": [
- "main#7258 broadcast-boundary producercoalescing defaultoff: другойэтап, неполныйаналог.",
- "PR#7354/#7418 decoder/readbufferreuse: частичнаяцель, неglobalrow+bytecoalescer."
+ "main#7258 broadcast-boundary producer coalescing defaults off: a different stage, not a full equivalent.",
+ "PR#7354/#7418 decoder/read-buffer reuse: a partial goal, not a global row+byte coalescer."
],
"config": [
"spark.comet.shuffle.read.coalesce.enabled=true",
@@ -4485,7 +3938,7 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/Native.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"native/shuffle/src/read_coalescer.rs",
"native/core/src/execution/operators/shuffle_scan.rs",
@@ -4495,61 +3948,48 @@
"spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometBlockStoreShuffleReader.scala",
"spark/src/main/scala/org/apache/comet/Native.scala"
],
- "proposed_action": "Перенести native/JVM/proto/JNI sides в одномсогласованномPR.",
+ "proposed_action": "Port native/JVM/proto/JNI sides in one coherent PR.",
"upstream": {
- "base": "Base1.2 не имеетJoomreadcoalescer/flags.",
+ "base": "Base1.2 lacks Joom read-coalescer/flags.",
"outside_base": [
- "main#7258 broadcast-boundary producercoalescing defaultoff: другойэтап, неполныйаналог.",
- "PR#7354/#7418 decoder/readbufferreuse: частичнаяцель, неglobalrow+bytecoalescer."
+ "main#7258 broadcast-boundary producer coalescing defaults off: a different stage, not a full equivalent.",
+ "PR#7354/#7418 decoder/read-buffer reuse: a partial goal, not a global row+byte coalescer."
]
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "J14",
- "J22",
- "J23"
- ],
+ "original_dependency_notes": ["J14", "J22", "J23"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "J14",
- "J22",
- "J23",
- "J24"
- ],
+ "prerequisites": ["J14", "J22", "J23", "J24"],
"phase": 6,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G07"
- ],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести native/JVM/proto/JNI sides в одномсогласованномPR.",
+ "approved_decision": "Port native/JVM/proto/JNI sides in one coherent PR.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J05",
- "title": "Scalar subqueries как runtime-аргументы JVM codegen dispatcher",
- "contract": "Lower ScalarSubquery→BoundReference; resolved result передавать native argument; length-one Arrow vectors broadcast через rowMask, включая nested/null checks. Cached kernel не захватывает старый результат.",
- "source_commits": [
- "65be8b751"
- ],
+ "title": "Scalar subqueries as runtime arguments to the JVM codegen dispatcher",
+ "contract": "Lower ScalarSubquery→BoundReference; pass the resolved result as a native argument; broadcast length-one Arrow vectors through rowMask, including nested/null checks. A cached kernel must not capture the old result.",
+ "source_commits": ["65be8b751"],
"source_files": [
"spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala",
"spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala",
"spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenInput.scala"
],
- "upstream_base": "В1.2 CometBatchKernelCodegen.canHandle отвергает subqueries; Joom runtime lowering/rowMask отсутствует.",
- "proposal": "Адаптировать к новому dispatcher/DispatchOccurrence, сохранив возможность native plans вокруг JVM expression.",
+ "upstream_base": "In1.2 CometBatchKernelCodegen.canHandle rejects subqueries; Joom runtime lowering/rowMask are absent.",
+ "proposal": "Adapt to the new dispatcher/DispatchOccurrence while preserving native plans around the JVM expression.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
"Spark4 ScalarSubquery lifecycle/AQE/reuse; nondeterministic expression state; cached kernel result isolation.",
- "Это не полное решение ограничения struct-typed native scalar subquery Spark4.2 UNION."
+ "This is not a complete solution to the Spark4.2 UNION limitation for struct-typed native scalar subqueries."
],
"dependencies": [],
"acceptance": [
@@ -4557,7 +3997,7 @@
],
"implementation_questions": [],
"other_upstream_candidates": [
- "PR #5889 draft struct scalar subqueries: частично пересекается по native coverage, не является прямой заменойdispatcher lowering."
+ "Draft PR#5889 struct scalar subqueries partially overlaps in native coverage, but is not a direct replacement for dispatcher lowering."
],
"config": [],
"source_commit_details": [
@@ -4589,68 +4029,59 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenInput.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala",
"spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala",
"spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenInput.scala"
],
- "proposed_action": "Адаптировать к новому dispatcher/DispatchOccurrence, сохранив возможность native plans вокруг JVM expression.",
+ "proposed_action": "Adapt to the new dispatcher/DispatchOccurrence while preserving native plans around the JVM expression.",
"upstream": {
- "base": "В1.2 CometBatchKernelCodegen.canHandle отвергает subqueries; Joom runtime lowering/rowMask отсутствует.",
+ "base": "In1.2 CometBatchKernelCodegen.canHandle rejects subqueries; Joom runtime lowering/rowMask are absent.",
"outside_base": [
- "PR #5889 draft struct scalar subqueries: частично пересекается по native coverage, не является прямой заменойdispatcher lowering."
+ "Draft PR#5889 struct scalar subqueries partially overlaps in native coverage, but is not a direct replacement for dispatcher lowering."
]
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 7,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G08"
- ],
+ "implementation_plan_groups": ["G08"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать к новому dispatcher/DispatchOccurrence, сохранив возможность native plans вокруг JVM expression.",
+ "approved_decision": "Adapt to the new dispatcher/DispatchOccurrence while preserving native plans around the JVM expression.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J06",
- "title": "Короткое замыкание JVM-dispatched conjuncts в фильтре",
- "contract": "Детерминированные top-level AND conjuncts исполняют JVM dispatcher только на строках, прошедших предыдущие части; SQL null semantics и nondeterministic behavior сохраняются.",
- "source_commits": [
- "764adf614",
- "092d70af2"
- ],
+ "title": "Short-circuit JVM-dispatched conjuncts in filters",
+ "contract": "Deterministic top-level AND conjuncts run the JVM dispatcher only on rows passing previous parts; preserve SQL null semantics and nondeterministic behavior.",
+ "source_commits": ["764adf614", "092d70af2"],
"source_files": [
"spark/src/main/scala/org/apache/spark/sql/comet/operators.scala",
"spark/src/main/scala/org/apache/comet/CometConf.scala"
],
- "upstream_base": "Joom CASE-WHEN rewrite/flag вbase1.2 отсутствует.",
- "proposal": "Адаптировать собственную реализацию с отдельной проверкой возможного upstream backport.",
+ "upstream_base": "Joom CASE-WHEN rewrite/flag is absent in base1.2.",
+ "proposal": "Adapt our implementation and separately assess a possible upstream backport.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "ANSI exceptions/evaluation order; не применить rewrite к nondeterministic conditions; dispatcher annotations/changeoccurrence."
- ],
- "dependencies": [
- "J05 if subqueries in dispatched predicates"
+ "ANSI exceptions/evaluation order; do not apply the rewrite to nondeterministic conditions; dispatcher annotations/changed occurrences."
],
+ "dependencies": ["J05 if subqueries in dispatched predicates"],
"acceptance": [
"CometFilterShortCircuitSuite: dangerouscasts/failingUDF onexcludedrows, NULLs,nondeterminism, vectorrowmasks."
],
"implementation_questions": [],
"other_upstream_candidates": [
- "main #7326 lazy AND/OR/IN selected-rows evaluation: существенный partial analogue, сравнить отдельно до решения backport vsownrewrite."
+ "main#7326 lazy AND/OR/IN selected-rows evaluation is a substantial partial equivalent; compare separately before deciding between backport and our rewrite."
],
"config": [
"spark.comet.exec.filter.shortCircuitJvmDispatch.enabled=true"
@@ -4680,50 +4111,43 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/spark/sql/comet/operators.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/CometConf.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/spark/sql/comet/operators.scala",
"spark/src/main/scala/org/apache/comet/CometConf.scala"
],
- "proposed_action": "Адаптировать собственную реализацию с отдельной проверкой возможного upstream backport.",
+ "proposed_action": "Adapt our implementation and separately assess a possible upstream backport.",
"upstream": {
- "base": "Joom CASE-WHEN rewrite/flag вbase1.2 отсутствует.",
+ "base": "Joom CASE-WHEN rewrite/flag is absent in base1.2.",
"outside_base": [
- "main #7326 lazy AND/OR/IN selected-rows evaluation: существенный partial analogue, сравнить отдельно до решения backport vsownrewrite."
+ "main#7326 lazy AND/OR/IN selected-rows evaluation is a substantial partial equivalent; compare separately before deciding between backport and our rewrite."
]
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [
"J05 if subqueries in dispatched predicates"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "J05"
- ],
+ "prerequisites": ["J05"],
"phase": 7,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G08"
- ],
+ "implementation_plan_groups": ["G08"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать собственную реализацию с отдельной проверкой возможного upstream backport.",
+ "approved_decision": "Adapt our implementation and separately assess a possible upstream backport.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J07",
- "title": "ScalaUDF function pools: отправка closures один раз на native plan",
- "contract": "Сериализовать функции UDF в одном shared stream, ссылки изexpressions; preserve reference sharing, task-local functionstate,nondeterministic occurrence identity; cache без unshadedGuava dependency.",
- "source_commits": [
- "df0b1b9ff",
- "ab179fa2e"
- ],
+ "title": "ScalaUDF function pools: send closures once per native plan",
+ "contract": "Serialize UDF functions in one shared stream, referenced by expressions; preserve reference sharing, task-local function state and nondeterministic occurrence identity; cache without an unshaded Guava dependency.",
+ "source_commits": ["df0b1b9ff", "ab179fa2e"],
"source_files": [
"spark/src/main/scala/org/apache/comet/udf/codegen/CometUdfFunctionPool.scala",
"spark/src/main/scala/org/apache/comet/udf/codegen/CometSharedFunctionRef.scala",
@@ -4731,23 +4155,20 @@
"spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala",
"spark/src/main/scala/org/apache/comet/CometExecIterator.scala"
],
- "upstream_base": "Вbase1.2 bound expressions serialise independently; shared pools отсутствуют.",
- "proposal": "Адаптировать к новым DispatchOccurrence/digest и lifecycle; запускать именно shaded releaseJAR tests.",
+ "upstream_base": "In base1.2 bound expressions serialize independently; shared pools are absent.",
+ "proposal": "Adapt to new DispatchOccurrence/digest and lifecycle; run tests specifically against the shaded release JAR.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Высокий риск ошибочного sharing stateful/nondeterministic closures междуoccurrences/tasks; new1.2cache/lifecycle.",
- "Ранее Guava вызвалошибку shadedartifact; обычныхunshadedunittests недостаточно."
- ],
- "dependencies": [
- "J05 commoncodegenpaths",
- "J16 lifecycle"
+ "High risk of incorrect sharing of stateful/nondeterministic closures between occurrences/tasks; new1.2 cache/lifecycle.",
+ "Guava previously caused a shaded-artifact failure; ordinary unshaded unit tests are insufficient."
],
+ "dependencies": ["J05 commoncodegenpaths", "J16 lifecycle"],
"acceptance": [
"CometCodegenSuite sharedclosures identity/serializedsize/nondeterminism/failedserialization fallback; remoteexecutorclasspath + shadedJAR."
],
"implementation_questions": [],
"other_upstream_candidates": [
- "PR #5027 JVMArrow memory accounting — другой механизм, functionpools не заменяет."
+ "PR#5027 JVMArrow memory accounting is a different mechanism and does not replace function pools."
],
"config": [],
"source_commit_details": [
@@ -4781,7 +4202,7 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/CometExecIterator.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/comet/udf/codegen/CometUdfFunctionPool.scala",
"spark/src/main/scala/org/apache/comet/udf/codegen/CometSharedFunctionRef.scala",
@@ -4789,48 +4210,36 @@
"spark/src/main/scala/org/apache/comet/serde/CometScalaUDF.scala",
"spark/src/main/scala/org/apache/comet/CometExecIterator.scala"
],
- "proposed_action": "Адаптировать к новым DispatchOccurrence/digest и lifecycle; запускать именно shaded releaseJAR tests.",
+ "proposed_action": "Adapt to new DispatchOccurrence/digest and lifecycle; run tests specifically against the shaded release JAR.",
"upstream": {
- "base": "Вbase1.2 bound expressions serialise independently; shared pools отсутствуют.",
+ "base": "In base1.2 bound expressions serialize independently; shared pools are absent.",
"outside_base": [
- "PR #5027 JVMArrow memory accounting — другой механизм, functionpools не заменяет."
+ "PR#5027 JVMArrow memory accounting is a different mechanism and does not replace function pools."
]
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "J05 commoncodegenpaths",
- "J16 lifecycle"
- ],
+ "original_dependency_notes": ["J05 commoncodegenpaths", "J16 lifecycle"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "J05"
- ],
+ "prerequisites": ["J05"],
"phase": 7,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G08"
- ],
+ "implementation_plan_groups": ["G08"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать к новым DispatchOccurrence/digest и lifecycle; запускать именно shaded releaseJAR tests.",
+ "approved_decision": "Adapt to new DispatchOccurrence/digest and lifecycle; run tests specifically against the shaded release JAR.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J17",
- "title": "Размер task binaries: исключение тяжёлых планов из Java serialization",
- "contract": "NativeShuffleSpec несериализуется вdownstream readerstages; operatorprotobufs transient; native childplan остаётсядоступнымwriter; querytextsинтернируютсятакжевscancommon/Delta opaquepayload/shufflewriterroot.",
- "source_commits": [
- "a7170c287",
- "d0c3e79d2",
- "af29142a3",
- "61590a84a"
- ],
+ "title": "Task-binary size: exclude heavy plans from Java serialization",
+ "contract": "NativeShuffleSpec is not serialized into downstream reader stages; operator protobufs are transient; the native child plan remains available to the writer; query texts are also interned in scan common/Delta opaque payload/shuffle-writer root.",
+ "source_commits": ["a7170c287", "d0c3e79d2", "af29142a3", "61590a84a"],
"source_files": [
"spark/src/main/scala/org/apache/comet/serde/QueryContextInterner.scala",
"spark/src/main/scala/org/apache/spark/sql/comet/operators.scala",
@@ -4839,16 +4248,14 @@
"spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala",
"spark/src/main/scala/org/apache/comet/ContribServices.scala"
],
- "upstream_base": "UpstreamQueryContextInterner ужеестьдляrootnativeblock; residualJoom scanpayload/shufflepool/transientnativeShuffleSpec/nativeOp нет. Base1.2 nativeShuffleSpec nontransient.",
- "proposal": "Адаптироватьтолькоresidualсохраняязапросныйконтекстдляошибок; coherentDeltaPlanDataInjector changesвместеDelta.",
+ "upstream_base": "Upstream QueryContextInterner already exists for the root native block; residual Joom scan-payload/shuffle-pool/transient nativeShuffleSpec/nativeOp changes are absent. Base1.2 nativeShuffleSpec is nontransient.",
+ "proposal": "Adapt only residual changes while retaining query context for errors; keep coherent DeltaPlanDataInjector changes together with Delta.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "1.2CapturedSQLConf/NativeShuffleSpec/Celebornfallback contracts изменены; transientfieldне долженбытьнуженexecutorreader.",
- "ANSIerrorquerycontext/sourceKeyhash должнасовпадатьдо/послеinterning."
- ],
- "dependencies": [
- "Delta integration дляDelta opaquepayload branch"
+ "1.2 CapturedSQLConf/NativeShuffleSpec/Celeborn fallback contracts changed; a transient field must not be needed by the executor reader.",
+ "ANSI error query context/sourceKey hash must match before/after interning."
],
+ "dependencies": ["Delta integration for the Delta opaque-payload branch"],
"acceptance": [
"CometTaskBinarySizeSuite,QueryContextInternerSuite,CometNativeShuffleInputRDDSuite; Java/Kryo serialization, longquerymanyscans/union; jobnativeplans anderrorSQLpositions."
],
@@ -4929,7 +4336,7 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/ContribServices.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/comet/serde/QueryContextInterner.scala",
"spark/src/main/scala/org/apache/spark/sql/comet/operators.scala",
@@ -4938,53 +4345,45 @@
"spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometNativeShuffleWriter.scala",
"spark/src/main/scala/org/apache/comet/ContribServices.scala"
],
- "proposed_action": "Адаптироватьтолькоresidualсохраняязапросныйконтекстдляошибок; coherentDeltaPlanDataInjector changesвместеDelta.",
+ "proposed_action": "Adapt only residual changes while retaining query context for errors; keep coherent DeltaPlanDataInjector changes together with Delta.",
"upstream": {
- "base": "UpstreamQueryContextInterner ужеестьдляrootnativeblock; residualJoom scanpayload/shufflepool/transientnativeShuffleSpec/nativeOp нет. Base1.2 nativeShuffleSpec nontransient.",
+ "base": "Upstream QueryContextInterner already exists for the root native block; residual Joom scan-payload/shuffle-pool/transient nativeShuffleSpec/nativeOp changes are absent. Base1.2 nativeShuffleSpec is nontransient.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [
- "Delta integration дляDelta opaquepayload branch"
+ "Delta integration for the Delta opaque-payload branch"
],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "I01",
- "J07"
- ],
+ "prerequisites": ["I01", "J07"],
"phase": 7,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G08"
- ],
+ "implementation_plan_groups": ["G08"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптироватьтолькоresidualсохраняязапросныйконтекстдляошибок; coherentDeltaPlanDataInjector changesвместеDelta.",
+ "approved_decision": "Adapt only residual changes while retaining query context for errors; keep coherent DeltaPlanDataInjector changes together with Delta.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J08",
- "title": "Binary borrowing в non-codegen ColumnarToRow",
- "contract": "Физически скопироватьBinaryType вUnsafeRow через borrowedUTF8Stringspan, избегаяпромежуточного byte[]; план/данные остаютсяBinary; unsupportedvector/nested/dictionary paths сохраняютобычныйfallback.",
- "source_commits": [
- "c0b0448c7",
- "3d99265f0"
- ],
+ "title": "Binary borrowing in non-codegen ColumnarToRow",
+ "contract": "Physically copy BinaryType into UnsafeRow through a borrowed UTF8String span, avoiding an intermediate byte[]; plans/data remain Binary; unsupported vector/nested/dictionary paths retain ordinary fallback.",
+ "source_commits": ["c0b0448c7", "3d99265f0"],
"source_files": [
"spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala",
"spark/src/test/java/org/apache/spark/sql/benchmark/CometBinaryRowCopyBenchmark.java"
],
- "upstream_base": "В1.2 classcodegencache есть, borrowedBinary нет; experimental directnativeC2R defaultfalse не автоматическизаменяет этуоптимизацию.",
- "proposal": "Адаптировать поверх CometUnsafeProjection/new C2R path, сохранивобычныйfallback.",
+ "upstream_base": "1.2 has a class-codegen cache but no borrowed Binary; experimental direct native C2R default=false is not an automatic substitute for this optimization.",
+ "proposal": "Adapt on top of CometUnsafeProjection/new C2R path while retaining ordinary fallback.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Spark4 UnsafeRow/codegen/ArrowJava vector APIs; lifetime borrowedbytes только внутри немедленнойкопии; недопустим UTF8decode."
+ "Spark4 UnsafeRow/codegen/ArrowJava vector APIs; borrowed-byte lifetime is limited to the immediate copy; UTF8 decoding is forbidden."
],
"dependencies": [],
"acceptance": [
@@ -5019,64 +4418,55 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/test/java/org/apache/spark/sql/benchmark/CometBinaryRowCopyBenchmark.java"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala",
"spark/src/test/java/org/apache/spark/sql/benchmark/CometBinaryRowCopyBenchmark.java"
],
- "proposed_action": "Адаптировать поверх CometUnsafeProjection/new C2R path, сохранивобычныйfallback.",
+ "proposed_action": "Adapt on top of CometUnsafeProjection/new C2R path while retaining ordinary fallback.",
"upstream": {
- "base": "В1.2 classcodegencache есть, borrowedBinary нет; experimental directnativeC2R defaultfalse не автоматическизаменяет этуоптимизацию.",
+ "base": "1.2 has a class-codegen cache but no borrowed Binary; experimental direct native C2R default=false is not an automatic substitute for this optimization.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 7,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G07"
- ],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать поверх CometUnsafeProjection/new C2R path, сохранивобычныйfallback.",
+ "approved_decision": "Adapt on top of CometUnsafeProjection/new C2R path while retaining ordinary fallback.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J09",
- "title": "Повторное использование экземпляров C2R projections",
- "contract": "Пулprojection instances ограниченschema/count/bufferbytes; возвращать только после завершения всех читателейtask, чтобы reusedUnsafeRowbuffer не портил ещёчитаемыеданные.",
- "source_commits": [
- "9560d4a8e",
- "8e065270a"
- ],
+ "title": "Reuse C2R projection instances",
+ "contract": "Bound the projection-instance pool by schema/count/buffer bytes; return instances only after all task readers finish so a reused UnsafeRow buffer cannot corrupt data still being read.",
+ "source_commits": ["9560d4a8e", "8e065270a"],
"source_files": [
"spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala"
],
- "upstream_base": "Upstream#6858 CometUnsafeProjection кэширует generatedclass, создаётnewinstance; частично покрываетцель, не sameinstance lifecycle.",
- "proposal": "Обсудить отдельно: сохранитьupstreamclasscache + остаточныйinstancepool либо доказатьдостаточностьupstream измерениями; ничего не исключено.",
+ "upstream_base": "Upstream#6858 CometUnsafeProjection caches the generated class and creates a new instance; it partially covers the goal, not same-instance lifecycle.",
+ "proposal": "Discuss separately: retain upstream class cache plus residual instance pool, or demonstrate upstream sufficiency through measurement; nothing has been excluded.",
"approval": "RESOLVED_BY_APPROVED_POLICY",
"risks": [
"Cross-task mutableUnsafeRow reuse/data corruption if release tooearly; Spark4 readercompletion.",
- "Пользаinstancepool поверхновогоcache требует измерения; bounded1MiBbuffers guard сохранитьприport."
- ],
- "dependencies": [
- "J08 if commonprojectionwrapper"
+ "Benefit from the instance pool above the new cache requires measurement; preserve the bounded1MiB-buffer guard when porting."
],
+ "dependencies": ["J08 if commonprojectionwrapper"],
"acceptance": [
"CometBatchRowProjectionSuite pooledprojections/retainedrows/tworeaders/completionlifecycle; codegencount + allocationbenchmarks."
],
"implementation_questions": [
- "Утвердить вариант: upstreamclasscache плюс нашinstancepool (предлагаетсядлясохраненияфункциональности) либо согласоватьзаменуupstream после измерений."
+ "Approve either upstream class cache plus our instance pool (recommended to preserve functionality), or an upstream replacement after measurement."
],
"other_upstream_candidates": [],
"config": [],
@@ -5104,38 +4494,32 @@
"source_urls": [
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala"
],
- "proposed_action": "Сохранить Joom projection-instance pool поверх upstream generated-class cache, соблюдая lifetime/bounds guards. Частичный class cache сам по себе не заменяет pool; упрощение возможно только при подтверждённой эквивалентности по утверждённому правилу.",
+ "proposed_action": "Preserve the Joom projection-instance pool on top of upstream generated-class caching, respecting lifetime/bounds guards. A partial class cache alone does not replace the pool; simplify only with demonstrated equivalence under the approved rule.",
"upstream": {
- "base": "Upstream#6858 CometUnsafeProjection кэширует generatedclass, создаётnewinstance; частично покрываетцель, не sameinstance lifecycle.",
+ "base": "Upstream#6858 CometUnsafeProjection caches the generated class and creates a new instance; it partially covers the goal, not same-instance lifecycle.",
"outside_base": []
},
- "open_decisions": "Решение зафиксировано; дополнительных вопросов о требуемом поведении нет. Приёмочные проверки ещё не выполнены.",
+ "open_decisions": "The decision is recorded; no additional questions about required behavior remain. Acceptance checks have not run yet.",
"approval_status": "RESOLVED_BY_APPROVED_POLICY",
- "original_dependency_notes": [
- "J08 if commonprojectionwrapper"
- ],
+ "original_dependency_notes": ["J08 if commonprojectionwrapper"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "J08"
- ],
+ "prerequisites": ["J08"],
"phase": 7,
"requires_upstream_discussion": false,
- "decision_category": "определено утверждённым общим правилом",
+ "decision_category": "defined by the approved general rule",
"same_pr_or_co_review": [],
- "approved_decision": "Сохранить Joom projection-instance pool поверх upstream generated-class cache, соблюдая lifetime/bounds guards. Частичный class cache сам по себе не заменяет pool; упрощение возможно только при подтверждённой эквивалентности по утверждённому правилу.",
+ "approved_decision": "Preserve the Joom projection-instance pool on top of upstream generated-class caching, respecting lifetime/bounds guards. A partial class cache alone does not replace the pool; simplify only with demonstrated equivalence under the approved rule.",
"decision_date": "2026-11-16",
"decision_source": "decisions.json",
- "original_proposal": "Обсудить отдельно: сохранитьupstreamclasscache + остаточныйinstancepool либо доказатьдостаточностьupstream измерениями; ничего не исключено.",
+ "original_proposal": "Discuss separately: retain upstream class cache plus residual instance pool, or demonstrate upstream sufficiency through measurement; nothing has been excluded.",
"prior_open_decisions": [
- "Утвердить вариант: upstreamclasscache плюс нашinstancepool (предлагаетсядлясохраненияфункциональности) либо согласоватьзаменуupstream после измерений."
- ],
- "implementation_plan_groups": [
- "G07"
+ "Approve either upstream class cache plus our instance pool (recommended to preserve functionality), or an upstream replacement after measurement."
],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
@@ -5143,27 +4527,23 @@
},
{
"id": "J25",
- "title": "Запрет whole-stage codegen для C2R поверх Comet broadcast",
- "contract": "CometColumnarToRowExec.supportCodegen=false надCometBroadcastExchangeчерезBroadcastQueryStage/ReusedExchange, чтобынеисполнятьbroadcastкакобычныйRDD.",
- "source_commits": [
- "3adfb1ecc"
- ],
+ "title": "Disable whole-stage codegen for C2R above Comet broadcast",
+ "contract": "CometColumnarToRowExec.supportCodegen=false above CometBroadcastExchange through BroadcastQueryStage/ReusedExchange so broadcast is not executed as an ordinary RDD.",
+ "source_commits": ["3adfb1ecc"],
"source_files": [
"spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala"
],
- "upstream_base": "Полная цепочка target93+Spark4.2.1: EliminateRedundantTransitions114–122/328–345 создаёт обычный CometColumnarToRowExec, поскольку nativeC2R flag по умолчанию false (CometConf489–498). Класс имеет CodegenSupport без override supportCodegen; default Spark4.2.1 CodegenSupport=true. CollapseCodegenStages925–984 при не-codegen широком join рекурсивно оборачивает его узкий C2R build child в отдельный WholeStageCodegenExec. BroadcastHashJoinExec.doExecute вызывает buildPlan.executeBroadcast (Spark4.2.1:136), но WholeStageCodegenExec broadcast override не имеет; SparkPlan350–351 бросает unsupported error. C2R собственный doExecuteBroadcast есть, но wrapper мешает дойти до него. Upstream native/direct converters и DPP subquery-specific cleanup не закрывают эту ordinary C2R ветку.",
- "proposal": "Перенести целевой supportCodegen=false guard для C2R, чей child — CometBroadcastExchangeExec, BroadcastQueryStageExec или ReusedExchangeExec над ним; оставить новый CometBroadcastValue/IdentityBroadcastMode copy и оба C2R converters. Port исходный regression с широким Spark join/узкой broadcast build side.",
+ "upstream_base": "Full target93+Spark4.2.1 chain: EliminateRedundantTransitions114–122/328–345 creates ordinary CometColumnarToRowExec because nativeC2R defaults false (CometConf489–498). The class has CodegenSupport without overriding supportCodegen; default Spark4.2.1 CodegenSupport=true. For a non-codegen wide join, CollapseCodegenStages925–984 recursively wraps its narrow C2R build child in a separate WholeStageCodegenExec. BroadcastHashJoinExec.doExecute calls buildPlan.executeBroadcast (Spark4.2.1:136), but WholeStageCodegenExec has no broadcast override; SparkPlan350–351 throws an unsupported error. C2R has its own doExecuteBroadcast, but the wrapper prevents reaching it. Upstream native/direct converters and DPP subquery-specific cleanup do not cover this ordinary C2R branch.",
+ "proposal": "Port the targeted supportCodegen=false guard for C2R whose child is CometBroadcastExchangeExec, BroadcastQueryStageExec or ReusedExchangeExec above it; retain new CometBroadcastValue/IdentityBroadcastMode copying and both C2R converters. Port the original regression with a wide Spark join/narrow broadcast build side.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Spark4.2.1 CollapseCodegenStages сохраняет исходную проблемную форму плана; это source-level подтверждение необходимости guard для такой формы, не сообщение о выполненном runtime воспроизведении.",
- "AQE/reused broadcast wrappers и optional native/direct C2R paths требуют тестов. Не откатить upstream IdentityBroadcastMode rows.copy correctness."
- ],
- "dependencies": [
- "J02"
+ "Spark4.2.1 CollapseCodegenStages preserves the original problematic plan shape; this is source-level confirmation that the guard is needed for that shape, not a report of an executed runtime reproduction.",
+ "AQE/reused-broadcast wrappers and optional native/direct C2R paths need tests. Do not revert upstream IdentityBroadcastMode rows.copy correctness."
],
+ "dependencies": ["J02"],
"acceptance": [
- "Исходный CometJoinSuite regression: Spark broadcast join over AQE skewed Spark SMJ, codegen.maxFields=3/100, costBasedEngines=true/false.",
- "Утверждать Spark broadcast join/native broadcast сохранены и build не WholeStageCodegenExec; результат совпадает Spark. Дополнительно reused exchange и обычный JVM C2R default path."
+ "Original CometJoinSuite regression: Spark broadcast join over AQE skewed Spark SMJ, codegen.maxFields=3/100, costBasedEngines=true/false.",
+ "Assert Spark broadcast join/native broadcast are retained and build is not WholeStageCodegenExec; result matches Spark. Add reused-exchange and ordinary JVM C2R default-path cases."
],
"implementation_questions": [],
"other_upstream_candidates": [],
@@ -5182,120 +4562,85 @@
"source_urls": [
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"upstream_source_evidence": [
{
"ref": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
"path": "spark/src/main/scala/org/apache/comet/rules/EliminateRedundantTransitions.scala",
- "lines": [
- 114,
- 122,
- 328,
- 345
- ]
+ "lines": [114, 122, 328, 345]
},
{
"ref": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
"path": "spark/src/main/scala/org/apache/comet/CometConf.scala",
- "lines": [
- 489,
- 498
- ]
+ "lines": [489, 498]
},
{
"ref": "93cc6f9d71136bd0d87f77dd901989144d8c09fb",
"path": "spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala",
- "lines": [
- 54,
- 60,
- 175,
- 195
- ]
+ "lines": [54, 60, 175, 195]
},
{
"repo": "apache/spark",
"ref": "v4.2.1",
"path": "sql/core/src/main/scala/org/apache/spark/sql/execution/WholeStageCodegenExec.scala",
- "lines": [
- 77,
- 925,
- 984
- ]
+ "lines": [77, 925, 984]
},
{
"repo": "apache/spark",
"ref": "v4.2.1",
"path": "sql/core/src/main/scala/org/apache/spark/sql/execution/joins/BroadcastHashJoinExec.scala",
- "lines": [
- 133,
- 136
- ]
+ "lines": [133, 136]
},
{
"repo": "apache/spark",
"ref": "v4.2.1",
"path": "sql/core/src/main/scala/org/apache/spark/sql/execution/SparkPlan.scala",
- "lines": [
- 350,
- 351
- ]
+ "lines": [350, 351]
}
],
"source_paths": [
"spark/src/main/scala/org/apache/spark/sql/comet/CometColumnarToRowExec.scala"
],
- "proposed_action": "Перенести целевой supportCodegen=false guard для C2R, чей child — CometBroadcastExchangeExec, BroadcastQueryStageExec или ReusedExchangeExec над ним; оставить новый CometBroadcastValue/IdentityBroadcastMode copy и оба C2R converters. Port исходный regression с широким Spark join/узкой broadcast build side.",
+ "proposed_action": "Port the targeted supportCodegen=false guard for C2R whose child is CometBroadcastExchangeExec, BroadcastQueryStageExec or ReusedExchangeExec above it; retain new CometBroadcastValue/IdentityBroadcastMode copying and both C2R converters. Port the original regression with a wide Spark join/narrow broadcast build side.",
"upstream": {
- "base": "Полная цепочка target93+Spark4.2.1: EliminateRedundantTransitions114–122/328–345 создаёт обычный CometColumnarToRowExec, поскольку nativeC2R flag по умолчанию false (CometConf489–498). Класс имеет CodegenSupport без override supportCodegen; default Spark4.2.1 CodegenSupport=true. CollapseCodegenStages925–984 при не-codegen широком join рекурсивно оборачивает его узкий C2R build child в отдельный WholeStageCodegenExec. BroadcastHashJoinExec.doExecute вызывает buildPlan.executeBroadcast (Spark4.2.1:136), но WholeStageCodegenExec broadcast override не имеет; SparkPlan350–351 бросает unsupported error. C2R собственный doExecuteBroadcast есть, но wrapper мешает дойти до него. Upstream native/direct converters и DPP subquery-specific cleanup не закрывают эту ordinary C2R ветку.",
+ "base": "Full target93+Spark4.2.1 chain: EliminateRedundantTransitions114–122/328–345 creates ordinary CometColumnarToRowExec because nativeC2R defaults false (CometConf489–498). The class has CodegenSupport without overriding supportCodegen; default Spark4.2.1 CodegenSupport=true. For a non-codegen wide join, CollapseCodegenStages925–984 recursively wraps its narrow C2R build child in a separate WholeStageCodegenExec. BroadcastHashJoinExec.doExecute calls buildPlan.executeBroadcast (Spark4.2.1:136), but WholeStageCodegenExec has no broadcast override; SparkPlan350–351 throws an unsupported error. C2R has its own doExecuteBroadcast, but the wrapper prevents reaching it. Upstream native/direct converters and DPP subquery-specific cleanup do not cover this ordinary C2R branch.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "J02"
- ],
+ "original_dependency_notes": ["J02"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01"
- ],
+ "prerequisites": ["P01"],
"phase": 7,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G07"
- ],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести целевой supportCodegen=false guard для C2R, чей child — CometBroadcastExchangeExec, BroadcastQueryStageExec или ReusedExchangeExec над ним; оставить новый CometBroadcastValue/IdentityBroadcastMode copy и оба C2R converters. Port исходный regression с широким Spark join/узкой broadcast build side.",
+ "approved_decision": "Port the targeted supportCodegen=false guard for C2R whose child is CometBroadcastExchangeExec, BroadcastQueryStageExec or ReusedExchangeExec above it; retain new CometBroadcastValue/IdentityBroadcastMode copying and both C2R converters. Port the original regression with a wide Spark join/narrow broadcast build side.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J27",
- "title": "Покрытие default Kryo broadcast payload",
- "contract": "СохранитьисходныерегрессионныесценарииdefaultKryobroadcastpayload; этоvalidation, не отдельнаяалгоритмическаяфича.",
- "source_commits": [
- "a52c1685f"
- ],
+ "title": "Default Kryo broadcast-payload coverage",
+ "contract": "Preserve original default-Kryo broadcast-payload regression scenarios; this is validation, not a separate algorithmic feature.",
+ "source_commits": ["a52c1685f"],
"source_files": [
"spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastKryoPayloadSuite.scala"
],
- "upstream_base": "ИсходныйJoomsuiteнеравенподтверждённомупрохождениюupstream.",
- "proposal": "Перенести/адаптироватьsuiteвновыйCItarget, связатьсbroadcast/C2R/cards.",
+ "upstream_base": "An original Joom suite is not proof of upstream passing.",
+ "proposal": "Port/adapt the suite into the new CI target and link it with broadcast/C2R/cards.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Spark4.2.1Kryo/defaultserializer/classloader; новыеupstreambroadcastcontracts."
- ],
- "dependencies": [
- "J02",
- "J08",
- "J25"
+ "Spark4.2.1 Kryo/default serializer/classloader; new upstream broadcast contracts."
],
+ "dependencies": ["J02", "J08", "J25"],
"acceptance": [
- "CometBroadcastKryoPayloadSuite именноdefaultserializer иshadedreleaseartifact."
+ "CometBroadcastKryoPayloadSuite specifically with default serializer and shaded release artifact."
],
"implementation_questions": [],
"other_upstream_candidates": [],
@@ -5315,73 +4660,59 @@
"source_urls": [
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastKryoPayloadSuite.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/test/scala/org/apache/spark/sql/comet/CometBroadcastKryoPayloadSuite.scala"
],
- "proposed_action": "Перенести/адаптироватьsuiteвновыйCItarget, связатьсbroadcast/C2R/cards.",
+ "proposed_action": "Port/adapt the suite into the new CI target and link it with broadcast/C2R/cards.",
"upstream": {
- "base": "ИсходныйJoomsuiteнеравенподтверждённомупрохождениюupstream.",
+ "base": "An original Joom suite is not proof of upstream passing.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "J02",
- "J08",
- "J25"
- ],
+ "original_dependency_notes": ["J02", "J08", "J25"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "J08",
- "J21",
- "J25"
- ],
+ "prerequisites": ["J08", "J21", "J25"],
"phase": 7,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G07"
- ],
+ "implementation_plan_groups": ["G07"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести/адаптироватьsuiteвновыйCItarget, связатьсbroadcast/C2R/cards.",
+ "approved_decision": "Port/adapt the suite into the new CI target and link it with broadcast/C2R/cards.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J02",
- "title": "Выбор форматов shuffle/broadcast по producer и consumer",
- "contract": "Сохранить BoundaryFormats/ChooseBoundaryFormats: Spark↔Spark boundary остаётся Spark; mixed/native consumers получают корректный формат; co-partitioned join inputs используют совместимые hash functions; root columnar shuffle с Spark producer работает.",
- "source_commits": [
- "01df400a3",
- "b5fdf8668",
- "34a841191"
- ],
+ "title": "Choose shuffle/broadcast formats by producer and consumer",
+ "contract": "Preserve BoundaryFormats/ChooseBoundaryFormats: Spark↔Spark boundaries remain Spark; mixed/native consumers receive the correct format; co-partitioned join inputs use compatible hash functions; root columnar shuffle with a Spark producer works.",
+ "source_commits": ["01df400a3", "b5fdf8668", "34a841191"],
"source_files": [
"spark/src/main/scala/org/apache/comet/rules/BoundaryFormats.scala",
"spark/src/main/scala/org/apache/comet/rules/ChooseBoundaryFormats.scala",
"spark/src/main/scala/org/apache/comet/rules/CometRule.scala"
],
- "upstream_base": "Полного аналога нет; стандартные transition rules не выбирают форматы тем же whole-plan алгоритмом.",
- "proposal": "Адаптировать как общий слой для J01 и отдельно включаемого boundaryFormats режима.",
+ "upstream_base": "No full equivalent exists; standard transition rules do not choose formats through the same whole-plan algorithm.",
+ "proposal": "Adapt as a shared layer for J01 and the separately enabled boundaryFormats mode.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
"Hash compatibility Spark4/nested keys/decimals; reused exchanges/subquery stages/AQE tags.",
- "Сохранить fresh upstream requiredChildDistribution и coalesce behavior (#7231/#7510).",
- "NativeShuffleSpec/CapturedSQLConf и remote shuffle paths в1.2 изменены; не откатить их."
+ "Preserve fresh upstream requiredChildDistribution and coalesce behavior (#7231/#7510).",
+ "NativeShuffleSpec/CapturedSQLConf and remote-shuffle paths changed in1.2; do not revert them."
],
"dependencies": [],
"acceptance": [
- "ChooseBoundaryFormatsSuite + BoundaryTestHelpers; paired join keys with differing hash semantics; Spark-Spark shuffle остаетсяSpark; AQE/reusedbroadcast/rootshuffle."
+ "ChooseBoundaryFormatsSuite plus BoundaryTestHelpers; paired join keys with differing hash semantics; Spark-Spark shuffle remains Spark; AQE/reused broadcast/root shuffle."
],
"implementation_questions": [],
"other_upstream_candidates": [],
"config": [
- "spark.comet.exec.boundaryFormats.enabled=false; J01 выполняет analogous selection даже без отдельного flag"
+ "spark.comet.exec.boundaryFormats.enabled=false; J01 performs analogous selection even without the separate flag"
],
"source_commit_details": [
{
@@ -5427,66 +4758,53 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/rules/ChooseBoundaryFormats.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/rules/CometRule.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/comet/rules/BoundaryFormats.scala",
"spark/src/main/scala/org/apache/comet/rules/ChooseBoundaryFormats.scala",
"spark/src/main/scala/org/apache/comet/rules/CometRule.scala"
],
- "proposed_action": "Адаптировать как общий слой для J01 и отдельно включаемого boundaryFormats режима.",
+ "proposed_action": "Adapt as a shared layer for J01 and the separately enabled boundaryFormats mode.",
"upstream": {
- "base": "Полного аналога нет; стандартные transition rules не выбирают форматы тем же whole-plan алгоритмом.",
+ "base": "No full equivalent exists; standard transition rules do not choose formats through the same whole-plan algorithm.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "I07",
- "J21",
- "J25"
- ],
+ "prerequisites": ["I07", "J21", "J25"],
"phase": 8,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_staging": "Сначала standalone BoundaryFormats/ConversionCount и shared LeafColumns без ссылок на ещё отсутствующий EngineCostModel. Cost-aware вызов в ChooseBoundaryFormats добавляется вместе с J01; иначе ранний PR не скомпилируется.",
- "implementation_plan_groups": [
- "G09"
- ],
+ "implementation_staging": "First standalone BoundaryFormats/ConversionCount and shared LeafColumns without references to the still-absent EngineCostModel. Add the cost-aware call in ChooseBoundaryFormats together with J01; otherwise the early PR will not compile.",
+ "implementation_plan_groups": ["G09"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать как общий слой для J01 и отдельно включаемого boundaryFormats режима.",
+ "approved_decision": "Adapt as a shared layer for J01 and the separately enabled boundaryFormats mode.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J03",
- "title": "Отдельная эвристика fallback сортировки широких строк",
- "contract": "Публичная возможность остаётся в tag13: Spark consumer + wide payload по числу leaf columns возвращает sort Spark; stable initial/AQE decision; игнорируется при costBasedEngines.",
- "source_commits": [
- "2d3cc8914",
- "909335e3f",
- "9ff373647",
- "b83e9172b"
- ],
+ "title": "Separate wide-row sort fallback heuristic",
+ "contract": "The public feature remains in tag13: Spark consumer plus wide payload measured by leaf-column count returns sort to Spark; stable initial/AQE decision; ignored when costBasedEngines is active.",
+ "source_commits": ["2d3cc8914", "909335e3f", "9ff373647", "b83e9172b"],
"source_files": [
"spark/src/main/scala/org/apache/comet/rules/WideRowSortFallback.scala",
"spark/src/main/scala/org/apache/comet/rules/LeafColumns.scala",
"spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala"
],
- "upstream_base": "Отдельной equivalent schema-width fallback rule нет.",
- "proposal": "Перенести как сохраняемый альтернативный режим; не считать удалённой из-за наличия cost model.",
+ "upstream_base": "No separate equivalent schema-width fallback rule exists.",
+ "proposal": "Port as a retained alternative mode; do not treat it as removed because a cost model exists.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "AQE engine tag/reversion integration; consumers/transition wrappers изменились."
- ],
- "dependencies": [
- "J02"
+ "AQE engine-tag/reversion integration; consumers/transition wrappers changed."
],
+ "dependencies": ["J02"],
"acceptance": [
"WideRowSortFallbackSuite; flagon/off,costmodelon/off,nestedwidth,initial/AQE; no unexpected fallback fornative consumer."
],
@@ -5553,63 +4871,51 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/rules/LeafColumns.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/comet/rules/WideRowSortFallback.scala",
"spark/src/main/scala/org/apache/comet/rules/LeafColumns.scala",
"spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala"
],
- "proposed_action": "Перенести как сохраняемый альтернативный режим; не считать удалённой из-за наличия cost model.",
+ "proposed_action": "Port as a retained alternative mode; do not treat it as removed because a cost model exists.",
"upstream": {
- "base": "Отдельной equivalent schema-width fallback rule нет.",
+ "base": "No separate equivalent schema-width fallback rule exists.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "J02"
- ],
+ "original_dependency_notes": ["J02"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01",
- "J02"
- ],
+ "prerequisites": ["P01", "J02"],
"phase": 8,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G09"
- ],
+ "implementation_plan_groups": ["G09"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести как сохраняемый альтернативный режим; не считать удалённой из-за наличия cost model.",
+ "approved_decision": "Port as a retained alternative mode; do not treat it as removed because a cost model exists.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J04",
- "title": "Отдельная эвристика fallback shuffle широких строк",
- "contract": "При заданном threshold оставлять wide payload shuffle вSpark; leaf counting исключаетpartitionkey; игнорировать приcostmodel.",
- "source_commits": [
- "c2562fb89",
- "b83e9172b"
- ],
+ "title": "Separate wide-row shuffle fallback heuristic",
+ "contract": "At the configured threshold, keep wide-payload shuffle in Spark; leaf counting excludes the partition key; ignore when the cost model is active.",
+ "source_commits": ["c2562fb89", "b83e9172b"],
"source_files": [
"spark/src/main/scala/org/apache/comet/rules/WideRowShuffleFallback.scala",
"spark/src/main/scala/org/apache/comet/rules/LeafColumns.scala"
],
- "upstream_base": "Полного аналога нет.",
- "proposal": "Перенести флаг/правило и тесты как отдельный alternative mode.",
+ "upstream_base": "No full equivalent exists.",
+ "proposal": "Port the flag/rule and tests as a separate alternative mode.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Новые local/remote native shuffle selection contracts; hash/range keys иcolumnar boundary interplay."
- ],
- "dependencies": [
- "J02"
+ "New local/remote native-shuffle selection contracts; hash/range keys and columnar-boundary interplay."
],
+ "dependencies": ["J02"],
"acceptance": [
"WideRowShuffleFallbackSuite, hash/range/round-robin modes, nested keys, AQE,costmode precedence."
],
@@ -5648,45 +4954,38 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/rules/WideRowShuffleFallback.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/rules/LeafColumns.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/comet/rules/WideRowShuffleFallback.scala",
"spark/src/main/scala/org/apache/comet/rules/LeafColumns.scala"
],
- "proposed_action": "Перенести флаг/правило и тесты как отдельный alternative mode.",
+ "proposed_action": "Port the flag/rule and tests as a separate alternative mode.",
"upstream": {
- "base": "Полного аналога нет.",
+ "base": "No full equivalent exists.",
"outside_base": []
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
- "original_dependency_notes": [
- "J02"
- ],
+ "original_dependency_notes": ["J02"],
"scope_kind": "source_line_contract_or_support",
- "prerequisites": [
- "P01",
- "J02"
- ],
+ "prerequisites": ["P01", "J02"],
"phase": 8,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G09"
- ],
+ "implementation_plan_groups": ["G09"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Перенести флаг/правило и тесты как отдельный alternative mode.",
+ "approved_decision": "Port the flag/rule and tests as a separate alternative mode.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Требуемое поведение определено; необходимо утвердить предложенное действие.",
+ "prior_open_decisions": "Required behavior is defined; the proposed action needs approval.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
},
{
"id": "J01",
- "title": "Калиброванный выбор движка каждого оператора по стоимости всего плана",
- "contract": "Сохранить whole-plan выбор Spark/Comet с ценами по leaf columns, row widths, nested fraction, shuffle partition count; решения на initial/AQE plans, cost overrides/explain, array-key и SMJ condition pricing. Сохраняется итоговая таблица EngineCostTable тега13: calib29 с последующими поправками, включая prod-sized sort calibration. Перекалибрование обсуждается отдельно.",
+ "title": "Calibrated per-operator engine selection by whole-plan cost",
+ "contract": "Preserve whole-plan Spark/Comet selection with costs based on leaf columns, row widths, nested fraction and shuffle partition count; initial/AQE plan decisions, cost overrides/explain, array-key and SMJ-condition pricing. Preserve the final tag13 EngineCostTable: calib29 with subsequent adjustments, including production-sized sort calibration. Recalibration requires separate discussion.",
"source_commits": [
"fd190a66f",
"d1148b208",
@@ -5713,29 +5012,29 @@
"spark/src/main/scala/org/apache/comet/rules/CometRule.scala",
"spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala"
],
- "upstream_base": "В выбранной branch-1.2 отсутствует Joom solver/calibrated table; upstream transition-heavy-stage rules другой алгоритм.",
- "proposal": "Адаптировать свою реализацию, сохранив конфигурационный контракт и исходные регрессии; квалифицировать после переноса native capabilities.",
+ "upstream_base": "The selected branch-1.2 lacks the Joom solver/calibrated table; upstream transition-heavy-stage rules use a different algorithm.",
+ "proposal": "Adapt our implementation while preserving configuration contracts and original regressions; qualify after native capabilities are ported.",
"approval": "APPROVED_IN_IMPLEMENTATION_PLAN",
"risks": [
- "Высокий интеграционный риск: новый split planner, operator conversion/fallback и AQE Spark4.2.1. Не копировать CometRule целиком.",
- "Новая upstream distribution fix #7231/#7510: сохранить CometDelegatesRequiredChildDistribution и корректное CoalesceShufflePartitions при own rewrites.",
- "Calib29 снималась на прежних runtime/operators; правильность выбора и performance parity требуют отдельных измерений."
+ "High integration risk: new split planner, operator conversion/fallback and Spark4.2.1 AQE. Do not copy the entire CometRule.",
+ "New upstream distribution fix #7231/#7510: preserve CometDelegatesRequiredChildDistribution and correct CoalesceShufflePartitions during our rewrites.",
+ "Calib29 was measured on previous runtimes/operators; decision correctness and performance parity require separate measurements."
],
"dependencies": [
"J02",
"J03/J04 compatibility",
- "перенесённые native sort/window/aggregate/SMJ paths"
+ "ported native sort/window/aggregate/SMJ paths"
],
"acceptance": [
- "CostBasedEngineChoiceSuite полностью; initial/executed AQE on/off plans + mixed Spark/Comet stages; cost table parser overrides/explain.",
- "Сохранение native coverage и row correctness; benchmark production-like wide/nested sorts/shuffles/join filters."
+ "Full CostBasedEngineChoiceSuite; initial/executed AQE on/off plans plus mixed Spark/Comet stages; cost-table parser overrides/explain.",
+ "Preserve native coverage and row correctness; benchmark production-like wide/nested sorts/shuffles/join filters."
],
"implementation_questions": [],
"other_upstream_candidates": [
- "PR #6609 draft stage-level model: частичный аналог цели, без наших калибровок/whole-plan boundary solver; не base1.2."
+ "Draft PR#6609 stage-level model is a partial equivalent in goal, without our calibrations/whole-plan boundary solver; not in base1.2."
],
"config": [
- "costBasedEngines.enabled=false в библиотеке; datasci включает отдельно",
+ "costBasedEngines.enabled=false in the library; datasci enables it separately",
"costTable, log.enabled, cometOperatorWeight=-1, sparkOperatorWeight=0, cometOperatorWeights"
],
"source_commit_details": [
@@ -5935,7 +5234,7 @@
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/rules/CometRule.scala",
"https://github.com/joomcode/datafusion-comet/blob/joom-1.1-13/spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala"
],
- "confidence": "Исходники/историяпроверены; сборка/runtime/performanceнепроверены; всеproposalтребуютутверждения.",
+ "confidence": "Sources/history inspected; build/runtime/performance not verified; all proposals require approval.",
"source_paths": [
"spark/src/main/scala/org/apache/comet/rules/CostBasedEngineChoice.scala",
"spark/src/main/scala/org/apache/comet/rules/EngineCostTable.scala",
@@ -5943,19 +5242,19 @@
"spark/src/main/scala/org/apache/comet/rules/CometRule.scala",
"spark/src/main/scala/org/apache/comet/rules/CometExecRule.scala"
],
- "proposed_action": "Адаптировать свою реализацию, сохранив конфигурационный контракт и исходные регрессии; квалифицировать после переноса native capabilities.",
+ "proposed_action": "Adapt our implementation while preserving configuration contracts and original regressions; qualify after native capabilities are ported.",
"upstream": {
- "base": "В выбранной branch-1.2 отсутствует Joom solver/calibrated table; upstream transition-heavy-stage rules другой алгоритм.",
+ "base": "The selected branch-1.2 lacks the Joom solver/calibrated table; upstream transition-heavy-stage rules use a different algorithm.",
"outside_base": [
- "PR #6609 draft stage-level model: частичный аналог цели, без наших калибровок/whole-plan boundary solver; не base1.2."
+ "Draft PR#6609 stage-level model is a partial equivalent in goal, without our calibrations/whole-plan boundary solver; not in base1.2."
]
},
- "open_decisions": "Объём и границы работы утверждены планом9PR+резерв10. Код ещё не пишется; технические адаптации проверяются при реализации.",
+ "open_decisions": "Scope and boundaries are approved in the9PR+reserve10 plan. No code is being written yet; technical adaptations will be verified during implementation.",
"approval_status": "APPROVED_IN_IMPLEMENTATION_PLAN",
"original_dependency_notes": [
"J02",
"J03/J04 compatibility",
- "перенесённые native sort/window/aggregate/SMJ paths"
+ "ported native sort/window/aggregate/SMJ paths"
],
"scope_kind": "source_line_contract_or_support",
"prerequisites": [
@@ -5978,15 +5277,13 @@
],
"phase": 8,
"requires_upstream_discussion": false,
- "decision_category": "объём утверждён в плане реализации",
+ "decision_category": "scope approved in the implementation plan",
"same_pr_or_co_review": [],
- "implementation_plan_groups": [
- "G09"
- ],
+ "implementation_plan_groups": ["G09"],
"implementation_plan_approval": "APPROVED_EXPLICITLY",
- "approved_decision": "Адаптировать свою реализацию, сохранив конфигурационный контракт и исходные регрессии; квалифицировать после переноса native capabilities.",
+ "approved_decision": "Adapt our implementation while preserving configuration contracts and original regressions; qualify after native capabilities are ported.",
"decision_source": "decisions.json#implementation_plan",
- "prior_open_decisions": "Утвердить перенос собственного cost model; сохранить итоговые coefficients/overrides тега13 для первой версии. Изменение калибровок после измерений обсуждать отдельно.",
+ "prior_open_decisions": "Approve porting our cost model; preserve final tag13 coefficients/overrides for the first version. Discuss calibration changes separately after measurement.",
"implementation_status": "NOT_STARTED",
"validation_status": "NOT_RUN",
"correctness_policy_ref": "decisions.json#correctness_policy"
@@ -6006,94 +5303,91 @@
"N08",
"P03"
],
- "policy_resolved_entry_ids": [
- "I03",
- "J09"
- ],
+ "policy_resolved_entry_ids": ["I03", "J09"],
"user_decisions": {
"date": "2026-11-16",
- "source": "Явные решения пользователя в текущем диалоге; пакет актуализирован2026-11-17. Историческое утверждение плана2026-11-16 сохранено.",
+ "source": "Explicit user decisions in this conversation; package updated2026-11-17. Historical plan approval2026-11-16 is preserved.",
"general_policy": {
"status": "APPROVED",
- "user_quote": "По максимому используем upstream реализацию и добавляем наши исправления , где имеет смысл.",
- "rule": "Максимально использовать upstream в качестве основы; сохранять все недостающие Joom-возможности, исправления и оптимизации. При потере поведения или оптимизации сохранять/адаптировать нашу часть.",
+ "user_quote": "[English translation] Use the upstream implementation as much as possible and add our fixes where it makes sense.",
+ "rule": "Use upstream as the base as much as possible; preserve all missing Joom capabilities, fixes and optimizations. If behavior or optimization is lost, retain/adapt our part.",
"conditions": [
- "Функциональную эквивалентность подтвердить Joom regressions и проверкой native execution/планов.",
- "Для оптимизаций проверить memory/spill lifecycle и по возможности выполнить доступные локальные performance measurements; непроверенную производительность явно отметить. Полные GCP benchmarks отложены на пользовательскую проверку.",
- "Частичный аналог не означает полноценную замену Joom-возможности.",
- "Не исключать Joom-поведение только потому, что в upstream существует похожий механизм."
+ "Confirm functional equivalence through Joom regressions and actual native execution/plan checks.",
+ "For optimizations, verify memory/spill lifecycle and perform available local performance measurements where possible; explicitly identify unverified performance. Full GCP benchmarks are deferred to user validation.",
+ "A partial equivalent does not imply a complete replacement of the Joom capability.",
+ "Do not exclude Joom behavior merely because upstream has a similar mechanism."
]
},
"explicitly_approved": {
"I01": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать native Delta scanner Comet1.2; сохранить GCP ADC, сериализацию и защиту от planning deadlock, адаптируя недостающие Joom-особенности."
+ "decision": "Use the Comet1.2 native Delta scanner; preserve GCP ADC, serialization and planning-deadlock protection, adapting missing Joom specifics."
},
"D10": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream Single/Final+MergeAsPartial и новые Spark aggregate-state codecs; сохранить устойчивость к spill и перенести Joom memory/offset guards, ordered drain и low-copy collect_list. Не переносить старую Joom-маршрутизацию PartialReduce без необходимости."
+ "decision": "Use upstream Single/Final+MergeAsPartial and new Spark aggregate-state codecs; preserve spill robustness and port Joom memory/offset guards, ordered drain and low-copy collect_list. Do not port old Joom PartialReduce routing unless necessary."
},
"N08": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Сохранить upstream SortAggregate architecture и добавить недостающие Joom-оптимизации порядка/streaming и необходимые correctness guards."
+ "decision": "Preserve upstream SortAggregate architecture and add missing Joom ordering/streaming optimizations and required correctness guards."
},
"I06": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream timestamp producer/CASE fixes и Spark4.2 semantics; переносить нашу дополнительную нормализацию сравнений там, где она ещё необходима."
+ "decision": "Use upstream timestamp producer/CASE fixes and Spark4.2 semantics; port our additional comparison normalization where still necessary."
},
"I07": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream native wide-decimal hashing вместо возвращения старого precision>18 запрета; проверить совместимость со Spark, включая mixed boundaries."
+ "decision": "Use upstream native wide-decimal hashing instead of restoring the old precision>18 restriction; verify Spark compatibility, including mixed boundaries."
},
"N06": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream native string MIN/MAX; сохранить Joom cases, native coverage и недостающие guards."
+ "decision": "Use upstream native string MIN/MAX; preserve Joom cases, native coverage and missing guards."
},
"I04": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream DPP/empty-scan lifecycle; сохранить соответствующие Joom регрессии и недостающие исправления."
+ "decision": "Use the upstream DPP/empty-scan lifecycle; preserve corresponding Joom regressions and missing fixes."
},
"I05": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream recursive boolean FFI repair; сохранить дополнительные Joom nested/sliced regressions."
+ "decision": "Use upstream recursive boolean FFI repair; preserve additional Joom nested/sliced regressions."
},
"J18": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream однократную загрузку bundled native library; проверить исходный Joom reload-сценарий."
+ "decision": "Use upstream one-time bundled native-library loading; verify the original Joom reload scenario."
},
"J21": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream нормализацию sliced broadcast offsets; сохранить Joom регрессии и добавить недостающие исправления при обнаружении потери поведения."
+ "decision": "Use upstream sliced-broadcast offset normalization; retain Joom regressions and add missing fixes if behavior is lost."
},
"J24": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Использовать upstream small-batch/effective JVM shuffle batch cap; сохранить наши проверки."
+ "decision": "Use the upstream small-batch/effective JVM shuffle batch cap; preserve our checks."
},
"P03": {
"status": "APPROVED_EXPLICITLY",
- "decision": "Сохранить ANSI=true. Приоритет — отсутствие успешных jobs с некорректными результатами. Сравнить Spark4.2.1 Comet-off/on на одинаковых входах/ANSI; проверить результаты и обязательные ошибки; сохранить SUM/AVG overflow guards, исправлять несовместимости или заранее ограничивать native-путь. По возможности добавить автоматические regression/differential tests при реализации. PR#5889 в текущую миграцию не включать."
+ "decision": "Keep ANSI=true. Prioritize preventing successful jobs with incorrect results. Compare Spark4.2.1 Comet-off/on with identical inputs/ANSI; verify results and mandatory errors; preserve SUM/AVG overflow guards, fix incompatibilities or restrict the native path before execution. Add automated regression/differential tests during implementation where possible. Do not include PR#5889 in this migration."
}
},
"covered_by_general_policy": {
"I03": {
"status": "RESOLVED_BY_APPROVED_POLICY",
- "decision": "Сохранить защиту от planning deadlock через актуальный upstream lifecycle с корректным execution partition count; старое безусловное UnknownPartitioning(0) не копировать. Проверить исходные Joom reproductions."
+ "decision": "Preserve planning-deadlock protection through the current upstream lifecycle with the correct execution partition count; do not copy the old unconditional UnknownPartitioning(0). Verify the original Joom reproductions."
},
"J09": {
"status": "RESOLVED_BY_APPROVED_POLICY",
- "decision": "Сохранить Joom projection-instance pool поверх upstream generated-class cache, соблюдая lifetime/bounds guards. Частичный class cache сам по себе не заменяет pool; упрощение возможно только при подтверждённой эквивалентности по утверждённому правилу."
+ "decision": "Preserve the Joom projection-instance pool on top of upstream generated-class caching, respecting lifetime/bounds guards. A partial class cache alone does not replace the pool; simplify only with demonstrated equivalence under the approved rule."
}
},
"not_decided": [],
"authorization_boundary": {
"repository_operations_authorized": true,
- "implementation_authorized": false,
+ "implementation_authorized": true,
"git_mutation_authorized": true,
"repository_scope": "/Users/skotlov/d/projects/datafusion-comet",
"git_scope": "/Users/skotlov/d/projects/datafusion-comet/.git",
- "user_quote": "Отныне ты имеешь мое разрешение делать все необходимые манипуляции в /Users/skotlov/d/projects/datafusion-comet (в т.ч. в /Users/skotlov/d/projects/datafusion-comet/.git)",
- "interpretation": "Разрешение охватывает необходимые локальные изменения репозитория, ветки, коммиты, интеграцию и локальные теги. Подготовка передачи работы не означает, что реализация G01 уже начата.",
- "implementation_status": "NOT_STARTED_USER_CODE_HOLD",
+ "user_quote": "[English translation] From now on, you have my permission to perform all necessary operations in /Users/skotlov/d/projects/datafusion-comet (including /Users/skotlov/d/projects/datafusion-comet/.git).",
+ "interpretation": "Permission covers necessary local repository changes, branches, commits, integration and local tags. Preparing a handoff does not mean G01 implementation has already started.",
+ "implementation_status": "G01_IN_PROGRESS",
"network_build_dependencies_authorized": true,
"network_scope_status": "APPROVED_WITH_USER_SAFETY_CONDITION",
"github_api_read_only_audit_authorized": true,
@@ -6101,275 +5395,170 @@
"automatic_cross_chat_messaging_authorized": true,
"publishing_authorized": false,
"remote_push_status": "USER_REPORTED_403; LOCAL_WORK_ONLY",
- "note": "Git/документация и создание/ведение рабочих сессий разрешены; сеть разрешена с условием безопасности. Последнее прямое указание запрещает писать код до нового поручения. Push, публикация и production-операции исключены.",
+ "note": "Only G01 implementation is authorized. Each subsequent group requires a separate direct approval after review of the previous one. Network and Git retain their previous scope; push/publication/production are prohibited.",
"recorded_on": "2026-11-17",
"network_approval": {
"recorded_on": "2026-11-17",
"status": "APPROVED_WITH_USER_SAFETY_CONDITION",
- "source": "Прямой ответ пользователя на request_user_input_async в координаторском чате.",
- "user_quote": "Разрешаю на всю миграцию Comet1.2/Spark4.2.1 сетевые операции, перечисленные в docs/joom-migration/NETWORK.md: Maven/SBT, Rustup/Cargo, закреплённые git-зависимости и build tools с GitHub, Docker Hub и Ubuntu packages, включая дочерние процессы внутри контейнеров. При условии, что все скачиваемое на мой ноут из сети и запускаемое - является известными и широкоиспользуемыми артефактами, в безопасности которых для моей машины не приходится сомневаться. Без push, внешней публикации и production операций.",
+ "source": "Direct user reply to request_user_input_async in the coordinator chat.",
+ "user_quote": "[English translation] For the entire Comet1.2/Spark4.2.1 migration, I authorize the network operations listed in docs/joom-migration/NETWORK.md: Maven/SBT, Rustup/Cargo, pinned Git dependencies and build tools from GitHub, Docker Hub and Ubuntu packages, including child processes inside containers. All artifacts downloaded from the network to my laptop and executed must be well-known and widely used, with no reason to doubt their safety for my machine. No push, external publication or production operations.",
"scope_document": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/NETWORK.md",
"approved_proposal_commit": "fbc250a9867515958c6a33920c1ac15b5d2d62b1",
"approved_proposal_sha256": "3bedf4a5dc5d14b1793d25c4ae9073b697a1691a614aa69494c37486bc8dac02",
- "scope": "На всю утверждённую миграцию: Maven/SBT, Rustup/Cargo, закреплённые git-зависимости и build tools с GitHub, Docker Hub и Ubuntu packages, включая дочерние процессы внутри контейнеров.",
- "user_safety_condition": "Все скачиваемые на ноутбук и запускаемые артефакты должны быть известными и широко используемыми, без обоснованных сомнений в их безопасности для машины пользователя.",
- "execution_rule": "Использовать известные зависимости выбранного upstream-проекта и официальные источники/реестры, закреплённые версии/commit и доступные publisher checksums. Перед запуском внешнего installer/build script проверить его происхождение и содержание. Наличие пакета в registry или lockfile само по себе не отменяет условие пользователя. Не добавлять и не запускать нестандартный, непроверенный или сомнительный инструмент/источник на основании одного этого разрешения; при такой необходимости согласовать отдельно.",
+ "scope": "For the entire approved migration: Maven/SBT, Rustup/Cargo, pinned Git dependencies and build tools from GitHub, Docker Hub and Ubuntu packages, including child processes inside containers.",
+ "user_safety_condition": "All artifacts downloaded to the laptop and executed must be well-known and widely used, with no reasonable doubt about their safety for the user machine.",
+ "execution_rule": "Use known dependencies of the selected upstream project, official sources/registries, pinned versions/commits and available publisher checksums. Before executing an external installer/build script, check its origin and content. Presence in a registry or lockfile alone does not waive the user condition. Do not add or execute unusual, unverified or questionable tools/sources under this permission alone; obtain separate agreement if required.",
"excluded": [
"Git push",
- "Внешняя публикация артефактов/PR/images",
- "Production-операции",
- "Остальные исключения исходного NETWORK.md"
+ "External publication of artifacts/PRs/images",
+ "Production operations",
+ "Other exclusions in the original NETWORK.md"
],
"network_operations_performed_during_recording": false
},
"implementation_hold": {
"active": true,
- "user_quote": "Код не пиши до моего укзания.",
- "recorded_on": "2026-11-17",
- "allowed_now": "Документация и read-only подготовка; реализацию не начинать."
+ "applies_to_groups": [
+ "G02",
+ "G03",
+ "G04",
+ "G05",
+ "G06",
+ "G07",
+ "G08",
+ "G09",
+ "G10"
+ ],
+ "allowed_group": "G01",
+ "user_quote": "[English translation] Start G01 implementation according to the approved plan. Report after completion and checks. Do not proceed to G02 yet: I will explicitly authorize each subsequent one of the10items after reviewing the previous one."
},
"session_orchestration_authorization": {
"recorded_on": "2026-11-17",
- "user_quote": "Проект Comet добавлен (/Users/skotlov/d/projects/datafusion-comet). Создавай и веди рабочие сессии по утверждённому плану, передавай им задания и уточнения, проверяй результаты.",
+ "user_quote": "[English translation] The Comet project has been added (/Users/skotlov/d/projects/datafusion-comet). Create and manage working sessions under the approved plan, pass tasks and clarifications to them, and review their results.",
"project_id": "79ec4996-572a-4070-96f7-2344b555eb2c",
"code_hold_preserved": true
- }
+ },
+ "implementation_authorized_groups": ["G01"]
},
"ansi_policy": {
"status": "APPROVED_EXPLICITLY",
"spark_sql_ansi_enabled": true,
- "user_quote": "Давай сохраним `ansi=true`.",
- "rule": "Целевой режим Spark4.2.1 — ANSI=true. Подтверждённые несовместимости исправлять либо заранее ограничивать соответствующий native-путь; не отключать ANSI глобально для обхода расхождений."
+ "user_quote": "[English translation] Let us keep `ansi=true`.",
+ "rule": "The target Spark4.2.1 mode is ANSI=true. Fix confirmed incompatibilities or restrict the corresponding native path before execution; do not globally disable ANSI to work around differences."
},
"correctness_policy": {
"status": "APPROVED_EXPLICITLY",
- "priority": "Исключить ситуацию, когда job завершается без ошибки и публикует некорректный результат.",
- "user_quote": "Самое главное, мы тут должны сделать все, чтобы исключить ситуацию, когда джоба завершается без ошибки, но создает некорректный результат.",
+ "priority": "Prevent a job from completing without error and publishing an incorrect result.",
+ "user_quote": "[English translation] Above all, we must do everything we can to prevent a job from completing without error while producing an incorrect result.",
"release_criteria": [
- "Известные небезопасные native-пути исправлены либо исключены плановым fallback до исполнения; зелёный статус job/CI сам по себе недостаточен.",
- "Сравнивать Spark4.2.1 безComet и сComet на одном snapshot входов, одинаковом ANSI=true и остальных значимых настройках.",
- "Сравнивать сами результаты, типы, NULL, duplicate rows и гарантированный SQL порядок; не ограничиваться count или отдельными суммами.",
- "Проверять обе стороны контракта: обязан завершиться/обязан упасть. Comet-success при обязательной Spark ANSI-ошибке считается нарушением корректности.",
- "Сохранять overflow detection и guards mixedSpark/native partial-final SUM/AVG, включая новый state codec.",
- "Для подтверждённой несовместимости fallback определяется до исполнения/публикации; не полагаться на retry вSpark после записи или ошибки.",
- "Проверять actual native execution/планы, чтобы fallback не маскировал отсутствие тестирования custom-пути.",
- "Не скрывать расхождения общей нормализацией, потерей duplicates или произвольными float tolerances; учитывать определённую Spark семантику и разрешённую недетерминированность."
- ],
- "automated_regressions": "По возможности добавить автоматизированные regression/differential tests при реализации; сейчас фиксируется план без написания кода.",
+ "Known unsafe native paths are fixed or excluded through planning-time fallback before execution; green job/CI status alone is insufficient.",
+ "Compare Spark4.2.1 without and with Comet on the same input snapshot, identical ANSI=true and other relevant settings.",
+ "Compare actual results, types, NULL, duplicate rows and guaranteed SQL order; do not rely only on count or individual sums.",
+ "Check both sides of the contract: must succeed/must fail. Comet success where Spark requires an ANSI error is a correctness violation.",
+ "Preserve overflow detection and guards for mixed Spark/native partial-final SUM/AVG, including the new state codec.",
+ "For a confirmed incompatibility, select fallback before execution/publication; do not rely on retry in Spark after writing or an error.",
+ "Verify actual native execution/plans so fallback does not hide lack of testing of the custom path.",
+ "Do not hide differences with general normalization, loss of duplicates or arbitrary float tolerances; respect defined Spark semantics and permitted nondeterminism."
+ ],
+ "automated_regressions": "Add automated regression/differential tests during implementation where possible; this step records the plan without writing code.",
"priority_cases": [
- "Mixed Spark/native decimal SUM/AVG и промежуточное переполнение, forced spill и новый PartialMerge codec.",
- "Отдельно runtime codegen fallback decimal SUM; статических NO_CODEGEN guards недостаточно.",
- "GroupeddecimalAVG + LIMIT, malformedcasts, arithmeticoverflow, divisionbyzero и lazy AND/OR/CASE/JVM dispatch.",
- "SMJ hoisting: canthrow expressions/условные ветки/outer join, ANSI=true.",
- "Custom RANGE windows на DATE/DECIMAL границах и существующие fallback guards.",
- "Результаты с activeJoom cost/boundary flags, mixedshuffle, AQEon/off; проверки native-планов.",
- "Известные silent-result edgecases независимоANSI, включая BINARY→STRING/invalidUTF8, где применимо."
- ],
- "limits": "Набор тестов не является математическим доказательством отсутствия всех ошибок. Непокрытые риски явно фиксировать; не объявлять совместимость доказанной без проверки."
+ "Mixed Spark/native decimal SUM/AVG and intermediate overflow, forced spill and the new PartialMerge codec.",
+ "Separately test runtime-codegen-fallback decimal SUM; static NO_CODEGEN guards are insufficient.",
+ "Grouped decimal AVG+LIMIT, malformed casts, arithmetic overflow, division by zero and lazy AND/OR/CASE/JVM dispatch.",
+ "SMJ hoisting: can-throw expressions/conditional branches/outer join, ANSI=true.",
+ "Custom RANGE windows at DATE/DECIMAL boundaries and existing fallback guards.",
+ "Results with active Joom cost/boundary flags, mixed shuffle, AQE on/off; native-plan checks.",
+ "Known silent-result edge cases regardless of ANSI, including BINARY→STRING/invalidUTF8 where applicable."
+ ],
+ "limits": "A test suite is not a mathematical proof that all bugs are absent. Explicitly record uncovered risks; do not claim compatibility proven without validation."
},
"excluded_upstream_changes": {
"PR5889": {
"status": "EXCLUDED_FROM_CURRENT_MIGRATION",
- "user_quote": "Этот PR #5889 пока не трогаем и не включаем.",
- "rule": "Не переносить, не включать и не продолжать отдельную работу над PR#5889 без нового указания пользователя. Сохранять штатный безопасный fallback выбранной базы для неподдержанных struct scalar subqueries."
+ "user_quote": "[English translation] Do not touch or include PR#5889 for now.",
+ "rule": "Do not port, include or continue separate work on PR#5889 without a new user instruction. Preserve the selected base safe fallback for unsupported struct scalar subqueries."
}
},
"implementation_plan": {
"status": "APPROVED_EXPLICITLY",
"date": "2026-11-16",
- "user_quote": "Согласен, зафиксируй план из 9 PR с резервом десятого.",
+ "user_quote": "[English translation] Agreed; record the plan of9PRs with a tenth in reserve.",
"main_pr_count": 9,
"reserve_pr_count": 1,
"maximum_planned_pr_count": 10,
"path": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/approved-plan.json",
"readable_path": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/approved-plan-ru.txt",
"step2_owner": "G01 joom/1.2-dev",
- "step2_order": "Spark4.2.1/Delta4.4.1 на registryDF55.2 → pristinevendor55.2 отдельнымкоммитом → rootpatch/sharedlock/explicitvendorunitCI → повторныепроверки и release tooling.",
+ "step2_order": "Spark4.2.1/Delta4.4.1 on registryDF55.2 → pristine vendor55.2 in a separate commit → root patch/shared lock/explicit vendor-unit CI → repeated checks and release tooling.",
"step2_separate_prs_superseded": true,
"card_groups": {
- "P01": [
- "G01"
- ],
- "I10": [
- "G01"
- ],
- "P03": [
- "G01"
- ],
- "D01": [
- "G01"
- ],
- "P02": [
- "G01"
- ],
- "I01": [
- "G01"
- ],
- "I02": [
- "G01"
- ],
- "I03": [
- "G01"
- ],
- "I04": [
- "G01"
- ],
- "I08": [
- "G01"
- ],
- "J18": [
- "G01"
- ],
- "I09": [
- "G01"
- ],
- "J14": [
- "G02"
- ],
- "I05": [
- "G02"
- ],
- "J16": [
- "G02"
- ],
- "J22": [
- "G02"
- ],
- "J15": [
- "G02"
- ],
- "D10": [
- "G04"
- ],
- "N06": [
- "G04"
- ],
- "D11": [
- "G03"
- ],
- "D02": [
- "G03"
- ],
- "D04": [
- "G03"
- ],
- "D03": [
- "G03"
- ],
- "D05": [
- "G05"
- ],
- "D06": [
- "G04"
- ],
- "D07": [
- "G04"
- ],
- "D08": [
- "G04"
- ],
- "N09": [
- "G04"
- ],
- "D09": [
- "G04"
- ],
- "N04": [
- "G04"
- ],
- "N05": [
- "G04"
- ],
- "N08": [
- "G04"
- ],
- "N07": [
- "G04"
- ],
- "N01": [
- "G06"
- ],
- "N02": [
- "G06"
- ],
- "N03": [
- "G06"
- ],
- "I06": [
- "G02"
- ],
- "J08": [
- "G07"
- ],
- "J09": [
- "G07"
- ],
- "J21": [
- "G07"
- ],
- "J25": [
- "G07"
- ],
- "J27": [
- "G07"
- ],
- "J10": [
- "G07"
- ],
- "J13": [
- "G07"
- ],
- "J11": [
- "G07"
- ],
- "J23": [
- "G07"
- ],
- "J12": [
- "G07"
- ],
- "J24": [
- "G07"
- ],
- "J26": [
- "G07"
- ],
- "J05": [
- "G08"
- ],
- "J06": [
- "G08"
- ],
- "J07": [
- "G08"
- ],
- "J17": [
- "G08"
- ],
- "J02": [
- "G09"
- ],
- "I07": [
- "G09"
- ],
- "J03": [
- "G09"
- ],
- "J04": [
- "G09"
- ],
- "J01": [
- "G09"
- ]
+ "P01": ["G01"],
+ "I10": ["G01"],
+ "P03": ["G01"],
+ "D01": ["G01"],
+ "P02": ["G01"],
+ "I01": ["G01"],
+ "I02": ["G01"],
+ "I03": ["G01"],
+ "I04": ["G01"],
+ "I08": ["G01"],
+ "J18": ["G01"],
+ "I09": ["G01"],
+ "J14": ["G02"],
+ "I05": ["G02"],
+ "J16": ["G02"],
+ "J22": ["G02"],
+ "J15": ["G02"],
+ "D10": ["G04"],
+ "N06": ["G04"],
+ "D11": ["G03"],
+ "D02": ["G03"],
+ "D04": ["G03"],
+ "D03": ["G03"],
+ "D05": ["G05"],
+ "D06": ["G04"],
+ "D07": ["G04"],
+ "D08": ["G04"],
+ "N09": ["G04"],
+ "D09": ["G04"],
+ "N04": ["G04"],
+ "N05": ["G04"],
+ "N08": ["G04"],
+ "N07": ["G04"],
+ "N01": ["G06"],
+ "N02": ["G06"],
+ "N03": ["G06"],
+ "I06": ["G02"],
+ "J08": ["G07"],
+ "J09": ["G07"],
+ "J21": ["G07"],
+ "J25": ["G07"],
+ "J27": ["G07"],
+ "J10": ["G07"],
+ "J13": ["G07"],
+ "J11": ["G07"],
+ "J23": ["G07"],
+ "J12": ["G07"],
+ "J24": ["G07"],
+ "J26": ["G07"],
+ "J05": ["G08"],
+ "J06": ["G08"],
+ "J07": ["G08"],
+ "J17": ["G08"],
+ "J02": ["G09"],
+ "I07": ["G09"],
+ "J03": ["G09"],
+ "J04": ["G09"],
+ "J01": ["G09"]
},
"quality_requirements_unchanged": true,
- "no_implementation_authorization": true,
- "execution_not_started": true,
+ "no_implementation_authorization": false,
+ "execution_not_started": false,
"original_research_path": "/Users/skotlov/d/temp/20261116_comet12-consolidated-plan/consolidated-plan.json"
},
"delivery_scope": {
"status": "APPROVED_EXPLICITLY",
- "goal": "Реализовать согласованный перенос и получить локальные JAR-артефакты Comet и Delta integration для включения пользователем в Spark4.2 image.",
+ "goal": "Implement the agreed migration and produce local Comet and Delta-integration JARs for the user to include in the Spark4.2 image.",
"targets": [
"Spark4.2.1",
"Scala2.13.18",
@@ -6377,59 +5566,71 @@
"Delta4.4.1",
"GCP"
],
- "linux_release_architectures": [
- "amd64",
- "arm64"
- ],
- "checks": "Обязательные correctness/ANSI/regression/integration и packaging проверки сохраняются. Финальные JAR должны иметь SHA256, Git SHA и сведения о сборке. Полную проверку на production GCP jobs пользователь проведёт отдельно.",
- "user_quote": "Нам сейчас нужно сделать все задуманное, чтобы я получил jar артефакты для включения в наш spark 4.2 image. Проверять производительность я буду позже."
+ "linux_release_architectures": ["amd64", "arm64"],
+ "checks": "Mandatory correctness/ANSI/regression/integration and packaging checks remain. Final JARs must have SHA256, Git SHA and build details. The user will separately perform full validation on production GCP jobs.",
+ "user_quote": "[English translation] We now need to implement everything planned so I can obtain JAR artifacts for inclusion in our Spark4.2 image. I will evaluate performance later."
},
"performance_policy": {
"status": "APPROVED_EXPLICITLY",
- "user_quote": "ну по возможности ты ее конечно проверяй как сможешь во время реализации.",
- "during_implementation": "Существующие Scala microbenchmarks/Spark Benchmark, Rust Criterion и доступные TPC-H/DS runners в принятом стиле. При необходимости обычное расширение соответствующего benchmark; отдельную инфраструктуру ради миграции не создавать.",
- "production_benchmarks": "Полные замеры на реальных GCP jobs отложены на пользовательскую проверку и не являются текущим условием передачи локальных JAR.",
+ "user_quote": "[English translation] Of course, check performance where you can during implementation.",
+ "during_implementation": "Use existing Scala microbenchmarks/Spark Benchmark, Rust Criterion and available TPC-H/DS runners in the established style. Extend the corresponding benchmark normally if needed; do not create separate infrastructure for the migration.",
+ "production_benchmarks": "Full measurements on real GCP jobs are deferred to user validation and are not a current condition for delivering local JARs.",
"correctness_unchanged": true,
- "latest_user_quote": "Я хочу сказать, что с т.з. оценки производительности ты должен действовать точно так , как уже принято в этом проекте datafusion-comet. Не надо создавать специальных тестов, если там подобного никогда не было.",
+ "latest_user_quote": "[English translation] For performance evaluation, work exactly as this datafusion-comet project already does. Do not create special tests if nothing similar has existed there.",
"reference": "PERFORMANCE.md",
"correctness_tests_unchanged": true
},
"last_updated_on": "2026-11-17",
"execution_documentation_policy": {
- "user_quote": "вся подобная документация для целей выполнения миграции, будет только в /Users/skotlov/d/projects/datafusion-comet/docs/joom-migration",
+ "user_quote": "[English translation] All such migration-execution documentation must be kept only in /Users/skotlov/d/projects/datafusion-comet/docs/joom-migration.",
"directory": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration",
- "rule": "В репозитории миграционные инструкции/решения/handoffs/прогресс/cleanup только здесь. Корневой AGENTS.md восстановлен. Raw logs/tool outputs — в датированных temp по общим правилам и в cleanup ledger."
+ "rule": "Keep repository migration instructions/decisions/handoffs/progress/cleanup only here. Root AGENTS.md is restored. Raw logs/tool outputs go into dated temp directories under the general rules and cleanup ledger."
},
"environment_blocker_policy": {
- "user_quote": "если ты видишь, что тебе что-то не хватает в окружении (тулов, прав,...), то ты не пытаешься это обойти трудными и долгими путями, а говоришь мне и мы решаем вопрос.",
- "rule": "Сообщать конкретный blocker; не строить длительные обходы. Обычная per-command PATH/JAVA_HOME настройка и штатный Maven wrapper допустимы."
+ "user_quote": "[English translation] If you find something missing from the environment (tools, permissions, etc.), do not attempt difficult and lengthy workarounds; tell me and we will resolve it.",
+ "rule": "Report the specific blocker; do not build lengthy workarounds. Normal per-command PATH/JAVA_HOME configuration and the standard Maven wrapper are allowed."
},
"cleanup_policy": {
- "user_quote": "Хотелось бы в конце получить перечень, что и куда было сохранено и создано, чтобы я потом мог почистить. Т.е. мне нужна инструкция как мне чистить более ненужное после окончания твоей работы.",
+ "user_quote": "[English translation] At the end, I would like an inventory of what was saved/created and where so I can clean it up later. I need instructions for removing what is no longer needed after your work.",
"ledger": "cleanup-ledger.json",
"baseline": "resource-baseline.json",
"final_guide": "CLEANUP.md",
- "rule": "Записывать точные paths/IDs, размеры, происхождение и shared/task-owned статус; в конце точечные cleanup-команды с сохранением final JAR/checksums/build manifests."
+ "rule": "Record exact paths/IDs, sizes, provenance and shared/task-owned status; finish with targeted cleanup commands that preserve final JARs/checksums/build manifests."
},
"execution_model": {
"selected_on": "2026-11-17",
"status": "SELECTED_BY_COORDINATOR_WITHIN_USER_AUTHORIZED_ORCHESTRATION",
"primary": "ONE_MAIN_CHAT_EXECUTES_AND_COORDINATES",
- "main_chat_responsibility": "Последовательная реализация G01–G09 в утверждённых feature branches, интеграция, запуск необходимых проверок, состояние/решения/cleanup учёт; единственный пишущий исполнитель checkout.",
- "internal_helpers": "Ограниченные исследования, сверка совместимости, независимый review diff/тестового покрытия. По умолчанию read-only; не запускать параллельную запись в общий checkout или все тяжёлые builds одновременно.",
- "delegation_owner": "Координатор самостоятельно выбирает задачи, границы и время делегирования, передаёт контекст и проверяет результат. Пользователь не должен управлять помощниками вручную.",
+ "main_chat_responsibility": "Sequential implementation of G01–G09 in the approved feature branches, integration, required checks and state/decisions/cleanup tracking; sole writing executor of the checkout.",
+ "internal_helpers": "Bounded research, compatibility checks and independent diff/test-coverage review. Read-only by default; do not write concurrently to the shared checkout or run all heavy builds simultaneously.",
+ "delegation_owner": "The coordinator independently chooses delegation tasks, boundaries and timing, passes context and verifies results. The user should not have to manage helpers manually.",
"delegation_checkpoints": {
- "G01": "Dependency graph/vendor provenance, Spark/Delta compatibility и Linux JAR packaging.",
- "G02_G07": "JNI/FFI, ownership/cancellation и buffer lifetime.",
- "G03_G06": "Sort/aggregate/SMJ/window memory/spill, outer-join/window correctness и ANSI semantics.",
+ "G01": "Dependency graph/vendor provenance, Spark/Delta compatibility and Linux JAR packaging.",
+ "G02_G07": "JNI/FFI, ownership/cancellation and buffer lifetime.",
+ "G03_G06": "Sort/aggregate/SMJ/window memory/spill, outer-join/window correctness and ANSI semantics.",
"G08": "Expression evaluation order, UDF/serialization.",
- "G09_FINAL": "Полнота 58 карточек, engine boundaries/AQE, совместные regressions и артефакты."
+ "G09_FINAL": "Completeness of58cards, engine boundaries/AQE, combined regressions and artifacts."
},
- "new_user_visible_chat_policy": "Создавать по решению координатора при конкретной пользе от длительной изолированной задачи/отдельного ограниченного контекста или технической необходимости передачи продолжения. Новый PR/большой diff/compaction сами по себе не требуют нового чата. Сначала сообщить причину, сохранить checkpoint (actual HEAD, tree, tests, next step), передать контекст и проверить результат; повторное разрешение на каждую сессию не требуется в существующем scope.",
- "existing_g01_preparation_chat": "Оставить idle; подготовка завершена. Реализацию в нём автоматически не начинать; использовать при реальной необходимости отдельной проверки.",
- "user_involvement": "Явное начало кода, новые решения о поведении/объёме/разрешениях и реальные environment blockers. Не повседневное распределение задач.",
+ "new_user_visible_chat_policy": "Create a new user-visible session at the coordinator discretion when a long isolated task/separate bounded context or a technical handoff genuinely benefits. A new PR/large diff/compaction alone does not require a new chat. First explain the reason, save a checkpoint (actual HEAD, tree, tests, next step), transfer context and verify the result; repeated approval per session is not required within existing scope.",
+ "existing_g01_preparation_chat": "Leave idle; preparation is complete. Do not automatically begin implementation there; use it when a separate check is genuinely necessary.",
+ "user_involvement": "Explicit implementation start, new behavior/scope/permission decisions and real environment blockers; not everyday task allocation.",
"implementation_hold_preserved": true,
"approved_feature_groups_and_release_milestones_unchanged": true
+ },
+ "stage_start_policy": {
+ "explicit_user_approval_per_group": true,
+ "currently_authorized": ["G01"],
+ "groups_not_authorized": [
+ "G02",
+ "G03",
+ "G04",
+ "G05",
+ "G06",
+ "G07",
+ "G08",
+ "G09",
+ "G10"
+ ]
}
},
"plan_approved_entry_ids": [
@@ -6495,190 +5696,74 @@
"approved_implementation_plan": {
"status": "APPROVED_EXPLICITLY",
"date": "2026-11-16",
- "user_quote": "Согласен, зафиксируй план из 9 PR с резервом десятого.",
+ "user_quote": "[English translation] Agreed; record the plan of9PRs with a tenth in reserve.",
"main_pr_count": 9,
"reserve_pr_count": 1,
"maximum_planned_pr_count": 10,
"path": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/approved-plan.json",
"readable_path": "/Users/skotlov/d/projects/datafusion-comet/docs/joom-migration/approved-plan-ru.txt",
"step2_owner": "G01 joom/1.2-dev",
- "step2_order": "Spark4.2.1/Delta4.4.1 на registryDF55.2 → pristinevendor55.2 отдельнымкоммитом → rootpatch/sharedlock/explicitvendorunitCI → повторныепроверки и release tooling.",
+ "step2_order": "Spark4.2.1/Delta4.4.1 on registryDF55.2 → pristine vendor55.2 in a separate commit → root patch/shared lock/explicit vendor-unit CI → repeated checks and release tooling.",
"step2_separate_prs_superseded": true,
"card_groups": {
- "P01": [
- "G01"
- ],
- "I10": [
- "G01"
- ],
- "P03": [
- "G01"
- ],
- "D01": [
- "G01"
- ],
- "P02": [
- "G01"
- ],
- "I01": [
- "G01"
- ],
- "I02": [
- "G01"
- ],
- "I03": [
- "G01"
- ],
- "I04": [
- "G01"
- ],
- "I08": [
- "G01"
- ],
- "J18": [
- "G01"
- ],
- "I09": [
- "G01"
- ],
- "J14": [
- "G02"
- ],
- "I05": [
- "G02"
- ],
- "J16": [
- "G02"
- ],
- "J22": [
- "G02"
- ],
- "J15": [
- "G02"
- ],
- "D10": [
- "G04"
- ],
- "N06": [
- "G04"
- ],
- "D11": [
- "G03"
- ],
- "D02": [
- "G03"
- ],
- "D04": [
- "G03"
- ],
- "D03": [
- "G03"
- ],
- "D05": [
- "G05"
- ],
- "D06": [
- "G04"
- ],
- "D07": [
- "G04"
- ],
- "D08": [
- "G04"
- ],
- "N09": [
- "G04"
- ],
- "D09": [
- "G04"
- ],
- "N04": [
- "G04"
- ],
- "N05": [
- "G04"
- ],
- "N08": [
- "G04"
- ],
- "N07": [
- "G04"
- ],
- "N01": [
- "G06"
- ],
- "N02": [
- "G06"
- ],
- "N03": [
- "G06"
- ],
- "I06": [
- "G02"
- ],
- "J08": [
- "G07"
- ],
- "J09": [
- "G07"
- ],
- "J21": [
- "G07"
- ],
- "J25": [
- "G07"
- ],
- "J27": [
- "G07"
- ],
- "J10": [
- "G07"
- ],
- "J13": [
- "G07"
- ],
- "J11": [
- "G07"
- ],
- "J23": [
- "G07"
- ],
- "J12": [
- "G07"
- ],
- "J24": [
- "G07"
- ],
- "J26": [
- "G07"
- ],
- "J05": [
- "G08"
- ],
- "J06": [
- "G08"
- ],
- "J07": [
- "G08"
- ],
- "J17": [
- "G08"
- ],
- "J02": [
- "G09"
- ],
- "I07": [
- "G09"
- ],
- "J03": [
- "G09"
- ],
- "J04": [
- "G09"
- ],
- "J01": [
- "G09"
- ]
+ "P01": ["G01"],
+ "I10": ["G01"],
+ "P03": ["G01"],
+ "D01": ["G01"],
+ "P02": ["G01"],
+ "I01": ["G01"],
+ "I02": ["G01"],
+ "I03": ["G01"],
+ "I04": ["G01"],
+ "I08": ["G01"],
+ "J18": ["G01"],
+ "I09": ["G01"],
+ "J14": ["G02"],
+ "I05": ["G02"],
+ "J16": ["G02"],
+ "J22": ["G02"],
+ "J15": ["G02"],
+ "D10": ["G04"],
+ "N06": ["G04"],
+ "D11": ["G03"],
+ "D02": ["G03"],
+ "D04": ["G03"],
+ "D03": ["G03"],
+ "D05": ["G05"],
+ "D06": ["G04"],
+ "D07": ["G04"],
+ "D08": ["G04"],
+ "N09": ["G04"],
+ "D09": ["G04"],
+ "N04": ["G04"],
+ "N05": ["G04"],
+ "N08": ["G04"],
+ "N07": ["G04"],
+ "N01": ["G06"],
+ "N02": ["G06"],
+ "N03": ["G06"],
+ "I06": ["G02"],
+ "J08": ["G07"],
+ "J09": ["G07"],
+ "J21": ["G07"],
+ "J25": ["G07"],
+ "J27": ["G07"],
+ "J10": ["G07"],
+ "J13": ["G07"],
+ "J11": ["G07"],
+ "J23": ["G07"],
+ "J12": ["G07"],
+ "J24": ["G07"],
+ "J26": ["G07"],
+ "J05": ["G08"],
+ "J06": ["G08"],
+ "J07": ["G08"],
+ "J17": ["G08"],
+ "J02": ["G09"],
+ "I07": ["G09"],
+ "J03": ["G09"],
+ "J04": ["G09"],
+ "J01": ["G09"]
},
"quality_requirements_unchanged": true,
"no_implementation_authorization": false,
@@ -6687,19 +5772,20 @@
},
"global_pending_decisions": [],
"out_of_scope": [
- "joom/1.1-join-filter-lazy: три коммита после joom-1.1-13, не входят в зафиксированный исходный тег.",
- "Spark4.3, Cometmain1.3/DF56 и произвольное включение draft PR не являются выбранной базой.",
- "Публикация, теги, PR и изменение datasci/runtime не выполняются на шаге1.",
- "PR#5889: пользователь явно поручил пока не трогать и не включать в текущую миграцию; сохранить штатный fallback выбранной базы."
+ "joom/1.1-join-filter-lazy: three commits after joom-1.1-13, outside the pinned source tag.",
+ "Spark4.3, Cometmain1.3/DF56 and arbitrary inclusion of draft PRs are not the selected base.",
+ "No publication, tags, PRs or datasci/runtime changes at step1.",
+ "PR#5889: the user explicitly instructed not to touch or include it in this migration for now; preserve the selected base fallback."
],
- "verification_limits": "Только исходники, Git history и read-only GitHub API; сборки, тесты, benchmarks и production qualification не выполнялись.",
+ "verification_limits": "Sources, Git history and read-only GitHub API only; no builds, tests, benchmarks or production qualification were performed.",
"last_updated_on": "2026-11-17",
"snapshot_metadata": {
"imported_on": "2026-11-17",
- "purpose": "Утверждённый состав до начала реализации; implementation_status/validation_status фиксируют исходный момент, не текущий прогресс.",
+ "purpose": "Approved scope before implementation; implementation_status/validation_status record that initial point, not current progress.",
"current_decisions": "decisions.json",
"current_execution_state": "STATE.json",
"original_research_file": "/Users/skotlov/d/temp/20261116_comet12-migration-register/migration-register.json",
"latest_permissions_ref": "decisions.json#authorization_boundary"
- }
+ },
+ "snapshot_note": "Approved scope snapshot; original planning statuses are historical. Track implementation in STATE.json and resolve final G01 acceptance through G01-CLOSEOUT.md, the annotated RC1 tag and delivered artifact receipts. No later group is automatically authorized."
}
diff --git a/docs/joom-migration/resource-baseline.json b/docs/joom-migration/resource-baseline.json
index bfc9fb326f8..f1a21ca823d 100644
--- a/docs/joom-migration/resource-baseline.json
+++ b/docs/joom-migration/resource-baseline.json
@@ -36,13 +36,7 @@
"stderr": ""
},
"docker_volumes": {
- "command": [
- "docker",
- "volume",
- "ls",
- "--format",
- "{{json .}}"
- ],
+ "command": ["docker", "volume", "ls", "--format", "{{json .}}"],
"exit_code": 0,
"stdout": "{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/0e88c655dea9164b191c6603c1f71749842fc72b0b806ed3e840fa93630a021a/_data\",\"Name\":\"0e88c655dea9164b191c6603c1f71749842fc72b0b806ed3e840fa93630a021a\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/1e376663e8c746a83773f07d5e234505bb0d32d3a27875ad4ab31548f5f2d9a8/_data\",\"Name\":\"1e376663e8c746a83773f07d5e234505bb0d32d3a27875ad4ab31548f5f2d9a8\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/5e9825b3c53661ea5a9dab788f9017364c9cf3ab9886d932f5eaf40d8871dfb3/_data\",\"Name\":\"5e9825b3c53661ea5a9dab788f9017364c9cf3ab9886d932f5eaf40d8871dfb3\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/5eb026d403f42dcf3c68023aa4cf417b62d8df2d3933068eeb1c8d6a86f7c7b3/_data\",\"Name\":\"5eb026d403f42dcf3c68023aa4cf417b62d8df2d3933068eeb1c8d6a86f7c7b3\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/6b04f9dc78a635783b7b331bdd6b55045589e83dec7663b8368c716a500edada/_data\",\"Name\":\"6b04f9dc78a635783b7b331bdd6b55045589e83dec7663b8368c716a500edada\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/6d92e30b69254153f3edbc59e544373cb36e098d5e9aaeb23debab2065276d37/_data\",\"Name\":\"6d92e30b69254153f3edbc59e544373cb36e098d5e9aaeb23debab2065276d37\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/6dbb5f61b78cca43b99b45f9b30274d72b58a360e6250c8fa89734ab8ba30fd7/_data\",\"Name\":\"6dbb5f61b78cca43b99b45f9b30274d72b58a360e6250c8fa89734ab8ba30fd7\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/7b5637b230f708e7dd096caee1207a9b7826ef89996ea55fb94f57e300df743d/_data\",\"Name\":\"7b5637b230f708e7dd096caee1207a9b7826ef89996ea55fb94f57e300df743d\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/8b86764a12cd34ab404aa2753f2a78edf8764eef5bc0cb3ad6218145b1d64dfb/_data\",\"Name\":\"8b86764a12cd34ab404aa2753f2a78edf8764eef5bc0cb3ad6218145b1d64dfb\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/12c9c3210904c50a4c8a4976885077a0bec21d7302bc918de72aaaa10880dec9/_data\",\"Name\":\"12c9c3210904c50a4c8a4976885077a0bec21d7302bc918de72aaaa10880dec9\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/21cdf241fea25b11d89f25e132c4c43c6f62d6ee015e22ff8af2b40df56a752e/_data\",\"Name\":\"21cdf241fea25b11d89f25e132c4c43c6f62d6ee015e22ff8af2b40df56a752e\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/28c198ea12c9b7803f86dc2d7b7dc33149af073bfba5eef21405e41b6ba08c60/_data\",\"Name\":\"28c198ea12c9b7803f86dc2d7b7dc33149af073bfba5eef21405e41b6ba08c60\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/31d1ebdde65780fb7ababf66f4c0598c84659334039a6a93e6ee5feac52d10df/_data\",\"Name\":\"31d1ebdde65780fb7ababf66f4c0598c84659334039a6a93e6ee5feac52d10df\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/90ff02f987c45949a3c37323b826027263be8615322a47734d9cdd77fbb5916f/_data\",\"Name\":\"90ff02f987c45949a3c37323b826027263be8615322a47734d9cdd77fbb5916f\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/585dcb10f295ad574762054a1564b9adf927c671fa1fd556f7eff87aeaa6420b/_data\",\"Name\":\"585dcb10f295ad574762054a1564b9adf927c671fa1fd556f7eff87aeaa6420b\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/04525faeba18c8c3bc6518a78179a56a812fd5df0ae6a462fe461bea57d0282c/_data\",\"Name\":\"04525faeba18c8c3bc6518a78179a56a812fd5df0ae6a462fe461bea57d0282c\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/6759c170dda97aeee9cb4c210c5b646890399f88440a9d9d4134181af40b4922/_data\",\"Name\":\"6759c170dda97aeee9cb4c210c5b646890399f88440a9d9d4134181af40b4922\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/9446afdf148731e34b427d322b3db024e06db87671a4b9443271b696baa46ebe/_data\",\"Name\":\"9446afdf148731e34b427d322b3db024e06db87671a4b9443271b696baa46ebe\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/9954d150d9564ea2e92417abc728bd7b50478408e3208d98e9040c7f332777d0/_data\",\"Name\":\"9954d150d9564ea2e92417abc728bd7b50478408e3208d98e9040c7f332777d0\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/65692d2d6be6b8529d92e30fa3973d4f21002d0867799423d02b9540cffe8bbd/_data\",\"Name\":\"65692d2d6be6b8529d92e30fa3973d4f21002d0867799423d02b9540cffe8bbd\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/76681e98462b5c9b029f86aa33d3f1f6c7b6b4a38c1830c1748c0a2cd3a27da5/_data\",\"Name\":\"76681e98462b5c9b029f86aa33d3f1f6c7b6b4a38c1830c1748c0a2cd3a27da5\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/534603f2525568cc5a48568ae67694d9fdac9196b47df40130f8dc4acbe2553b/_data\",\"Name\":\"534603f2525568cc5a48568ae67694d9fdac9196b47df40130f8dc4acbe2553b\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/a0d82c9420ea1c77c2e2d525da5c6ac5fcaba05ee5272435117ac9d31a1a0b3e/_data\",\"Name\":\"a0d82c9420ea1c77c2e2d525da5c6ac5fcaba05ee5272435117ac9d31a1a0b3e\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/ab49f9178991b52363f582500ef896e8899f2adb3510beacf401698d7aeed34b/_data\",\"Name\":\"ab49f9178991b52363f582500ef896e8899f2adb3510beacf401698d7aeed34b\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/bad7c2bfabffc109361643d74471ba79f1ceadbb1e64ead862f2940663d08b97/_data\",\"Name\":\"bad7c2bfabffc109361643d74471ba79f1ceadbb1e64ead862f2940663d08b97\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/buildx_buildkit_spark-adhoc-builder_state/_data\",\"Name\":\"buildx_buildkit_spark-adhoc-builder_state\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/buildx_buildkit_spark-builder0_state/_data\",\"Name\":\"buildx_buildkit_spark-builder0_state\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/c0a30dd49d4e163c0341ea0996afba755b4d2d06876e877211be05927e365d97/_data\",\"Name\":\"c0a30dd49d4e163c0341ea0996afba755b4d2d06876e877211be05927e365d97\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/c0cd7e13a5cea33565760bd7d75548f9e9fc1f1a1c601703ad81d357d13dce3a/_data\",\"Name\":\"c0cd7e13a5cea33565760bd7d75548f9e9fc1f1a1c601703ad81d357d13dce3a\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/c9a5bcbe527c62d757cf17736e034622956168f111448c3f98adb9fc843309f2/_data\",\"Name\":\"c9a5bcbe527c62d757cf17736e034622956168f111448c3f98adb9fc843309f2\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/d5c86776dcb3b2debffd7bbf6f2330ffd9e3a480f0b9418cdd0513f3093f32ca/_data\",\"Name\":\"d5c86776dcb3b2debffd7bbf6f2330ffd9e3a480f0b9418cdd0513f3093f32ca\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/d6f5d149179cca1a9d3e8d4cf14caf3fb74daa45741b7f3b339ebbb90ae59f7c/_data\",\"Name\":\"d6f5d149179cca1a9d3e8d4cf14caf3fb74daa45741b7f3b339ebbb90ae59f7c\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/d63bd83d73b7f0d30a3e91095262ca70cfe9c5d7175b2a01ace30253e5d43319/_data\",\"Name\":\"d63bd83d73b7f0d30a3e91095262ca70cfe9c5d7175b2a01ace30253e5d43319\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/d882842a8c39320da60a5c08cee768d33efa311cf1f4323dd68d79fbb03658d2/_data\",\"Name\":\"d882842a8c39320da60a5c08cee768d33efa311cf1f4323dd68d79fbb03658d2\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/dbf3b90f850a2b58d961804d1aa8130a2aa22be4d7198633c0184e8c5cd02a02/_data\",\"Name\":\"dbf3b90f850a2b58d961804d1aa8130a2aa22be4d7198633c0184e8c5cd02a02\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/e912941fdce51dd17d3492b48c3236e2101c6f514af9fb9a9547b0316b94ad51/_data\",\"Name\":\"e912941fdce51dd17d3492b48c3236e2101c6f514af9fb9a9547b0316b94ad51\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/f0517d657e2ddb43a76b58ee000430a190f252f4678167bae02b92f5176f6835/_data\",\"Name\":\"f0517d657e2ddb43a76b58ee000430a190f252f4678167bae02b92f5176f6835\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/fa06a75cc3dcf915a10d5f15a22d2edcd8c201b53460813cda383c4963a58ee6/_data\",\"Name\":\"fa06a75cc3dcf915a10d5f15a22d2edcd8c201b53460813cda383c4963a58ee6\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/fcef38130484b3294328d1b5e56c94e818f036693334a017a6d30d4f777c8df2/_data\",\"Name\":\"fcef38130484b3294328d1b5e56c94e818f036693334a017a6d30d4f777c8df2\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/fd075ca2f674f5d6e797364e3318dfcfbf272043177e1c2bb096afa01493525e/_data\",\"Name\":\"fd075ca2f674f5d6e797364e3318dfcfbf272043177e1c2bb096afa01493525e\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n{\"Availability\":\"N/A\",\"Driver\":\"local\",\"Group\":\"N/A\",\"Labels\":\"com.docker.volume.anonymous=\",\"Links\":\"N/A\",\"Mountpoint\":\"/var/lib/docker/volumes/fe40236a14943fbb5d1e56e05bf83a03e469838cfb812a4d7a12471d10ba706a/_data\",\"Name\":\"fe40236a14943fbb5d1e56e05bf83a03e469838cfb812a4d7a12471d10ba706a\",\"Scope\":\"local\",\"Size\":\"N/A\",\"Status\":\"N/A\"}\n",
"stderr": ""
@@ -61,23 +55,13 @@
"stderr": ""
},
"docker_builders": {
- "command": [
- "docker",
- "buildx",
- "ls"
- ],
+ "command": ["docker", "buildx", "ls"],
"exit_code": 0,
"stdout": "NAME/NODE DRIVER/ENDPOINT STATUS BUILDKIT PLATFORMS\nspark-adhoc-builder docker-container \n \\_ spark-adhoc-builder \\_ desktop-linux running v0.30.0 linux/amd64* (+2), linux/arm64*, linux/arm (+2), linux/ppc64le, (3 more)\nspark-builder* docker-container \n \\_ spark-builder0 \\_ desktop-linux running v0.31.2 linux/amd64* (+2), linux/arm64*, linux/arm (+2), linux/ppc64le, (3 more)\ndesktop-linux docker \n \\_ desktop-linux \\_ desktop-linux running v0.17.3 linux/amd64 (+2), linux/arm64, linux/arm (+2), linux/ppc64le, (3 more)\ndefault error \n",
"stderr": "\nCannot load builder default: Cannot connect to the Docker daemon at unix:///var/run/docker.sock. Is the docker daemon running?\n"
},
"docker_disk_usage": {
- "command": [
- "docker",
- "system",
- "df",
- "--format",
- "{{json .}}"
- ],
+ "command": ["docker", "system", "df", "--format", "{{json .}}"],
"exit_code": 0,
"stdout": "{\"Active\":\"2\",\"Reclaimable\":\"3.307GB (87%)\",\"Size\":\"3.79GB\",\"TotalCount\":\"13\",\"Type\":\"Images\"}\n{\"Active\":\"2\",\"Reclaimable\":\"0B\",\"Size\":\"0B\",\"TotalCount\":\"2\",\"Type\":\"Containers\"}\n{\"Active\":\"2\",\"Reclaimable\":\"6.665GB (24%)\",\"Size\":\"27.77GB\",\"TotalCount\":\"41\",\"Type\":\"Local Volumes\"}\n{\"Active\":\"0\",\"Reclaimable\":\"3.199GB\",\"Size\":\"3.199GB\",\"TotalCount\":\"85\",\"Type\":\"Build Cache\"}\n",
"stderr": ""
diff --git a/docs/joom-migration/vendor-provenance.json b/docs/joom-migration/vendor-provenance.json
index 979ed3e67d2..4dccaa06e42 100644
--- a/docs/joom-migration/vendor-provenance.json
+++ b/docs/joom-migration/vendor-provenance.json
@@ -735,5 +735,25 @@
"source": "registry+https://github.com/rust-lang/crates.io-index",
"checksum": "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97"
},
- "target_manifest_requirement": "Use standalone normalizedpublishedcrate manifest; preserve display.rs/serde55.2.0, license/notice; do not inherit Cometworkspace version1.2.0."
+ "target_manifest_requirement": "Use standalone normalizedpublishedcrate manifest; preserve display.rs/serde55.2.0, license/notice; do not inherit Cometworkspace version1.2.0.",
+ "target_vendor_execution": {
+ "pristine_import_commit": "b7f25aa37569ad50b1dee2f8204f57ac219ddc91",
+ "integration_commit": "a7b044913fa411cf8130876b9f2e1cbe7f734f35",
+ "published_archive_sha256": "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97",
+ "production_algorithm_changes": "NONE_IN_G01",
+ "test_only_adaptations": [
+ "src/spill/mod.rs: flush and compare metric to physical file bytes at batch/finish checkpoints, preserve lifetime cleanup checks",
+ "src/spill/spill_pool.rs: compare metric to physical file bytes before/after writer drop"
+ ],
+ "why": "Comet Arrow59.3.2 omits all-valid IPC bitmap buffers; pristine DF55.2 lock used59.2.0. Runtime spill behavior is unchanged.",
+ "additional_format_config": "rustfmt.toml from DataFusion tag55.2.0; edition2024,width90",
+ "workspace_lock": "native/Cargo.lock; nested published lockfile is provenance only",
+ "default_unit_tests_passed": 1695,
+ "test_utils_statistics_passed": 42,
+ "historical_pre_cleanup_pristine_import_commit": "98c9f2f195c52d1b291e933be0c91624afece67a",
+ "history_cleanup_mapping": "G01-results.json",
+ "historical_pre_pr_import_commit": "0a76d68a33f6b4bcbce9fbb412b90350c5fed80b",
+ "historical_pre_pr_integration_commit": "15a466a09abf6106bbd80402e1967ba96076fbcc",
+ "pr_alignment_mapping": "G01-PR-preparation.json"
+ }
}
diff --git a/native/Cargo.lock b/native/Cargo.lock
index 939de12ce4e..61ceb5ce1db 100644
--- a/native/Cargo.lock
+++ b/native/Cargo.lock
@@ -105,6 +105,15 @@ dependencies = [
"alloc-no-stdlib",
]
+[[package]]
+name = "alloca"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4"
+dependencies = [
+ "cc",
+]
+
[[package]]
name = "allocator-api2"
version = "0.2.21"
@@ -249,6 +258,8 @@ dependencies = [
"arrow-schema",
"arrow-select",
"arrow-string",
+ "half",
+ "rand 0.9.5",
]
[[package]]
@@ -1247,6 +1258,16 @@ dependencies = [
"tinyvec",
]
+[[package]]
+name = "bstr"
+version = "1.13.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6bb31b46c14244e20ee9984b11bf5c992b91fb6939fea616e3512c8baecdbe5f"
+dependencies = [
+ "memchr",
+ "serde_core",
+]
+
[[package]]
name = "bumpalo"
version = "3.20.3"
@@ -1543,6 +1564,17 @@ dependencies = [
"crossbeam-utils",
]
+[[package]]
+name = "console"
+version = "0.16.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e96a4956774c13c126a8b5af4daa79384f4d826534c95a02d76afb39e2ab64e3"
+dependencies = [
+ "encode_unicode",
+ "libc",
+ "windows-sys 0.61.2",
+]
+
[[package]]
name = "const-oid"
version = "0.9.6"
@@ -1663,7 +1695,7 @@ dependencies = [
"cast",
"ciborium",
"clap",
- "criterion-plot",
+ "criterion-plot 0.6.0",
"itertools 0.13.0",
"num-traits",
"oorandom",
@@ -1677,6 +1709,32 @@ dependencies = [
"walkdir",
]
+[[package]]
+name = "criterion"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3"
+dependencies = [
+ "alloca",
+ "anes",
+ "cast",
+ "ciborium",
+ "clap",
+ "criterion-plot 0.8.2",
+ "futures",
+ "itertools 0.13.0",
+ "num-traits",
+ "oorandom",
+ "page_size",
+ "plotters",
+ "rayon",
+ "regex",
+ "serde",
+ "serde_json",
+ "tinytemplate",
+ "walkdir",
+]
+
[[package]]
name = "criterion-plot"
version = "0.6.0"
@@ -1687,6 +1745,16 @@ dependencies = [
"itertools 0.13.0",
]
+[[package]]
+name = "criterion-plot"
+version = "0.8.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea"
+dependencies = [
+ "cast",
+ "itertools 0.13.0",
+]
+
[[package]]
name = "crossbeam-channel"
version = "0.5.17"
@@ -1997,7 +2065,7 @@ dependencies = [
"comet-test-udfs",
"comet-udf-sdk",
"crc32fast",
- "criterion",
+ "criterion 0.7.0",
"datafusion",
"datafusion-comet-common",
"datafusion-comet-jni-bridge",
@@ -2088,7 +2156,7 @@ name = "datafusion-comet-operators"
version = "1.2.0"
dependencies = [
"arrow",
- "criterion",
+ "criterion 0.7.0",
"datafusion",
"datafusion-comet-common",
"datafusion-comet-spark-expr",
@@ -2117,7 +2185,7 @@ dependencies = [
"clap",
"crc32c",
"crc32fast",
- "criterion",
+ "criterion 0.7.0",
"datafusion",
"datafusion-comet-common",
"datafusion-comet-jni-bridge",
@@ -2142,7 +2210,7 @@ dependencies = [
"arrow",
"base64 0.23.1",
"chrono",
- "criterion",
+ "criterion 0.7.0",
"datafusion",
"datafusion-comet-common",
"datafusion-comet-jni-bridge",
@@ -2585,6 +2653,7 @@ dependencies = [
"datafusion-expr-common",
"datafusion-functions-aggregate-common",
"datafusion-physical-expr-common",
+ "datafusion-proto-models",
"half",
"hashbrown 0.17.1",
"indexmap 2.14.2",
@@ -2619,6 +2688,7 @@ dependencies = [
"chrono",
"datafusion-common",
"datafusion-expr-common",
+ "datafusion-proto-models",
"hashbrown 0.17.1",
"indexmap 2.14.2",
"itertools 0.15.0",
@@ -2648,8 +2718,6 @@ dependencies = [
[[package]]
name = "datafusion-physical-plan"
version = "55.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b1451d6817e5ac0dd21819b19949d1c420ab51d748f55dd9a96ecb5418a24c97"
dependencies = [
"arrow",
"arrow-data",
@@ -2658,29 +2726,60 @@ dependencies = [
"arrow-schema",
"async-trait",
"bytes",
+ "criterion 0.8.2",
"datafusion-common",
"datafusion-common-runtime",
"datafusion-execution",
"datafusion-expr",
"datafusion-functions",
+ "datafusion-functions-aggregate",
"datafusion-functions-aggregate-common",
+ "datafusion-functions-window",
"datafusion-functions-window-common",
"datafusion-physical-expr",
"datafusion-physical-expr-common",
+ "datafusion-proto-common",
+ "datafusion-proto-models",
"futures",
"half",
"hashbrown 0.17.1",
"indexmap 2.14.2",
+ "insta",
"itertools 0.15.0",
"log",
"num-traits",
"parking_lot",
"pin-project-lite",
+ "rand 0.9.5",
+ "rstest",
+ "rstest_reuse",
"serde",
"serde_json",
"tokio",
]
+[[package]]
+name = "datafusion-proto-common"
+version = "55.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a95763d5fa64a88b658ba4ec7314d9c538e2aea749dc88d5ce2afaafbb4c69b8"
+dependencies = [
+ "arrow",
+ "datafusion-common",
+ "prost",
+]
+
+[[package]]
+name = "datafusion-proto-models"
+version = "55.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3c4982d05dee1e0b90b7f3b39845dca7eee5635c550db09158d385b8181bdc6a"
+dependencies = [
+ "datafusion-common",
+ "datafusion-proto-common",
+ "prost",
+]
+
[[package]]
name = "datafusion-pruning"
version = "55.2.0"
@@ -2946,6 +3045,12 @@ version = "1.18.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "252afb9ae5eaa683babdc6a068b3f5726eb19e05070c731f9b2a23a7c3e8ed34"
+[[package]]
+name = "encode_unicode"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "34aa73646ffb006b8f5147f3dc182bd4bcb190227ce861fc4a4844bf8e3cb2c0"
+
[[package]]
name = "equator"
version = "0.4.2"
@@ -3199,6 +3304,12 @@ version = "0.3.34"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cd417de3d1d015fc3bfd2b1ea46dfc7bab72ef86f1cc7cc9c78e728b34a6d1fd"
+[[package]]
+name = "futures-timer"
+version = "3.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af43fadb8a98512d547e37b4e92e0ced13e205c061b87b4623eff01d918d6968"
+
[[package]]
name = "futures-util"
version = "0.3.34"
@@ -3287,6 +3398,19 @@ version = "0.3.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e4eba85ea1d0a966a983acd07deee566e67395d2d96b6fb39e62b5a833f1eb0b"
+[[package]]
+name = "globset"
+version = "0.4.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f98b0d85522dc0a824cfebbc9594ff4a9b289c568bea0370319dfbdf20ccdd2d"
+dependencies = [
+ "aho-corasick",
+ "bstr",
+ "log",
+ "regex-automata",
+ "regex-syntax",
+]
+
[[package]]
name = "gloo-timers"
version = "0.3.0"
@@ -3327,6 +3451,8 @@ dependencies = [
"cfg-if",
"crunchy",
"num-traits",
+ "rand 0.9.5",
+ "rand_distr",
"zerocopy",
]
@@ -3837,6 +3963,22 @@ dependencies = [
"generic-array",
]
+[[package]]
+name = "insta"
+version = "1.49.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b67d3d2e287e4b86c10b3f3b641033d1f89b74bdb39d05f34952e2b9a6fe21cd"
+dependencies = [
+ "console",
+ "globset",
+ "once_cell",
+ "regex",
+ "similar",
+ "strip-ansi-escapes",
+ "tempfile",
+ "walkdir",
+]
+
[[package]]
name = "inventory"
version = "0.3.24"
@@ -4889,6 +5031,16 @@ version = "0.5.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1a80800c0488c3a21695ea981a54918fbb37abf04f4d0720c453632255e2ff0e"
+[[package]]
+name = "page_size"
+version = "0.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
[[package]]
name = "parking"
version = "2.2.1"
@@ -5296,6 +5448,15 @@ dependencies = [
"syn 3.0.5",
]
+[[package]]
+name = "proc-macro-crate"
+version = "3.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e67ba7e9b2b56446f1d419b1d807906278ffa1a658a8a5d8a39dcb1f5a78614f"
+dependencies = [
+ "toml_edit",
+]
+
[[package]]
name = "proc-macro2"
version = "1.0.107"
@@ -5570,6 +5731,7 @@ version = "0.8.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e058c7de0b26af77780c769414d6257830bb240f3c38477dbc2c16e5f54d6d4c"
dependencies = [
+ "libc",
"rand_chacha 0.3.1",
"rand_core 0.6.4",
]
@@ -5639,6 +5801,16 @@ version = "0.10.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69"
+[[package]]
+name = "rand_distr"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6a8615d50dcf34fa31f7ab52692afec947c4dd0ab803cc87cb3b0b4570ff7463"
+dependencies = [
+ "num-traits",
+ "rand 0.9.5",
+]
+
[[package]]
name = "rand_pcg"
version = "0.10.2"
@@ -5732,6 +5904,12 @@ version = "0.8.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6f6ff9a378485b298a5286656da665ba74413d36db0979633275d2e708145d4"
+[[package]]
+name = "relative-path"
+version = "1.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ba39f3699c378cd8970968dcbff9c43159ea4cfbd88d43c00b22f2ef10a435d2"
+
[[package]]
name = "reqsign-aliyun-oss"
version = "3.1.5"
@@ -5995,6 +6173,46 @@ dependencies = [
"zeroize",
]
+[[package]]
+name = "rstest"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5a3193c063baaa2a95a33f03035c8a72b83d97a54916055ba22d35ed3839d49"
+dependencies = [
+ "futures-timer",
+ "futures-util",
+ "rstest_macros",
+]
+
+[[package]]
+name = "rstest_macros"
+version = "0.26.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9c845311f0ff7951c5506121a9ad75aec44d083c31583b2ea5a30bcb0b0abba0"
+dependencies = [
+ "cfg-if",
+ "glob",
+ "proc-macro-crate",
+ "proc-macro2",
+ "quote",
+ "regex",
+ "relative-path",
+ "rustc_version",
+ "syn 2.0.119",
+ "unicode-ident",
+]
+
+[[package]]
+name = "rstest_reuse"
+version = "0.7.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ac1bd675701e1d74aef2f1b332d28182700f1bc7f2371209ac5eb26a238f422"
+dependencies = [
+ "quote",
+ "rand 0.8.8",
+ "syn 2.0.119",
+]
+
[[package]]
name = "rust-ini"
version = "0.21.3"
@@ -6464,6 +6682,12 @@ version = "0.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e3a9fe34e3e7a50316060351f37187a3f546bce95496156754b601a5fa71b76e"
+[[package]]
+name = "similar"
+version = "2.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa"
+
[[package]]
name = "siphasher"
version = "1.0.3"
@@ -6562,6 +6786,15 @@ version = "0.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7f446288b699d66d0fd2e30d1cfe7869194312524b3b9252594868ed26ef056a"
+[[package]]
+name = "strip-ansi-escapes"
+version = "0.2.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2a8f8038e7e7969abb3f1b7c2a811225e9296da208539e0f79c5251d6cac0025"
+dependencies = [
+ "vte",
+]
+
[[package]]
name = "strsim"
version = "0.11.1"
@@ -6850,6 +7083,7 @@ dependencies = [
"bytes",
"libc",
"mio",
+ "parking_lot",
"pin-project-lite",
"signal-hook-registry",
"socket2",
@@ -6904,6 +7138,36 @@ dependencies = [
"tokio",
]
+[[package]]
+name = "toml_datetime"
+version = "1.1.3+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "173f25c883cd2176a2d8212155638c0a60d484a9d84546d270866f40f56b6e49"
+dependencies = [
+ "serde_core",
+]
+
+[[package]]
+name = "toml_edit"
+version = "0.25.19+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "dd43688c4858d7264ae1cf9a9f874a67c3feb74a883b1cd7bebd6a4ea8307218"
+dependencies = [
+ "indexmap 2.14.2",
+ "toml_datetime",
+ "toml_parser",
+ "winnow",
+]
+
+[[package]]
+name = "toml_parser"
+version = "1.1.5+spec-1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baa693a8032d7e1cada7d0041e96126df243179ff061456783ac7f12bda4744c"
+dependencies = [
+ "winnow",
+]
+
[[package]]
name = "tower"
version = "0.5.3"
@@ -7175,6 +7439,15 @@ version = "0.8.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5c3082ca00d5a5ef149bb8b555a72ae84c9c59f7250f013ac822ac2e49b19c64"
+[[package]]
+name = "vte"
+version = "0.14.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "231fdcd7ef3037e8330d8e17e61011a2c244126acc0a982f4040ac3f9f0bc077"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "walkdir"
version = "2.5.0"
@@ -7491,6 +7764,15 @@ version = "0.52.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec"
+[[package]]
+name = "winnow"
+version = "1.0.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c3b93199df0ab874cdf0c6a94f0a0dc3423fe41f1cafb62b8bd7b1deb277aef"
+dependencies = [
+ "memchr",
+]
+
[[package]]
name = "wit-bindgen"
version = "0.57.1"
diff --git a/native/Cargo.toml b/native/Cargo.toml
index 498e3c7414e..3cd94ccd9ff 100644
--- a/native/Cargo.toml
+++ b/native/Cargo.toml
@@ -17,12 +17,17 @@
[workspace]
default-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "operators", "comet-udf-sdk", "comet-test-udfs"]
-members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "operators", "comet-udf-sdk", "comet-test-udfs"]
+members = ["core", "spark-expr", "common", "proto", "jni-bridge", "shuffle", "operators", "comet-udf-sdk", "comet-test-udfs", "vendor/datafusion-physical-plan"]
# Crates under ../contrib are intentionally NOT workspace members. Core pulls them in as
# optional path dependencies when their corresponding contrib feature is enabled.
exclude = ["../contrib"]
resolver = "2"
+# Keep the published 55.2.0 crate in this workspace so its unit tests use the
+# same patch and lockfile as Comet. Joom algorithm patches are added separately.
+[patch.crates-io]
+datafusion-physical-plan = { path = "vendor/datafusion-physical-plan" }
+
[workspace.package]
version = "1.2.0"
homepage = "https://datafusion.apache.org/comet"
diff --git a/native/common/src/error.rs b/native/common/src/error.rs
index 23f92b46393..bc0b26b6522 100644
--- a/native/common/src/error.rs
+++ b/native/common/src/error.rs
@@ -248,6 +248,15 @@ pub enum SparkError {
matched_fields: String,
},
+ /// SPARK-60262: two requested fields select the same physical Parquet column.
+ /// Carries raw names so the JVM's own factory quotes identifiers and supplies
+ /// the version's exact error parameters.
+ #[error("[PARQUET_FIELD_REQUESTED_MORE_THAN_ONCE] Requested fields {requested_fields:?} select Parquet field {parquet_field} more than once")]
+ ParquetFieldRequestedMoreThanOnce {
+ requested_fields: Vec,
+ parquet_field: String,
+ },
+
/// The read schema carries Parquet field ids but the file carries none. Mirrors the runtime
/// error raised in Spark's `ParquetReadSupport` when
/// `spark.sql.parquet.fieldId.read.ignoreMissing` is false. Spark's message names no file,
@@ -389,6 +398,9 @@ impl SparkError {
SparkError::FileNotFound { .. } => "FileNotFound",
SparkError::DuplicateFieldCaseInsensitive { .. } => "DuplicateFieldCaseInsensitive",
SparkError::DuplicateFieldByFieldId { .. } => "DuplicateFieldByFieldId",
+ SparkError::ParquetFieldRequestedMoreThanOnce { .. } => {
+ "ParquetFieldRequestedMoreThanOnce"
+ }
SparkError::ParquetMissingFieldIds { .. } => "ParquetMissingFieldIds",
SparkError::ParquetSchemaConvert { .. } => "ParquetSchemaConvert",
SparkError::CannotReadFile { .. } => "CannotReadFile",
@@ -650,6 +662,13 @@ impl SparkError {
"matchedFields": matched_fields,
})
}
+ SparkError::ParquetFieldRequestedMoreThanOnce {
+ requested_fields,
+ parquet_field,
+ } => serde_json::json!({
+ "requestedFields": requested_fields,
+ "parquetField": parquet_field,
+ }),
SparkError::ParquetMissingFieldIds { file_path } => {
serde_json::json!({
"filePath": file_path,
@@ -768,7 +787,10 @@ impl SparkError {
// DuplicateFieldByFieldId - converted to SparkRuntimeException by the shim
// (Spark's `foundDuplicateFieldInFieldIdLookupModeError` returns SparkRuntimeException)
- SparkError::DuplicateFieldByFieldId { .. } => "org/apache/spark/SparkRuntimeException",
+ SparkError::DuplicateFieldByFieldId { .. }
+ | SparkError::ParquetFieldRequestedMoreThanOnce { .. } => {
+ "org/apache/spark/SparkRuntimeException"
+ }
// ParquetMissingFieldIds - converted to the plain RuntimeException Spark's
// ParquetReadSupport throws when the file lacks field ids and
@@ -883,6 +905,9 @@ impl SparkError {
// Duplicate field id in id-lookup mode
SparkError::DuplicateFieldByFieldId { .. } => Some("_LEGACY_ERROR_TEMP_2094"),
+ SparkError::ParquetFieldRequestedMoreThanOnce { .. } => {
+ Some("PARQUET_FIELD_REQUESTED_MORE_THAN_ONCE")
+ }
// ParquetMissingFieldIds is a plain RuntimeException with no error class. The 4.x
// shim supplies FAILED_READ_FILE itself, so none is exposed here.
@@ -1020,6 +1045,25 @@ impl From for DataFusionError {
mod tests {
use super::*;
+ #[test]
+ fn repeated_parquet_field_error_preserves_raw_identifiers() {
+ let error = SparkError::ParquetFieldRequestedMoreThanOnce {
+ requested_fields: vec!["x`y".into(), "a".into()],
+ parquet_field: "A`B".into(),
+ };
+ let parsed: serde_json::Value = serde_json::from_str(&error.to_json()).unwrap();
+ assert_eq!(parsed["errorType"], "ParquetFieldRequestedMoreThanOnce");
+ assert_eq!(
+ parsed["errorClass"],
+ "PARQUET_FIELD_REQUESTED_MORE_THAN_ONCE"
+ );
+ assert_eq!(
+ parsed["params"]["requestedFields"],
+ serde_json::json!(["x`y", "a"])
+ );
+ assert_eq!(parsed["params"]["parquetField"], "A`B");
+ }
+
#[test]
fn test_divide_by_zero_json() {
let error = SparkError::DivideByZero;
diff --git a/native/core/build.rs b/native/core/build.rs
index da42b85a5c1..b4c968d1005 100644
--- a/native/core/build.rs
+++ b/native/core/build.rs
@@ -47,12 +47,14 @@ fn main() {
// from OUT_DIR.
if let Ok(out_dir) = std::env::var("OUT_DIR") {
let out_path = std::path::PathBuf::from(out_dir);
- // OUT_DIR is .../target//build//out
+ // Cargo 1.100 adds a package-name component below `build`. Locate the
+ // profile directory rather than assuming a fixed OUT_DIR depth; this
+ // also preserves target triples and custom profiles/target directories.
let target_profile_dir = out_path
.ancestors()
- .nth(3)
- .map(|p| p.to_path_buf())
- .unwrap_or_default();
+ .find(|p| p.file_name().is_some_and(|name| name == "build"))
+ .and_then(Path::parent)
+ .expect("OUT_DIR must be inside a Cargo profile build directory");
let dylib_ext = if cfg!(target_os = "macos") {
"dylib"
} else if cfg!(target_os = "windows") {
diff --git a/native/core/src/execution/delta_dv.rs b/native/core/src/execution/delta_dv.rs
index 982c4434207..b515772dbc2 100644
--- a/native/core/src/execution/delta_dv.rs
+++ b/native/core/src/execution/delta_dv.rs
@@ -1550,6 +1550,7 @@ mod tests {
false,
"",
"",
+ false,
)
.unwrap();
let batches: Vec<_> = scan
diff --git a/native/core/src/execution/operators/dynamic_filter/join/tests.rs b/native/core/src/execution/operators/dynamic_filter/join/tests.rs
index 54ae6737edf..7e545b919af 100644
--- a/native/core/src/execution/operators/dynamic_filter/join/tests.rs
+++ b/native/core/src/execution/operators/dynamic_filter/join/tests.rs
@@ -673,6 +673,7 @@ fn parquet_probe(
false,
"CORRECTED",
"CORRECTED",
+ false,
)
.unwrap();
(file, scan)
@@ -827,6 +828,7 @@ async fn reader_filter_crosses_null_check_conjunction_and_retains_residual() {
false,
"CORRECTED",
"CORRECTED",
+ false,
)
.unwrap();
let checks = [("key", 0), ("payload", 1), ("other", 2)].map(|(name, index)| {
diff --git a/native/core/src/execution/operators/dynamic_filter/join/tests/schema_errors.rs b/native/core/src/execution/operators/dynamic_filter/join/tests/schema_errors.rs
index f37b78ad2c6..dca0cbe5942 100644
--- a/native/core/src/execution/operators/dynamic_filter/join/tests/schema_errors.rs
+++ b/native/core/src/execution/operators/dynamic_filter/join/tests/schema_errors.rs
@@ -89,6 +89,7 @@ fn scan(
false,
"CORRECTED",
"CORRECTED",
+ false,
)
.unwrap()
}
diff --git a/native/core/src/execution/operators/dynamic_filter/join/tests/schema_errors/partition_columns.rs b/native/core/src/execution/operators/dynamic_filter/join/tests/schema_errors/partition_columns.rs
index f6b00114f10..63df42cc026 100644
--- a/native/core/src/execution/operators/dynamic_filter/join/tests/schema_errors/partition_columns.rs
+++ b/native/core/src/execution/operators/dynamic_filter/join/tests/schema_errors/partition_columns.rs
@@ -74,6 +74,7 @@ fn partitioned_scan(
false,
"CORRECTED",
"CORRECTED",
+ false,
)
.unwrap()
}
diff --git a/native/core/src/execution/operators/dynamic_filter/join/tests/timestamp_errors.rs b/native/core/src/execution/operators/dynamic_filter/join/tests/timestamp_errors.rs
index aa982280517..757d1a0bc7c 100644
--- a/native/core/src/execution/operators/dynamic_filter/join/tests/timestamp_errors.rs
+++ b/native/core/src/execution/operators/dynamic_filter/join/tests/timestamp_errors.rs
@@ -108,6 +108,7 @@ async fn assert_timestamp_overflow_preserved(nested: bool) {
false,
"CORRECTED",
"CORRECTED",
+ false,
)
.unwrap();
let join = single_key_join_plans(
diff --git a/native/core/src/execution/operators/dynamic_filter/topk/tests.rs b/native/core/src/execution/operators/dynamic_filter/topk/tests.rs
index 58a9866bca0..394cddb6885 100644
--- a/native/core/src/execution/operators/dynamic_filter/topk/tests.rs
+++ b/native/core/src/execution/operators/dynamic_filter/topk/tests.rs
@@ -199,6 +199,7 @@ fn parquet_scan(
false,
"CORRECTED",
"CORRECTED",
+ false,
)
.unwrap()
}
diff --git a/native/core/src/execution/operators/dynamic_filter/topk/tests/timestamp.rs b/native/core/src/execution/operators/dynamic_filter/topk/tests/timestamp.rs
index ed5b7267e0a..15a13d37fb3 100644
--- a/native/core/src/execution/operators/dynamic_filter/topk/tests/timestamp.rs
+++ b/native/core/src/execution/operators/dynamic_filter/topk/tests/timestamp.rs
@@ -98,6 +98,7 @@ fn timestamp_input(
false,
"CORRECTED",
"CORRECTED",
+ false,
)
.unwrap();
(file, scan)
diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs
index 268f35570a6..885ca267fba 100644
--- a/native/core/src/execution/planner.rs
+++ b/native/core/src/execution/planner.rs
@@ -485,6 +485,7 @@ impl PhysicalPlanner {
rebase_from_file_metadata,
datetime_rebase_mode_in_read,
int96_rebase_mode_in_read,
+ common.reject_duplicate_field_selection,
)?;
Ok(Arc::new(SparkPlan::new(plan_id, scan, vec![])))
}
@@ -791,6 +792,7 @@ impl PhysicalPlanner {
expr.is_spark4_plus,
);
cast_options.decimal_zero_any_exponent = expr.decimal_zero_any_exponent;
+ cast_options.decimal_tiny_values_to_zero = expr.decimal_tiny_values_to_zero;
Ok(Arc::new(Cast::new(
child,
datatype,
diff --git a/native/core/src/parquet/eager_page_index_reader_factory.rs b/native/core/src/parquet/eager_page_index_reader_factory.rs
index 8ebe639bc87..f1a77ce7f09 100644
--- a/native/core/src/parquet/eager_page_index_reader_factory.rs
+++ b/native/core/src/parquet/eager_page_index_reader_factory.rs
@@ -93,6 +93,7 @@
//! bring it back.
use crate::parquet::datetime_rebase::stamp_int96_leaves;
+use crate::parquet::field_resolver::validate_field_selection;
use crate::parquet::parquet_support::SparkParquetOptions;
use crate::parquet::schema_adapter::{
check_file_conversions, RejectOnNonEmpty, REPEATED_PRIMITIVE_KEY,
@@ -233,6 +234,7 @@ pub struct EagerPageIndexReaderFactory {
require_field_ids: bool,
stamp_int96_leaves: bool,
conversion_check: Option<(SchemaRef, SparkParquetOptions, bool)>,
+ reject_duplicate_field_selection: bool,
deferred_rejections: Arc>>,
next_file_prefetch: Option>,
}
@@ -266,6 +268,7 @@ impl EagerPageIndexReaderFactory {
require_field_ids: false,
stamp_int96_leaves: false,
conversion_check: None,
+ reject_duplicate_field_selection: false,
deferred_rejections: Arc::new(Mutex::new(HashMap::new())),
next_file_prefetch: None,
}
@@ -283,6 +286,13 @@ impl EagerPageIndexReaderFactory {
self
}
+ /// Apply the actual Spark version's repeated-field contract to the required
+ /// projection. Older producers/versions keep the previous behavior.
+ pub(crate) fn with_reject_duplicate_field_selection(mut self, enabled: bool) -> Self {
+ self.reject_duplicate_field_selection = enabled;
+ self
+ }
+
pub fn with_spark_variant_schema(mut self, enabled: bool) -> Self {
self.spark_variant_schema = enabled;
self
@@ -371,6 +381,7 @@ impl ParquetFileReaderFactory for EagerPageIndexReaderFactory {
require_field_ids: self.require_field_ids,
stamp_int96_leaves: self.stamp_int96_leaves,
conversion_check: self.conversion_check.clone(),
+ reject_duplicate_field_selection: self.reject_duplicate_field_selection,
deferred_rejections: Arc::clone(&self.deferred_rejections),
deferred_rejection,
next_file_prefetch: self.next_file_prefetch.clone(),
@@ -392,6 +403,7 @@ struct EagerPageIndexReader {
require_field_ids: bool,
stamp_int96_leaves: bool,
conversion_check: Option<(SchemaRef, SparkParquetOptions, bool)>,
+ reject_duplicate_field_selection: bool,
deferred_rejections: Arc>>,
deferred_rejection: Option,
next_file_prefetch: Option>,
@@ -750,6 +762,20 @@ impl AsyncFileReader for EagerPageIndexReader {
self.conversion_check.as_ref()
{
let file = metadata.file_metadata();
+ if self.reject_duplicate_field_selection {
+ let schema =
+ parquet_to_arrow_schema(file.schema_descr(), file.key_value_metadata())?;
+ validate_field_selection(
+ schema.fields(),
+ required.fields(),
+ conversion_options.case_sensitive,
+ conversion_options.use_field_id,
+ )
+ .map_err(|error| match error {
+ DataFusionError::External(source) => ParquetError::External(source),
+ other => ParquetError::External(Box::new(other)),
+ })?;
+ }
let needs_check = *row_filters
|| file.schema_descr().columns().iter().any(|column| {
column.self_type().get_basic_info().repetition()
@@ -1649,6 +1675,57 @@ mod tests {
assert!(metadata_for(false, without_ids).await.is_ok());
}
+ #[tokio::test]
+ async fn required_field_selection_is_checked_before_data_only_for_the_new_contract() {
+ let store = Arc::new(InMemory::new());
+ let file = put_int_file(&store, "alias.parquet", Some("1")).await;
+ let runtime = datafusion::execution::runtime_env::RuntimeEnv::default();
+ let metrics = ExecutionPlanMetricsSet::new();
+ let by_id = arrow::datatypes::Field::new("x", DataType::Int32, true)
+ .with_metadata([(PARQUET_FIELD_ID_META_KEY.to_string(), "1".to_string())].into());
+ let by_name = arrow::datatypes::Field::new("a", DataType::Int32, true);
+ for enabled in [false, true] {
+ for pruned in [false, true] {
+ let required = Arc::new(Schema::new(if pruned {
+ vec![by_id.clone()]
+ } else {
+ vec![by_id.clone(), by_name.clone()]
+ }));
+ let mut options =
+ SparkParquetOptions::new(datafusion_comet_spark_expr::EvalMode::Legacy, "UTC");
+ options.use_field_id = true;
+ let factory = EagerPageIndexReaderFactory::new(
+ Arc::clone(&store) as Arc,
+ runtime.cache_manager.get_file_metadata_cache(),
+ ScanIoSource::Local,
+ &metrics,
+ )
+ .with_conversion_check(required, options, false)
+ .with_reject_duplicate_field_selection(enabled);
+ let mut reader = factory
+ .create_reader(0, file.clone(), None, &metrics)
+ .unwrap();
+ let result = reader.get_metadata(None).await;
+ if enabled && !pruned {
+ let ParquetError::External(source) = result.unwrap_err() else {
+ panic!("expected the structured repeated-field error");
+ };
+ assert!(matches!(
+ source.downcast_ref::(),
+ Some(SparkError::ParquetFieldRequestedMoreThanOnce {
+ requested_fields, parquet_field,
+ }) if requested_fields == &["x", "a"] && parquet_field == "a"
+ ));
+ } else {
+ assert!(
+ result.is_ok(),
+ "legacy or projected single-field read was rejected: {result:?}"
+ );
+ }
+ }
+ }
+ }
+
#[tokio::test]
async fn conversion_check_preserves_structured_spark_errors() {
let store = Arc::new(InMemory::new());
diff --git a/native/core/src/parquet/field_resolver.rs b/native/core/src/parquet/field_resolver.rs
index 060fe5c8286..ef51c2b7972 100644
--- a/native/core/src/parquet/field_resolver.rs
+++ b/native/core/src/parquet/field_resolver.rs
@@ -29,8 +29,9 @@
//! [`FieldResolver`], so a change to the precedence rules reaches the root and nested fields
//! together.
+use crate::parquet::name_fold::fold_names;
use crate::parquet::parquet_support::{duplicate_parquet_field_error, field_id};
-use arrow::datatypes::FieldRef;
+use arrow::datatypes::{DataType, FieldRef};
use datafusion::common::{DataFusionError, Result as DataFusionResult};
use datafusion_comet_common::SparkError;
use std::collections::HashMap;
@@ -206,6 +207,121 @@ impl<'a> FieldResolver<'a> {
}
}
+/// Validate the fields a scan actually requests, before any pages are decoded.
+/// Spark's SPARK-60262 clipping guard rejects one physical field selected twice,
+/// including an id match beside a name match. Validate the projected schema,
+/// not the table's full schema: an unreferenced alias does not read the column.
+pub(crate) fn validate_field_selection(
+ file: &[FieldRef],
+ requested: &[FieldRef],
+ case_sensitive: bool,
+ use_field_id: bool,
+) -> DataFusionResult<()> {
+ let names: Vec<&str> = file
+ .iter()
+ .chain(requested)
+ .map(|field| field.name().as_str())
+ .collect();
+ let folded = fold_names(&names, case_sensitive)?;
+ let (file_folded, requested_folded) = folded.split_at(file.len());
+ let resolver = FieldResolver::new(
+ requested,
+ requested_folded,
+ file,
+ file_folded,
+ case_sensitive,
+ use_field_id,
+ );
+ // Spark clips each field recursively in request order, then checks this
+ // group's repetitions. Preserve that order: a nested failure must not be
+ // hidden by a root repetition or a later sibling's failed lookup.
+ let mut selected = Vec::with_capacity(requested.len());
+ for (index, field) in requested.iter().enumerate() {
+ let physical = resolver.resolve(index)?;
+ if let Some(physical) = physical {
+ validate_nested_selection(
+ file[physical].data_type(),
+ field.data_type(),
+ case_sensitive,
+ use_field_id,
+ )?;
+ }
+ selected.push(physical);
+ }
+ let mut seen = std::collections::HashSet::new();
+ for &physical in selected.iter().flatten() {
+ // Spark checks the exact clipped Parquet name, not the ordinal or
+ // folded name. Distinct-id fields named A/A repeat, while A/a do not.
+ let physical_name = file[physical].name();
+ if !seen.insert(physical_name.as_str()) {
+ return Err(SparkError::ParquetFieldRequestedMoreThanOnce {
+ requested_fields: requested
+ .iter()
+ .zip(&selected)
+ .filter(|(_, selection)| {
+ selection.is_some_and(|index| file[index].name() == physical_name)
+ })
+ .map(|(field, _)| field.name().clone())
+ .collect(),
+ parquet_field: file[physical].name().clone(),
+ }
+ .into());
+ }
+ }
+ Ok(())
+}
+
+fn validate_nested_selection(
+ file: &DataType,
+ requested: &DataType,
+ case_sensitive: bool,
+ use_field_id: bool,
+) -> DataFusionResult<()> {
+ match (file, requested) {
+ (DataType::Dictionary(_, value), _) => {
+ validate_nested_selection(value, requested, case_sensitive, use_field_id)
+ }
+ (DataType::Struct(file), DataType::Struct(requested)) => {
+ validate_field_selection(file, requested, case_sensitive, use_field_id)
+ }
+ (
+ DataType::List(file)
+ | DataType::LargeList(file)
+ | DataType::FixedSizeList(file, _)
+ | DataType::ListView(file)
+ | DataType::LargeListView(file),
+ DataType::List(requested)
+ | DataType::LargeList(requested)
+ | DataType::FixedSizeList(requested, _)
+ | DataType::ListView(requested)
+ | DataType::LargeListView(requested),
+ ) => validate_nested_selection(
+ file.data_type(),
+ requested.data_type(),
+ case_sensitive,
+ use_field_id,
+ ),
+ (DataType::Map(file, _), DataType::Map(requested, _)) => {
+ // Map key/value wrappers pair positionally, as Spark clips them;
+ // only structs inside a key or value resolve fields by name/id.
+ if let (DataType::Struct(file), DataType::Struct(requested)) =
+ (file.data_type(), requested.data_type())
+ {
+ for (file, requested) in file.iter().zip(requested) {
+ validate_nested_selection(
+ file.data_type(),
+ requested.data_type(),
+ case_sensitive,
+ use_field_id,
+ )?;
+ }
+ }
+ Ok(())
+ }
+ _ => Ok(()), // Existing conversion validation handles differing shapes.
+ }
+}
+
/// The error for requested field `name` when the file fields at `matches` all share its folded
/// name: Spark's `foundDuplicateFieldInCaseInsensitiveModeError` when the read is
/// case-insensitive, and Comet's duplicate field name error for a repeated exact name, where Spark
@@ -260,9 +376,11 @@ impl Indices {
#[cfg(test)]
mod test {
- use super::{FieldResolver, Selection};
+ use super::{validate_field_selection, FieldResolver, Selection};
use crate::parquet::name_fold::fold_names;
use arrow::datatypes::{DataType, Field, FieldRef};
+ use datafusion::common::DataFusionError;
+ use datafusion_comet_common::SparkError;
use parquet::arrow::PARQUET_FIELD_ID_META_KEY;
use std::collections::HashMap;
use std::sync::Arc;
@@ -349,4 +467,180 @@ mod test {
assert_eq!(resolver.select(0).unwrap(), Selection::ByName(&[1]));
});
}
+
+ fn repeated_selection(error: DataFusionError) -> (Vec, String) {
+ let DataFusionError::External(source) = error else {
+ panic!("expected structured repeated-field error, got {error}");
+ };
+ let Some(SparkError::ParquetFieldRequestedMoreThanOnce {
+ requested_fields,
+ parquet_field,
+ }) = source.downcast_ref::()
+ else {
+ panic!("unexpected error: {source}");
+ };
+ (requested_fields.clone(), parquet_field.clone())
+ }
+
+ #[test]
+ fn repeated_field_id_and_name_selection_is_rejected_in_both_orders_and_case_modes() {
+ let file = [field("A", Some(1))];
+ for (case_sensitive, by_name) in [(true, "A"), (false, "a")] {
+ for reverse in [false, true] {
+ let mut requested = vec![field("x", Some(1)), field(by_name, None)];
+ if reverse {
+ requested.reverse();
+ }
+ let error =
+ validate_field_selection(&file, &requested, case_sensitive, true).unwrap_err();
+ assert_eq!(
+ repeated_selection(error),
+ (
+ requested.iter().map(|field| field.name().clone()).collect(),
+ "A".into()
+ )
+ );
+ // An unreferenced alias must not reject a projected read.
+ for projected in requested.chunks(1) {
+ validate_field_selection(&file, projected, case_sensitive, true).unwrap();
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn repeated_field_id_selection_is_rejected_but_missing_and_distinct_fields_are_valid() {
+ let file = [field("A", Some(1)), field("B", Some(2))];
+ let repeated = [field("x", Some(1)), field("y", Some(1))];
+ assert_eq!(
+ repeated_selection(
+ validate_field_selection(&file, &repeated, false, true).unwrap_err()
+ ),
+ (vec!["x".into(), "y".into()], "A".into())
+ );
+ let distinct = [
+ field("x", Some(1)),
+ field("b", None),
+ field("missing", Some(9)),
+ ];
+ validate_field_selection(&file, &distinct, false, true).unwrap();
+ // With id reads off, x is missing and a reads the file by name once.
+ let name_only = [field("x", Some(1)), field("a", None)];
+ validate_field_selection(&file, &name_only, false, false).unwrap();
+ }
+
+ #[test]
+ fn repeated_case_folded_name_selection_matches_the_running_spark_contract() {
+ let file = [field("A", None)];
+ let requested = [field("a", None), field("A", None)];
+ assert_eq!(
+ repeated_selection(
+ validate_field_selection(&file, &requested, false, false).unwrap_err()
+ ),
+ (vec!["a".into(), "A".into()], "A".into())
+ );
+ validate_field_selection(&file, &requested, true, false).unwrap();
+ }
+
+ #[test]
+ fn repeated_selection_is_checked_below_struct_list_and_map_without_rejecting_pruning() {
+ let physical = DataType::Struct(vec![field("A", Some(1))].into());
+ let repeated = DataType::Struct(vec![field("x", Some(1)), field("a", None)].into());
+ let pruned = DataType::Struct(vec![field("x", Some(1))].into());
+ let wrappers: Vec DataType>> = vec![
+ Box::new(|value| value),
+ Box::new(|value| DataType::List(Arc::new(Field::new("element", value, true)))),
+ Box::new(|value| DataType::LargeList(Arc::new(Field::new("element", value, true)))),
+ Box::new(|value| {
+ DataType::Map(
+ Arc::new(Field::new(
+ "entries",
+ DataType::Struct(
+ vec![
+ Arc::new(Field::new("key", DataType::Utf8, false)),
+ Arc::new(Field::new("value", value, true)),
+ ]
+ .into(),
+ ),
+ false,
+ )),
+ false,
+ )
+ }),
+ ];
+ for wrap in wrappers {
+ let file = [Arc::new(Field::new("s", wrap(physical.clone()), true))];
+ let requested = [Arc::new(Field::new("s", wrap(repeated.clone()), true))];
+ assert_eq!(
+ repeated_selection(
+ validate_field_selection(&file, &requested, false, true).unwrap_err()
+ ),
+ (vec!["x".into(), "a".into()], "A".into())
+ );
+ let requested = [Arc::new(Field::new("s", wrap(pruned.clone()), true))];
+ validate_field_selection(&file, &requested, false, true).unwrap();
+ }
+ }
+
+ #[test]
+ fn nested_selection_errors_precede_group_repetitions_and_later_lookup_errors() {
+ let physical_struct = Arc::new(
+ Field::new(
+ "s",
+ DataType::Struct(vec![field("A", Some(1))].into()),
+ true,
+ )
+ .with_metadata(field("s", Some(10)).metadata().clone()),
+ );
+ let requested_struct = Arc::new(
+ Field::new(
+ "s",
+ DataType::Struct(vec![field("nested_x", Some(1)), field("a", None)].into()),
+ true,
+ )
+ .with_metadata(field("s", Some(10)).metadata().clone()),
+ );
+ // The current group repeats B before visiting the struct. Spark still
+ // reports the struct's A collision, since clipping recurses before the
+ // current group's repeated-column check.
+ let file = [field("B", Some(20)), Arc::clone(&physical_struct)];
+ let requested = [
+ field("root_x", Some(20)),
+ field("b", None),
+ Arc::clone(&requested_struct),
+ ];
+ assert_eq!(
+ repeated_selection(
+ validate_field_selection(&file, &requested, false, true).unwrap_err()
+ ),
+ (vec!["nested_x".into(), "a".into()], "A".into())
+ );
+ // A later sibling has an ambiguous field-id lookup, which must not
+ // hide the earlier struct's collision either.
+ let file = [physical_struct, field("B", Some(20)), field("C", Some(20))];
+ let requested = [requested_struct, field("root_x", Some(20))];
+ assert_eq!(
+ repeated_selection(
+ validate_field_selection(&file, &requested, false, true).unwrap_err()
+ ),
+ (vec!["nested_x".into(), "a".into()], "A".into())
+ );
+ }
+
+ #[test]
+ fn repeated_selection_uses_exact_physical_names_not_ordinals_or_folded_names() {
+ let requested = [field("x", Some(1)), field("y", Some(2))];
+ let repeated_names = [field("A", Some(1)), field("A", Some(2))];
+ assert_eq!(
+ repeated_selection(
+ validate_field_selection(&repeated_names, &requested, false, true).unwrap_err()
+ ),
+ (vec!["x".into(), "y".into()], "A".into())
+ );
+ // ID matches are unambiguous despite the case-insensitive name index.
+ // Spark's clipped-name repetition check is case-sensitive.
+ let distinct_names = [field("A", Some(1)), field("a", Some(2))];
+ validate_field_selection(&distinct_names, &requested, false, true).unwrap();
+ validate_field_selection(&distinct_names, &requested, true, true).unwrap();
+ }
}
diff --git a/native/core/src/parquet/parquet_exec.rs b/native/core/src/parquet/parquet_exec.rs
index 88f94cb18cf..56f7eb64e57 100644
--- a/native/core/src/parquet/parquet_exec.rs
+++ b/native/core/src/parquet/parquet_exec.rs
@@ -105,6 +105,7 @@ pub(crate) fn init_datasource_exec(
rebase_from_file_metadata: bool,
datetime_rebase_mode_in_read: &str,
int96_rebase_mode_in_read: &str,
+ reject_duplicate_field_selection: bool,
) -> Result, ExecutionError> {
// Computed once and reused below for `try_pushdown_filters`. `copied_config()` clones only
// `SessionConfig` (an `Arc` plus a small extensions map); `SessionContext::
@@ -264,6 +265,7 @@ pub(crate) fn init_datasource_exec(
scan_io_source,
parquet_source.metrics(),
)
+ .with_reject_duplicate_field_selection(reject_duplicate_field_selection)
.with_spark_variant_schema(projects_variant)
.with_require_field_ids(require_field_ids)
.with_int96_leaf_stamp(rebase_from_file_metadata)
@@ -572,6 +574,7 @@ mod tests {
false,
"",
"",
+ false,
)
.unwrap()
}
@@ -709,6 +712,7 @@ mod tests {
rebase_from_file_metadata,
"",
"",
+ false,
)
.unwrap();
@@ -785,6 +789,7 @@ mod tests {
true,
datetime_rebase_mode,
datetime_rebase_mode,
+ false,
)
.unwrap();
@@ -894,6 +899,7 @@ mod tests {
true,
"CORRECTED",
"CORRECTED",
+ false,
)
.unwrap();
@@ -990,6 +996,7 @@ mod tests {
true,
datetime_rebase_mode,
int96_rebase_mode,
+ false,
)
.unwrap();
@@ -1137,6 +1144,7 @@ mod tests {
true,
datetime_rebase_mode,
int96_rebase_mode,
+ false,
)
.unwrap();
@@ -1406,6 +1414,7 @@ mod tests {
false,
"",
"",
+ false,
)
.unwrap();
diff --git a/native/core/src/parquet/parquet_exec/variant_tests.rs b/native/core/src/parquet/parquet_exec/variant_tests.rs
index 0bfdcecd222..4d8e799a659 100644
--- a/native/core/src/parquet/parquet_exec/variant_tests.rs
+++ b/native/core/src/parquet/parquet_exec/variant_tests.rs
@@ -166,6 +166,7 @@ async fn scan_variant_file(filename: PathBuf) -> VariantArray {
false,
"",
"",
+ false,
)
.unwrap();
let mut stream = scan.execute(0, session_ctx.task_ctx()).unwrap();
@@ -234,6 +235,7 @@ async fn unread_variant_does_not_override_arrow_schema_hint() {
false,
"",
"",
+ false,
)
.unwrap();
let mut stream = scan.execute(0, session.task_ctx()).unwrap();
@@ -267,6 +269,7 @@ fn encrypted_projected_variant_is_rejected_before_reader_creation() {
false,
"",
"",
+ false,
);
assert!(result
.unwrap_err()
diff --git a/native/proto/src/proto/expr.proto b/native/proto/src/proto/expr.proto
index 25abcd2f24f..209fbb6c607 100644
--- a/native/proto/src/proto/expr.proto
+++ b/native/proto/src/proto/expr.proto
@@ -465,6 +465,8 @@ message Cast {
// True when the running Spark casts a zero string with any exponent ("0e39") to decimal as
// 0, instead of applying the integral-digit limit to it (SPARK-60118).
bool decimal_zero_any_exponent = 7;
+ // SPARK-60119 avoids BigDecimal scale overflow for values that round to zero.
+ bool decimal_tiny_values_to_zero = 8;
}
// Spark's internal PreciseTimestampConversion, used by time-window grouping to convert
diff --git a/native/proto/src/proto/operator.proto b/native/proto/src/proto/operator.proto
index 5bc1ac12aa7..c23cb14cb7f 100644
--- a/native/proto/src/proto/operator.proto
+++ b/native/proto/src/proto/operator.proto
@@ -185,6 +185,9 @@ message NativeScanCommon {
// True when Spark supplied a Parquet data filter, including when Comet could
// not serialize any of those filters into data_filters.
bool has_data_filters = 19;
+ // SPARK-60262: reject multiple requested fields selecting the same physical
+ // Parquet column. Enabled only when the running Spark has this read contract.
+ bool reject_duplicate_field_selection = 20;
}
message NativeScan {
diff --git a/native/spark-expr/src/conversion_funcs/cast.rs b/native/spark-expr/src/conversion_funcs/cast.rs
index 3cff045c5d4..cb78fa0f9ce 100644
--- a/native/spark-expr/src/conversion_funcs/cast.rs
+++ b/native/spark-expr/src/conversion_funcs/cast.rs
@@ -172,6 +172,9 @@ pub struct SparkCastOptions {
/// True when the running Spark casts a zero string with any exponent ("0e39") to decimal as
/// 0 instead of applying the integral-digit limit to it (SPARK-60118).
pub decimal_zero_any_exponent: bool,
+ /// SPARK-60119 rounds extremely small decimals to zero instead of overflowing
+ /// BigDecimal's scale adjustment. This is separate from the zero-input fix.
+ pub decimal_tiny_values_to_zero: bool,
/// Support casting unsigned ints to signed ints (used by Parquet SchemaAdapter)
pub allow_cast_unsigned_ints: bool,
/// We also use the cast logic for adapting Parquet schemas, so this flag is used
@@ -191,6 +194,7 @@ impl SparkCastOptions {
timezone: timezone.to_string(),
is_spark4_plus: false,
decimal_zero_any_exponent: false,
+ decimal_tiny_values_to_zero: false,
allow_cast_unsigned_ints: false,
is_adapting_schema: false,
null_string: "null".to_string(),
@@ -204,6 +208,7 @@ impl SparkCastOptions {
timezone: "".to_string(),
is_spark4_plus: false,
decimal_zero_any_exponent: false,
+ decimal_tiny_values_to_zero: false,
allow_cast_unsigned_ints: false,
is_adapting_schema: false,
null_string: "null".to_string(),
@@ -316,6 +321,7 @@ pub(crate) fn cast_array(
scale,
eval_mode,
cast_options.decimal_zero_any_exponent,
+ cast_options.decimal_tiny_values_to_zero,
),
(Utf8 | LargeUtf8, Decimal256(precision, scale)) => cast_string_to_decimal(
&array,
@@ -324,6 +330,7 @@ pub(crate) fn cast_array(
scale,
eval_mode,
cast_options.decimal_zero_any_exponent,
+ cast_options.decimal_tiny_values_to_zero,
),
(Decimal128(input_precision, input_scale), Decimal128(output_precision, output_scale))
if eval_mode == EvalMode::Ansi =>
diff --git a/native/spark-expr/src/conversion_funcs/string.rs b/native/spark-expr/src/conversion_funcs/string.rs
index 981cc0e5ced..6b2599dc0d9 100644
--- a/native/spark-expr/src/conversion_funcs/string.rs
+++ b/native/spark-expr/src/conversion_funcs/string.rs
@@ -335,14 +335,25 @@ pub(crate) fn cast_string_to_decimal(
scale: &i8,
eval_mode: EvalMode,
zero_any_exponent: bool,
+ tiny_values_to_zero: bool,
) -> SparkResult {
match to_type {
- DataType::Decimal128(_, _) => {
- cast_string_to_decimal128_impl(array, eval_mode, *precision, *scale, zero_any_exponent)
- }
- DataType::Decimal256(_, _) => {
- cast_string_to_decimal256_impl(array, eval_mode, *precision, *scale, zero_any_exponent)
- }
+ DataType::Decimal128(_, _) => cast_string_to_decimal128_impl(
+ array,
+ eval_mode,
+ *precision,
+ *scale,
+ zero_any_exponent,
+ tiny_values_to_zero,
+ ),
+ DataType::Decimal256(_, _) => cast_string_to_decimal256_impl(
+ array,
+ eval_mode,
+ *precision,
+ *scale,
+ zero_any_exponent,
+ tiny_values_to_zero,
+ ),
_ => Err(SparkError::Internal(format!(
"Unexpected type in cast_string_to_decimal: {:?}",
to_type
@@ -356,6 +367,7 @@ fn cast_string_to_decimal128_impl(
precision: u8,
scale: i8,
zero_any_exponent: bool,
+ tiny_values_to_zero: bool,
) -> SparkResult {
let string_array = array
.as_any()
@@ -369,7 +381,13 @@ fn cast_string_to_decimal128_impl(
decimal_builder.append_null();
} else {
let str_value = string_array.value(i);
- match parse_string_to_decimal(str_value, precision, scale, zero_any_exponent) {
+ match parse_string_to_decimal(
+ str_value,
+ precision,
+ scale,
+ zero_any_exponent,
+ tiny_values_to_zero,
+ ) {
Ok(Some(decimal_value)) => {
decimal_builder.append_value(decimal_value);
}
@@ -420,6 +438,7 @@ fn cast_string_to_decimal256_impl(
precision: u8,
scale: i8,
zero_any_exponent: bool,
+ tiny_values_to_zero: bool,
) -> SparkResult {
let string_array = array
.as_any()
@@ -433,7 +452,13 @@ fn cast_string_to_decimal256_impl(
decimal_builder.append_null();
} else {
let str_value = string_array.value(i);
- match parse_string_to_decimal(str_value, precision, scale, zero_any_exponent) {
+ match parse_string_to_decimal(
+ str_value,
+ precision,
+ scale,
+ zero_any_exponent,
+ tiny_values_to_zero,
+ ) {
Ok(Some(decimal_value)) => {
// Convert i128 to i256
let i256_value = i256::from_i128(decimal_value);
@@ -674,11 +699,14 @@ fn is_special_value(trimmed: &str) -> bool {
/// apply the integral-digit limit to zero too, so such a string returned NULL (or raised
/// NUMERIC_OUT_OF_SUPPORTED_RANGE under ANSI). Since SPARK-60118 Spark skips that limit for zero,
/// which fits any decimal type, and casts it to 0.
+/// `tiny_values_to_zero` selects SPARK-60119's rescaling fast path, independently
+/// of the zero-input fix, so older Spark releases retain their error behavior.
fn parse_string_to_decimal(
input_str: &str,
precision: u8,
scale: i8,
zero_any_exponent: bool,
+ tiny_values_to_zero: bool,
) -> SparkResult