diff --git a/.github/CI-ARCHITECTURE.md b/.github/CI-ARCHITECTURE.md
index ebdcaf541c4ae..9e0279944e496 100644
--- a/.github/CI-ARCHITECTURE.md
+++ b/.github/CI-ARCHITECTURE.md
@@ -106,7 +106,7 @@ The script also:
- Generates a unified PR comment with all test information
- Parses Maven reactor output from `incremental-test.log` and reports **per-module elapsed time**, total reactor duration, and the top 20 slowest modules (see `reactor_timing.sh`)
-Unit tests for reactor timing parsing live in `reactor_timing_test.sh`.
+Unit tests for reactor timing parsing live in `reactor_timing_test.sh`, and for getting the PR diff in `pr_diff_test.sh`. The PR CI scripts validation workflow runs every `*_test.sh`.
#### Recovered flake reporting (`collect-flakes.py`)
@@ -238,7 +238,7 @@ The comparison is apples-to-apples: the current approach's reactor is filtered t
The script overrides `fullBuildTriggers` to empty (`-Dscalpel.fullBuildTriggers=`) because Scalpel's default (`.mvn/**`) would trigger a full build whenever `.mvn/extensions.xml` itself changes (e.g., Dependabot bumping Scalpel).
-The grep-based script fetches the PR diff via the GitHub REST API (unchanged). Scalpel uses local git history to compare effective POM models and detect source file changes — the CI workflow progressively deepens the shallow clone (`50 → 200 → 1000 → full`) until the merge-base is reachable. Scalpel runs for **all PRs** (not just POM changes) so the shadow comparison covers source-only changes too, building confidence for a future switch to Scalpel-driven builds. Scalpel disables its built-in JGit fetch (`-Dscalpel.fetchBaseBranch=false`) because the CI workflow already pre-fetches the base branch with native git (Scalpel uses JGit 5.x which has limited shallow clone support).
+The grep-based script fetches the PR diff via the GitHub REST API. When the API does not return it (HTTP 406 for a PR beyond the GitHub diff limits, such as more than 300 files), the diff is computed from the local checkout against its merge base with the base branch, fetching the base branch when the checkout does not have it (camel-spring-boot runs this action after a depth-1 checkout); without either, the step fails rather than pass without testing (`pr_diff.sh`). Scalpel uses local git history to compare effective POM models and detect source file changes — the CI workflow progressively deepens the shallow clone (`50 → 200 → 1000 → full`, `fetchBaseBranch` in `pr_diff.sh`) until the merge-base is reachable. Scalpel runs for **all PRs** (not just POM changes) so the shadow comparison covers source-only changes too, building confidence for a future switch to Scalpel-driven builds. Scalpel disables its built-in JGit fetch (`-Dscalpel.fetchBaseBranch=false`) because the CI workflow already pre-fetches the base branch with native git (Scalpel uses JGit 5.x which has limited shallow clone support).
## Manual Integration Test Advisories
diff --git a/.github/actions/incremental-build/incremental-build.sh b/.github/actions/incremental-build/incremental-build.sh
index fe96bed265a40..bd311a37a0dcb 100755
--- a/.github/actions/incremental-build/incremental-build.sh
+++ b/.github/actions/incremental-build/incremental-build.sh
@@ -39,6 +39,8 @@ trap '' PIPE
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
# shellcheck source=reactor_timing.sh
source "${SCRIPT_DIR}/reactor_timing.sh"
+# shellcheck source=pr_diff.sh
+source "${SCRIPT_DIR}/pr_diff.sh"
echo "Using MVND_OPTS=$MVND_OPTS"
echo "Using MAVEN_EXTRA_ARGS=${MAVEN_EXTRA_ARGS:-}"
@@ -118,29 +120,6 @@ hasLabel() {
"https://api.github.com/repos/${repository}/issues/${issueNumber}/labels" | jq -r '.[].name' | { grep -c "$label" || true; }
}
-# Fetch the PR diff from the GitHub API. Returns the full unified diff.
-fetchDiff() {
- local prId="$1"
- local repository="$2"
-
- local diff_output
- diff_output=$(curl -s -w "\n%{http_code}" \
- -H "Authorization: Bearer ${GITHUB_TOKEN}" \
- -H "Accept: application/vnd.github.v3.diff" \
- "https://api.github.com/repos/${repository}/pulls/${prId}")
-
- local http_code
- http_code=$(echo "$diff_output" | tail -n 1)
- local diff_body
- diff_body=$(echo "$diff_output" | sed '$d')
-
- if [[ "$http_code" -lt 200 || "$http_code" -ge 300 || -z "$diff_body" ]]; then
- echo "WARNING: Failed to fetch PR diff (HTTP $http_code). Falling back to full build." >&2
- return
- fi
- echo "$diff_body"
-}
-
# ── POM dependency analysis (previously detect-dependencies) ───────────
#
# Uses the pre-#22022 approach: grep for ${property-name} references in
@@ -744,19 +723,23 @@ main() {
fi
fi
- # Fetch the diff (PR diff via API, or git diff for push builds)
+ # Fetch the diff (PR diff via API or the local checkout, or git diff for push builds)
local diff_body
if [ -n "$prId" ]; then
echo "Fetching PR #${prId} diff..."
- diff_body=$(fetchDiff "$prId" "$repository")
+ diff_body=$(fetchDiff "$prId" "$repository" "${GITHUB_BASE_REF:-main}")
+ if [ -z "$diff_body" ]; then
+ # Without the diff no module is selected: fail rather than pass without testing anything
+ echo "ERROR: Could not get the diff of PR #${prId}, so the modules to test are unknown"
+ exit 1
+ fi
else
echo "No PR ID, using git diff HEAD~1..."
diff_body=$(git diff HEAD~1 2>/dev/null || true)
- fi
-
- if [ -z "$diff_body" ]; then
- echo "Could not fetch diff, skipping tests"
- exit 0
+ if [ -z "$diff_body" ]; then
+ echo "Could not get the diff of HEAD~1, skipping tests"
+ exit 0
+ fi
fi
# ── Step 1: File-path analysis ──
diff --git a/.github/actions/incremental-build/pr_diff.sh b/.github/actions/incremental-build/pr_diff.sh
new file mode 100644
index 0000000000000..7e574e0e5bb23
--- /dev/null
+++ b/.github/actions/incremental-build/pr_diff.sh
@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+# Helpers that give the unified diff of a pull request.
+
+# The PR diff from the GitHub API, or nothing when the API does not return it.
+# The API refuses the diff of a PR beyond the GitHub diff limits (HTTP 406, for
+# example more than 300 files).
+fetchApiDiff() {
+ local prId="$1"
+ local repository="$2"
+
+ local diff_output
+ diff_output=$(curl -s -w "\n%{http_code}" \
+ -H "Authorization: Bearer ${GITHUB_TOKEN}" \
+ -H "Accept: application/vnd.github.v3.diff" \
+ "https://api.github.com/repos/${repository}/pulls/${prId}" || true)
+
+ local http_code
+ http_code=$(echo "$diff_output" | tail -n 1)
+ local diff_body
+ diff_body=$(echo "$diff_output" | sed '$d')
+
+ if [[ ! "$http_code" =~ ^2[0-9][0-9]$ || -z "$diff_body" ]]; then
+ local message
+ message=$(echo "$diff_body" | jq -r '.message // empty' 2>/dev/null || true)
+ echo "WARNING: The GitHub API did not return the PR diff (HTTP ${http_code:-none})${message:+: $message}" >&2
+ return 0
+ fi
+ echo "$diff_body"
+}
+
+# Fetches origin/ and deepens a shallow checkout until the merge
+# base with HEAD is reachable: 50, 200, 1000 commits, then the full history.
+# Never fails, the caller checks the merge base.
+fetchBaseBranch() {
+ local baseRef="$1"
+
+ local depth
+ for depth in 50 200 1000; do
+ git fetch --no-tags --deepen="$depth" origin "$baseRef" 2>/dev/null || true
+ git fetch --no-tags --depth="$depth" origin "${baseRef}:refs/remotes/origin/${baseRef}" 2>/dev/null || true
+ if git merge-base HEAD "origin/${baseRef}" >/dev/null 2>&1; then
+ echo "Merge base reachable at depth $depth" >&2
+ return 0
+ fi
+ echo "Merge base not reachable at depth $depth, deepening..." >&2
+ done
+ echo "Merge base still not reachable, fetching full history" >&2
+ git fetch --no-tags --unshallow origin "$baseRef" 2>/dev/null || true
+ git fetch --no-tags origin "${baseRef}:refs/remotes/origin/${baseRef}" 2>/dev/null || true
+}
+
+# The diff of HEAD against its merge base with origin/, or nothing
+# when the merge base cannot be fetched. Fetches the base branch when the
+# checkout does not have the merge base (callers that skip the fetch, such as
+# camel-spring-boot, which runs this action from main).
+localDiff() {
+ local baseRef="$1"
+
+ if ! git merge-base HEAD "origin/${baseRef}" >/dev/null 2>&1; then
+ fetchBaseBranch "$baseRef"
+ fi
+ local merge_base
+ if ! merge_base=$(git merge-base HEAD "origin/${baseRef}" 2>/dev/null); then
+ echo "WARNING: No merge base between HEAD and origin/${baseRef} in the local history" >&2
+ return 0
+ fi
+ git diff "$merge_base" HEAD
+}
+
+# The PR diff: from the GitHub API, else from the local checkout.
+fetchDiff() {
+ local prId="$1"
+ local repository="$2"
+ local baseRef="${3:-main}"
+
+ local diff_body
+ diff_body=$(fetchApiDiff "$prId" "$repository")
+ if [ -z "$diff_body" ]; then
+ echo "Computing the PR diff from the local checkout against origin/${baseRef}" >&2
+ diff_body=$(localDiff "$baseRef")
+ fi
+ echo "$diff_body"
+}
diff --git a/.github/actions/incremental-build/pr_diff_test.sh b/.github/actions/incremental-build/pr_diff_test.sh
new file mode 100755
index 0000000000000..0ce766856671b
--- /dev/null
+++ b/.github/actions/incremental-build/pr_diff_test.sh
@@ -0,0 +1,121 @@
+#!/usr/bin/env bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+set -euo pipefail
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=pr_diff.sh
+source "${SCRIPT_DIR}/pr_diff.sh"
+
+pass=0
+fail=0
+
+assert_eq() {
+ local expected="$1"
+ local actual="$2"
+ local message="$3"
+ if [[ "$expected" == "$actual" ]]; then
+ pass=$((pass + 1))
+ else
+ echo "FAIL: $message"
+ echo " expected: [$expected]"
+ echo " actual: [$actual]"
+ fail=$((fail + 1))
+ fi
+}
+
+changed_files() {
+ sed -n -e '/^diff --git a/p' | awk '{print $3}' | cut -b 3- | sort
+}
+
+# A repository whose origin/main moved on after the PR branch left it
+repo=$(mktemp -d)
+clone=$(mktemp -d)
+trap 'rm -rf "$repo" "$clone"' EXIT
+cd "$repo"
+git init -q -b main
+git config user.email test@example.com
+git config user.name test
+mkdir -p components/a components/b
+echo a > components/a/A.java
+git add -A && git commit -q -m base
+git checkout -q -b pr
+echo b > components/b/B.java
+echo a2 > components/a/A.java
+git add -A && git commit -q -m pr
+git checkout -q main
+echo c > main-only.txt
+git add -A && git commit -q -m "main moves on"
+git update-ref refs/remotes/origin/main main
+git checkout -q pr
+
+assert_eq "$(printf 'components/a/A.java\ncomponents/b/B.java')" "$(localDiff main | changed_files)" \
+ "local diff has the PR changes only, not the later changes of the base branch"
+assert_eq "" "$(localDiff other 2>/dev/null)" "no local diff without the base branch"
+
+# A pull_request build checks out the merge of the PR into the base branch
+git checkout -q --detach main
+git merge -q --no-edit pr
+assert_eq "$(printf 'components/a/A.java\ncomponents/b/B.java')" "$(localDiff main | changed_files)" \
+ "local diff of the merge commit has the PR changes only"
+git checkout -q pr
+
+# A PR that removes a component
+git checkout -q -b removal main
+git rm -q -r components/a
+git commit -q -m "remove a"
+assert_eq "components/a/A.java" "$(localDiff main | changed_files)" "local diff has the removed files"
+git checkout -q pr
+
+# A depth=1 checkout of the PR without the base branch, as camel-spring-boot does
+git clone -q --depth 1 --branch pr "file://${repo}" "$clone"
+assert_eq "" "$(git -C "$clone" rev-parse -q --verify origin/main)" "the shallow clone has no origin/main"
+assert_eq "$(printf 'components/a/A.java\ncomponents/b/B.java')" "$(cd "$clone" && localDiff main 2>/dev/null | changed_files)" \
+ "local diff fetches the base branch and deepens the checkout to the merge base"
+
+export GITHUB_TOKEN=test-token
+
+# The API refuses the diff of a PR beyond the diff limits
+curl() {
+ printf '%s\n%s' '{"message": "Sorry, the diff exceeded the maximum number of files (300).", "status": "406"}' 406
+}
+assert_eq "" "$(fetchApiDiff 1 apache/camel 2>/dev/null)" "no API diff on HTTP 406"
+assert_eq "WARNING: The GitHub API did not return the PR diff (HTTP 406): Sorry, the diff exceeded the maximum number of files (300)." \
+ "$(fetchApiDiff 1 apache/camel 2>&1 >/dev/null)" "the warning names the HTTP status and the API message"
+assert_eq "$(printf 'components/a/A.java\ncomponents/b/B.java')" "$(fetchDiff 1 apache/camel main 2>/dev/null | changed_files)" \
+ "falls back to the local diff on HTTP 406"
+assert_eq "" "$(fetchDiff 1 apache/camel other 2>/dev/null)" "no diff when neither the API nor the local history has it"
+
+# The API returns the diff
+curl() {
+ printf '%s\n%s' 'diff --git a/components/x/X.java b/components/x/X.java' 200
+}
+assert_eq "components/x/X.java" "$(fetchDiff 1 apache/camel main 2>/dev/null | changed_files)" "uses the API diff when there is one"
+
+# The API cannot be reached
+curl() {
+ return 6
+}
+assert_eq "$(printf 'components/a/A.java\ncomponents/b/B.java')" "$(fetchDiff 1 apache/camel main 2>/dev/null | changed_files)" \
+ "falls back to the local diff when curl fails"
+
+echo ""
+echo "pr_diff_test.sh: ${pass} passed, ${fail} failed"
+if [[ "$fail" -ne 0 ]]; then
+ exit 1
+fi
diff --git a/.github/workflows/pr-build-main.yml b/.github/workflows/pr-build-main.yml
index 9bb582972e30d..575f0e2f405a2 100644
--- a/.github/workflows/pr-build-main.yml
+++ b/.github/workflows/pr-build-main.yml
@@ -87,25 +87,11 @@ jobs:
- name: Fetch base branch for Scalpel change detection
if: ${{ !inputs.skip_full_build }}
run: |
- # Scalpel needs the merge base between HEAD and the base branch.
- # The checkout is depth=1; progressively deepen: 50 → 200 → 1000 → full.
- # Scalpel is observational — fetch failures must not break the build.
- BASE_REF="${GITHUB_BASE_REF:-main}"
- for depth in 50 200 1000; do
- git fetch --no-tags --deepen=$depth origin ${BASE_REF} 2>/dev/null || true
- git fetch --no-tags --depth=$depth origin "${BASE_REF}:refs/remotes/origin/${BASE_REF}" 2>/dev/null || true
- if git merge-base HEAD "origin/${BASE_REF}" >/dev/null 2>&1; then
- echo "Merge base reachable at depth $depth"
- break
- fi
- echo "Merge base not reachable at depth $depth, deepening..."
- done
- # If still not reachable, fetch full history as last resort
- if ! git merge-base HEAD "origin/${BASE_REF}" >/dev/null 2>&1; then
- echo "Merge base still not reachable, fetching full history"
- git fetch --no-tags --unshallow origin ${BASE_REF} 2>/dev/null || true
- git fetch --no-tags origin "${BASE_REF}:refs/remotes/origin/${BASE_REF}" 2>/dev/null || true
- fi
+ # Scalpel and the PR diff fallback (when the GitHub API refuses the
+ # diff) need the merge base between HEAD and the base branch.
+ # Fetch failures must not break the build: the diff fallback fetches again.
+ source .github/actions/incremental-build/pr_diff.sh
+ fetchBaseBranch "${GITHUB_BASE_REF:-main}"
- id: install-packages
uses: ./.github/actions/install-packages
- id: install-mvnd
@@ -267,21 +253,8 @@ jobs:
- name: Fetch base branch for Scalpel change detection
if: ${{ !inputs.skip_full_build }}
run: |
- BASE_REF="${GITHUB_BASE_REF:-main}"
- for depth in 50 200 1000; do
- git fetch --no-tags --deepen=$depth origin ${BASE_REF} 2>/dev/null || true
- git fetch --no-tags --depth=$depth origin "${BASE_REF}:refs/remotes/origin/${BASE_REF}" 2>/dev/null || true
- if git merge-base HEAD "origin/${BASE_REF}" >/dev/null 2>&1; then
- echo "Merge base reachable at depth $depth"
- break
- fi
- echo "Merge base not reachable at depth $depth, deepening..."
- done
- if ! git merge-base HEAD "origin/${BASE_REF}" >/dev/null 2>&1; then
- echo "Merge base still not reachable, fetching full history"
- git fetch --no-tags --unshallow origin ${BASE_REF} 2>/dev/null || true
- git fetch --no-tags origin "${BASE_REF}:refs/remotes/origin/${BASE_REF}" 2>/dev/null || true
- fi
+ source .github/actions/incremental-build/pr_diff.sh
+ fetchBaseBranch "${GITHUB_BASE_REF:-main}"
- id: install-packages
uses: ./.github/actions/install-packages
- id: install-mvnd
diff --git a/.github/workflows/pr-ci-scripts-validation.yml b/.github/workflows/pr-ci-scripts-validation.yml
index 1d1f9c8bdd958..987e475eb83d5 100644
--- a/.github/workflows/pr-ci-scripts-validation.yml
+++ b/.github/workflows/pr-ci-scripts-validation.yml
@@ -65,3 +65,11 @@ jobs:
run: |
cd .github/actions/incremental-build
uv run --with defusedxml python3 -m unittest discover --verbose
+
+ - name: Test the incremental build shell scripts
+ shell: bash
+ run: |
+ cd .github/actions/incremental-build
+ for test in *_test.sh; do
+ bash "$test"
+ done