Skip to content
CI

Preserve retrieval response fields in public contracts #1114

Sign in to view logs

Preserve retrieval response fields in public contracts

Preserve retrieval response fields in public contracts #1114

Workflow file for this run

.github/workflows/ci.yml at 26a3e56
name: CI
on:
push:
branches:
- main
pull_request:
branches:
- main
schedule:
- cron: "23 7 * * *"
workflow_dispatch:
permissions:
contents: read
jobs:
tests:
# No custom 'name' — GitHub auto-appends matrix values (e.g. "tests (ubuntu-latest)").
# Using ${{ matrix.os }} in name shows the raw expression when the job is skipped.
runs-on: ${{ matrix.os }}
timeout-minutes: 45
if: github.event_name != 'schedule'
env:
# Keep the general CI test matrix on the JS indexer path; native coverage
# stays isolated in native-build because hosted runners have hit addon bugs.
SDL_MCP_DISABLE_NATIVE_ADDON: "1"
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
node-version:
- 24.x
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: ${{ matrix.node-version }}
cache: "npm"
- name: Cache node_modules
id: cache-node-modules
uses: actions/cache@v5
with:
path: node_modules
key: ${{ runner.os }}-node-${{ matrix.node-version }}-modules-${{ hashFiles('package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-${{ matrix.node-version }}-modules-
- name: Cache dist build artifacts
id: cache-dist
uses: actions/cache@v5
with:
path: |
dist
key: ${{ runner.os }}-dist-${{ hashFiles('src/**/*.ts', 'tsconfig*.json') }}
restore-keys: |
${{ runner.os }}-dist-
- name: Cache indexed memory
if: github.event_name == 'pull_request'
id: cache-memory
uses: actions/cache@v5
with:
path: |
data
.sdl-sync
key: ${{ runner.os }}-memory-${{ github.sha }}
restore-keys: |
${{ runner.os }}-memory-${{ github.base_ref }}-
- name: Install dependencies
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: npm ci --ignore-scripts --legacy-peer-deps
- name: Setup tree-sitter
uses: ./.github/actions/setup-tree-sitter
- name: Setup LadybugDB native binary
shell: bash
run: |
# Split packages (@sdl-mcp/ladybug-*): binary resolves via require.resolve(), no setup needed.
# Upstream kuzu (@ladybugdb/core): needs install.js to copy prebuilt binary.
if [ -f node_modules/kuzu/install.js ] && [ ! -f node_modules/kuzu/lbugjs.node ]; then
echo "Running upstream kuzu install.js to copy platform binary..."
cd node_modules/kuzu && node install.js
else
echo "LadybugDB binary available (split package or already installed)"
fi
- name: Build all
run: npm run build:all
- name: Typecheck
run: npm run typecheck
- name: Lint
run: npm run lint
- name: Check config sync (types.ts vs types.js defaults)
run: npm run check:config-sync
- name: Check tool inventory sync
run: npm run docs:tools:check
- name: Test tool output contract
run: npm run test:tool-output-contract
- name: Security audit
run: npm audit --audit-level=moderate
- name: Run tests (including 6 new language adapters + cross-platform paths)
shell: bash
run: |
TEST_OUT="${RUNNER_TEMP}/test-output.txt"
# Keep the last started file visible when a platform-specific child hangs.
set +e
npm test 2>&1 | tee "$TEST_OUT"
TEST_EXIT=${PIPESTATUS[0]}
set -e
if grep -q "^not ok" "$TEST_OUT"; then
FAILURES=true
else
FAILURES=false
fi
# Node 24 changed TAP output prefix from "# " to "ℹ " — accept both
if [ "$TEST_EXIT" -ne 0 ] || [ "$FAILURES" = "true" ] || ! grep -qE "(#|ℹ) fail 0" "$TEST_OUT"; then
echo "## Test Failures (${{ matrix.os }}, Node ${{ matrix.node-version }})" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
SUMMARY_LINES="${RUNNER_TEMP}/test-summary-lines.txt"
grep -E "not ok|failureType|error:.*'|# fail|ℹ fail|# tests|ℹ tests|# pass|ℹ pass" "$TEST_OUT" > "$SUMMARY_LINES" || true
head -80 "$SUMMARY_LINES" >> $GITHUB_STEP_SUMMARY
echo '```' >> $GITHUB_STEP_SUMMARY
exit 1
fi
- name: Upload test output on failure
if: failure()
uses: actions/upload-artifact@v7
with:
name: test-output-${{ matrix.os }}
path: ${{ runner.temp }}/test-output.txt
retention-days: 3
- name: Run test harness
run: npm run test:harness
- name: Verify runtime entrypoints
shell: bash
run: |
echo "Verifying CLI help exits cleanly..."
node dist/cli/index.js --help
echo "Verifying version command works..."
node dist/cli/index.js version
- name: Smoke test serve --stdio stays alive (Linux)
if: runner.os == 'Linux'
shell: bash
run: |
echo "Testing that serve --stdio does not exit prematurely on Linux..."
# Pipe stdin from a long-running process so the server has a live
# stdin handle (mirrors a real MCP client connection).
node dist/cli/index.js serve --stdio \
2>/tmp/sdl-mcp-serve-stderr.log \
< <(sleep 15) &
SERVER_PID=$!
# Give the server 2 seconds to fully start up
sleep 2
# Must still be running — silent exit is the reported bug
if ! kill -0 "${SERVER_PID}" 2>/dev/null; then
echo "[FAIL] serve --stdio exited prematurely on Linux"
echo "--- stderr output ---"
cat /tmp/sdl-mcp-serve-stderr.log || true
exit 1
fi
echo "[PASS] serve --stdio is still running after 2 s"
# Verify the startup message reached stderr
if grep -q "Starting MCP server on stdio transport" /tmp/sdl-mcp-serve-stderr.log; then
echo "[PASS] Startup message confirmed in stderr"
else
echo "[FAIL] Expected startup message not found in stderr"
echo "--- stderr output ---"
cat /tmp/sdl-mcp-serve-stderr.log || true
kill "${SERVER_PID}" 2>/dev/null || true
exit 1
fi
# Graceful teardown
kill "${SERVER_PID}" 2>/dev/null || true
wait "${SERVER_PID}" 2>/dev/null || true
rm -f /tmp/sdl-mcp-serve-stderr.log
echo "Smoke test passed"
- name: Display test summary
if: always()
shell: bash
run: |
echo "Tests completed on ${{ matrix.os }} with Node ${{ matrix.node-version }}"
echo "This validates cross-platform behavior for all language adapters."
benchmarks:
runs-on: ${{ matrix.os }}
if: github.event_name != 'schedule'
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
# Windows hosted runners can segfault in LadybugDB during benchmark:ci
# even with the Rust indexer addon disabled. Keep benchmark guardrails
# on Linux; Windows remains covered by tests and native parity.
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 24.x
cache: "npm"
- name: Cache node_modules
id: cache-node-modules
uses: actions/cache@v5
with:
path: node_modules
key: ${{ runner.os }}-node-24.x-modules-${{ hashFiles('package-lock.json') }}
restore-keys: |
${{ runner.os }}-node-24.x-modules-
- name: Cache dist build artifacts
id: cache-dist
uses: actions/cache@v5
with:
path: |
dist
key: ${{ runner.os }}-dist-${{ hashFiles('src/**/*.ts', 'tsconfig*.json') }}
restore-keys: |
${{ runner.os }}-dist-
- name: Install dependencies
if: steps.cache-node-modules.outputs.cache-hit != 'true'
run: npm ci --ignore-scripts --legacy-peer-deps
- name: Setup tree-sitter
uses: ./.github/actions/setup-tree-sitter
- name: Setup LadybugDB native binary
shell: bash
run: |
if [ -f node_modules/kuzu/install.js ] && [ ! -f node_modules/kuzu/lbugjs.node ]; then
echo "Running upstream kuzu install.js to copy platform binary..."
cd node_modules/kuzu && node install.js
else
echo "LadybugDB binary available (split package or already installed)"
fi
- name: Build all
run: npm run build:all
- name: Cache locked benchmark repos
id: cache-benchmark-repos
uses: actions/cache@v5
with:
path: .tmp/external-benchmarks
key: ${{ runner.os }}-benchmark-repos-${{ hashFiles('scripts/benchmark/matrix-external-repos.lock.json') }}
restore-keys: |
${{ runner.os }}-benchmark-repos-
- name: Setup locked benchmark repos
if: steps.cache-benchmark-repos.outputs.cache-hit != 'true'
run: npm run benchmark:setup-external
- name: Run Benchmark CI Guardrails (locked OSS repo)
shell: bash
env:
SDL_CONFIG: config/sdlmcp.config.json
# Keep benchmark:ci as a stable JS/LadybugDB regression guardrail.
# Native parser crash coverage lives in native-build and sync-memory;
# enabling it here has repeatedly produced Linux exit 139 flakes.
# benchmark:ci also forces legacy indexing and skips native SCIP generation.
SDL_MCP_DISABLE_NATIVE_ADDON: "1"
# Avoid overlapping pass-1 LadybugDB writes with native tree-sitter
# parsing in the same Node process; hosted runners have hit exit 139
# at that boundary even when the Rust addon is disabled.
SDL_MCP_PASS1_STABLE_DB_WRITES: "1"
run: |
echo "Running benchmark CI against locked OSS repo (zod-oss)..."
THRESHOLD_PATH="config/benchmark.ci.config.json"
echo "Using threshold config: ${THRESHOLD_PATH}"
npm run build
set +e
# Retry only the known native crash; threshold failures remain final.
for BENCHMARK_ATTEMPT in 1 2; do
SDL_GRAPH_DB_PATH="${RUNNER_TEMP}/benchmark-ci-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${BENCHMARK_ATTEMPT}.lbug" \
node dist/cli/index.js benchmark:ci --repo-id zod-oss --threshold-path "${THRESHOLD_PATH}" --json --out .benchmark/latest-result.json
BENCHMARK_EXIT_CODE=$?
if [ "$BENCHMARK_EXIT_CODE" -ne 139 ] || [ "$BENCHMARK_ATTEMPT" -eq 2 ]; then
break
fi
echo "[WARN] benchmark:ci exited 139; retrying once with a fresh graph"
done
set -e
echo "Benchmark exit code: ${BENCHMARK_EXIT_CODE}"
if [ $BENCHMARK_EXIT_CODE -ne 0 ]; then
echo "[FAIL] Benchmark guardrails failed"
echo "Review the results in .benchmark/latest-result.json"
exit 1
fi
echo "[PASS] Benchmark guardrails passed"
- name: Upload Benchmark CI result
if: always()
uses: actions/upload-artifact@v7
with:
name: benchmark-ci-result
path: .benchmark/latest-result.json
if-no-files-found: warn
retention-days: 7
- name: Auto-update benchmark baseline on main
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
shell: bash
run: |
if [ "${{ runner.os }}" = "Windows" ]; then
BASELINE_PATH=".benchmark/baseline.windows.json"
else
BASELINE_PATH=".benchmark/baseline.zod-oss.json"
fi
if [ -f .benchmark/latest-result.json ]; then
mkdir -p "$(dirname "$BASELINE_PATH")"
cp .benchmark/latest-result.json "$BASELINE_PATH"
echo "Baseline updated: $BASELINE_PATH"
else
echo "No result file found, skipping baseline update"
fi
- name: Setup external benchmark repos (matrix smoke)
if: matrix.os == 'ubuntu-latest'
run: npm run benchmark:setup-external
- name: Build benchmark matrix config (matrix smoke)
if: matrix.os == 'ubuntu-latest'
shell: bash
run: |
node -e "
const fs = require('fs');
const path = require('path');
const base = JSON.parse(fs.readFileSync('config/sdlmcp.config.json', 'utf8'));
const ext = JSON.parse(fs.readFileSync('benchmarks/real-world/external-repos.config.json', 'utf8'));
const cwd = process.cwd();
const resolved = (ext.repos || []).map(r => ({ ...r, rootPath: path.resolve(cwd, r.rootPath) }));
const merged = { ...base, repos: [...(base.repos || []), ...resolved] };
fs.writeFileSync('benchmarks/real-world/benchmark.config.json', JSON.stringify(merged, null, 2) + '\\n');
"
- name: Run benchmark matrix smoke
if: matrix.os == 'ubuntu-latest'
run: npm run benchmark:matrix -- --matrix benchmarks/real-world/matrix.json --config benchmarks/real-world/benchmark.config.json --out-dir .benchmark/matrix-smoke --limit-runs 2 --skip-index
- name: Validate benchmark matrix smoke claims
if: matrix.os == 'ubuntu-latest'
run: node --experimental-strip-types scripts/check-benchmark-claims.ts --in .benchmark/matrix-smoke/aggregate.json --profile smoke
background-integrity-benchmark:
runs-on: ${{ matrix.os }}
timeout-minutes: 30
if: github.event_name != 'schedule'
env:
SDL_MCP_DISABLE_NATIVE_ADDON: "1"
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 24.x
cache: "npm"
- name: Install dependencies
run: npm ci --ignore-scripts --legacy-peer-deps
- name: Setup tree-sitter
uses: ./.github/actions/setup-tree-sitter
- name: Setup LadybugDB native binary
shell: bash
run: |
if [ -f node_modules/kuzu/install.js ] && [ ! -f node_modules/kuzu/lbugjs.node ]; then
echo "Running upstream kuzu install.js to copy platform binary..."
cd node_modules/kuzu && node install.js
else
echo "LadybugDB binary available (split package or already installed)"
fi
- name: Build all
run: npm run build:all
- name: Run background graph integrity benchmark
run: npm run benchmark:background-graph-integrity -- --out .benchmark/background-graph-integrity/${{ runner.os }}-${{ github.sha }}.json
- name: Upload background graph integrity benchmark
if: always()
uses: actions/upload-artifact@v7
with:
name: background-graph-integrity-${{ runner.os }}
path: .benchmark/background-graph-integrity/${{ runner.os }}-${{ github.sha }}.json
if-no-files-found: error
native-build:
runs-on: ${{ matrix.os }}
if: github.event_name != 'schedule'
strategy:
fail-fast: false
matrix:
include:
- target: x86_64-pc-windows-msvc
os: windows-latest
run-tests: true
addon-file: sdl-mcp-native.win32-x64-msvc.node
- target: x86_64-apple-darwin
os: macos-latest
run-tests: false
addon-file: sdl-mcp-native.darwin-x64.node
- target: aarch64-apple-darwin
os: macos-latest
run-tests: true
addon-file: sdl-mcp-native.darwin-arm64.node
- target: x86_64-unknown-linux-gnu
os: ubuntu-latest
run-tests: true
addon-file: sdl-mcp-native.linux-x64-gnu.node
- target: x86_64-unknown-linux-musl
os: ubuntu-latest
run-tests: false
addon-file: sdl-mcp-native.linux-x64-musl.node
- target: aarch64-unknown-linux-gnu
os: ubuntu-latest
run-tests: false
addon-file: sdl-mcp-native.linux-arm64-gnu.node
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 24.x
cache: "npm"
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: ${{ matrix.target }}
- name: Install cross-compilation tools (aarch64-linux-gnu)
if: matrix.target == 'aarch64-unknown-linux-gnu'
run: |
sudo apt-get update
sudo apt-get install -y gcc-aarch64-linux-gnu g++-aarch64-linux-gnu
- name: Install cross-compilation tools (x86_64-linux-musl)
if: matrix.target == 'x86_64-unknown-linux-musl'
run: |
sudo apt-get update
sudo apt-get install -y musl-tools
- name: Cache Cargo registry
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
native/target
key: ${{ runner.os }}-${{ matrix.target }}-cargo-${{ hashFiles('native/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-${{ matrix.target }}-cargo-
- name: Install dependencies
run: npm ci --ignore-scripts --legacy-peer-deps
- name: Setup tree-sitter
uses: ./.github/actions/setup-tree-sitter
- name: Setup LadybugDB native binary
if: matrix.run-tests
shell: bash
run: |
if [ -f node_modules/kuzu/install.js ] && [ ! -f node_modules/kuzu/lbugjs.node ]; then
echo "Running upstream kuzu install.js to copy platform binary..."
cd node_modules/kuzu && node install.js
else
echo "LadybugDB binary available (split package or already installed)"
fi
- name: Build native addon
run: npx napi build --release --cargo-cwd native --config native/package.json --target ${{ matrix.target }} native
env:
CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
- name: Normalize native addon artifact name
shell: bash
run: |
SOURCE_ADDON_PATH="native/sdl-mcp-native.node"
TARGET_ADDON_PATH="native/${{ matrix.addon-file }}"
if [ ! -f "$SOURCE_ADDON_PATH" ]; then
echo "[ERROR] Built native addon not found at $SOURCE_ADDON_PATH"
find native -maxdepth 3 -name "*.node" -print || true
exit 1
fi
cp "$SOURCE_ADDON_PATH" "$TARGET_ADDON_PATH"
echo "Normalized native addon artifact: $TARGET_ADDON_PATH"
- name: Configure built native addon path
shell: bash
run: |
RELATIVE_ADDON_PATH="native/${{ matrix.addon-file }}"
if [ ! -f "$RELATIVE_ADDON_PATH" ]; then
echo "[ERROR] Built native addon not found at $RELATIVE_ADDON_PATH"
find native -maxdepth 3 -name "*.node" -print || true
exit 1
fi
ADDON_PATH=$(node -e "const path=require('path'); console.log(path.resolve(process.cwd(), process.argv[1]).replace(/\\\\/g, '/'))" "$RELATIVE_ADDON_PATH")
echo "SDL_MCP_NATIVE_ADDON_PATH=$ADDON_PATH" >> "$GITHUB_ENV"
echo "Using built native addon: $ADDON_PATH"
- name: Build TypeScript
if: matrix.run-tests
run: npm run build:all
- name: Run native parity tests
if: matrix.run-tests
run: npm run test:native-parity
- name: Run native full-index smoke
if: matrix.run-tests
run: npm run test:native-index-smoke
- name: Engine parity harness
if: matrix.run-tests
run: npm run test:parity
- name: Upload native addon
uses: actions/upload-artifact@v7
with:
name: native-addon-${{ matrix.target }}
path: native/${{ matrix.addon-file }}
retention-days: 7
benchmark-matrix-nightly:
name: Benchmark Matrix Nightly
runs-on: ubuntu-latest
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 24.x
cache: "npm"
- name: Install dependencies
run: npm ci --ignore-scripts --legacy-peer-deps
- name: Setup tree-sitter
uses: ./.github/actions/setup-tree-sitter
- name: Setup Rust toolchain
uses: dtolnay/rust-toolchain@stable
with:
targets: x86_64-unknown-linux-gnu
- name: Cache Cargo registry
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
native/target
key: ${{ runner.os }}-x86_64-unknown-linux-gnu-cargo-${{ hashFiles('native/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-x86_64-unknown-linux-gnu-cargo-
- name: Setup LadybugDB native binary
shell: bash
run: |
if [ -f node_modules/kuzu/install.js ] && [ ! -f node_modules/kuzu/lbugjs.node ]; then
echo "Running upstream kuzu install.js to copy platform binary..."
cd node_modules/kuzu && node install.js
else
echo "LadybugDB binary available (split package or already installed)"
fi
- name: Build native addon
run: npx napi build --release --cargo-cwd native --config native/package.json --target x86_64-unknown-linux-gnu native
- name: Normalize native addon artifact name
shell: bash
run: |
SOURCE_ADDON_PATH="native/sdl-mcp-native.node"
TARGET_ADDON_PATH="native/sdl-mcp-native.linux-x64-gnu.node"
if [ ! -f "$SOURCE_ADDON_PATH" ]; then
echo "[ERROR] Built native addon not found at $SOURCE_ADDON_PATH"
find native -maxdepth 3 -name "*.node" -print || true
exit 1
fi
cp "$SOURCE_ADDON_PATH" "$TARGET_ADDON_PATH"
echo "Normalized native addon artifact: $TARGET_ADDON_PATH"
- name: Configure Rust indexer addon path
shell: bash
run: |
RELATIVE_ADDON_PATH="native/sdl-mcp-native.linux-x64-gnu.node"
if [ ! -f "$RELATIVE_ADDON_PATH" ]; then
echo "[ERROR] Native addon not found at $RELATIVE_ADDON_PATH"
find native -maxdepth 3 -name "*.node" -print || true
exit 1
fi
ADDON_PATH=$(node -e "const path=require('path'); console.log(path.resolve(process.cwd(), process.argv[1]).replace(/\\\\/g, '/'))" "$RELATIVE_ADDON_PATH")
echo "SDL_MCP_NATIVE_ADDON_PATH=$ADDON_PATH" >> "$GITHUB_ENV"
echo "Using native addon: $ADDON_PATH"
- name: Build all
run: npm run build:all
- name: Cache benchmark matrix repos
id: cache-nightly-benchmark-repos
uses: actions/cache@v5
with:
path: .tmp/external-benchmarks
key: ${{ runner.os }}-benchmark-repos-${{ hashFiles('scripts/benchmark/matrix-external-repos.lock.json') }}
restore-keys: |
${{ runner.os }}-benchmark-repos-
- name: Setup external benchmark repos
run: npm run benchmark:setup-external
- name: Build benchmark matrix config
shell: bash
run: |
node -e "
const fs = require('fs');
const path = require('path');
const base = JSON.parse(fs.readFileSync('config/sdlmcp.config.json', 'utf8'));
const ext = JSON.parse(fs.readFileSync('benchmarks/real-world/external-repos.config.json', 'utf8'));
const cwd = process.cwd();
const resolved = (ext.repos || []).map(r => ({ ...r, rootPath: path.resolve(cwd, r.rootPath) }));
const merged = { ...base, repos: [...(base.repos || []), ...resolved] };
fs.writeFileSync('benchmarks/real-world/benchmark.config.json', JSON.stringify(merged, null, 2) + '\\n');
"
- name: Run full benchmark matrix
shell: bash
env:
# Keep benchmark matrix claim validation on the stable JS/LadybugDB path.
# Native parser crash coverage lives in native-build and sync-memory.
SDL_MCP_DISABLE_NATIVE_ADDON: "1"
# Serialize pass-1 DB writes against parser work to avoid hosted-runner exit 139 flakes.
SDL_MCP_PASS1_STABLE_DB_WRITES: "1"
run: npm run benchmark:matrix -- --force-legacy-index --matrix benchmarks/real-world/matrix.json --config benchmarks/real-world/benchmark.config.json --out-dir .benchmark/matrix-nightly
- name: Audit realism benchmark claims (non-blocking)
continue-on-error: true
run: node --experimental-strip-types scripts/check-benchmark-claims.ts --in .benchmark/matrix-nightly/aggregate.json --profile realism
- name: Summarize realism claim audit
if: always()
shell: bash
run: |
echo "Nightly benchmark matrix completed." >> "$GITHUB_STEP_SUMMARY"
echo "" >> "$GITHUB_STEP_SUMMARY"
echo "- External benchmark repos are pinned via \`scripts/benchmark/matrix-external-repos.lock.json\`." >> "$GITHUB_STEP_SUMMARY"
echo "- Realism claim validation is audit-only on the scheduled workflow until the benchmark matrix is retuned to satisfy the published claim floors again." >> "$GITHUB_STEP_SUMMARY"
- name: Upload matrix artifacts
if: always()
uses: actions/upload-artifact@v7
with:
name: benchmark-matrix-nightly
path: .benchmark/matrix-nightly/**
if-no-files-found: warn
retention-days: 14
sync-memory:
# Keep the job name static because skipped matrix jobs can surface the raw
# `${{ matrix.* }}` expression in the Actions UI before matrix expansion.
name: Sync Indexed Memory
runs-on: ${{ matrix.os }}
needs: [tests, native-build]
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
strategy:
fail-fast: false
matrix:
include:
- os: ubuntu-latest
node-version: 24.x
native-artifact: native-addon-x86_64-unknown-linux-gnu
native-addon-file: sdl-mcp-native.linux-x64-gnu.node
- os: windows-latest
node-version: 24.x
native-artifact: native-addon-x86_64-pc-windows-msvc
native-addon-file: sdl-mcp-native.win32-x64-msvc.node
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Setup Node.js ${{ matrix.node-version }}
uses: actions/setup-node@v6
with:
node-version: ${{ matrix.node-version }}
cache: "npm"
- name: Cache dist build artifacts
uses: actions/cache@v5
with:
path: |
dist
key: ${{ runner.os }}-dist-${{ hashFiles('src/**/*.ts', 'tsconfig*.json') }}
restore-keys: |
${{ runner.os }}-dist-
- name: Install dependencies
run: npm ci --ignore-scripts --legacy-peer-deps
- name: Setup tree-sitter
uses: ./.github/actions/setup-tree-sitter
- name: Setup LadybugDB native binary
shell: bash
run: |
if [ -f node_modules/kuzu/install.js ] && [ ! -f node_modules/kuzu/lbugjs.node ]; then
echo "Running upstream kuzu install.js to copy platform binary..."
cd node_modules/kuzu && node install.js
else
echo "LadybugDB binary available (split package or already installed)"
fi
- name: Download native addon artifact
uses: actions/download-artifact@v7
with:
name: ${{ matrix.native-artifact }}
path: native
- name: Configure Rust indexer addon path
shell: bash
run: |
RELATIVE_ADDON_PATH="native/${{ matrix.native-addon-file }}"
if [ ! -f "$RELATIVE_ADDON_PATH" ]; then
echo "[ERROR] Native addon not found at $RELATIVE_ADDON_PATH"
find native -maxdepth 3 -name "*.node" -print || true
exit 1
fi
ADDON_PATH=$(node -e "const path=require('path'); console.log(path.resolve(process.cwd(), process.argv[1]).replace(/\\\\/g, '/'))" "$RELATIVE_ADDON_PATH")
echo "SDL_MCP_NATIVE_ADDON_PATH=$ADDON_PATH" >> "$GITHUB_ENV"
echo "Using native addon: $ADDON_PATH"
- name: Build all
run: npm run build:all
- name: Cache locked benchmark repos
id: cache-sync-benchmark-repos
uses: actions/cache@v5
with:
path: .tmp/external-benchmarks
key: ${{ runner.os }}-benchmark-repos-${{ hashFiles('scripts/benchmark/matrix-external-repos.lock.json') }}
restore-keys: |
${{ runner.os }}-benchmark-repos-
- name: Setup locked benchmark repos
if: steps.cache-sync-benchmark-repos.outputs.cache-hit != 'true'
run: npm run benchmark:setup-external
- name: Get Git commit info
id: git-info
shell: bash
run: |
echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT
echo "branch=$(git rev-parse --abbrev-ref HEAD)" >> $GITHUB_OUTPUT
- name: Initialize SDL-MCP
run: |
npm run init || node dist/cli/index.js init --force
- name: Configure Linux core dump capture
if: runner.os == 'Linux'
shell: bash
run: |
sudo sysctl -w "kernel.core_pattern=${RUNNER_TEMP}/core.%e.%p" || \
echo "::warning::Unable to override kernel.core_pattern"
echo "core_pattern=$(cat /proc/sys/kernel/core_pattern)"
- name: Index repository for CI sync
id: index
shell: bash
env:
# Linux hosted runners still hit exit 139 under serialized native
# pass 1. Keep sync artifact generation on the stable TS path there;
# native crash coverage stays in native-build's native-index smoke.
SDL_MCP_DISABLE_NATIVE_ADDON: ${{ runner.os == 'Linux' && '1' || '' }}
# Windows keeps native sync coverage, but without chunk prefetch.
SDL_MCP_NATIVE_PASS1_SERIAL: "1"
# Serialize pass-1 DB flushes against parsing so sync-memory avoids
# the known tree-sitter / LadybugDB overlap crash path.
SDL_MCP_PASS1_STABLE_DB_WRITES: "1"
# Emit transaction-sized boundaries only in this diagnostic CI job.
SDL_MCP_SNAPSHOT_DIAGNOSTICS: "1"
run: |
echo "Starting CI memory index on ${{ runner.os }}..."
if [ "${RUNNER_OS}" = "Linux" ]; then
ulimit -c unlimited
fi
START_TIME=$(date +%s%3N)
set +e
SUCCESSFUL_DB_PATH=""
for INDEX_ATTEMPT in 1 2; do
# A native crash can leave its LadybugDB family unsafe to reopen.
SDL_GRAPH_DB_PATH="${RUNNER_TEMP}/sync-index-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}-${INDEX_ATTEMPT}.lbug"
export SDL_GRAPH_DB_PATH
node dist/cli/index.js index
INDEX_EXIT_CODE=$?
if [ "${INDEX_EXIT_CODE}" -eq 0 ]; then
SUCCESSFUL_DB_PATH="${SDL_GRAPH_DB_PATH}"
break
fi
if [ "${INDEX_EXIT_CODE}" -ne 139 ] || [ "${INDEX_ATTEMPT}" -eq 2 ]; then
break
fi
echo "[WARN] Index exited 139; retrying once with a fresh graph"
done
set -e
if [ "${INDEX_EXIT_CODE}" -ne 0 ]; then
END_TIME=$(date +%s%3N)
DURATION=$((END_TIME - START_TIME))
echo "duration_ms=${DURATION}" >> $GITHUB_OUTPUT
echo "[FAIL] Index failed in ${DURATION}ms with exit code ${INDEX_EXIT_CODE}"
exit "${INDEX_EXIT_CODE}"
fi
echo "SDL_GRAPH_DB_PATH=${SUCCESSFUL_DB_PATH}" >> "${GITHUB_ENV}"
END_TIME=$(date +%s%3N)
DURATION=$((END_TIME - START_TIME))
echo "duration_ms=${DURATION}" >> $GITHUB_OUTPUT
echo "[OK] Index completed in ${DURATION}ms"
- name: Collect Linux snapshot crash metadata
if: failure() && runner.os == 'Linux'
shell: bash
run: |
{
echo "node_version=$(node --version)"
LADYBUG_CORE_VERSION=$(node -p "require('kuzu/package.json').version" 2>/dev/null || true)
echo "ladybug_core_version=${LADYBUG_CORE_VERSION}"
echo "core_pattern=$(cat /proc/sys/kernel/core_pattern)"
echo "core_limit=$(ulimit -c)"
find "${RUNNER_TEMP}" -maxdepth 1 -type f \
\( -name 'sync-index-*' -o -name 'core*' \) \
-printf '%f %s bytes\n'
} > "${RUNNER_TEMP}/sync-crash-metadata.txt"
- name: Upload Linux snapshot crash diagnostics
if: failure() && runner.os == 'Linux'
uses: actions/upload-artifact@v7
with:
name: sync-snapshot-crash-${{ github.run_id }}-${{ github.run_attempt }}
path: |
${{ runner.temp }}/sync-index-${{ github.run_id }}-${{ github.run_attempt }}-*.lbug*
${{ runner.temp }}/core.*
${{ runner.temp }}/sync-crash-metadata.txt
if-no-files-found: warn
retention-days: 3
compression-level: 6
- name: Export sync artifact
id: export
shell: bash
run: |
echo "Exporting sync artifact for CI..."
START_TIME=$(date +%s%3N)
node dist/cli/index.js export \
--commit-sha "${{ steps.git-info.outputs.sha }}" \
--branch "${{ steps.git-info.outputs.branch }}" \
--output ./ci-artifacts/sync-${{ runner.os }}-${{ github.sha }}.sdl-artifact.json
END_TIME=$(date +%s%3N)
DURATION=$((END_TIME - START_TIME))
echo "duration_ms=${DURATION}" >> $GITHUB_OUTPUT
echo "[OK] Export completed in ${DURATION}ms"
- name: Validate artifact structure
shell: bash
run: |
echo "Validating artifact integrity on ${{ runner.os }}..."
ARTIFACT_FILE="./ci-artifacts/sync-${{ runner.os }}-${{ github.sha }}.sdl-artifact.json"
if [ -f "$ARTIFACT_FILE" ]; then
node -e "
const fs = require('fs');
const crypto = require('crypto');
const path = process.argv[1];
const payload = JSON.parse(fs.readFileSync(path, 'utf8'));
const required = ['artifact_id', 'repo_id', 'version_id', 'artifact_hash', 'compressed_data', 'commit_sha'];
for (const key of required) {
if (!(key in payload)) {
throw new Error('Missing required field: ' + key);
}
}
if (typeof payload.compressed_data !== 'string' || payload.compressed_data.length === 0) {
throw new Error('compressed_data must be a non-empty string');
}
const sha256 = crypto.createHash('sha256').update(fs.readFileSync(path)).digest('hex');
console.log('[OK] Artifact structure validated');
console.log('[INFO] artifact_id=' + payload.artifact_id);
console.log('[INFO] repo_id=' + payload.repo_id);
console.log('[INFO] file_sha256=' + sha256);
" "$ARTIFACT_FILE"
else
echo "[ERROR] Artifact file not found: $ARTIFACT_FILE"
exit 1
fi
- name: Upload sync artifact
uses: actions/upload-artifact@v7
with:
name: sync-artifact-${{ matrix.os }}
path: ./ci-artifacts/*.sdl-artifact.json
retention-days: 30
- name: Display sync metrics
if: always()
shell: bash
run: |
echo "=== CI Sync Metrics (${{ runner.os }}) ==="
echo "Commit SHA: ${{ steps.git-info.outputs.sha }}"
echo "Branch: ${{ steps.git-info.outputs.branch }}"
INDEX_DUR="${{ steps.index.outputs.duration_ms }}"
EXPORT_DUR="${{ steps.export.outputs.duration_ms }}"
echo "Index Duration: ${INDEX_DUR:-0}ms"
echo "Export Duration: ${EXPORT_DUR:-0}ms"
TOTAL_DURATION=$((${INDEX_DUR:-0} + ${EXPORT_DUR:-0}))
echo "Total Sync Duration: ${TOTAL_DURATION}ms"
- name: Validate sync performance budget (non-blocking)
if: always()
shell: bash
run: |
if [ "${{ runner.os }}" = "Windows" ]; then
# Windows runners have higher variance for filesystem-heavy indexing.
MAX_INDEX_MS=1000000
MAX_EXPORT_MS=60000
MAX_TOTAL_MS=1100000
else
MAX_INDEX_MS=750000
MAX_EXPORT_MS=60000
MAX_TOTAL_MS=820000
fi
INDEX_MS="${{ steps.index.outputs.duration_ms }}"
INDEX_MS="${INDEX_MS:-0}"
EXPORT_MS="${{ steps.export.outputs.duration_ms }}"
EXPORT_MS="${EXPORT_MS:-0}"
TOTAL_MS=$((INDEX_MS + EXPORT_MS))
BUDGET_WARN=false
echo "Validating performance budget..."
if [ $INDEX_MS -gt $MAX_INDEX_MS ]; then
echo "::warning::Index duration exceeded budget: ${INDEX_MS}ms > ${MAX_INDEX_MS}ms"
BUDGET_WARN=true
fi
if [ $EXPORT_MS -gt $MAX_EXPORT_MS ]; then
echo "::warning::Export duration exceeded budget: ${EXPORT_MS}ms > ${MAX_EXPORT_MS}ms"
BUDGET_WARN=true
fi
if [ $TOTAL_MS -gt $MAX_TOTAL_MS ]; then
echo "::warning::Total sync duration exceeded budget: ${TOTAL_MS}ms > ${MAX_TOTAL_MS}ms"
BUDGET_WARN=true
fi
if [ "$BUDGET_WARN" = "true" ]; then
echo "[WARN] Performance budget exceeded; recorded as non-blocking warning"
else
echo "[OK] Performance budget validated"
fi
echo " - Index: ${INDEX_MS}ms / ${MAX_INDEX_MS}ms"
echo " - Export: ${EXPORT_MS}ms / ${MAX_EXPORT_MS}ms"
echo " - Total: ${TOTAL_MS}ms / ${MAX_TOTAL_MS}ms"
sync-validation:
name: Validate Cross-Platform Sync
runs-on: ubuntu-latest
needs: sync-memory
if: github.event_name == 'push' && github.ref == 'refs/heads/main'
steps:
- name: Checkout code
uses: actions/checkout@v6
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 24.x
cache: "npm"
- name: Install dependencies
run: npm ci --ignore-scripts --legacy-peer-deps
- name: Setup tree-sitter
uses: ./.github/actions/setup-tree-sitter
- name: Setup LadybugDB native binary
shell: bash
run: |
if [ -f node_modules/kuzu/install.js ] && [ ! -f node_modules/kuzu/lbugjs.node ]; then
echo "Running upstream kuzu install.js to copy platform binary..."
cd node_modules/kuzu && node install.js
else
echo "LadybugDB binary available (split package or already installed)"
fi
- name: Build all
run: npm run build:all
- name: Download Linux artifact
uses: actions/download-artifact@v7
with:
name: sync-artifact-ubuntu-latest
path: ./artifacts/linux/
- name: Download Windows artifact
uses: actions/download-artifact@v7
with:
name: sync-artifact-windows-latest
path: ./artifacts/windows/
- name: Compare artifacts
shell: bash
run: |
echo "Comparing Linux and Windows sync artifacts..."
LINUX_ARTIFACT=$(find ./artifacts/linux -name "*.sdl-artifact.json" | head -n 1)
WINDOWS_ARTIFACT=$(find ./artifacts/windows -name "*.sdl-artifact.json" | head -n 1)
if [ -z "$LINUX_ARTIFACT" ] || [ -z "$WINDOWS_ARTIFACT" ]; then
echo "[ERROR] Missing artifacts for comparison"
exit 1
fi
echo "Linux artifact: $LINUX_ARTIFACT"
echo "Windows artifact: $WINDOWS_ARTIFACT"
node -e "
const fs = require('fs');
const linux = JSON.parse(fs.readFileSync('${LINUX_ARTIFACT}', 'utf-8'));
const windows = JSON.parse(fs.readFileSync('${WINDOWS_ARTIFACT}', 'utf-8'));
console.log('Artifact comparison:');
console.log(' Linux artifact_hash:', linux.artifact_hash);
console.log(' Windows artifact_hash:', windows.artifact_hash);
console.log(' Linux files extracted:', linux.compressed_data.length);
console.log(' Windows files extracted:', windows.compressed_data.length);
if (linux.artifact_hash === windows.artifact_hash) {
console.log('[OK] Artifacts match across platforms');
} else {
console.log('[WARN] Artifacts differ (expected for different commit SHAs)');
console.log(' Both artifacts are valid for their respective commits');
}
"
context-quality-scheduled:
runs-on: ${{ matrix.os }}
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
timeout-minutes: 180
strategy:
fail-fast: false
matrix:
os:
- ubuntu-latest
- windows-latest
steps:
- name: Checkout code
uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Checkout pinned context-quality corpus
uses: actions/checkout@v6
with:
ref: fcf4f2e11c5a1bb9b301a245af42b28556414b8e
path: .context-quality-corpus
fetch-depth: 1
persist-credentials: false
- name: Setup Node.js
uses: actions/setup-node@v6
with:
node-version: 24.x
cache: "npm"
- name: Install dependencies
run: npm ci --ignore-scripts --legacy-peer-deps
- name: Setup tree-sitter
uses: ./.github/actions/setup-tree-sitter
- name: Setup LadybugDB native binary
shell: bash
run: |
if [ -f node_modules/kuzu/install.js ] && [ ! -f node_modules/kuzu/lbugjs.node ]; then
echo "Running upstream kuzu install.js to copy platform binary..."
cd node_modules/kuzu && node install.js
else
echo "LadybugDB binary available (split package or already installed)"
fi
- name: Setup Windows Rust toolchain for FTS preloader
if: runner.os == 'Windows'
uses: dtolnay/rust-toolchain@stable
with:
targets: x86_64-pc-windows-msvc
- name: Cache Windows FTS preloader build
if: runner.os == 'Windows'
uses: actions/cache@v5
with:
path: |
~/.cargo/registry
~/.cargo/git
native/target
key: ${{ runner.os }}-x86_64-pc-windows-msvc-context-fts-${{ hashFiles('native/Cargo.lock') }}
restore-keys: |
${{ runner.os }}-x86_64-pc-windows-msvc-context-fts-
- name: Build and configure Windows FTS preloader
if: runner.os == 'Windows'
shell: pwsh
run: |
& npx napi build --release --cargo-cwd native --config native/package.json --target x86_64-pc-windows-msvc native
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
$addonPath = (Resolve-Path "native/sdl-mcp-native.node").Path
"SDL_MCP_NATIVE_ADDON_PATH=$addonPath" >> $env:GITHUB_ENV
- name: Build
run: npm run build
- name: Resolve immutable cache inputs
id: context-inputs
shell: pwsh
run: |
$inputJson = & node --input-type=module -e 'import { getModelCacheDir, getRequiredModelSetDigest } from "./scripts/postinstall-models.mjs"; import { LADYBUG_SCHEMA_VERSION } from "./dist/db/ladybug-schema.js"; console.log(JSON.stringify({ modelCache: getModelCacheDir(), modelSet: getRequiredModelSetDigest(), schema: LADYBUG_SCHEMA_VERSION }));'
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
$inputs = $inputJson | ConvertFrom-Json
"model-cache=$($inputs.modelCache)" >> $env:GITHUB_OUTPUT
"model-set=$($inputs.modelSet)" >> $env:GITHUB_OUTPUT
"schema=$($inputs.schema)" >> $env:GITHUB_OUTPUT
- name: Cache immutable embedding models
uses: actions/cache@v5
with:
path: ${{ steps.context-inputs.outputs.model-cache }}
key: ${{ runner.os }}-${{ runner.arch }}-context-models-${{ steps.context-inputs.outputs.model-set }}
- name: Verify required embedding models
run: node scripts/postinstall-models.mjs --strict
- name: Cache verified context-quality graph families
id: context-graph-cache
uses: actions/cache@v5
with:
path: .benchmark/context-quality-cache
key: ${{ runner.os }}-${{ runner.arch }}-schema${{ steps.context-inputs.outputs.schema }}-models-${{ steps.context-inputs.outputs.model-set }}-corpus-fcf4f2e11c5a1bb9b301a245af42b28556414b8e-configs-${{ hashFiles('tests/benchmark/config/context-quality-sdl-mcp.json', 'tests/benchmark/config/context-quality-neutral.json') }}
- name: Build both isolated graph families on cache miss
if: steps.context-graph-cache.outputs.cache-hit != 'true'
shell: pwsh
env:
SDL_MCP_PASS1_STABLE_DB_WRITES: "1"
run: |
$corpusRoot = (Resolve-Path ".context-quality-corpus").Path
$sdlConfig = (Resolve-Path "tests/benchmark/config/context-quality-sdl-mcp.json").Path
$neutralConfig = (Resolve-Path "tests/benchmark/config/context-quality-neutral.json").Path
$sdlSource = Join-Path $env:RUNNER_TEMP "context-quality-source-sdl-mcp.lbug"
$neutralSource = Join-Path $env:RUNNER_TEMP "context-quality-source-neutral.lbug"
$env:SDL_CONTEXT_QUALITY_REPO_ROOT = $corpusRoot
$env:SDL_CONFIG = $sdlConfig
& node dist/cli/index.js index --force --safe-rebuild $sdlSource
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
$env:SDL_CONTEXT_QUALITY_REPO_ROOT = Join-Path $corpusRoot "sdlbench/tests/fixtures/repo"
$env:SDL_CONFIG = $neutralConfig
& node dist/cli/index.js index --force --safe-rebuild $neutralSource
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
"SDL_CONTEXT_QUALITY_SDL_SOURCE=$sdlSource" >> $env:GITHUB_ENV
"SDL_CONTEXT_QUALITY_NEUTRAL_SOURCE=$neutralSource" >> $env:GITHUB_ENV
- name: Publish both verified graph families on cache miss
if: steps.context-graph-cache.outputs.cache-hit != 'true'
shell: pwsh
env:
SDL_CONTEXT_QUALITY_REPO_SHA: fcf4f2e11c5a1bb9b301a245af42b28556414b8e
SDL_CONTEXT_QUALITY_CACHE_PREPARE_ONLY: "1"
run: |
$corpusRoot = (Resolve-Path ".context-quality-corpus").Path
$cacheRoot = Join-Path $env:GITHUB_WORKSPACE ".benchmark/context-quality-cache"
$env:SDL_CONTEXT_QUALITY_REPO_ROOT = $corpusRoot
$env:SDL_CONTEXT_QUALITY_REPO_ID = "sdl-mcp"
$env:SDL_CONFIG = (Resolve-Path "tests/benchmark/config/context-quality-sdl-mcp.json").Path
$env:SDL_CONTEXT_QUALITY_MANIFEST_CONFIG_PATH = $env:SDL_CONFIG
$env:SDL_CONTEXT_QUALITY_CACHE_SOURCE_PATH = $env:SDL_CONTEXT_QUALITY_SDL_SOURCE
$env:SDL_CONTEXT_QUALITY_CACHE_PRIMARY_PATH = Join-Path $cacheRoot "sdl-mcp/graph.lbug"
$env:SDL_CONTEXT_QUALITY_CACHE_READY_PATH = Join-Path $cacheRoot "sdl-mcp/ready.json"
$env:SDL_CONTEXT_QUALITY_WORKING_DB_PATH = Join-Path $env:RUNNER_TEMP "context-quality-publish-sdl-mcp.lbug"
& node tests/benchmark/context-quality-runner.mjs
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
$env:SDL_CONTEXT_QUALITY_REPO_ROOT = Join-Path $corpusRoot "sdlbench/tests/fixtures/repo"
$env:SDL_CONTEXT_QUALITY_REPO_ID = "sdl-neutral-fixture"
$env:SDL_CONFIG = (Resolve-Path "tests/benchmark/config/context-quality-neutral.json").Path
$env:SDL_CONTEXT_QUALITY_MANIFEST_CONFIG_PATH = $env:SDL_CONFIG
$env:SDL_CONTEXT_QUALITY_CACHE_SOURCE_PATH = $env:SDL_CONTEXT_QUALITY_NEUTRAL_SOURCE
$env:SDL_CONTEXT_QUALITY_CACHE_PRIMARY_PATH = Join-Path $cacheRoot "neutral/graph.lbug"
$env:SDL_CONTEXT_QUALITY_CACHE_READY_PATH = Join-Path $cacheRoot "neutral/ready.json"
$env:SDL_CONTEXT_QUALITY_WORKING_DB_PATH = Join-Path $env:RUNNER_TEMP "context-quality-publish-neutral.lbug"
& node tests/benchmark/context-quality-runner.mjs
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
- name: Run paired context-quality evaluation from validated copies
shell: pwsh
env:
SDL_CONTEXT_QUALITY_REPO_SHA: fcf4f2e11c5a1bb9b301a245af42b28556414b8e
SDL_CONTEXT_QUALITY_REQUIRE_INDEX: "1"
SDL_CONTEXT_QUALITY_V2_SHADOW: "1"
run: |
$corpusRoot = (Resolve-Path ".context-quality-corpus").Path
$cacheRoot = Join-Path $env:GITHUB_WORKSPACE ".benchmark/context-quality-cache"
$resultRoot = Join-Path $env:GITHUB_WORKSPACE ".benchmark/context-quality-results"
New-Item -ItemType Directory -Path $resultRoot -Force | Out-Null
Remove-Item Env:SDL_CONTEXT_QUALITY_CACHE_SOURCE_PATH -ErrorAction SilentlyContinue
Remove-Item Env:SDL_CONTEXT_QUALITY_CACHE_PREPARE_ONLY -ErrorAction SilentlyContinue
$env:SDL_CONTEXT_QUALITY_CORPUS = "sdl-mcp"
$env:SDL_CONTEXT_QUALITY_REPO_ROOT = $corpusRoot
$env:SDL_CONTEXT_QUALITY_REPO_ID = "sdl-mcp"
$env:SDL_CONFIG = (Resolve-Path "tests/benchmark/config/context-quality-sdl-mcp.json").Path
$env:SDL_CONTEXT_QUALITY_MANIFEST_CONFIG_PATH = $env:SDL_CONFIG
$env:SDL_CONTEXT_QUALITY_CACHE_PRIMARY_PATH = Join-Path $cacheRoot "sdl-mcp/graph.lbug"
$env:SDL_CONTEXT_QUALITY_CACHE_READY_PATH = Join-Path $cacheRoot "sdl-mcp/ready.json"
$env:SDL_CONTEXT_QUALITY_WORKING_DB_PATH = Join-Path $env:RUNNER_TEMP "context-quality-run-sdl-mcp.lbug"
$env:SDL_CONTEXT_QUALITY_OUTPUT_PATH = Join-Path $resultRoot "sdl-mcp.json"
& node tests/benchmark/context-quality-runner.mjs
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
$env:SDL_CONTEXT_QUALITY_CORPUS = "neutral"
$env:SDL_CONTEXT_QUALITY_REPO_ROOT = Join-Path $corpusRoot "sdlbench/tests/fixtures/repo"
$env:SDL_CONTEXT_QUALITY_REPO_ID = "sdl-neutral-fixture"
$env:SDL_CONFIG = (Resolve-Path "tests/benchmark/config/context-quality-neutral.json").Path
$env:SDL_CONTEXT_QUALITY_MANIFEST_CONFIG_PATH = $env:SDL_CONFIG
$env:SDL_CONTEXT_QUALITY_CACHE_PRIMARY_PATH = Join-Path $cacheRoot "neutral/graph.lbug"
$env:SDL_CONTEXT_QUALITY_CACHE_READY_PATH = Join-Path $cacheRoot "neutral/ready.json"
$env:SDL_CONTEXT_QUALITY_WORKING_DB_PATH = Join-Path $env:RUNNER_TEMP "context-quality-run-neutral.lbug"
$env:SDL_CONTEXT_QUALITY_OUTPUT_PATH = Join-Path $resultRoot "neutral.json"
& node tests/benchmark/context-quality-runner.mjs
if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
- name: Upload context-quality artifacts
if: always()
uses: actions/upload-artifact@v7
with:
name: context-quality-${{ matrix.os }}
path: .benchmark/context-quality-results/**
if-no-files-found: warn
retention-days: 14