ci.yml

name: CI

on:
  pull_request:
  push:
    branches: [main]

permissions:
  contents: read

jobs:
  memcheck:
    name: Memcheck public boundaries
    runs-on: ubuntu-24.04
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Setup Zig
        uses: mlugg/setup-zig@v2
        with:
          version: 0.17.0

      - uses: actions/setup-python@v6
        with:
          python-version: "3.13"

      - name: Install memory-check and OCR tools
        run: |
          sudo apt-get update
          sudo apt-get install -y valgrind libc6-dbg cmake g++ libleptonica-dev tesseract-ocr-eng poppler-utils
          python -m pip install cffi

      # libgomp's initialization allocations survive dlclose even after
      # omp_pause_resource_all. Exercise unloading with Tesseract's supported
      # serial build instead of suppressing those leaks or retaining the DSO.
      - name: Checkout pinned Tesseract for memory checks
        uses: actions/checkout@v7
        with:
          repository: tesseract-ocr/tesseract
          ref: 8ee020e14cf5be4e3f0e9beb09b6b050a1871854 # 5.3.4
          path: benchmark/eval/raw_cache/tesseract-memcheck
          persist-credentials: false

      - name: Build Tesseract with unloadable dependencies
        run: |
          cmake -S benchmark/eval/raw_cache/tesseract-memcheck -B "$RUNNER_TEMP/tesseract-build" \
            -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX="$RUNNER_TEMP/memcheck-tesseract" \
            -DCMAKE_INSTALL_LIBDIR=lib -DBUILD_SHARED_LIBS=ON -DOPENMP_BUILD=OFF \
            -DBUILD_TRAINING_TOOLS=OFF -DBUILD_TESTS=OFF
          cmake --build "$RUNNER_TEMP/tesseract-build" --parallel 4
          cmake --install "$RUNNER_TEMP/tesseract-build"
          echo "TESSDATA_PREFIX=/usr/share/tesseract-ocr/5/tessdata" >> "$GITHUB_ENV"
          echo "LD_LIBRARY_PATH=$RUNNER_TEMP/memcheck-tesseract/lib:$LD_LIBRARY_PATH" >> "$GITHUB_ENV"
          echo "PKG_CONFIG_PATH=$RUNNER_TEMP/memcheck-tesseract/lib/pkgconfig:$PKG_CONFIG_PATH" >> "$GITHUB_ENV"

      - name: Build normal baseline
        run: zig build --summary all

      - name: Build isolated diagnostic artifacts
        run: zig build memcheck-build -Dtesseract-c=true -Dtesseract-include="$RUNNER_TEMP/memcheck-tesseract/include" -Dtesseract-lib="$RUNNER_TEMP/memcheck-tesseract/lib" --summary all

      - name: Qualify Memcheck and exercise CLI, shared library, Python, and OCR
        run: |
          python benchmark/eval/memcheck_runner.py \
            --prefix zig-out/memcheck \
            --baseline-prefix zig-out \
            --python "$(command -v python)" \
            --python-leak-policy parser-allocations \
            --ocr \
            --output benchmark/eval/outputs/memcheck-ci

      - name: Upload memory-check evidence
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: memcheck
          path: benchmark/eval/outputs/memcheck-ci
          if-no-files-found: warn

  native:
    name: Native parser gates
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Setup Zig
        uses: mlugg/setup-zig@v2
        with:
          version: 0.17.0

      - name: Checkout pinned Adobe CMap resources
        uses: actions/checkout@v7
        with:
          repository: adobe-type-tools/cmap-resources
          ref: f5cf3bca7fdfeaceb77aa82847e974f2306c20b4
          path: benchmark/eval/raw_cache/cmap-resources
          sparse-checkout: |
            Adobe-CNS1-7
            Adobe-GB1-6
            Adobe-Japan1-7
            Adobe-Korea1-2

      - name: Checkout pinned Adobe CID mappings
        uses: actions/checkout@v7
        with:
          repository: adobe-type-tools/mapping-resources-pdf
          ref: 2dd5e53fb74a01718b9dfd448a0d1cce6fff2aa5
          path: benchmark/eval/raw_cache/mapping-resources-pdf
          sparse-checkout: pdf2unicode

      - name: Verify generated Adobe CMap resources
        run: |
          python3 benchmark/eval/generate_cmap_resources.py \
            --cmap-root benchmark/eval/raw_cache/cmap-resources \
            --mapping-root benchmark/eval/raw_cache/mapping-resources-pdf \
            --output src/cmap_resources.zig \
            --binary-output src/cmap_resources.bin \
            --check

      - name: Check formatting
        run: zig fmt --check $(git ls-files '*.zig')

      - name: Run tests
        run: zig build test --summary all

      - name: Run tiny corpus scorecard
        run: zig build benchmark-eval --summary all

      - name: Build native artifacts
        run: zig build --summary all

      - name: Build shared library
        run: zig build shared --summary all

      - name: Verify frozen topology and emitted-order controls
        run: |
          python3 -m unittest discover -s benchmark/eval -p 'test_table_compare.py'
          python3 -m unittest discover -s benchmark/eval -p 'test_bounded_zig.py'
          python3 -m unittest discover -s benchmark/eval/table_topology -p 'test_*.py'
          python3 -m unittest discover -s benchmark/eval/table_topology_real -p 'test_*.py'
          python3 -m unittest discover -s benchmark/eval/reading_order_next -p 'test_*.py'
          python3 benchmark/eval/table_topology/generate.py --check
          python3 benchmark/eval/reading_order_next/generate.py --check
          for split in dev holdout; do
            python3 benchmark/eval/table_topology/verify.py \
              --cli zig-out/bin/pdf-parser --split "$split" \
              --output "benchmark/eval/outputs/topology-ci/$split"
            python3 benchmark/eval/reading_order_next/verify.py \
              --cli zig-out/bin/pdf-parser --split "$split" \
              --output "benchmark/eval/outputs/reading-order-ci/$split"
          done

      - name: Upload topology and emitted-order evidence
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: topology-and-emitted-order
          path: |
            benchmark/eval/outputs/topology-ci
            benchmark/eval/outputs/reading-order-ci
          if-no-files-found: warn

      - uses: actions/setup-node@v6
        with:
          node-version: "22"

      - name: Test public JavaScript adapters against current CLI
        run: |
          sudo apt-get update
          sudo apt-get install -y tesseract-ocr tesseract-ocr-eng poppler-utils
          tesseract --version
          pdftoppm -v
          node --test --test-concurrency=1 integrations/javascript/*.test.mjs

  python:
    name: Python ${{ matrix.python-version }} bindings and wheel
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.13"]
    steps:
      - uses: actions/checkout@v7
        with:
          persist-credentials: false

      - name: Setup Zig
        uses: mlugg/setup-zig@v2
        with:
          version: 0.17.0

      - uses: actions/setup-python@v6
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install test and packaging tools
        run: python -m pip install "pytest<9" cffi "build<2"

      - name: Build current shared library and test fixture
        run: |
          zig build shared --summary all
          mkdir -p test
          zig run gen_test.zig -- test/test.pdf

      - name: Test bindings against current library
        env:
          PDF_PARSER_LIB: ${{ github.workspace }}/zig-out/lib/libpdf_parser.so
        run: PYTHONPATH=python python -m pytest python/tests -q

      - name: Build platform wheel from current Zig source
        run: python -m build --wheel python

      - name: Verify installed wheel and adaptive ABI in isolation
        run: |
          python -m venv "$RUNNER_TEMP/pdf-parser-wheel-test"
          "$RUNNER_TEMP/pdf-parser-wheel-test/bin/pip" install python/dist/*.whl
          "$RUNNER_TEMP/pdf-parser-wheel-test/bin/python" -I \
            python/tests/wheel_smoke.py \
            python/dist/*.whl \
            benchmark/eval/corpus/clean_born_digital/clean-native.pdf \
            libpdf_parser.so

  wasm:
    name: Build WASM
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v7

      - name: Setup Zig
        uses: mlugg/setup-zig@v2
        with:
          version: 0.17.0

      - name: Build WASM
        run: zig build wasm

      - name: Enforce WASM size budget
        run: |
          wasm_size=$(wc -c < zig-out/bin/pdf_parser.wasm)
          if [ "$wasm_size" -gt 3250000 ]; then
            echo "WASM artifact exceeds 3,250,000-byte budget: $wasm_size"
            exit 1
          fi

  adaptive-ocr:
    name: Adaptive OCR quality gate
    runs-on: ubuntu-24.04
    env:
      OCR_TESSERACT_PACKAGE_VERSION: 5.3.4-1build5
      OCR_TESSDATA_ENG_PACKAGE_VERSION: 1:4.1.0-2
      OCR_POPPLER_PACKAGE_VERSION: 24.02.0-1ubuntu9.10
      OCR_TESSERACT_VERSION: tesseract 5.3.4
      OCR_POPPLER_VERSION: pdftoppm version 24.02.0
    steps:
      - uses: actions/checkout@v7

      - name: Install OCR and rasterization tools
        run: |
          sudo apt-get update
          sudo apt-get install --no-install-recommends -y \
            "poppler-utils=${OCR_POPPLER_PACKAGE_VERSION}" \
            "tesseract-ocr=${OCR_TESSERACT_PACKAGE_VERSION}" \
            "tesseract-ocr-eng=${OCR_TESSDATA_ENG_PACKAGE_VERSION}"
          test "$(dpkg-query -W -f='${Version}' tesseract-ocr)" = "$OCR_TESSERACT_PACKAGE_VERSION"
          test "$(dpkg-query -W -f='${Version}' tesseract-ocr-eng)" = "$OCR_TESSDATA_ENG_PACKAGE_VERSION"
          test "$(dpkg-query -W -f='${Version}' poppler-utils)" = "$OCR_POPPLER_PACKAGE_VERSION"

      - name: Setup Zig
        uses: mlugg/setup-zig@v2
        with:
          version: 0.17.0

      - name: Build parser
        run: zig build --summary all

      - name: Enforce scanned-form text and numeric floors
        run: |
          python3 benchmark/eval/ocr_form_quality.py \
            --expected-ocr-version "$OCR_TESSERACT_VERSION" \
            --expected-rasterizer-version "$OCR_POPPLER_VERSION" \
            --output /tmp/ocr-form-quality.json

      - name: Enforce real JBIG2 scan quality
        run: |
          python3 benchmark/eval/ocr_hard_document_quality.py \
            --expected-ocr-version "$OCR_TESSERACT_VERSION" \
            --expected-rasterizer-version "$OCR_POPPLER_VERSION" \
            --output /tmp/ocr-hard-document-quality.json

      - name: Enforce real JPX scan quality
        run: |
          python3 benchmark/eval/ocr_hard_document_quality.py \
            --pdf benchmark/eval/corpus/scanned_typewritten/jpx-public-domain-map-cover.pdf \
            --truth benchmark/eval/ground_truth/ocr_text/scanned_typewritten/jpx-public-domain-map-cover.json \
            --expected-ocr-version "$OCR_TESSERACT_VERSION" \
            --expected-rasterizer-version "$OCR_POPPLER_VERSION" \
            --output /tmp/ocr-jpx-hard-document-quality.json

      - name: Upload OCR quality report
        if: always()
        uses: actions/upload-artifact@v6
        with:
          name: ocr-quality
          path: |
            /tmp/ocr-form-quality.json
            /tmp/ocr-hard-document-quality.json
            /tmp/ocr-jpx-hard-document-quality.json
          if-no-files-found: ignore