Accept pandas DataFrames in Python bindings

.github/workflows/ci.yml

@@ -67,6 +67,41 @@ jobs:
       - name: Test
         run: python -m pytest python/tests
 
+  # The wheel's configuration: no conda, no Arrow, the one header-only
+  # dependency fetched. The library and the command line build, and the Python
+  # package passes its suite with PyPI's pandas and pyarrow, reading and writing
+  # parquet through them.
+  no-arrow:
+    name: without Arrow on ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest, macos-latest]
+    defaults:
+      run:
+        shell: bash
+    steps:
+      - uses: actions/checkout@v4
+
+      - uses: actions/setup-python@v5
+        with:
+          python-version: "3.12"
+
+      - name: Configure and build the command line
+        run: |
+          cmake -S . -B build-noarrow -DCPPLINK_WITH_ARROW=OFF
+          cmake --build build-noarrow -j "$(nproc 2>/dev/null || sysctl -n hw.ncpu)"
+          ./build-noarrow/cpplink --version
+
+      - name: Install the package
+        run: pip install . -Ccmake.define.CPPLINK_WITH_ARROW=OFF
+
+      - name: Test
+        run: |
+          pip install pytest pandas pyarrow
+          cd "$RUNNER_TEMP" && python -m pytest "$GITHUB_WORKSPACE/python/tests"
+
   lint:
     name: format and lint
     runs-on: ubuntu-latest

CMakeLists.txt

@@ -6,6 +6,15 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
 option(BUILD_TESTING "Enable building of tests" OFF)
 option(CPPLINK_BUILD_PYTHON "Build the Python extension module" OFF)
+# Arrow C++ is what reads and writes parquet, and nothing else: every column
+# crosses into the core through the C Data Interface, whose structs are vendored.
+# Off, the parquet entry points fail with the reason and the library links no
+# Arrow, which is how the Python wheel is built -- its input and output are data
+# frames. AUTO takes Arrow where find_package finds it, which is the conda
+# environment, and does without it where it does not, which is a wheel build.
+# The tests need parquet fixtures, so they need it on.
+set(CPPLINK_WITH_ARROW AUTO CACHE STRING "Read and write parquet through Arrow C++: ON, OFF or AUTO")
+set_property(CACHE CPPLINK_WITH_ARROW PROPERTY STRINGS AUTO ON OFF)
 
 # Without this the default build has no -O flag at all, which costs roughly two
 # orders of magnitude in the pair-evaluation loop. Debug builds must be asked for.
@@ -27,23 +36,51 @@ if(CPPLINK_SANITIZE)
     endif()
 endif()
 
-find_package(Arrow REQUIRED)
-find_package(Parquet REQUIRED)
-find_package(nlohmann_json REQUIRED)
+# nlohmann_json is header-only: the environment's where it has one, else the
+# pinned release, fetched, which is what a wheel build on a bare runner does.
+find_package(nlohmann_json QUIET)
+if(NOT nlohmann_json_FOUND)
+    include(FetchContent)
+    set(JSON_BuildTests OFF CACHE INTERNAL "")
+    FetchContent_Declare(nlohmann_json
+        URL https://github.com/nlohmann/json/releases/download/v3.12.0/json.tar.xz
+        URL_HASH SHA256=42f6e95cad6ec532fd372391373363b62a14af6d771056dbfc86160e6dfff7aa)
+    FetchContent_MakeAvailable(nlohmann_json)
+    message(STATUS "nlohmann_json: fetched v3.12.0")
+endif()
+
+if(CPPLINK_WITH_ARROW STREQUAL "AUTO")
+    find_package(Arrow QUIET)
+    find_package(Parquet QUIET)
+    if(Arrow_FOUND AND Parquet_FOUND)
+        set(CPPLINK_WITH_ARROW ON)
+    else()
+        set(CPPLINK_WITH_ARROW OFF)
+    endif()
+    message(STATUS "Arrow: ${CPPLINK_WITH_ARROW} (found automatically)")
+elseif(CPPLINK_WITH_ARROW)
+    find_package(Arrow REQUIRED)
+    find_package(Parquet REQUIRED)
+endif()
 
 # Core logic lives in a library so both the CLI and the tests can link against it.
 file(GLOB_RECURSE CPPLINK_SOURCES CONFIGURE_DEPENDS src/cpplink/*.cpp src/cpplink/*.hpp)
+# `parquet_io.hpp` has two implementations and a build compiles one of them.
+if(CPPLINK_WITH_ARROW)
+    list(FILTER CPPLINK_SOURCES EXCLUDE REGEX "parquet_io_stub\\.cpp$")
+else()
+    list(FILTER CPPLINK_SOURCES EXCLUDE REGEX "parquet_io_arrow\\.cpp$")
+endif()
 
 add_library(cpplink_core ${CPPLINK_SOURCES})
 target_include_directories(cpplink_core PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/src)
 # The Python extension is a shared module linking this static library, so its
 # objects must be relocatable.
 set_target_properties(cpplink_core PROPERTIES POSITION_INDEPENDENT_CODE ON)
-target_link_libraries(cpplink_core PUBLIC
-    Arrow::arrow_shared
-    Parquet::parquet_shared
-    nlohmann_json::nlohmann_json
-)
+target_link_libraries(cpplink_core PUBLIC nlohmann_json::nlohmann_json)
+if(CPPLINK_WITH_ARROW)
+    target_link_libraries(cpplink_core PUBLIC Arrow::arrow_shared Parquet::parquet_shared)
+endif()
 
 add_executable(cpplink src/main.cpp)
 target_link_libraries(cpplink PRIVATE cpplink_core)
@@ -53,6 +90,9 @@ if(CPPLINK_BUILD_PYTHON)
 endif()
 
 if(BUILD_TESTING)
+    if(NOT CPPLINK_WITH_ARROW)
+        message(FATAL_ERROR "the tests write parquet fixtures: BUILD_TESTING needs CPPLINK_WITH_ARROW=ON")
+    endif()
     enable_testing()
 
     find_package(GTest REQUIRED)

CONTRIBUTING.md

@@ -51,6 +51,10 @@ python -m pytest python/tests
 The editable install redirects the Python sources to `python/cpplink/`, so a change there needs no rebuild; a change to `python/bindings/` or to the core needs the `pip install` line again.
 The binding is a second front end over the same stage functions, not a second implementation, and `python/tests/test_parity.py` holds it to the command line's output byte for byte.
 
+The same package builds with `CPPLINK_WITH_ARROW=OFF`, which is what the wheels workflow ([.github/workflows/wheels.yml](.github/workflows/wheels.yml)) does through cibuildwheel and what CI's `no-arrow` job checks: the module then links nothing but the C++ standard library and a parquet file is read and written through pyarrow and pandas.
+`python/tests/conftest.py` marks the tests that need the core to read parquet itself `needs_core_parquet`, and they are skipped in that build; a new test over a parquet path the core opens takes the same mark.
+`test_reports.py` records a field count per bound report class, so a struct gaining a field the binding forgot is caught when the count is bumped deliberately.
+
 ## Code style
 
 C++ follows the [Google C++ Style Guide](https://google.github.io/styleguide/cppguide.html) with two customizations, a 4-space indent and a 90-column limit, both in [.clang-format](.clang-format) and [CPPLINT.cfg](CPPLINT.cfg).

README.md

@@ -101,11 +101,11 @@ The same pipeline from Python:
 ```python
 import cpplink
 
-linker = cpplink.Linker("schema.json", "sample.parquet")
+linker = cpplink.Linker("schema.json", "sample.parquet")     # or a pandas frame
 model, report = linker.estimate(out="model.json")
-linker.predict(model, "predictions.parquet", threshold=20)
-result = linker.cluster("predictions.parquet", truth="sample.truth.csv")
-print(result.quality)
+predictions = linker.predict(model, threshold=20)            # a pandas frame
+clusters = linker.cluster(truth="sample.truth.csv")          # another
+print(linker.last_cluster.quality)
 ```
 
 Between `init` and `estimate` sit the diagnostics that cost seconds and decide the quality of the result: `profile` says what each column can be worth before any model exists, `levels` places the fuzzy thresholds from the data, `explain-blocking` prices every source without enumerating a pair, and `recall` measures what blocking reaches.
@@ -130,6 +130,8 @@ The Python package builds into the same environment, against the Arrow it holds:
 pip install -e . --no-build-isolation -Ccmake.define.CMAKE_PREFIX_PATH=$CONDA_PREFIX
 ```
 
+Outside conda it builds with no Arrow at all, `pip install . -Ccmake.define.CPPLINK_WITH_ARROW=OFF`, and reads and writes parquet through pandas instead; [From Python](https://4ment.github.io/cpplink/python/) says what that build leaves out.
+
 Building with the tests, the sanitizer presets, the lint tools and the documentation are in [CONTRIBUTING.md](CONTRIBUTING.md).
 
 ## License