Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
45 changes: 44 additions & 1 deletion .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,49 @@ jobs:
- name: Test
run: ctest --preset ${{ matrix.preset }} -j "$(nproc 2>/dev/null || sysctl -n hw.ncpu)"

# MSVC, against the conda-forge Arrow for win-64. The Visual Studio generator is
# multi-config, so the preset names the configuration at build and test time
# rather than through CMAKE_BUILD_TYPE; conda puts every library under
# Library/, which is what the preset's prefix path points at. cmd rather than
# bash so that CONDA_PREFIX reaches CMake as a Windows path. The build preset
# names its targets so that the build never enters the RUN_TESTS project
# enable_testing() defines, which runs ctest and so would test the tree from
# inside the build that is meant to produce it. Those names are CMake target
# names, not file names: MSBuild matches them against the generated project
# files, which are case-insensitive here, so a target called run_tests would
# select RUN_TESTS.
windows:
name: msvc on windows-latest
runs-on: windows-latest
defaults:
run:
shell: cmd /C call {0}
steps:
- uses: actions/checkout@v4

- uses: mamba-org/setup-micromamba@v2
with:
environment-file: environment.yml
init-shell: cmd.exe
cache-environment: true

- name: Configure
run: cmake --preset windows

- name: Build
run: cmake --build --preset windows -j %NUMBER_OF_PROCESSORS%

- name: Test
run: ctest --preset windows -j %NUMBER_OF_PROCESSORS%

- name: Install the package
run: >-
pip install -e . --no-build-isolation
-Ccmake.define.CMAKE_PREFIX_PATH=%CONDA_PREFIX%/Library

- name: Test the package
run: python -m pytest python/tests

python:
name: python on ${{ matrix.os }}
runs-on: ${{ matrix.os }}
Expand Down Expand Up @@ -116,7 +159,7 @@ jobs:
- name: clang-format
run: >-
clang-format --dry-run --Werror
src/main.cpp src/cpplink/* tests/*.cpp python/bindings/*
src/main.cpp src/cpplink/* tests/*.cpp tests/*.hpp python/bindings/*

- name: cpplint
run: cpplint --recursive src tests python/bindings
Expand Down
18 changes: 14 additions & 4 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -99,16 +99,26 @@ if(BUILD_TESTING)
include(GoogleTest)

file(GLOB TEST_SOURCES CONFIGURE_DEPENDS tests/*.cpp)
add_executable(run_tests ${TEST_SOURCES})
target_link_libraries(run_tests PRIVATE cpplink_core GTest::gtest_main)
# The target is cpplink_tests and the binary is run_tests: enable_testing()
# defines a target called RUN_TESTS, and the Visual Studio generator names a
# project file after its target, so a target called run_tests and that one
# are the same file on a case-insensitive filesystem. Whichever is written
# second wins, and building the test executable then runs ctest against a
# tree with no test executable in it.
add_executable(cpplink_tests ${TEST_SOURCES})
set_target_properties(cpplink_tests PROPERTIES OUTPUT_NAME run_tests)
target_link_libraries(cpplink_tests PRIVATE cpplink_core GTest::gtest_main)
# Test-only headers are included as "tests/x.hpp", rooted at the project
# directory, the way the library's are rooted at src/.
target_include_directories(cpplink_tests PRIVATE ${CMAKE_CURRENT_SOURCE_DIR})
# A test reading a checked-in fixture names it from the source tree, so the
# suite passes from ctest's working directory as well as from the root.
target_compile_definitions(run_tests PRIVATE
target_compile_definitions(cpplink_tests PRIVATE
CPPLINK_SOURCE_DIR="${CMAKE_CURRENT_SOURCE_DIR}")

# Arrow pulls in a large shared library graph; discovery needs longer than the
# 5 s default to load it.
gtest_discover_tests(run_tests DISCOVERY_TIMEOUT 120)
gtest_discover_tests(cpplink_tests DISCOVERY_TIMEOUT 120)
endif()

# clang-format target
Expand Down
22 changes: 21 additions & 1 deletion CMakePresets.json
Original file line number Diff line number Diff line change
Expand Up @@ -42,13 +42,27 @@
"CMAKE_BUILD_TYPE": "RelWithDebInfo",
"CPPLINK_SANITIZE": "thread"
}
},
{
"name": "windows",
"displayName": "MSVC x64, with tests",
"inherits": "base",
"generator": "Visual Studio 18 2026",
"architecture": {"value": "x64", "strategy": "set"},
"cacheVariables": {"CMAKE_PREFIX_PATH": "$env{CONDA_PREFIX}/Library"}
}
],
"buildPresets": [
{"name": "release", "configurePreset": "release"},
{"name": "debug", "configurePreset": "debug"},
{"name": "asan", "configurePreset": "asan"},
{"name": "tsan", "configurePreset": "tsan"}
{"name": "tsan", "configurePreset": "tsan"},
{
"name": "windows",
"configurePreset": "windows",
"configuration": "Release",
"targets": ["cpplink", "cpplink_tests"]
}
],
"testPresets": [
{
Expand All @@ -73,6 +87,12 @@
"inherits": "base",
"configurePreset": "tsan",
"environment": {"TSAN_OPTIONS": "halt_on_error=1:second_deadlock_stack=1"}
},
{
"name": "windows",
"inherits": "base",
"configurePreset": "windows",
"configuration": "Release"
}
]
}
3 changes: 2 additions & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -36,9 +36,10 @@ Two of them run the suite under a sanitizer, which is what checks the claim that
```sh
cmake --preset asan && cmake --build --preset asan && ctest --preset asan # ASan + UBSan, about 30 s
cmake --preset tsan && cmake --build --preset tsan && ctest --preset tsan # ThreadSanitizer, about 5 min
cmake --preset windows && cmake --build --preset windows && ctest --preset windows # MSVC, from an activated conda prompt
```

Both are clean over the whole suite.
The two sanitizer presets are clean over the whole suite; `windows` is the same suite under MSVC, and CI runs it on `windows-latest`.
CI ([.github/workflows/ci.yml](.github/workflows/ci.yml)) runs `release`, `asan` and `tsan` on Linux and macOS, then the Python suite and the format and lint checks below.

## The Python package
Expand Down
3 changes: 2 additions & 1 deletion docs/getting-started.md
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,8 @@ ctest --preset asan
```

`release` and `debug` are the plain builds with tests on, and `ctest --preset <name>` runs the suite with output on failure.
The GitHub Actions workflow runs `release`, `asan` and `tsan` on Linux and macOS, then the Python suite and the format and lint checks.
On Windows the `windows` preset builds with Visual Studio 2026 against the conda-forge Arrow, which lives under `%CONDA_PREFIX%\Library`, and the build and test presets of the same name select the `Release` configuration, since the Visual Studio generator holds every configuration in one tree.
The GitHub Actions workflow runs `release`, `asan` and `tsan` on Linux and macOS and `windows` on Windows, then the Python suite and the format and lint checks.

## The whole pipeline in seven commands

Expand Down
2 changes: 1 addition & 1 deletion python/bindings/stages.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -332,7 +332,7 @@ MergeOutcome MergePredictions(const std::string& shards, const std::string& out,
options.out_path = out;
options.batch_rows = batch_rows;
if (format.is_none()) {
const std::string suffix = std::filesystem::path(out).extension();
const std::string suffix = std::filesystem::path(out).extension().string();
if (suffix == ".parquet" || suffix == ".pq")
options.format = MergeFormat::kParquet;
} else {
Expand Down
35 changes: 31 additions & 4 deletions src/cpplink/app.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,6 @@
#include <vector>

#include <nlohmann/json.hpp>
#include <sys/ioctl.h>
#include <unistd.h>

#include "cpplink/blocking.hpp"
#include "cpplink/cluster.hpp"
Expand Down Expand Up @@ -44,6 +42,25 @@
#include "cpplink/simplify.hpp"
#include "cpplink/waterfall.hpp"

// The terminal query below is the one platform-specific call. windows.h comes after
// every project header so that its macros -- and it defines several hundred, from
// ERROR to far -- rewrite nothing the project declares.
#ifdef _WIN32
#ifndef NOMINMAX
#define NOMINMAX // windows.h's min/max macros would break std::min below.
#endif
#ifndef WIN32_LEAN_AND_MEAN
#define WIN32_LEAN_AND_MEAN
#endif
#include <cstdio>

#include <io.h>
#include <windows.h>
#else
#include <sys/ioctl.h>
#include <unistd.h>
#endif

namespace cpplink {

const char* const kVersion = "0.1.0";
Expand Down Expand Up @@ -910,10 +927,19 @@ int RunExplain(const std::vector<std::string>& args, std::ostream& out,
// string stream -- it is printed as lines. The stream is compared to the process's
// own stderr because an ostream carries no descriptor to ask.
unsigned TerminalColumns(const std::ostream& stream) {
if (&stream != &std::cerr || isatty(STDERR_FILENO) == 0) return 0;
if (&stream != &std::cerr) return 0;
#ifdef _WIN32
if (_isatty(_fileno(stderr)) == 0) return 0;
CONSOLE_SCREEN_BUFFER_INFO info{};
if (GetConsoleScreenBufferInfo(GetStdHandle(STD_ERROR_HANDLE), &info) == 0) return 80;
const int columns = info.srWindow.Right - info.srWindow.Left + 1;
return columns > 0 ? static_cast<unsigned>(columns) : 80;
#else
if (isatty(STDERR_FILENO) == 0) return 0;
winsize size{};
if (ioctl(STDERR_FILENO, TIOCGWINSZ, &size) != 0 || size.ws_col == 0) return 80;
return size.ws_col;
#endif
}

int RunExplainBlocking(const std::vector<std::string>& args, std::ostream& out,
Expand Down Expand Up @@ -1625,7 +1651,8 @@ int RunMergeEdges(const std::vector<std::string>& args, std::ostream& out,
// The extension is what a user means by the format; --format is for a name
// that does not carry one.
if (!format_given) {
const std::string suffix = std::filesystem::path(options.out_path).extension();
const std::string suffix =
std::filesystem::path(options.out_path).extension().string();
if (suffix == ".parquet" || suffix == ".pq") {
options.format = MergeFormat::kParquet;
}
Expand Down
6 changes: 6 additions & 0 deletions src/cpplink/dictionary.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,12 @@ inline constexpr uint32_t kNullId = 0xFFFFFFFFu;
// by the index stay valid as the arena grows.
class Dictionary {
public:
Dictionary() = default;
Dictionary(const Dictionary&) = delete;
Dictionary& operator=(const Dictionary&) = delete;
Dictionary(Dictionary&&) = default;
Dictionary& operator=(Dictionary&&) = default;

uint32_t Intern(std::string_view value);
std::string_view Value(uint32_t id) const { return entries_[id]; }
uint32_t Size() const { return static_cast<uint32_t>(entries_.size()); }
Expand Down
1 change: 1 addition & 0 deletions src/cpplink/inspect.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
#include "cpplink/inspect.hpp"

#include <algorithm>
#include <cstdio>
#include <iomanip>
#include <ostream>
#include <string>
Expand Down
1 change: 1 addition & 0 deletions src/cpplink/schema.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@

#include "cpplink/schema.hpp"

#include <algorithm>
#include <cmath>
#include <fstream>
#include <iomanip>
Expand Down
32 changes: 29 additions & 3 deletions src/cpplink/string_metrics.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,10 @@
#include <utility>
#include <vector>

#ifdef _MSC_VER
#include <intrin.h>
#endif

namespace cpplink {
namespace {

Expand Down Expand Up @@ -56,8 +60,30 @@ double ApplyWinkler(std::string_view a, std::string_view b, double jaro,

// The one instruction, asked for by name. `std::bitset<64>::count()` does not
// lower to it here -- it leaves a call to the generic bit-iterator count in the
// binary, which a profile of the scoring loop finds among the hot leaves.
int PopCount(uint64_t value) { return __builtin_popcountll(value); }
// binary, which a profile of the scoring loop finds among the hot leaves. MSVC's
// `__popcnt64` emits the instruction with no fallback, so an x64 build assumes
// POPCNT, which every processor Windows 11 runs on has; gcc and clang substitute
// a bit-trick sequence where the target lacks it.
int PopCount(uint64_t value) {
#if defined(_MSC_VER) && defined(_M_ARM64)
return static_cast<int>(_CountOneBits64(value));
#elif defined(_MSC_VER)
return static_cast<int>(__popcnt64(value));
#else
return __builtin_popcountll(value);
#endif
}

// The index of the lowest set bit; `value` must not be zero.
int TrailingZeros(uint64_t value) {
#ifdef _MSC_VER
unsigned long index = 0; // NOLINT(runtime/int): the intrinsic's own type
_BitScanForward64(&index, value);
return static_cast<int>(index);
#else
return __builtin_ctzll(value);
#endif
}

// Myers' bit-vector edit distance (1999). The DP's column of vertical deltas is
// carried in two words -- vp for +1, vn for -1 -- so a whole column costs a dozen
Expand Down Expand Up @@ -187,7 +213,7 @@ double JaroWords(std::string_view a, std::string_view b, double screen) {
uint64_t left = claimed;
uint64_t right = taken;
while (left != 0) {
if (a[__builtin_ctzll(left)] != b[__builtin_ctzll(right)]) ++transpositions;
if (a[TrailingZeros(left)] != b[TrailingZeros(right)]) ++transpositions;
left &= left - 1;
right &= right - 1;
}
Expand Down
7 changes: 4 additions & 3 deletions tests/arrow_export_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,12 +19,13 @@
#include <arrow/api.h>
#include <arrow/c/bridge.h>
#include <gtest/gtest.h>
#include <unistd.h>

#include "cpplink/batch_loader.hpp"
#include "cpplink/parquet_io.hpp"
#include "cpplink/record_store.hpp"
#include "cpplink/schema.hpp"
#include "tests/process_id.hpp"
#include "tests/temp_dir.hpp"

namespace {

Expand Down Expand Up @@ -223,7 +224,7 @@ TEST(ArrowExport, ABatchCanBeReusedAfterExport) {
// same batches, through nothing but the C structs on either side.
TEST(ArrowExport, ParquetRoundTrip) {
const auto dir = std::filesystem::temp_directory_path() /
("cpplink_arrow_export_" + std::to_string(::getpid()));
("cpplink_arrow_export_" + cpplink_test::ProcessId());
std::filesystem::create_directories(dir);
const std::string path = (dir / "rows.parquet").string();

Expand Down Expand Up @@ -274,5 +275,5 @@ TEST(ArrowExport, ParquetRoundTrip) {
ArrowArrayStream missing;
EXPECT_FALSE(cpplink::OpenParquetStream(path, {"gone"}, &missing, &error));
EXPECT_NE(error.find("\"gone\" is not in"), std::string::npos) << error;
std::filesystem::remove_all(dir);
cpplink_test::RemoveAll(dir);
}
10 changes: 7 additions & 3 deletions tests/batch_loader_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,12 @@
#include <arrow/io/api.h>
#include <gtest/gtest.h>
#include <parquet/arrow/writer.h>
#include <unistd.h>

#include "cpplink/parquet_loader.hpp"
#include "cpplink/record_store.hpp"
#include "cpplink/schema.hpp"
#include "tests/process_id.hpp"
#include "tests/temp_dir.hpp"

namespace {

Expand Down Expand Up @@ -505,19 +506,22 @@ TEST(BatchLoader, MatchesTheParquetPathMemberForMember) {
{true, true, false, true})});

const auto dir = std::filesystem::temp_directory_path() /
("cpplink_batch_loader_" + std::to_string(::getpid()));
("cpplink_batch_loader_" + cpplink_test::ProcessId());
std::filesystem::create_directories(dir);
const std::string path = (dir / "table.parquet").string();
auto sink = arrow::io::FileOutputStream::Open(path);
ASSERT_TRUE(sink.ok());
ASSERT_TRUE(parquet::arrow::WriteTable(*table, arrow::default_memory_pool(), *sink,
/*chunk_size=*/3)
.ok());
// `WriteTable` does not close a stream it was handed, and a file with a
// handle still on it is one Windows will not let the removal below delete.
ASSERT_TRUE((*sink)->Close().ok());

cpplink::RecordStore from_file(schema);
std::string error;
ASSERT_TRUE(cpplink::LoadParquet(path, schema, &from_file, nullptr, &error)) << error;
std::filesystem::remove_all(dir);
cpplink_test::RemoveAll(dir);

// The same table, decoded from its buffers in batches of two.
cpplink::RecordStore from_buffers(schema);
Expand Down
Loading
Loading