diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index cfb15da..51889b5 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -66,29 +66,29 @@ add_executable(teacher_eval target_link_libraries(teacher_eval PRIVATE engine) add_executable(engine_tests - tests/parquet_loader_test.cpp - tests/projection_test.cpp - tests/split/time_split_test.cpp + tests/allocation/size_proportional_test.cpp + tests/core/interpolation_test.cpp + tests/core/parallel_test.cpp + tests/core/projection_test.cpp + tests/core/simplification_test.cpp + tests/core/subset_test.cpp + tests/evaluator/combined_test.cpp + tests/evaluator/evaluator_test.cpp + tests/io/parquet_loader_test.cpp + tests/pipeline/config_test.cpp + tests/query/continuous_frechet_test.cpp + tests/query/join_test.cpp + tests/query/knn_test.cpp + tests/query/range_test.cpp + tests/query/route_clustering_test.cpp + tests/query/where_at_test.cpp + tests/selection/rdp_test.cpp + tests/selection/sed_test.cpp + tests/selection/simplify_database_test.cpp tests/split/random_split_test.cpp - tests/selectors/rdp_test.cpp - tests/selectors/sed_test.cpp - tests/size_proportional_test.cpp - tests/interpolation_test.cpp - tests/where_at_test.cpp - tests/evaluator_test.cpp - tests/subset_test.cpp - tests/simplification_test.cpp - tests/selectors/simplify_database_test.cpp - tests/range_test.cpp - tests/parallel_test.cpp - tests/knn_test.cpp - tests/config_test.cpp - tests/join_test.cpp - tests/combined_test.cpp - tests/route_clustering_test.cpp - tests/continuous_frechet_test.cpp - tests/teacher/curves_test.cpp + tests/split/time_split_test.cpp tests/teacher/certificate_test.cpp + tests/teacher/curves_test.cpp tests/teacher/teacher_test.cpp ) target_link_libraries(engine_tests PRIVATE engine GTest::gtest_main Arrow::arrow_shared Parquet::parquet_shared) diff --git a/engine/apps/teacher_eval.cpp b/engine/apps/teacher_eval.cpp index 6efe830..b37461e 100644 --- a/engine/apps/teacher_eval.cpp +++ b/engine/apps/teacher_eval.cpp @@ -226,6 +226,7 @@ struct Arm { std::string name; double share; KeptSets kept; + std::optional metrics; // the teacher's, if it computed them }; // PARTNER: RDP with weight 1 + lambda for trajectories that have a join partner @@ -244,19 +245,24 @@ std::vector baseline_arms(const core::TrajectoryDb &db, db, allocation::AllocatorType::SizeProportional, budget, alpha); }; const auto equal = geometric(0.0); + arms.push_back({"RDP_GLOBAL", + share, + core::to_kept(selection::rdp::simplify_database(db, budget)), + {}}); + arms.push_back({"RDP_a0", share, select(db, AlgorithmType::RDP, equal), {}}); arms.push_back( - {"RDP_GLOBAL", share, core::to_kept(selection::rdp::simplify_database(db, budget))}); - arms.push_back({"RDP_a0", share, select(db, AlgorithmType::RDP, equal)}); - arms.push_back({"RDP_a025", share, select(db, AlgorithmType::RDP, geometric(kRdpAlpha))}); - arms.push_back({"SED_a0", share, select(db, AlgorithmType::SED, equal)}); - arms.push_back({"UNI_a0", share, select(db, AlgorithmType::UNIFORM, equal)}); + {"RDP_a025", share, select(db, AlgorithmType::RDP, geometric(kRdpAlpha)), {}}); + arms.push_back({"SED_a0", share, select(db, AlgorithmType::SED, equal), {}}); + arms.push_back({"UNI_a0", share, select(db, AlgorithmType::UNIFORM, equal), {}}); for (const double lambda : kPartnerLambdas) { std::vector weights(db.size()); for (std::size_t i = 0; i < db.size(); ++i) weights[i] = 1.0 + (partnered[i] ? lambda : 0.0); - arms.push_back({std::format("PARTNER_l{:g}", lambda), share, + arms.push_back({std::format("PARTNER_l{:g}", lambda), + share, select(db, AlgorithmType::RDP, - weighted_budgets(db, weights, budget - 2 * db.size()))}); + weighted_budgets(db, weights, budget - 2 * db.size())), + {}}); } } return arms; @@ -541,7 +547,7 @@ int main(int argc, char **argv) try { for (const double share : config.budget_shares) { start = std::chrono::steady_clock::now(); for (auto &arm : model.run(allocation::total_budget(db, share), args.arms)) - arms.push_back({arm.name, share, core::to_kept(arm.simplification)}); + arms.push_back({arm.name, share, core::to_kept(arm.simplification), arm.metrics}); timings << std::format("{},teacher_{:g},{:.2f}\n", seed, share, core::seconds_since(start)); } @@ -550,7 +556,7 @@ int main(int argc, char **argv) try { arms.push_back(std::move(arm)); for (const double share : config.budget_shares) for (auto &arm : model.single_recipes(allocation::total_budget(db, share))) - arms.push_back({arm.name, share, core::to_kept(arm.simplification)}); + arms.push_back({arm.name, share, core::to_kept(arm.simplification), {}}); } for (const auto &arm : arms) { validate(arm, db); @@ -614,7 +620,7 @@ int main(int argc, char **argv) try { evaluator::Evaluator(db, compressed, test, scoring).evaluate().scores; const auto normalized = evaluator::normalize_scores(scores, floor); // The teacher's own estimate of each metric, next to the test's - const auto teacher_metrics = objective.metrics(arm.kept); + const auto teacher_metrics = arm.metrics ? *arm.metrics : objective.metrics(arm.kept); const auto &teacher_floors = objective.floors(); auto norm = [&](teacher::Objective::Term t) { return (teacher_metrics[t] - teacher_floors[t]) / (1.0 - teacher_floors[t]); @@ -632,7 +638,7 @@ int main(int argc, char **argv) try { scores.join.f1, scores.clustering.score, scores.where_at.within_radius, scores.join.micro_f1, scores.knn.kendalls_tau_topk, normalized.knn, normalized.range, normalized.where_at, normalized.join, normalized.clustering, - normalized.combined, objective.total(arm.kept), + normalized.combined, objective.total(teacher_metrics), scores.clustering.predicted_clusters, core::seconds_since(start), mode_name(config.join.mode), scores.self_join.f1, normalized.self_join, normalized.join_f1, normalized.clustering_score, scores.self_knn.recall_at_k, diff --git a/engine/include/backend/engine/query/knn.hpp b/engine/include/backend/engine/query/knn.hpp index 3d7bb2e..3206bf0 100644 --- a/engine/include/backend/engine/query/knn.hpp +++ b/engine/include/backend/engine/query/knn.hpp @@ -5,6 +5,7 @@ #include #include +#include #include namespace query { @@ -57,5 +58,9 @@ KnnWorkload generate_knn_queries(const core::TrajectoryDb &original, std::vector knn(const core::TrajectoryDb &trajectories, const core::TrajectoryView &query, std::size_t k, FrechetVariant variant = FrechetVariant::Continuous); +// The same continuous neighbours, each with its knn_distance +std::vector> +knn_with_distances(const core::TrajectoryDb &trajectories, const core::TrajectoryView &query, + std::size_t k); } // namespace query diff --git a/engine/include/backend/engine/teacher/objective.hpp b/engine/include/backend/engine/teacher/objective.hpp index f7b795d..6d11c7f 100644 --- a/engine/include/backend/engine/teacher/objective.hpp +++ b/engine/include/backend/engine/teacher/objective.hpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include @@ -44,8 +45,10 @@ class PreparedTrajectories { }; // Upper bound on the continuous Fréchet distance between a trajectory and a simplification -// keeping a subsequence of its points. Infinity if kept is not such a subsequence -double simplification_bound(const core::TrajectoryView &original, const core::TrajectoryView &kept); +// keeping a subsequence of its points. Infinity if kept is not such a subsequence. segments: +// the original's edge lengths, if already known +double simplification_bound(const core::TrajectoryView &original, const core::TrajectoryView &kept, + std::span segments = {}); inline constexpr std::size_t kTermCount = 6; // kNN, range, where-at, join, shape, margin using TermWeights = std::array; @@ -120,9 +123,9 @@ class Objective { friend class Objective; std::size_t trajectory = 0; core::TrajectoryView candidate{}; - std::span partners{}; Terms terms{}; double join = 0.0; + double deviation = 0.0; // simplification_bound of the candidate }; IndependentScore prepare_independent(std::size_t i, const core::TrajectoryView &candidate, std::span partners, @@ -130,8 +133,11 @@ class Objective { double independent_value(const IndependentScore &score, double beat = -std::numeric_limits::infinity()) const; - // Exact J_T + // Exact J_T, from the kept sets or from their metrics double total(const KeptSets &kept) const; + double total(const Terms &metrics) const; + // simplification_bound of candidate against original i, with i's edge lengths cached + double deviation(std::size_t i, const core::TrajectoryView &candidate) const; // Unscaled metrics Terms metrics(const KeptSets &kept) const; @@ -168,8 +174,11 @@ class Objective { bool match(const core::TrajectoryView &a, const core::TrajectoryView &b, std::size_t term = 0) const; + // frechet_within(original j, candidate, radius), first on a coarse copy of j whose deviation + // bound decides most tests + bool original_within(std::size_t j, const core::TrajectoryView &candidate, double radius) const; // match(original of link.other, candidate), skipping the test when the originals' distance - // plus the candidate's own error already proves it. own_error < 0: not yet computed + // bracket and the candidate's own error already decide it. own_error < 0: not yet computed bool held_out_match(std::size_t i, const JoinLink &link, const core::TrajectoryView &candidate, double &own_error) const; @@ -177,10 +186,18 @@ class Objective { struct KnnMembership { std::uint32_t query; double cutoff; + // Bracket on the continuous distance from the query to original i; 0, infinity if unknown + double lower = 0.0, upper = std::numeric_limits::infinity(); }; - - // Knn slot left 0 - Terms raw_separable(std::size_t i, const core::TrajectoryView &candidate) const; + // Whether the triangle bound alone decides d(query, candidate) <= cutoff, given a bracket on + // d(query, original) and deviation bounding d(original, candidate); nullopt if it does not + std::optional certify(double lower, double upper, double deviation, double cutoff) const; + // m_pair_lower and m_pair_upper, once per pair across terms + void bracket_pairs(); + + // Knn slot left 0. deviation < 0: computed here if needed + Terms raw_separable(std::size_t i, const core::TrajectoryView &candidate, + double deviation = -1.0) const; double independent_sum(const IndependentScore &score, double knn) const; double join_metric(std::span state) const; std::vector join_f1s(std::span state) const; @@ -201,8 +218,9 @@ class Objective { std::vector m_pair_i, m_pair_j; // pairs used, positives first std::vector m_pair_term; - std::vector m_pair_upper; // held-out positives: bound on the originals' distance - std::vector m_eps; // threshold of each join term + // Per pair, a bracket on the originals' continuous distance; 0, infinity if unknown + std::vector m_pair_lower, m_pair_upper; + std::vector m_eps; // threshold of each join term std::size_t m_positive_count = 0; std::vector> m_links; // per trajectory std::vector m_truth; // positives per slot @@ -210,6 +228,13 @@ class Objective { std::vector m_join_floor; // per join term, macro F1 with endpoints only std::vector m_join_weight; // per join term, 1 / (terms * (1 - floor)) + // Continuous Fréchet only: brackets and deviation bounds decide tests, coarse copies screen + // the rest + bool m_certify = false; + std::vector m_coarse; // per trajectory, RDP to about kCoarseM + std::vector m_coarse_error; // its simplification_bound; < 0: not used + std::vector> m_segments; // per trajectory, its edge lengths + Terms m_floor{}; Terms m_scale{}; // weight / (weight sum * (1 - floor)) std::vector m_endpoint_separable; // scaled separable terms, endpoints only @@ -296,11 +321,15 @@ class KnnState { // From every query watching i, in the order propose and apply expect std::vector distances(std::size_t i, const core::TrajectoryView &candidate, bool parallel = false) const; + // i's current distances, as distances() gives them for i's current version + std::vector stored(std::size_t i) const; // Change in J_T double propose(std::size_t i, std::span distances) const; // The same change from i becoming candidate, computing a distance only near a query's k-th - // place - double propose_bounded(std::size_t i, const core::TrajectoryView &candidate) const; + // place. reach bounds d(i's current version, candidate), e.g. both deviation bounds: then + // the stored distance alone often places the candidate + double propose_bounded(std::size_t i, const core::TrajectoryView &candidate, + double reach = std::numeric_limits::infinity()) const; // Adds trajectory i to the list of every query that one of versions[i] comes within the query's // watch radius of, the distance of its watch-th nearest in state void watch(std::span> versions, diff --git a/engine/include/backend/engine/teacher/teacher.hpp b/engine/include/backend/engine/teacher/teacher.hpp index 013bdc3..16d843d 100644 --- a/engine/include/backend/engine/teacher/teacher.hpp +++ b/engine/include/backend/engine/teacher/teacher.hpp @@ -41,7 +41,8 @@ struct TeacherConfig { struct Arm { std::string name; core::Simplification simplification; - double objective = 0.0; // J_T + double objective = 0.0; // J_T + std::optional metrics; // the raw J_T metrics, when computed on the way }; // Sees only the database it is given and workloads built from it. Trained for held-out queries diff --git a/engine/src/query/knn.cpp b/engine/src/query/knn.cpp index 992cd38..ac2cc40 100644 --- a/engine/src/query/knn.cpp +++ b/engine/src/query/knn.cpp @@ -268,8 +268,9 @@ struct RankedNeighbor { } }; -std::vector continuous_knn(const core::TrajectoryDb &trajectory_db, - const core::TrajectoryView &query, std::size_t k) { +std::vector> continuous_knn(const core::TrajectoryDb &trajectory_db, + const core::TrajectoryView &query, + std::size_t k) { CandidateQueue queue(trajectory_db, query, k); std::priority_queue nearest; std::vector prev, curr; @@ -313,9 +314,9 @@ std::vector continuous_knn(const core::TrajectoryDb &trajectory_db, nearest.push(next); } } - std::vector result(nearest.size()); + std::vector> result(nearest.size()); for (std::size_t i = result.size(); i > 0; --i) { - result[i - 1] = nearest.top().idx; + result[i - 1] = {nearest.top().idx, nearest.top().distance}; nearest.pop(); } return result; @@ -323,6 +324,14 @@ std::vector continuous_knn(const core::TrajectoryDb &trajectory_db, } // namespace +std::vector> +knn_with_distances(const core::TrajectoryDb &trajectories, const core::TrajectoryView &query, + std::size_t k) { + if (k == 0) + throw std::invalid_argument("knn: k = 0, no neighbors"); + return continuous_knn(trajectories, query, k); +} + double knn_distance(const core::TrajectoryView &a, const core::TrajectoryView &b, FrechetVariant variant) { const double upper = frechet_distance(a, b); @@ -342,8 +351,13 @@ std::vector knn(const core::TrajectoryDb &trajectory_db, if (k == 0) { throw std::invalid_argument("knn: k = 0, no neighbors"); } - return variant == FrechetVariant::Discrete ? discrete_knn(trajectory_db, query_trajectory, k) - : continuous_knn(trajectory_db, query_trajectory, k); + if (variant == FrechetVariant::Discrete) + return discrete_knn(trajectory_db, query_trajectory, k); + const auto ranked = continuous_knn(trajectory_db, query_trajectory, k); + std::vector result(ranked.size()); + for (std::size_t i = 0; i < ranked.size(); ++i) + result[i] = ranked[i].first; + return result; } KnnWorkload generate_knn_queries(const core::TrajectoryDb &original, diff --git a/engine/src/teacher/README.md b/engine/src/teacher/README.md index f93b8f7..57a4c47 100644 --- a/engine/src/teacher/README.md +++ b/engine/src/teacher/README.md @@ -409,12 +409,28 @@ trajectory can't have a point in a box the original had none in. while checking kNN, so a candidate is not rescored just because its bound was promising. The prepared score borrows its candidate's coordinates and must be used before they change. -### `match(a, b)` and `held_out_match(i, link, candidate)` — public +### `match`, `held_out_match`, `original_within` — public - `match`: `query::frechet_within` with the term's threshold and the workload's Fréchet variant. -- `held_out_match`: whether original `link.other` finds candidate i. A positive pair whose - originals' discrete Fréchet distance plus `simplification_bound(i)` is below the threshold is - certified without the test (`certificate_test.cpp`). +- `held_out_match`: whether original `link.other` finds candidate i. Each pair's originals get a + distance bracket once (discrete Fréchet from above, exact below the largest threshold; Hausdorff + from below). With dev = `simplification_bound(i)`, upper + dev ≤ ε proves a hit and + lower − dev > ε a miss, without the test (`certificate_test.cpp`). kNN checks in the curves do + the same with the ranked neighbours' distances. +- `original_within(j, candidate, r)`: every remaining test of an original against a candidate + (join, curve kNN checks, `propose_bounded`, `watch`). Each original has a coarse RDP copy with + deviation μ ≤ 5 m (`kCoarseM`), used only if it at least halves the points. The coarse copy + within r − μ proves a hit, not within r + μ a miss; only the band between needs the full test. +- All of these are exact: they decide the same outcomes as the full test (continuous Fréchet + only), and `CoarseScreenAgreesWithTheFullTest` checks it. +- Speed, Geolife seed 42, 24 threads: these screens, one `CandidateMaker` per trajectory shared by + curves, candidates and trades, the k-th place read from the ranking in `propose_bounded` (and + the stored distance ± both deviation bounds tried first), the greedy where-at score skipping + `hypot` outside its ramp, cached edge lengths for the deviation bound, the loop reusing the + tracker's distances for each start candidate, and the tracker taking its list distances from + `query::knn_with_distances` took the teacher from 206 s to 104 s at 0.25 % and from 444 s to + 192 s at 1 %, with identical kept sets on seeds 42 and 7 (research notes, + `Teacher speedups 2026-11-22`). ### `join_metric(state)` — private @@ -552,12 +568,14 @@ simplified. - **Output:** one `Arm` per requested name, in the requested order. Each keeps exactly `total_budget` points, because every candidate keeps exactly its trajectory's budget. -### `make_candidates(objective, budgets, caps, selected)` — internal +### `make_candidates(objective, budgets, caps, makers, selected)` — internal - **Why:** each candidate is simplified once, then reused by every step. -- **Input:** objective, per-trajectory budgets and caps, and optionally selected method indices. -- **Computation:** in parallel, `CandidateMaker` builds all six recipes, or only the selected - recipe for each trajectory. Classic sequences and greedy sequences with the same seed are reused. +- **Input:** objective, per-trajectory budgets and caps, the shared makers, and optionally + selected method indices. +- **Computation:** in parallel, each trajectory's `CandidateMaker` builds all six recipes, or only + the selected one. A maker is created on first use and kept for the whole `run`, so its RDP, SED + and greedy sequences are built once and serve the curves, the candidates and the trades. - **Output:** `Candidates`. ### `assemble(candidates, method)` — internal @@ -644,8 +662,14 @@ trackers; `run` updates them in place, so it runs once. of each threshold's macro F1, normalised by its own endpoints-only floor. - `ShapeTermRampsTheDeviationBound`, `TwoSidedMarginScoresOutsidersToo`: shape and margin are 1 with every point, 0 at the floor, separable, and the margin term needs ranked neighbours. +- `CoarseScreenAgreesWithTheFullTest`: on dense trajectories, `original_within` gives the full + Fréchet test's answer at, near and far from the exact distance. +- `PreparedIndependentScoresMatchFullScoringAndSurvivePruning`: independent scores equal direct + scoring, with and without ranked-neighbour brackets for the kNN checks. +- `BoundedKnnProposalsMatchFullDistances`: `propose_bounded`, with and without a reach, equals + the proposal from exact distances. `engine/tests/teacher/certificate_test.cpp` checks `simplification_bound` never falls below the continuous Fréchet distance, alone and added to a discrete pair distance. -`engine/tests/config_test.cpp` has the three teacher config tests. +`engine/tests/pipeline/config_test.cpp` has the three teacher config tests. diff --git a/engine/src/teacher/greedy.cpp b/engine/src/teacher/greedy.cpp index 243ad30..5b56d64 100644 --- a/engine/src/teacher/greedy.cpp +++ b/engine/src/teacher/greedy.cpp @@ -76,6 +76,10 @@ std::vector Greedy::sequence(std::size_t limit, std::span(trajectory.t[from]), @@ -84,8 +88,14 @@ std::vector Greedy::sequence(std::size_t limit, std::span t_from ? std::clamp((m_probe_t[p] - t_from) / (t_to - t_from), 0.0, 1.0) : 0.0; const double x = trajectory.x[from] + fraction * (trajectory.x[to] - trajectory.x[from]); const double y = trajectory.y[from] + fraction * (trajectory.y[to] - trajectory.y[from]); - return ramp(std::hypot(x - m_probe_x[p], y - m_probe_y[p]), m_min_tolerance, - m_max_tolerance); + const double dx = x - m_probe_x[p], dy = y - m_probe_y[p]; + // Clearly outside the ramp it is 0 or 1 exactly, without the square root + const double squared = dx * dx + dy * dy; + if (squared >= far) + return 0.0; + if (squared <= near) + return 1.0; + return ramp(std::hypot(dx, dy), m_min_tolerance, m_max_tolerance); }; // Probes from t_a up to t_c, the last segment including its end auto probes_between = [&](std::size_t from, std::size_t to) { diff --git a/engine/src/teacher/objective.cpp b/engine/src/teacher/objective.cpp index a39e483..5735d99 100644 --- a/engine/src/teacher/objective.cpp +++ b/engine/src/teacher/objective.cpp @@ -6,12 +6,14 @@ #include "backend/engine/query/frechet_helper.hpp" #include "backend/engine/query/knn.hpp" #include "backend/engine/query/range_query.hpp" +#include "backend/engine/selection/rdp.hpp" #include #include #include #include #include +#include #include #include @@ -74,14 +76,77 @@ double range_marginal(std::size_t others, std::size_t expected, std::size_t scor constexpr std::size_t kMaxJoinTerms = 8; // Relative margin for certificates, far above the decision predicate's 1e-12 rounding slack constexpr double kCertificateMargin = 1e-9; +// Deviation of the coarse copies that pre-screen tests against originals +constexpr double kCoarseM = 5.0; + +// Largest distance from a vertex of a to the polyline b, squared: a lower bound on the continuous +// Fréchet distance. Each vertex searches outward from the previous vertex's nearest segment and +// stops once one is no farther than the running maximum +double squared_directed_hausdorff(const core::TrajectoryView &a, const core::TrajectoryView &b) { + const auto segment = [&](std::size_t i, std::size_t j) { + const double ax = b.x[j], ay = b.y[j], dx = b.x[j + 1] - ax, dy = b.y[j + 1] - ay; + const double px = a.x[i] - ax, py = a.y[i] - ay; + const double length = dx * dx + dy * dy; + const double along = px * dx + py * dy; + if (length == 0.0 || along <= 0.0) + return px * px + py * py; + if (along >= length) { + const double ex = a.x[i] - b.x[j + 1], ey = a.y[i] - b.y[j + 1]; + return ex * ex + ey * ey; + } + const double cross = px * dy - py * dx; + return cross * cross / length; + }; + if (a.size() == 0 || b.size() == 0) + return std::numeric_limits::infinity(); + double bound = 0.0; + if (b.size() == 1) { + for (std::size_t i = 0; i < a.size(); ++i) { + const double dx = a.x[i] - b.x[0], dy = a.y[i] - b.y[0]; + bound = std::max(bound, dx * dx + dy * dy); + } + return bound; + } + const std::size_t segments = b.size() - 1; + std::size_t start = 0; + for (std::size_t i = 0; i < a.size(); ++i) { + double nearest = segment(i, start); + std::size_t best = start; + for (std::size_t step = 1; nearest > bound && (step <= start || start + step < segments); + ++step) { + if (step <= start) { + const double d = segment(i, start - step); + if (d < nearest) { + nearest = d; + best = start - step; + } + } + if (start + step < segments) { + const double d = segment(i, start + step); + if (d < nearest) { + nearest = d; + best = start + step; + } + } + } + bound = std::max(bound, nearest); + start = best; + } + return bound; +} } // namespace // Upper bound on the continuous Fréchet distance between original and a simplification keeping // a subsequence of its points: each kept segment walks its original stretch at the same share // of arc length. Along each original edge the gap is convex, so its maximum is at a vertex -double simplification_bound(const core::TrajectoryView &original, - const core::TrajectoryView &kept) { +double simplification_bound(const core::TrajectoryView &original, const core::TrajectoryView &kept, + std::span segments) { + const auto edge = [&](std::size_t j) { + return segments.empty() ? std::hypot(original.x[j + 1] - original.x[j], + original.y[j + 1] - original.y[j]) + : segments[j]; + }; const std::size_t n = original.size(); if (kept.size() < 2 || n < 2) return std::numeric_limits::infinity(); @@ -100,13 +165,11 @@ double simplification_bound(const core::TrajectoryView &original, return std::numeric_limits::infinity(); double length = 0.0; for (std::size_t j = at; j < end; ++j) - length += - std::hypot(original.x[j + 1] - original.x[j], original.y[j + 1] - original.y[j]); + length += edge(j); double walked = 0.0; for (std::size_t j = at; j <= end; ++j) { if (j > at) - walked += std::hypot(original.x[j] - original.x[j - 1], - original.y[j] - original.y[j - 1]); + walked += edge(j - 1); const double u = length > 0 ? walked / length : 0.0; const double x = kept.x[k] + u * (kept.x[k + 1] - kept.x[k]); const double y = kept.y[k] + u * (kept.y[k + 1] - kept.y[k]); @@ -121,6 +184,41 @@ namespace { // Cached kNN distances are upper bounds on a fixed millimetre grid. constexpr double kKnnDistanceError = query::kKnnResolutionM; +// query::frechet_distance bisects the continuous distance to 1 mm from above; twice that brackets +// the ranked neighbours' distances from below +constexpr double kRankedDistanceError = 2e-3; + +// The shortest RDP prefix of original within max_error, if it at least halves the points +std::optional> coarse_copy(const core::TrajectoryView &original, + double max_error) { + if (original.size() < 8) + return std::nullopt; + const auto order = selection::rdp::sequence(original, original.size()); + auto prefix = [&](std::size_t b) { + std::vector kept(order.begin(), + order.begin() + static_cast(b)); + std::ranges::sort(kept); + return kept; + }; + auto within = [&](std::size_t b) { + return simplification_bound(original, PreparedTrajectory(original, prefix(b)).view()) <= + max_error; + }; + // Doubling, then bisection between the last miss and the first hit + const std::size_t most = original.size() / 2; + std::size_t lo = 2, hi = 2; + while (!within(hi)) { + if (hi == most) + return std::nullopt; + lo = hi; + hi = std::min(most, 2 * hi); + } + while (lo + 1 < hi) { + const auto mid = (lo + hi) / 2; + (within(mid) ? hi : lo) = mid; + } + return prefix(hi); +} double f1_of(std::size_t hits, std::size_t found, std::size_t expected) { return 2.0 * static_cast(hits) / static_cast(found + expected); @@ -175,6 +273,16 @@ Objective::Objective(const core::TrajectoryDb &db, const TeacherWorkloads &workl } m_knn_weight = valid ? 1.0 / (static_cast(workloads.k) * static_cast(valid)) : 0.0; + // A bracket on each membership's distance, from the ranked neighbours + if (workloads.ranked.size() == n) + for (std::size_t i = 0; i < n; ++i) + for (auto &membership : m_knn[i]) + for (const auto &neighbour : workloads.ranked[membership.query]) + if (neighbour.index == i) { + membership.upper = neighbour.distance; + membership.lower = std::max(0.0, neighbour.distance - kRankedDistanceError); + break; + } m_margins.resize(n); const std::size_t k = workloads.k; if (config.term_weights[Margin] > 0) { @@ -256,20 +364,32 @@ Objective::Objective(const core::TrajectoryDb &db, const TeacherWorkloads &workl for (std::size_t t = 0; t < m_truth.size(); ++t) m_with_truth[t / n] += m_truth[t] > 0; - // Positives: the originals' discrete Fréchet distance bounds the continuous one - m_pair_upper.assign(m_pair_i.size(), std::numeric_limits::infinity()); - core::parallel_for(m_positive_count, [&](std::size_t p) { - const auto k = m_pair_term[p]; - if (m_workloads.frechet != query::FrechetVariant::Continuous) - return; - std::vector prev, curr; - const double cutoff = m_eps[k] * m_eps[k]; - const double squared = query::get_squared_frechet_distance( - db.trajectory(m_pair_i[p]), db.trajectory(m_pair_j[p]), cutoff, prev, curr); - if (squared < cutoff) - m_pair_upper[p] = std::sqrt(squared); + m_segments.resize(n); + core::parallel_for(n, [&](std::size_t i) { + const auto t = db.trajectory(i); + auto &edges = m_segments[i]; + edges.resize(t.size() > 0 ? t.size() - 1 : 0); + for (std::size_t j = 0; j < edges.size(); ++j) + edges[j] = std::hypot(t.x[j + 1] - t.x[j], t.y[j + 1] - t.y[j]); }); + // Coarse copies, to pre-screen tests against originals + m_coarse.reserve(n); + for (std::size_t i = 0; i < n; ++i) + m_coarse.emplace_back(db.trajectory(i), std::vector{}); + m_coarse_error.assign(n, -1.0); + if (m_workloads.frechet == query::FrechetVariant::Continuous) + core::parallel_for(n, [&](std::size_t i) { + const auto original = db.trajectory(i); + if (const auto kept = coarse_copy(original, kCoarseM)) { + m_coarse[i].assign(original, *kept); + m_coarse_error[i] = simplification_bound(original, m_coarse[i].view()); + } + }); + + m_certify = m_workloads.frechet == query::FrechetVariant::Continuous; + bracket_pairs(); + // Floors: endpoints only, partners included, like the J4 floor. Each join term has its own, // so the join metric is 0 there. Terms without true pairs are left out, as J4 skips NaNs const auto endpoints = core::endpoints_only(db); @@ -299,16 +419,80 @@ Objective::Objective(const core::TrajectoryDb &db, const TeacherWorkloads &workl }); } +void Objective::bracket_pairs() { + // Each pair's originals, once per pair across terms: the discrete distance bounds the + // continuous one from above (exact below the largest threshold), Hausdorff from below + m_pair_lower.assign(m_pair_i.size(), 0.0); + m_pair_upper.assign(m_pair_i.size(), std::numeric_limits::infinity()); + if (!m_certify) + return; + std::vector order(m_pair_i.size()); + std::iota(order.begin(), order.end(), 0u); + std::ranges::sort(order, [&](std::uint32_t a, std::uint32_t b) { + return std::pair{m_pair_i[a], m_pair_j[a]} < std::pair{m_pair_i[b], m_pair_j[b]}; + }); + std::vector starts; + for (std::size_t r = 0; r < order.size(); ++r) + if (r == 0 || m_pair_i[order[r]] != m_pair_i[order[r - 1]] || + m_pair_j[order[r]] != m_pair_j[order[r - 1]]) + starts.push_back(r); + const double largest = *std::ranges::max_element(m_eps); + const double cutoff = largest * largest; + core::parallel_for(starts.size(), [&](std::size_t s) { + const auto first = order[starts[s]]; + const auto a = m_db.trajectory(m_pair_i[first]), b = m_db.trajectory(m_pair_j[first]); + std::vector prev, curr; + const double squared = query::get_squared_frechet_distance(a, b, cutoff, prev, curr); + const double upper = + squared < cutoff ? std::sqrt(squared) : std::numeric_limits::infinity(); + const double lower = std::sqrt(std::max( + {query::squared_endpoint_lower_bound(a, b), query::squared_box_extent_lower_bound(a, b), + squared_directed_hausdorff(a, b), squared_directed_hausdorff(b, a)})); + const std::size_t end = s + 1 < starts.size() ? starts[s + 1] : order.size(); + for (std::size_t r = starts[s]; r < end; ++r) { + m_pair_lower[order[r]] = lower; + m_pair_upper[order[r]] = upper; + } + }); +} + +std::optional Objective::certify(double lower, double upper, double deviation, + double cutoff) const { + // |d(query, candidate) - d(query, original)| <= d(original, candidate) <= deviation + if (upper + deviation <= cutoff * (1.0 - kCertificateMargin)) + return true; + if (lower - deviation > cutoff * (1.0 + kCertificateMargin)) + return false; + return std::nullopt; +} + bool Objective::held_out_match(std::size_t i, const JoinLink &link, const core::TrajectoryView &candidate, double &own_error) const { - // d(original partner, candidate) <= d(originals) + d(original i, candidate) - if (std::isfinite(m_pair_upper[link.pair])) { + if (m_certify) { if (own_error < 0) - own_error = simplification_bound(m_db.trajectory(i), candidate); - if (m_pair_upper[link.pair] + own_error <= m_eps[link.term] * (1.0 - kCertificateMargin)) + own_error = deviation(i, candidate); + if (const auto known = certify(m_pair_lower[link.pair], m_pair_upper[link.pair], own_error, + m_eps[link.term])) + return *known; + } + return original_within(link.other, candidate, m_eps[link.term]); +} + +bool Objective::original_within(std::size_t j, const core::TrajectoryView &candidate, + double radius) const { + if (m_coarse_error[j] >= 0) { + // d(original, coarse) <= error, so the coarse copy decides outside a band of 2 error + const auto coarse = m_coarse[j].view(); + const double error = m_coarse_error[j]; + if (radius - error > 0 && + query::continuous_frechet_within(coarse, candidate, + (radius - error) * (1.0 - kCertificateMargin))) return true; + if (!query::continuous_frechet_within(coarse, candidate, + (radius + error) * (1.0 + kCertificateMargin))) + return false; } - return match(m_db.trajectory(link.other), candidate, link.term); + return query::frechet_within(m_db.trajectory(j), candidate, radius, m_workloads.frechet); } bool Objective::match(const core::TrajectoryView &a, const core::TrajectoryView &b, @@ -316,8 +500,8 @@ bool Objective::match(const core::TrajectoryView &a, const core::TrajectoryView return query::frechet_within(a, b, m_eps[term], m_workloads.frechet); } -Objective::Terms Objective::raw_separable(std::size_t i, - const core::TrajectoryView &candidate) const { +Objective::Terms Objective::raw_separable(std::size_t i, const core::TrajectoryView &candidate, + double deviation) const { Terms terms{}; if (!m_probes[i].empty()) { double sum = 0; @@ -333,7 +517,8 @@ Objective::Terms Objective::raw_separable(std::size_t i, const bool shape = m_config.term_weights[Shape] > 0; const bool margin = m_config.term_weights[Margin] > 0 && !m_margins[i].empty(); if (shape || margin) { - const double deviation = simplification_bound(m_db.trajectory(i), candidate); + if (deviation < 0) + deviation = this->deviation(i, candidate); if (shape) terms[Shape] = m_shape_weight * ramp(deviation, m_config.min_shape_m, m_config.max_shape_m); @@ -386,8 +571,8 @@ Objective::prepare_independent(std::size_t i, const core::TrajectoryView &candid IndependentScore score; score.trajectory = i; score.candidate = candidate; - score.partners = partners; - score.terms = raw_separable(i, candidate); + score.deviation = deviation(i, candidate); + score.terms = raw_separable(i, candidate, score.deviation); const auto owned_weights = range_weights.empty() ? this->range_weights(i, partners) : std::vector{}; if (range_weights.empty()) @@ -398,7 +583,7 @@ Objective::prepare_independent(std::size_t i, const core::TrajectoryView &candid if (query::intersects(candidate, m_workloads.range.queries[m_range[i][k].box].box)) score.terms[Range] += range_weights[k]; std::array hits{}, found{}; - double own_error = -1.0; + double own_error = score.deviation; for (const auto &link : m_links[i]) { // The partner is the original query and i the database side const bool matched = held_out_match(i, link, candidate, own_error); @@ -434,8 +619,9 @@ double Objective::independent_value(const IndependentScore &score, double beat) if (value() + per_hit * static_cast(left) < beat - kBoundSlack) break; --left; - if (query::frechet_within(m_db.trajectory(m.query), score.candidate, m.cutoff, - m_workloads.frechet)) + const auto known = + m_certify ? certify(m.lower, m.upper, score.deviation, m.cutoff) : std::nullopt; + if (known ? *known : original_within(m.query, score.candidate, m.cutoff)) knn += m_knn_weight; } return value(); @@ -521,8 +707,13 @@ double Objective::knn_recall(const core::TrajectoryDb &compressed) const { return m_knn_weight * static_cast(std::accumulate(hits.begin(), hits.end(), 0ul)); } -double Objective::total(const KeptSets &kept) const { - const auto values = metrics(kept); +double Objective::deviation(std::size_t i, const core::TrajectoryView &candidate) const { + return simplification_bound(m_db.trajectory(i), candidate, m_segments[i]); +} + +double Objective::total(const KeptSets &kept) const { return total(metrics(kept)); } + +double Objective::total(const Terms &values) const { double sum = 0; for (std::size_t k = 0; k < TermCount; ++k) sum += m_scale[k] * (values[k] - m_floor[k]); @@ -755,31 +946,48 @@ KnnState::KnnState(const Objective &objective, const KeptSets &state, std::size_ m_hits.assign(n, 0); m_watchers.resize(n); const auto compressed = core::materialize(db, state); + const bool continuous = workloads.frechet == query::FrechetVariant::Continuous; + const PreparedTrajectories prepared(db, state); + const auto views = prepared.views(); + // Each list with its distances in state; the search already has its own neighbours' core::parallel_for(n, [&](std::size_t e) { const auto &entry = workloads.knn[e]; if (!entry.valid) return; auto &list = m_list[e]; list.assign(entry.top.begin(), entry.top.end()); - for (const auto j : - query::knn(compressed, db.trajectory(e), std::min(watch + 1, n), workloads.frechet)) - if (j != e && std::ranges::find(list, j) == list.end()) + constexpr double unknown = -1.0; + auto &distance = m_distance[e]; + distance.assign(list.size(), unknown); + const auto query = db.trajectory(e); + const std::size_t count = std::min(watch + 1, n); + std::vector> found; + if (continuous) + found = query::knn_with_distances(compressed, query, count); + else + for (const auto j : query::knn(compressed, query, count, workloads.frechet)) + found.push_back({j, unknown}); + for (const auto &[j, d] : found) { + if (j == e) + continue; + const auto at = std::ranges::find(list, j); + if (at == list.end()) { list.push_back(static_cast(j)); + distance.push_back(d); + } else { + distance[static_cast(at - list.begin())] = d; + } + } + for (std::size_t p = 0; p < list.size(); ++p) + if (distance[p] == unknown) + distance[p] = query::knn_distance(query, views[list[p]], workloads.frechet); }); for (std::size_t e = 0; e < n; ++e) { m_valid += workloads.knn[e].valid; - m_distance[e].assign(m_list[e].size(), 0.0); for (std::size_t p = 0; p < m_list[e].size(); ++p) m_watchers[m_list[e][p]].push_back( {static_cast(e), static_cast(p)}); } - const PreparedTrajectories prepared(db, state); - const auto views = prepared.views(); - std::vector> own(n); - core::parallel_for(n, [&](std::size_t i) { own[i] = distances(i, views[i]); }); - for (std::size_t i = 0; i < n; ++i) - for (std::size_t s = 0; s < m_watchers[i].size(); ++s) - m_distance[m_watchers[i][s].query][m_watchers[i][s].position] = own[i][s]; m_rank.resize(n); core::parallel_for(n, [&](std::size_t e) { if (!m_list[e].empty()) @@ -790,11 +998,10 @@ KnnState::KnnState(const Objective &objective, const KeptSets &state, std::size_ std::vector KnnState::distances(std::size_t i, const core::TrajectoryView &candidate, bool parallel) const { - const auto &db = m_objective.db(); const auto &watchers = m_watchers[i]; std::vector result(watchers.size()); auto one = [&](std::size_t s) { - result[s] = query::knn_distance(db.trajectory(watchers[s].query), candidate, + result[s] = query::knn_distance(m_objective.db().trajectory(watchers[s].query), candidate, m_objective.workloads().frechet); }; if (parallel) @@ -861,7 +1068,6 @@ void KnnState::refresh(std::size_t e) { void KnnState::watch(std::span> versions, std::span state) { const auto &db = m_objective.db(); - const auto frechet = m_objective.workloads().frechet; const std::size_t n = db.size(); std::vector radius(n, -1.0); core::parallel_for(n, [&](std::size_t e) { @@ -885,9 +1091,10 @@ void KnnState::watch(std::span> versions std::ranges::any_of(m_watchers[i], [&](const Watcher &w) { return w.query == e; })) continue; for (const auto &version : versions[i]) - if (query::frechet_within(db.trajectory(e), version, radius[e], frechet)) { + if (m_objective.original_within(e, version, radius[e])) { added[i].push_back({static_cast(e), - query::knn_distance(db.trajectory(e), state[i], frechet)}); + query::knn_distance(db.trajectory(e), state[i], + m_objective.workloads().frechet)}); break; } } @@ -908,41 +1115,56 @@ void KnnState::watch(std::span> versions } } -double KnnState::propose_bounded(std::size_t i, const core::TrajectoryView &candidate) const { - const auto &db = m_objective.db(); - const auto frechet = m_objective.workloads().frechet; +double KnnState::propose_bounded(std::size_t i, const core::TrajectoryView &candidate, + double reach) const { const std::size_t k = m_objective.workloads().k; // Clear of the k-th place by more than the stored distances' error constexpr double margin = 2.5 * kKnnDistanceError; long change = 0; - std::vector others; for (const auto &[e, position] : m_watchers[i]) { const auto inside = hits(e, position, -1.0); const auto outside = hits(e, position, std::numeric_limits::infinity()); std::size_t now = inside; if (inside != outside) { - // The k-th distance among the others: i is in clearly below it, out clearly above - others.clear(); - for (std::size_t p = 0; p < m_list[e].size(); ++p) - if (p != position) - others.push_back(m_distance[e][p]); - const auto kth = - others.begin() + static_cast(std::min(k, others.size()) - 1); - std::ranges::nth_element(others, kth); - const auto query = db.trajectory(e); - if (*kth - margin > 0 && - query::frechet_within(query, candidate, *kth - margin, frechet)) + // The k-th distance among the others, from the ranking (it has k + 1 entries here): + // the (k + 1)-th overall if i is in the top k, else the k-th. i is in clearly below + // it, out clearly above + const auto &rank = m_rank[e]; + const bool in_top = + std::find(rank.begin(), rank.begin() + static_cast(k), + static_cast(position)) != + rank.begin() + static_cast(k); + const double kth = m_distance[e][rank[in_top ? k : k - 1]]; + // |d(query, candidate) - d(query, current)| <= reach; the stored key is the current + // distance rounded up to the grid + const double at = m_distance[e][position]; + if (kth - margin > 0 && at + reach <= (kth - margin) * (1.0 - kCertificateMargin)) + now = inside; + else if (at - 2.0 * kKnnDistanceError - reach > + (kth + margin) * (1.0 + kCertificateMargin)) + now = outside; + else if (kth - margin > 0 && m_objective.original_within(e, candidate, kth - margin)) now = inside; - else if (!query::frechet_within(query, candidate, *kth + margin, frechet)) + else if (!m_objective.original_within(e, candidate, kth + margin)) now = outside; else - now = hits(e, position, query::knn_distance(query, candidate, frechet)); + now = hits(e, position, + query::knn_distance(m_objective.db().trajectory(e), candidate, + m_objective.workloads().frechet)); } change += static_cast(now) - static_cast(m_hits[e]); } return value(change); } +std::vector KnnState::stored(std::size_t i) const { + std::vector result; + result.reserve(m_watchers[i].size()); + for (const auto &watcher : m_watchers[i]) + result.push_back(m_distance[watcher.query][watcher.position]); + return result; +} + double KnnState::propose(std::size_t i, std::span distances) const { long change = 0; for (std::size_t s = 0; s < m_watchers[i].size(); ++s) { diff --git a/engine/src/teacher/teacher.cpp b/engine/src/teacher/teacher.cpp index 74d42cc..86362b4 100644 --- a/engine/src/teacher/teacher.cpp +++ b/engine/src/teacher/teacher.cpp @@ -62,14 +62,13 @@ template std::size_t first_equal(std::size_t c, Kept &&kept) { return first; } -// One trajectory's candidates up to cap. A candidate depends only on (i, c, b, cap), so curves -// and final kept sets agree +// One trajectory's candidates up to cap, from sequences built once. A candidate depends only on +// (i, c, b, cap), so curves and final kept sets agree class CandidateMaker { public: - CandidateMaker(const Objective &objective, std::size_t i, std::size_t cap, - bool reuse_sequences = false) + CandidateMaker(const Objective &objective, std::size_t i, std::size_t cap) : m_original(objective.db().trajectory(i)), m_cap(std::min(cap, m_original.size())), - m_objective(objective), m_index(i), m_reuse_sequences(reuse_sequences) {} + m_objective(objective), m_index(i) {} std::vector make(std::size_t candidate, std::size_t budget) { budget = std::min(budget, m_original.size()); @@ -98,7 +97,7 @@ class CandidateMaker { private: std::vector simplify(AlgorithmType type, std::size_t b) { - if (!m_reuse_sequences || type == AlgorithmType::UNIFORM) + if (type == AlgorithmType::UNIFORM) return selection::simplify_trajectory(m_original, type, b); auto &cached = type == AlgorithmType::RDP ? m_rdp : m_sed; if (!cached) @@ -134,23 +133,32 @@ class CandidateMaker { std::size_t m_cap; const Objective &m_objective; std::size_t m_index; - bool m_reuse_sequences; std::optional m_greedy; std::optional> m_rdp, m_sed; std::vector m_fill; std::map, std::vector> m_greedy_sequences; }; +// Per trajectory, built on first use and kept: its sequences serve the curves, the candidates and +// the trades alike +using Makers = std::vector>; + +CandidateMaker &maker_for(Makers &makers, const Objective &objective, std::size_t i, + std::size_t cap) { + if (!makers[i]) + makers[i].emplace(objective, i, cap); + return *makers[i]; +} + // All candidates at each trajectory's budget, or only the selected one Candidates make_candidates(const Objective &objective, const std::vector &budgets, - const std::vector &caps, + const std::vector &caps, Makers &makers, const Methods *selected = nullptr) { Candidates candidates; for (auto &kept : candidates) kept.resize(budgets.size()); core::parallel_for(budgets.size(), [&](std::size_t i) { - // Greedy needs the classic sequences for its seed and fill - CandidateMaker maker(objective, i, caps[i], !selected || (*selected)[i] >= kGreedy); + auto &maker = maker_for(makers, objective, i, caps[i]); if (selected) { const auto c = (*selected)[i]; candidates[c][i] = maker.make(c, budgets[i]); @@ -211,14 +219,15 @@ struct Curves { }; Curves build_curves(const core::TrajectoryDb &db, const Objective &objective, - const KeptSets &partners, const std::vector &caps, bool rdp_only) { + const KeptSets &partners, const std::vector &caps, Makers &makers, + bool rdp_only) { Curves curves{std::vector(db.size()), std::vector(db.size())}; const PreparedTrajectories prepared(db, partners); const auto partner_views = prepared.views(); const RangeState range(objective, partner_views); core::parallel_for(db.size(), [&](std::size_t i) { const auto range_weights = range.weights(i); - CandidateMaker maker(objective, i, caps[i], true); + auto &maker = maker_for(makers, objective, i, caps[i]); const std::size_t max_b = std::min(caps[i], db.trajectory(i).size()); auto &best = curves.best[i]; auto &rdp = curves.rdp[i]; @@ -335,7 +344,10 @@ class Search { first == c ? objective.separable(i, m_views[c][i]) : m_separable[i][first]; if (first != c) continue; - m_knn_distances[i][c] = m_knn->distances(i, m_views[c][i]); + // The start's distances are already in the tracker + m_knn_distances[i][c] = c == m_representative[i][method[i]] + ? m_knn->stored(i) + : m_knn->distances(i, m_views[c][i]); m_coverage[i][c] = m_range->coverage(i, m_views[c][i]); m_outcomes[i][c] = m_join->outcomes(i, m_views[c][i]); } @@ -439,6 +451,7 @@ Checked checked_loop(const Objective &objective, const Candidates &candidates, c struct Traded { KeptSets kept; double value; + std::optional metrics; }; // Moves delta points between trajectories while exact J_T rises. Each round scores every @@ -446,7 +459,8 @@ struct Traded { // buyers with the cheapest sellers, and keeps a pair only if its exact gain, rescored when applied, // is positive. Each round the kNN lists are extended to watch the offered versions Traded trade_budgets(const core::TrajectoryDb &db, const Objective &objective, - const std::vector &caps, KeptSets kept, double start_value) { + const std::vector &caps, Makers &makers, KeptSets kept, + double start_value) { const std::size_t n = db.size(); std::vector owned; owned.reserve(n); @@ -457,8 +471,11 @@ Traded trade_budgets(const core::TrajectoryDb &db, const Objective &objective, state[i] = owned[i].view(); JoinState join(objective, state); RangeState range(objective, state); - std::vector separable(n); - core::parallel_for(n, [&](std::size_t i) { separable[i] = objective.separable(i, state[i]); }); + std::vector separable(n), state_error(n); + core::parallel_for(n, [&](std::size_t i) { + separable[i] = objective.separable(i, state[i]); + state_error[i] = objective.deviation(i, state[i]); + }); // Built once: the trades keep it current KnnState knn(objective, kept, kKnnWatchFactor * objective.workloads().k); @@ -493,6 +510,7 @@ Traded trade_budgets(const core::TrajectoryDb &db, const Objective &objective, state[i] = owned[i].view(); kept[i] = std::move(p.kept); separable[i] = p.separable; + state_error[i] = objective.deviation(i, state[i]); }; struct Offer { @@ -503,7 +521,6 @@ Traded trade_budgets(const core::TrajectoryDb &db, const Objective &objective, double value = start_value; std::cout << std::format("teacher: trading from J_T {:.6f}\n", value); // Sequences depend only on the trajectory and its cap, so are built once for every round - std::vector> makers(n); for (std::size_t round = 1; round <= kMaxTradeRounds; ++round) { auto start = std::chrono::steady_clock::now(); // buy[d][i], sell[d][i]: i's best exact change at b_i + delta and b_i - delta @@ -515,9 +532,7 @@ Traded trade_budgets(const core::TrajectoryDb &db, const Objective &objective, core::parallel_for(n, [&](std::size_t i) { const std::size_t b = kept[i].size(); const std::size_t top = std::min(caps[i], db.trajectory(i).size()); - if (!makers[i]) - makers[i].emplace(objective, i, caps[i], true); - auto &maker = *makers[i]; + auto &maker = maker_for(makers, objective, i, caps[i]); auto best_at = [&](std::size_t budget) { Offer best; std::vector> seen; @@ -530,7 +545,8 @@ Traded trade_budgets(const core::TrajectoryDb &db, const Objective &objective, const auto view = prepared.view(); double gain = objective.separable(i, view) - separable[i] + join.propose(i, view).gain + range.propose(i, view).gain; - gain += knn.propose_bounded(i, view); + gain += + knn.propose_bounded(i, view, state_error[i] + objective.deviation(i, view)); if (gain > best.gain) best = {gain, std::move(candidate)}; } @@ -617,10 +633,7 @@ Traded trade_budgets(const core::TrajectoryDb &db, const Objective &objective, break; } const auto terms = objective.metrics(kept); - double exact = 0; - for (std::size_t m = 0; m < Objective::TermCount; ++m) - exact += - objective.scale(static_cast(m)) * (terms[m] - objective.floors()[m]); + const double exact = objective.total(terms); // Where-at is rescored on every commit, so only these three can drift std::cout << std::format( "teacher: trading J_T tracked {:.6f}, exact {:.6f}; tracked/exact kNN {:.6f}/{:.6f}, " @@ -629,8 +642,8 @@ Traded trade_budgets(const core::TrajectoryDb &db, const Objective &objective, join.metric(), terms[Objective::Join]); // The kNN tracker is not exact everywhere, so the trades could in principle lose if (exact < start_value) - return {start_kept, start_value}; - return {std::move(kept), exact}; + return {start_kept, start_value, {}}; + return {std::move(kept), exact, terms}; } std::string wins(const Methods &method) { @@ -709,9 +722,13 @@ std::vector Teacher::run(std::size_t total_budget, std::span known = {}) { auto kept = assemble(from, method); - const double j_t = known ? *known : objective.total(kept); + std::optional metrics; + if (!known) + metrics = objective.metrics(kept); + const double j_t = known ? *known : objective.total(*metrics); std::cout << std::format("teacher {}: {}, J_T {:.4f}\n", name, wins(method), j_t); - outputs[std::string(name)] = {std::string(name), core::to_simplification(kept), j_t}; + outputs[std::string(name)] = {std::string(name), core::to_simplification(kept), j_t, + metrics}; return j_t; }; auto start = std::chrono::steady_clock::now(); @@ -731,16 +748,19 @@ std::vector Teacher::run(std::size_t total_budget, std::span Teacher::run(std::size_t total_budget, std::span choice_value; if (want_choice) choice_value = keep("BUDGET_CHOICE", candidates, choice); @@ -764,10 +784,10 @@ std::vector Teacher::run(std::size_t total_budget, std::span Teacher::run(std::size_t total_budget, std::span Teacher::single_recipes(std::size_t total_budget) const { const auto budgets = allocation::create_trajectory_budgets( m_db, allocation::AllocatorType::SizeProportional, total_budget, 0.0); - const auto candidates = make_candidates(m_objective, budgets, budgets); + Makers makers(m_db.size()); + const auto candidates = make_candidates(m_objective, budgets, budgets, makers); std::vector arms; for (std::size_t c = 0; c < kCandidates.size(); ++c) { const auto name = std::format("RECIPE_{}", kCandidates[c]); arms.push_back( - {name, core::to_simplification(candidates[c]), m_objective.total(candidates[c])}); + {name, core::to_simplification(candidates[c]), m_objective.total(candidates[c]), {}}); std::cout << std::format("teacher {}: J_T {:.4f}\n", name, arms.back().objective); } return arms; diff --git a/engine/tests/size_proportional_test.cpp b/engine/tests/allocation/size_proportional_test.cpp similarity index 100% rename from engine/tests/size_proportional_test.cpp rename to engine/tests/allocation/size_proportional_test.cpp diff --git a/engine/tests/interpolation_test.cpp b/engine/tests/core/interpolation_test.cpp similarity index 100% rename from engine/tests/interpolation_test.cpp rename to engine/tests/core/interpolation_test.cpp diff --git a/engine/tests/parallel_test.cpp b/engine/tests/core/parallel_test.cpp similarity index 100% rename from engine/tests/parallel_test.cpp rename to engine/tests/core/parallel_test.cpp diff --git a/engine/tests/projection_test.cpp b/engine/tests/core/projection_test.cpp similarity index 100% rename from engine/tests/projection_test.cpp rename to engine/tests/core/projection_test.cpp diff --git a/engine/tests/simplification_test.cpp b/engine/tests/core/simplification_test.cpp similarity index 100% rename from engine/tests/simplification_test.cpp rename to engine/tests/core/simplification_test.cpp diff --git a/engine/tests/subset_test.cpp b/engine/tests/core/subset_test.cpp similarity index 100% rename from engine/tests/subset_test.cpp rename to engine/tests/core/subset_test.cpp diff --git a/engine/tests/combined_test.cpp b/engine/tests/evaluator/combined_test.cpp similarity index 100% rename from engine/tests/combined_test.cpp rename to engine/tests/evaluator/combined_test.cpp diff --git a/engine/tests/evaluator_test.cpp b/engine/tests/evaluator/evaluator_test.cpp similarity index 100% rename from engine/tests/evaluator_test.cpp rename to engine/tests/evaluator/evaluator_test.cpp diff --git a/engine/tests/parquet_loader_test.cpp b/engine/tests/io/parquet_loader_test.cpp similarity index 100% rename from engine/tests/parquet_loader_test.cpp rename to engine/tests/io/parquet_loader_test.cpp diff --git a/engine/tests/config_test.cpp b/engine/tests/pipeline/config_test.cpp similarity index 100% rename from engine/tests/config_test.cpp rename to engine/tests/pipeline/config_test.cpp diff --git a/engine/tests/continuous_frechet_test.cpp b/engine/tests/query/continuous_frechet_test.cpp similarity index 100% rename from engine/tests/continuous_frechet_test.cpp rename to engine/tests/query/continuous_frechet_test.cpp diff --git a/engine/tests/join_test.cpp b/engine/tests/query/join_test.cpp similarity index 100% rename from engine/tests/join_test.cpp rename to engine/tests/query/join_test.cpp diff --git a/engine/tests/knn_test.cpp b/engine/tests/query/knn_test.cpp similarity index 89% rename from engine/tests/knn_test.cpp rename to engine/tests/query/knn_test.cpp index b19b34c..78cf562 100644 --- a/engine/tests/knn_test.cpp +++ b/engine/tests/query/knn_test.cpp @@ -176,3 +176,20 @@ TEST(Knn, BreaksEqualDistanceTiesByDatabaseIndex) { const auto db = make_db(std::vector(80, {{39.9, 116.3}, {39.91, 116.31}})); EXPECT_EQ(query::knn(db, db.trajectory(70), 5), (std::vector{0, 1, 2, 3, 4})); } + +TEST(Knn, WithDistancesGivesTheSameNeighboursAndTheirKeys) { + std::mt19937_64 rng(31); + const auto db = make_db(random_tracks(rng, 200)); + const auto queries = make_db(random_tracks(rng, 30)); + for (std::size_t q = 0; q < queries.size(); ++q) + for (const std::size_t k : {1u, 5u, 20u}) { + const auto plain = query::knn(db, queries.trajectory(q), k); + const auto ranked = query::knn_with_distances(db, queries.trajectory(q), k); + ASSERT_EQ(ranked.size(), plain.size()); + for (std::size_t r = 0; r < plain.size(); ++r) { + EXPECT_EQ(ranked[r].first, plain[r]); + EXPECT_EQ(ranked[r].second, + query::knn_distance(queries.trajectory(q), db.trajectory(plain[r]))); + } + } +} diff --git a/engine/tests/range_test.cpp b/engine/tests/query/range_test.cpp similarity index 100% rename from engine/tests/range_test.cpp rename to engine/tests/query/range_test.cpp diff --git a/engine/tests/route_clustering_test.cpp b/engine/tests/query/route_clustering_test.cpp similarity index 100% rename from engine/tests/route_clustering_test.cpp rename to engine/tests/query/route_clustering_test.cpp diff --git a/engine/tests/where_at_test.cpp b/engine/tests/query/where_at_test.cpp similarity index 100% rename from engine/tests/where_at_test.cpp rename to engine/tests/query/where_at_test.cpp diff --git a/engine/tests/selectors/rdp_test.cpp b/engine/tests/selection/rdp_test.cpp similarity index 100% rename from engine/tests/selectors/rdp_test.cpp rename to engine/tests/selection/rdp_test.cpp diff --git a/engine/tests/selectors/sed_test.cpp b/engine/tests/selection/sed_test.cpp similarity index 100% rename from engine/tests/selectors/sed_test.cpp rename to engine/tests/selection/sed_test.cpp diff --git a/engine/tests/selectors/simplify_database_test.cpp b/engine/tests/selection/simplify_database_test.cpp similarity index 100% rename from engine/tests/selectors/simplify_database_test.cpp rename to engine/tests/selection/simplify_database_test.cpp diff --git a/engine/tests/teacher/teacher_test.cpp b/engine/tests/teacher/teacher_test.cpp index 83a1f9a..cec44a0 100644 --- a/engine/tests/teacher/teacher_test.cpp +++ b/engine/tests/teacher/teacher_test.cpp @@ -3,6 +3,7 @@ #include "backend/engine/core/projection.hpp" #include "backend/engine/core/trajectory_db.hpp" #include "backend/engine/evaluator/range.hpp" +#include "backend/engine/query/frechet_helper.hpp" #include "backend/engine/selection/rdp.hpp" #include "backend/engine/teacher/greedy.hpp" #include "backend/engine/teacher/objective.hpp" @@ -12,6 +13,7 @@ #include #include +#include #include #include #include @@ -696,14 +698,70 @@ TEST_F(TeacherTest, HeldOutJoinScoresOriginalQueriesAgainstCompressedPartners) { (sum / static_cast(counted) - floor) / (1.0 - floor), 1e-12); } +TEST_F(TeacherTest, CoarseScreenAgreesWithTheFullTest) { + // Dense, smooth trajectories, so each has a coarse copy far shorter than itself + std::mt19937_64 rng(12); + std::normal_distribution noise(0.0, 0.4); + std::vector offsets{0}; + std::vector ids; + std::vector t; + std::vector lat, lon; + for (std::size_t i = 0; i < 12; ++i) { + ids.push_back(static_cast(i)); + const double shift = 25.0 * static_cast(i), wave = 30.0 + 10.0 * (i % 3); + for (std::size_t k = 0; k < 200; ++k) { + const double x = 5.0 * static_cast(k) + noise(rng); + const double y = shift + wave * std::sin(static_cast(k) / 15.0) + noise(rng); + const auto ll = core::to_lat_lon(x, y); + t.push_back(static_cast(k)); + lat.push_back(ll.lat); + lon.push_back(ll.lon); + } + offsets.push_back(t.size()); + } + const core::TrajectoryDb db(offsets, ids, t, lat, lon); + auto config = small_config(); + config.k = 2; + const auto workloads = teacher::load_or_build_workloads(db, config, 5, m_dir); + const teacher::Objective objective(db, workloads); + std::size_t hits = 0, misses = 0; + for (std::size_t i = 0; i < db.size(); ++i) { + const auto original = db.trajectory(i); + for (const std::size_t b : {2u, 6u, 20u, 60u}) { + const auto kept = selection::rdp::simplify(original, b); + const teacher::PreparedTrajectory candidate(original, kept); + for (std::size_t j = 0; j < db.size(); ++j) { + const double d = query::frechet_distance(db.trajectory(j), candidate.view(), + query::FrechetVariant::Continuous); + for (const double r : {d - 6.0, d - 0.01, d + 0.01, d + 6.0, d * 0.5, d * 2.0}) { + if (!(r > 0)) + continue; + const bool full = query::frechet_within(db.trajectory(j), candidate.view(), r, + query::FrechetVariant::Continuous); + EXPECT_EQ(objective.original_within(j, candidate.view(), r), full) + << i << " " << j << " " << b << " " << r; + (full ? hits : misses) += 1; + } + } + } + } + EXPECT_GT(hits, 0u); + EXPECT_GT(misses, 0u); +} + TEST_F(TeacherTest, PreparedIndependentScoresMatchFullScoringAndSurvivePruning) { const auto db = random_db(2, 24, 20); - const auto workloads = teacher::load_or_build_workloads(db, small_config(), 9, m_dir); + const auto plain = teacher::load_or_build_workloads(db, small_config(), 9, m_dir); + // Ranked neighbours give the kNN checks distance brackets, so certificates decide most + auto ranked = plain; + teacher::add_ranked_neighbours(db, ranked, 3, 9, m_dir); std::mt19937_64 rng(74); std::size_t valid = 0; - for (const auto &entry : workloads.knn) + for (const auto &entry : plain.knn) valid += entry.valid; - { + for (const teacher::TeacherWorkloads *source : {&plain, &std::as_const(ranked)}) { + const auto &workloads = *source; + SCOPED_TRACE(source == &ranked ? "ranked" : "plain"); { const teacher::Objective objective(db, workloads); std::size_t with_truth = 0; @@ -912,6 +970,13 @@ TEST_F(TeacherTest, BoundedKnnProposalsMatchFullDistances) { const teacher::PreparedTrajectory candidate(db.trajectory(i), random_kept(i)); const double full = knn.propose(i, knn.distances(i, candidate.view())); EXPECT_DOUBLE_EQ(knn.propose_bounded(i, candidate.view()), full) << i; + // With the deviation bounds of the current version and the candidate as reach + const teacher::PreparedTrajectory current(db.trajectory(i), state[i]); + EXPECT_DOUBLE_EQ(knn.propose_bounded(i, candidate.view(), + objective.deviation(i, current.view()) + + objective.deviation(i, candidate.view())), + full) + << i; changed += full != 0.0; } EXPECT_GT(changed, 0u);