From 2ae736fe30242318bbe8a482cd87b9c53d6afaaf Mon Sep 17 00:00:00 2001 From: Kristofferjoha Date: Thu, 19 Nov 2026 15:47:19 +0100 Subject: [PATCH] added self vs held out j4 --- engine/apps/teacher_eval.cpp | 43 ++++-- engine/apps/teacher_stats.py | 40 ++++- .../backend/engine/evaluator/combined.hpp | 5 + .../backend/engine/evaluator/evaluator.hpp | 5 + .../engine/pipeline/test_workloads.hpp | 5 +- engine/include/backend/engine/query/knn.hpp | 3 + .../backend/engine/teacher/objective.hpp | 27 +++- .../backend/engine/teacher/teacher.hpp | 3 + .../backend/engine/teacher/workloads.hpp | 5 + engine/src/evaluator/combined.cpp | 9 ++ engine/src/evaluator/evaluator.cpp | 10 +- engine/src/evaluator/join.cpp | 2 +- engine/src/evaluator/knn.cpp | 10 +- engine/src/pipeline/config.cpp | 15 +- engine/src/pipeline/pipeline.cpp | 9 +- engine/src/pipeline/scorer.cpp | 8 +- engine/src/pipeline/test_workloads.cpp | 27 +++- engine/src/teacher/README.md | 27 ++-- engine/src/teacher/objective.cpp | 144 +++++++++++++++--- engine/src/teacher/teacher.cpp | 31 ++-- engine/src/teacher/workloads.cpp | 14 ++ engine/tests/combined_test.cpp | 17 +++ engine/tests/config_test.cpp | 11 ++ engine/tests/evaluator_test.cpp | 15 ++ engine/tests/teacher/teacher_test.cpp | 71 +++++++++ 25 files changed, 481 insertions(+), 75 deletions(-) diff --git a/engine/apps/teacher_eval.cpp b/engine/apps/teacher_eval.cpp index 448b0e1..0546e26 100644 --- a/engine/apps/teacher_eval.cpp +++ b/engine/apps/teacher_eval.cpp @@ -58,6 +58,7 @@ struct Args { std::optional> train_eps; // config: teacher.extra_join_eps_m std::optional held_out_term; // config: teacher.held_out_term std::optional ranking_knn; // config: teacher.ranking_knn + std::optional compressed_knn_query; // config: teacher.knn_query std::optional> term_weights; // config: teacher.term_weights std::vector eval_eps{kEvalEps.begin(), kEvalEps.end()}; bool knn_check = false; @@ -110,7 +111,12 @@ Args parse_args(int argc, char **argv) { args.ranking_knn = false; else if (key == "--held-out-term") args.held_out_term = true; - else if (key == "--knn-check") + else if (key == "--knn-query") { + const auto query = value(); + if (query != "Original" && query != "Compressed") + throw std::invalid_argument("--knn-query must be Original or Compressed"); + args.compressed_knn_query = query == "Compressed"; + } else if (key == "--knn-check") args.knn_check = true; else if (key == "--teacher-join") { const auto mode = value(); @@ -125,7 +131,8 @@ Args parse_args(int argc, char **argv) { throw std::invalid_argument("unknown argument " + key + " (--config --data --out --cache --seeds --shares --arms " "--teacher-join --train-eps --held-out-term " - "--independent-knn --term-weights --eval-eps --knn-check " + "--independent-knn --knn-query --term-weights --eval-eps " + "--knn-check " "--no-baselines --seen-pairs-only)"); } teacher::check_arms(args.arms); @@ -150,6 +157,8 @@ pipeline::Config effective_config(const Args &args) { config.teacher.held_out_term = *args.held_out_term; if (args.ranking_knn) config.teacher.ranking_knn = *args.ranking_knn; + if (args.compressed_knn_query) + config.teacher.compressed_knn_query = *args.compressed_knn_query; if (args.term_weights) std::ranges::copy(*args.term_weights, config.teacher.term_weights.begin()); config.self_join = true; @@ -357,10 +366,12 @@ int main(int argc, char **argv) try { "where_at_within_100,join_micro_f1,knn_tau,norm_knn,norm_range,norm_where_at,norm_join," "norm_clustering,J4,J_T,join_recall_seen,join_recall_unseen,predicted_clusters," "eval_s,join_mode,self_join_f1,norm_self_join,teacher_join_mode,norm_join_main," - "norm_clustering_main"); + "norm_clustering_main,self_knn_recall,norm_self_knn,norm_self_join_mean,J4_self," + "teacher_knn_query"); auto floors = open_csv(args.out / "floors.csv", - "seed,knn_recall,range_f1,where_at_avg_hit,join_f1,clustering,self_join_f1"); + "seed,knn_recall,range_f1,where_at_avg_hit,join_f1,clustering,self_join_f1," + "self_knn_recall"); auto thresholds = open_csv(args.out / "thresholds.csv", "seed,share,arm,join,eps_m,join_f1,norm_join,recall,teacher_threshold"); @@ -414,7 +425,9 @@ int main(int argc, char **argv) try { // Held-out data from here on start = std::chrono::steady_clock::now(); const auto pool = core::subset(all, parts.queries); - const pipeline::TestWorkloads test_workloads(db, pool, config); + const pipeline::TestWorkloads test_workloads( + db, pool, config, + teacher::self_knn_workload(model.objective().workloads(), config.knn.p)); // Both joins at other thresholds: does the join gain hold beyond the ones trained on? std::vector other_joins; for (const double eps : args.eval_eps) { @@ -433,9 +446,10 @@ int main(int argc, char **argv) try { const auto &scoring = config.where_at_scoring; const auto floor_db = core::materialize(db, core::endpoints_only(db)); const auto floor = evaluator::Evaluator(db, floor_db, test, scoring).evaluate().scores; - floors << std::format("{},{},{},{},{},{},{}\n", seed, floor.knn.recall_at_k, floor.range_f1, - floor.where_at.average_hit, floor.join.f1, floor.clustering.score, - floor.self_join.f1); + floors << std::format("{},{},{},{},{},{},{},{}\n", seed, floor.knn.recall_at_k, + floor.range_f1, floor.where_at.average_hit, floor.join.f1, + floor.clustering.score, floor.self_join.f1, + floor.self_knn.recall_at_k); std::vector other_floor; for (const auto &workload : other_joins) other_floor.push_back(evaluator::evaluate_join(floor_db, workload).f1); @@ -465,7 +479,7 @@ int main(int argc, char **argv) try { const auto recall = pair_recall(compressed, pairs, config.join); results << std::format( "{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{:.2f}," - "{},{},{},{},{},{}\n", + "{},{},{},{},{},{},{},{},{},{},{}\n", seed, arm.share, arm.name, allocation::total_budget(db, arm.share), scores.knn.recall_at_k, scores.range_f1, scores.where_at.average_hit, scores.join.f1, scores.clustering.score, scores.where_at.within_radius, @@ -475,7 +489,9 @@ int main(int argc, char **argv) try { scores.clustering.predicted_clusters, core::seconds_since(start), mode_name(config.join.mode), scores.self_join.f1, normalized.self_join, mode_name(objective.config().join_mode), normalized.join_f1, - normalized.clustering_score); + normalized.clustering_score, scores.self_knn.recall_at_k, normalized.self_knn, + normalized.self_join_mean, normalized.combined_self, + objective.config().compressed_knn_query ? "Compressed" : "Original"); for (std::size_t k = 0; k < other_joins.size(); ++k) { const auto &workload = other_joins[k]; @@ -526,9 +542,10 @@ int main(int argc, char **argv) try { db.trajectory(i).size(), arm.kept[i].size()); for (auto *out : {&results, &thresholds, &terms, &budgets}) out->flush(); - std::cerr << std::format("seed {} {} {:g}: J4 {:.4f}, join {:.4f}, self join {:.4f}\n", - seed, arm.name, arm.share, normalized.combined, - normalized.join, normalized.self_join); + std::cerr << std::format( + "seed {} {} {:g}: J4 {:.4f}, J4-self {:.4f}, join {:.4f}, self join {:.4f}\n", seed, + arm.name, arm.share, normalized.combined, normalized.combined_self, normalized.join, + normalized.self_join); } timings.flush(); floors.flush(); diff --git a/engine/apps/teacher_stats.py b/engine/apps/teacher_stats.py index 2be3679..f080b3c 100644 --- a/engine/apps/teacher_stats.py +++ b/engine/apps/teacher_stats.py @@ -1,10 +1,12 @@ """Summarises a teacher_eval run: python3 teacher_stats.py [summary.md] Per share: means and SDs over seeds, paired J4 differences with t intervals, the gate against -the best cheap baseline, the coordination checks, and the self join at 100, 150 and 300 m. +the best cheap baseline, the coordination checks, each step's gain in J_T (training) against J4 +and J4-self (test), and the self join at 100, 150 and 300 m. -norm_join is J4's join term (held-out when join.mode is HeldOut). norm_self_join is the database -self join, which the teacher trains on: an in-sample diagnostic that is not part of J4. +norm_join is J4's join term (held-out when join.mode is HeldOut). J4_self scores the in-database +setting: the database's own kNN and self join (compressed on both sides), with J4's range and +where-at. Its kNN and join queries are the ones the teacher trains on, by definition of the setting. """ import csv import math @@ -14,7 +16,16 @@ from collections import defaultdict T95 = {1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228} -METRICS = ["J4", "norm_knn", "norm_range", "norm_where_at", "norm_join", "norm_clustering", "norm_self_join"] +METRICS = ["J4", "norm_knn", "norm_range", "norm_where_at", "norm_join", "J4_self", "norm_self_knn", + "norm_self_join_mean", "norm_clustering"] +# Each step of the teacher, from the arm before it to the arm after it +STEPS = [("budget search", "RDP_a0", "BUDGET_RDP"), + ("recipe choice", "RDP_a0", "EQUAL_CHOICE"), + ("recipe choice at searched budgets", "BUDGET_RDP", "BUDGET_CHOICE"), + ("loop, equal budgets", "EQUAL_CHOICE_RDP_PARTNERS", "EQUAL_LOOP"), + ("loop", "BUDGET_CHOICE", "BUDGET_LOOP"), + ("looking once", "BUDGET_CHOICE", "BUDGET_LOOP_NO_UPDATE"), + ("reacting", "BUDGET_LOOP_NO_UPDATE", "BUDGET_LOOP")] CHEAP = ["RDP_a0", "RDP_a025", "PARTNER_l0.5", "PARTNER_l1"] @@ -106,6 +117,23 @@ def main(): if a in arms and b in arms: out.append(f"| {a} − {b} | {fmt(paired(arms[a]['J4'], arms[b]['J4']))} |") out.append("") + steps = [st for st in STEPS if st[1] in arms and st[2] in arms and arms[st[1]]["J_T"]] + if steps: + out.append("Gain of each step in training (J_T) against test (J4, J4-self), " + "mean over seeds. Kept = test gain / training gain:\n") + out.append("| Step | ΔJ_T | ΔJ4 | kept | ΔJ4_self | kept |") + out.append("|---|---:|---:|---:|---:|---:|") + for name, before, after in steps: + row = [f"{name} ({before} → {after})"] + train = paired(arms[after]["J_T"], arms[before]["J_T"]) + row.append(fmt(train)) + for m in ["J4", "J4_self"]: + test = paired(arms[after][m], arms[before][m]) + row.append(fmt(test)) + kept = test[0] / train[0] if test and train and abs(train[0]) > 1e-9 else None + row.append("–" if kept is None else f"{kept:.0%}") + out.append("| " + " | ".join(row) + " |") + out.append("") if share in other: out.append("Normalised self-join F1 at the teacher's 150 m and at thresholds it does not optimise:\n") out.append("| Arm | 100 m | 150 m | 300 m |") @@ -118,10 +146,6 @@ def main(): mean_sd(list(other[share][arm].get(300.0, {}).values()))] out.append(f"| {arm} | " + " | ".join(cells) + " |") out.append("") - pairs = [(arms[a]["J_T"][s], arms[a]["J4"][s]) for a in arms for s in arms[a]["J_T"] if s in arms[a]["J4"]] - if len(pairs) >= 3: - xs, ys = zip(*pairs) - out.append(f"Surrogate check: Pearson r(J_T, J4) = {statistics.correlation(xs, ys):.3f} over {len(pairs)} arm-seeds\n") text = "\n".join(out) if len(sys.argv) > 2: open(sys.argv[2], "w").write("# Teacher summary\n\n" + text + "\n") diff --git a/engine/include/backend/engine/evaluator/combined.hpp b/engine/include/backend/engine/evaluator/combined.hpp index 3094715..50abe17 100644 --- a/engine/include/backend/engine/evaluator/combined.hpp +++ b/engine/include/backend/engine/evaluator/combined.hpp @@ -12,6 +12,8 @@ namespace evaluator { // database) are diagnostics only. // join and clustering average over their threshold grids, each against its own floor; // join_f1 and clustering_score keep the main threshold alone. +// combined_self (J4-self) scores the in-database setting: self kNN, the same range and where-at, +// and the self join over its grid (self_join_mean). NaN unless all four are present. struct NormalizedScores { static constexpr double missing = std::numeric_limits::quiet_NaN(); @@ -30,8 +32,11 @@ struct NormalizedScores { double join = missing; double clustering = missing; double self_join = missing; + double self_knn = missing; + double self_join_mean = missing; double combined = missing; + double combined_self = missing; }; // NaN if an input is NaN or the floor is already 1 diff --git a/engine/include/backend/engine/evaluator/evaluator.hpp b/engine/include/backend/engine/evaluator/evaluator.hpp index 6d28998..c9b736d 100644 --- a/engine/include/backend/engine/evaluator/evaluator.hpp +++ b/engine/include/backend/engine/evaluator/evaluator.hpp @@ -29,6 +29,9 @@ struct Workloads { // Extra thresholds, averaged into the reported scores std::span join_grid = {}; std::span clustering_grid = {}; + // The database's own queries, for J4-self + const query::KnnWorkload *self_knn = nullptr; + std::span self_join_grid = {}; }; // Kendall's tau is in [-1, 1] and the others in [0, 1], or NaN if it could not be computed @@ -42,6 +45,8 @@ struct Scoretable { JoinScores self_join; std::vector join_grid; // in Workloads::join_grid order std::vector clustering_grid; // in Workloads::clustering_grid order + KnnScores self_knn; + std::vector self_join_grid; // in Workloads::self_join_grid order }; struct EvaluationResult { diff --git a/engine/include/backend/engine/pipeline/test_workloads.hpp b/engine/include/backend/engine/pipeline/test_workloads.hpp index a092efa..56767db 100644 --- a/engine/include/backend/engine/pipeline/test_workloads.hpp +++ b/engine/include/backend/engine/pipeline/test_workloads.hpp @@ -23,9 +23,12 @@ struct TestWorkloads { std::optional self_join; std::optional clustering; std::vector clustering_grid; // at config.join_score_eps_m + // For J4-self. The self join at config.join_score_eps_m, empty when join_grid already is + std::vector self_join_grid; + std::optional self_knn; // supplied, as its reference is slow to build TestWorkloads(const core::TrajectoryDb &database, const core::TrajectoryDb &queries, - const Config &config); + const Config &config, std::optional self_knn = std::nullopt); TestWorkloads(const TestWorkloads &) = delete; TestWorkloads &operator=(const TestWorkloads &) = delete; diff --git a/engine/include/backend/engine/query/knn.hpp b/engine/include/backend/engine/query/knn.hpp index 9a13661..05bec03 100644 --- a/engine/include/backend/engine/query/knn.hpp +++ b/engine/include/backend/engine/query/knn.hpp @@ -27,6 +27,9 @@ struct KnnWorkload { std::size_t k; double p = 0.5; FrechetVariant frechet = FrechetVariant::Continuous; + // query_index is a database trajectory, searched in its compressed form and excluded from its + // own answer and reference. queries stays empty + bool self = false; }; // Samples query_count trajectories from query_pool (the held-out split) and stores their k diff --git a/engine/include/backend/engine/teacher/objective.hpp b/engine/include/backend/engine/teacher/objective.hpp index 41b7065..385a8e3 100644 --- a/engine/include/backend/engine/teacher/objective.hpp +++ b/engine/include/backend/engine/teacher/objective.hpp @@ -56,6 +56,9 @@ struct ObjectiveConfig { // kNN ranked on the compressed database; false: per-trajectory check. Curves always use the // latter bool ranking_knn = false; + // The kNN query compressed too, the in-database setting; needs ranking_knn. Curves then share + // each hit between the query and its neighbour, both against the fixed partners + bool compressed_knn_query = false; TermWeights term_weights{1.0, 1.0, 1.0, 1.0}; }; @@ -106,6 +109,7 @@ class Objective { friend class Objective; std::size_t trajectory = 0; core::TrajectoryView candidate{}; + std::span partners{}; Terms terms{}; double join = 0.0; }; @@ -156,6 +160,8 @@ class Objective { Terms raw_separable(std::size_t i, const core::TrajectoryView &candidate, bool knn = true) const; double independent_sum(const IndependentScore &score, double knn) const; + std::size_t knn_checks(std::size_t i) const; + double knn_check_weight() const; double join_metric(std::span state) const; const core::TrajectoryDb &m_db; @@ -230,12 +236,26 @@ class KnnState { std::span watch_states, std::size_t watch); bool watched(std::size_t i) const { return !m_watchers[i].empty(); } - // From every query watching i, in the order propose and apply expect + // Whether i's choice can change the metric: watched, or with compressed queries a query + bool active(std::size_t i) const; + // Original queries only. From every query watching i, in the order propose and apply expect std::vector distances(std::size_t i, const core::TrajectoryView &candidate) const; // Change in J_T double propose(std::size_t i, std::span distances) const; void apply(std::size_t i, std::span distances); + // Either query kind. With compressed queries a distance depends on both sides, so a move also + // rescores i's own list, against the current state + struct Move { + std::size_t trajectory = 0; + std::vector watched; // as distances() + std::vector own; // i's own list; empty with original queries + }; + Move move(std::size_t i, const core::TrajectoryView &candidate, + std::span state) const; + double propose(const Move &move) const; + void apply(const Move &move); + double metric() const; // recall@k over the valid queries private: @@ -244,6 +264,11 @@ class KnnState { }; // True neighbours in e's top k, with the distance at position replaced std::size_t hits(std::size_t e, std::size_t position, double distance) const; + // The same with e's whole row replaced + std::size_t hits(std::size_t e, std::span row) const; + // Distances from query, e in some state, to e's list in state + std::vector own_row(std::size_t e, const core::TrajectoryView &query, + std::span state) const; const Objective &m_objective; std::size_t m_valid = 0; diff --git a/engine/include/backend/engine/teacher/teacher.hpp b/engine/include/backend/engine/teacher/teacher.hpp index e96ca03..a30f938 100644 --- a/engine/include/backend/engine/teacher/teacher.hpp +++ b/engine/include/backend/engine/teacher/teacher.hpp @@ -54,6 +54,9 @@ struct TeacherConfig { std::vector extra_join_eps_m; bool held_out_term = false; bool ranking_knn = true; + // ObjectiveConfig::compressed_knn_query. Unset: compressed with a Self join, the in-database + // setting + std::optional compressed_knn_query; TermWeights term_weights{1.0, 1.0, 1.0, 1.0}; }; diff --git a/engine/include/backend/engine/teacher/workloads.hpp b/engine/include/backend/engine/teacher/workloads.hpp index dce0223..fa88afa 100644 --- a/engine/include/backend/engine/teacher/workloads.hpp +++ b/engine/include/backend/engine/teacher/workloads.hpp @@ -2,6 +2,7 @@ #include "backend/engine/core/trajectory_db.hpp" #include "backend/engine/query/frechet_helper.hpp" +#include "backend/engine/query/knn.hpp" #include "backend/engine/query/range_query.hpp" #include "backend/engine/query/where_at_query.hpp" @@ -68,4 +69,8 @@ TeacherWorkloads load_or_build_workloads(const core::TrajectoryDb &db, const Wor std::uint64_t seed, const std::filesystem::path &cache_dir); +// The self-kNN test workload from the cached lists: every valid database query with its k nearest +// others. They depend on the database alone, so they are what a fresh build would give +query::KnnWorkload self_knn_workload(const TeacherWorkloads &workloads, double p); + } // namespace teacher diff --git a/engine/src/evaluator/combined.cpp b/engine/src/evaluator/combined.cpp index e2a7316..dc28f09 100644 --- a/engine/src/evaluator/combined.cpp +++ b/engine/src/evaluator/combined.cpp @@ -1,5 +1,6 @@ #include "backend/engine/evaluator/combined.hpp" +#include #include #include #include @@ -72,10 +73,18 @@ NormalizedScores normalize_scores(const Scoretable &scores, const Scoretable &fl result.clustering_score, scores.clustering_grid, floor.clustering_grid, [](const RouteClusteringScores &clustering) { return clustering.score; }, "clustering"); result.self_join = normalize(scores.self_join.f1, floor.self_join.f1); + result.self_knn = normalize(scores.self_knn.recall_at_k, floor.self_knn.recall_at_k); + result.self_join_mean = grid_mean( + result.self_join, scores.self_join_grid, floor.self_join_grid, + [](const JoinScores &join) { return join.f1; }, "self join"); // Clustering and the self join stay out (see combined.hpp) result.combined = mean_ignoring_nan(std::array{result.knn, result.range, result.where_at, result.join}); + const std::array internal{result.self_knn, result.range, result.where_at, + result.self_join_mean}; + if (std::ranges::none_of(internal, [](double v) { return std::isnan(v); })) + result.combined_self = mean_ignoring_nan(internal); return result; } diff --git a/engine/src/evaluator/evaluator.cpp b/engine/src/evaluator/evaluator.cpp index b79504c..430877c 100644 --- a/engine/src/evaluator/evaluator.cpp +++ b/engine/src/evaluator/evaluator.cpp @@ -78,6 +78,12 @@ EvaluationResult Evaluator::evaluate() const { std::vector clustering_grid; for (const auto &w : m_workloads.clustering_grid) clustering_grid.push_back(evaluate_route_clustering(m_simplified, w)); + const auto self_knn = m_workloads.self_knn + ? evaluate_knn(m_simplified, *m_workloads.self_knn).scores + : KnnScores{}; + std::vector self_join_grid; + for (const auto &w : m_workloads.self_join_grid) + self_join_grid.push_back(evaluate_join(m_simplified, w)); return {.scores = {.knn = knn.scores, .range_recall = range.recall, @@ -90,7 +96,9 @@ EvaluationResult Evaluator::evaluate() const { ? join.scores : get_self_join_scores(), .join_grid = std::move(join_grid), - .clustering_grid = std::move(clustering_grid)}, + .clustering_grid = std::move(clustering_grid), + .self_knn = self_knn, + .self_join_grid = std::move(self_join_grid)}, .knn = std::move(knn.answers), .range = std::move(range.answers), .join = std::move(join.pairs), diff --git a/engine/src/evaluator/join.cpp b/engine/src/evaluator/join.cpp index 7018670..86dbb1b 100644 --- a/engine/src/evaluator/join.cpp +++ b/engine/src/evaluator/join.cpp @@ -121,7 +121,7 @@ JoinEvaluation run_join(const core::TrajectoryDb &compressed, const query::JoinW result.micro_recall = expected == 0 ? nan : hits / expected; result.micro_f1 = found + expected == 0 ? nan : 2.0 * hits / (found + expected); - JoinEvaluation evaluation{.scores = result}; + JoinEvaluation evaluation{.scores = result, .pairs = {}}; evaluation.pairs.reserve(result.predicted_pairs); for (std::size_t i = 0; i < n; ++i) { const auto &entry = workload.entries[i]; diff --git a/engine/src/evaluator/knn.cpp b/engine/src/evaluator/knn.cpp index f4f1a42..849853d 100644 --- a/engine/src/evaluator/knn.cpp +++ b/engine/src/evaluator/knn.cpp @@ -101,8 +101,14 @@ KnnEvaluation evaluate_knn(const core::TrajectoryDb &compressed, std::vector topk_taus(workload.entries.size()); core::parallel_for(workload.entries.size(), [&](std::size_t i) { const auto &query = workload.entries[i]; - const auto trajectory = workload.queries.trajectory(query.query_index); - const auto simplified = query::knn(compressed, trajectory, workload.k, workload.frechet); + const auto trajectory = workload.self ? compressed.trajectory(query.query_index) + : workload.queries.trajectory(query.query_index); + auto simplified = query::knn(compressed, trajectory, workload.k + (workload.self ? 1 : 0), + workload.frechet); + if (workload.self) { + std::erase(simplified, query.query_index); + simplified.resize(std::min(simplified.size(), workload.k)); + } auto &answer = result.answers[i]; answer.query_trajectory_id = trajectory.trajectory_id; answer.neighbors.reserve(simplified.size()); diff --git a/engine/src/pipeline/config.cpp b/engine/src/pipeline/config.cpp index 0fbe693..8912b25 100644 --- a/engine/src/pipeline/config.cpp +++ b/engine/src/pipeline/config.cpp @@ -117,7 +117,7 @@ void parse_clustering_config(Config &config, const json &clustering) { void parse_teacher_config(Config &config, const json §ion) { validate_keys(section, {"enabled", "cache_dir", "arms", "range_query_count", "all_join_pairs", "join", - "extra_join_eps_m", "held_out_term", "ranking_knn", "term_weights"}, + "extra_join_eps_m", "held_out_term", "ranking_knn", "knn_query", "term_weights"}, "teacher"); auto &t = config.teacher; t.enabled = section.value("enabled", t.enabled); @@ -130,6 +130,12 @@ void parse_teacher_config(Config &config, const json §ion) { t.extra_join_eps_m = section.value("extra_join_eps_m", t.extra_join_eps_m); t.held_out_term = section.value("held_out_term", t.held_out_term); t.ranking_knn = section.value("ranking_knn", t.ranking_knn); + if (section.contains("knn_query")) { + const auto query = section["knn_query"].get(); + if (query != "Original" && query != "Compressed") + throw std::invalid_argument("config: teacher.knn_query must be Original or Compressed"); + t.compressed_knn_query = query == "Compressed"; + } t.term_weights = section.value("term_weights", t.term_weights); teacher::check_term_weights(t.term_weights); query::check_extra_thresholds(t.extra_join_eps_m, config.join.eps_m, @@ -221,13 +227,16 @@ teacher::WorkloadConfig teacher_workload_config(const Config &config) { } teacher::ObjectiveConfig teacher_objective_config(const Config &config) { + // Trains on the kind of join it is scored on, unless told otherwise + const auto join_mode = config.teacher.join.value_or(config.join.mode); return {.min_tolerance_m = config.where_at_scoring.min_tolerance_m, .max_tolerance_m = config.where_at_scoring.max_tolerance_m, .all_join_pairs = config.teacher.all_join_pairs, - // Trains on the kind of join it is scored on, unless told otherwise - .join_mode = config.teacher.join.value_or(config.join.mode), + .join_mode = join_mode, .held_out_term = config.teacher.held_out_term, .ranking_knn = config.teacher.ranking_knn, + .compressed_knn_query = config.teacher.compressed_knn_query.value_or( + join_mode == query::JoinMode::Self && config.teacher.ranking_knn), .term_weights = config.teacher.term_weights}; } diff --git a/engine/src/pipeline/pipeline.cpp b/engine/src/pipeline/pipeline.cpp index eb8491a..c21e8fd 100644 --- a/engine/src/pipeline/pipeline.cpp +++ b/engine/src/pipeline/pipeline.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include namespace pipeline { @@ -62,7 +63,12 @@ void run(const Config &config) { print_size("database", database); print_size("query pool", queries); - const TestWorkloads workloads(database, queries, config); + const auto teacher = make_teacher(database, config); + // The self-kNN reference comes from the teacher's cache; without the teacher J4-self is NaN + std::optional self_knn; + if (teacher && config.self_join) + self_knn = teacher::self_knn_workload(teacher->objective().workloads(), config.knn.p); + const TestWorkloads workloads(database, queries, config, std::move(self_knn)); if (config.export_results) { io::export_workloads(config.export_dir, database, workloads.range.queries, workloads.knn, workloads.join, @@ -71,7 +77,6 @@ void run(const Config &config) { } const Scorer scorer(database, workloads, config); - const auto teacher = make_teacher(database, config); for (const double share : config.budget_shares) { const std::size_t budget = allocation::total_budget(database, share); diff --git a/engine/src/pipeline/scorer.cpp b/engine/src/pipeline/scorer.cpp index 2ad2b83..2844908 100644 --- a/engine/src/pipeline/scorer.cpp +++ b/engine/src/pipeline/scorer.cpp @@ -23,6 +23,9 @@ void print_scores(const evaluator::Scoretable &scores) { std::cout << " self join F1: " << scores.self_join.f1 << ", precision: " << scores.self_join.precision << ", recall: " << scores.self_join.recall << '\n'; + if (scores.self_knn.query_count != 0) + std::cout << " self knn recall@k: " << scores.self_knn.recall_at_k << " (" + << scores.self_knn.query_count << " database queries)\n"; if (!std::isnan(scores.clustering.score) || scores.clustering.reference_clusters != 0) std::cout << " route clustering: " << scores.clustering.score << " (" << scores.clustering.predicted_clusters << " clusters, " @@ -36,7 +39,10 @@ void print_normalized(const evaluator::NormalizedScores &scores) { std::cout << ", clustering=" << scores.clustering; if (!std::isnan(scores.self_join)) std::cout << ", self join=" << scores.self_join; - std::cout << ", combined J4=" << scores.combined << '\n'; + std::cout << ", combined J4=" << scores.combined; + if (!std::isnan(scores.combined_self)) + std::cout << ", self knn=" << scores.self_knn << ", J4-self=" << scores.combined_self; + std::cout << '\n'; } } // namespace diff --git a/engine/src/pipeline/test_workloads.cpp b/engine/src/pipeline/test_workloads.cpp index 45427f6..ef4aa6d 100644 --- a/engine/src/pipeline/test_workloads.cpp +++ b/engine/src/pipeline/test_workloads.cpp @@ -99,16 +99,33 @@ std::vector make_clustering_grid(const core::Tra return grid; } +std::vector make_self_join_grid(const core::TrajectoryDb &database, + const Config &config) { + std::vector grid; + if (!config.self_join || config.join.mode == query::JoinMode::Self) + return grid; + for (const double eps : config.join_score_eps_m) { + grid.push_back(query::generate_self_join(database, eps, config.join.frechet)); + print_join(grid.back()); + } + return grid; +} + } // namespace TestWorkloads::TestWorkloads(const core::TrajectoryDb &database, const core::TrajectoryDb &queries, - const Config &config) + const Config &config, std::optional self_knn) : knn(make_knn(database, queries, config)), where_at(make_where_at(database, config)), range(make_range(database, config)), join(make_join(database, queries, config)), join_grid(make_join_grid(database, queries, config)), self_join(make_self_join(database, config)), clustering(make_clustering(database, self_join ? *self_join : join, config)), - clustering_grid(make_clustering_grid(database, config)) {} + clustering_grid(make_clustering_grid(database, config)), + self_join_grid(make_self_join_grid(database, config)), self_knn(std::move(self_knn)) { + if (this->self_knn) + std::cout << "self knn: " << this->self_knn->entries.size() + << " database queries, k=" << this->self_knn->k << '\n'; +} evaluator::Workloads TestWorkloads::view() const { const query::JoinWorkload *self = nullptr; @@ -123,7 +140,11 @@ evaluator::Workloads TestWorkloads::view() const { .clustering = clustering ? &*clustering : nullptr, .self_join = self, .join_grid = join_grid, - .clustering_grid = clustering_grid}; + .clustering_grid = clustering_grid, + .self_knn = self_knn ? &*self_knn : nullptr, + .self_join_grid = join.mode == query::JoinMode::Self + ? std::span(join_grid) + : std::span(self_join_grid)}; } } // namespace pipeline diff --git a/engine/src/teacher/README.md b/engine/src/teacher/README.md index 28b7b09..08f0a1a 100644 --- a/engine/src/teacher/README.md +++ b/engine/src/teacher/README.md @@ -56,16 +56,28 @@ Files: self join, whose pairs the teacher trains on, is then reported separately (`norm_self_join`, the seen / unseen recall and the 100 / 300 m thresholds) and is not part of J4. Results quoted below predate this and have the self join inside the score. +- **J4-self** (`J4_self`): the in-database setting, where the database's own trajectories are + the kNN and join queries. Its terms are self kNN (each database trajectory, compressed, searched + against the compressed database, itself excluded), J4's range and where-at, and the self join + averaged over `join.score_eps_m` like J4's join. Its kNN and join queries are the teacher's + training queries by definition, so only range, where-at and untrained thresholds are held out. + The self-kNN reference is read from the teacher's kNN cache (`self_knn_workload`); it depends + on the database alone. Without a teacher, `run` reports it as NaN. - **J5:** the earlier headline, J4 plus clustering. Results quoted below from earlier runs use it; `teacher_eval` no longer writes it. - **J_T:** the teacher's own score on its training workloads. It has the same shape as J4. kNN and range use training proxies. - **J_T's kNN term:** with `teacher.ranking_knn` (the default) it is recall@k of the teacher's - database queries answered on the compressed database, so competitors are compressed too, like - the test. The loop tracks it with `KnnState`. Value curves still use the per-trajectory check + database queries answered on the compressed database (`knn_recall`), so competitors are + compressed too, like the test. The loop tracks it with `KnnState`. Value curves still use the per-trajectory check against original competitors, which `teacher_eval --independent-knn` restores everywhere as the ablation. The per-trajectory check misleads the teacher: it credited the teacher with beating RDP on kNN where held-out queries showed it did not. + With a `Self` join the query is compressed too (`compressed_knn_query`, `teacher.knn_query` or + `teacher_eval --knn-query Original|Compressed` to override), as J4-self scores it. A + trajectory's choice then changes its own answer as well as the lists it is in, so `KnnState` + tracks both against the current state (`move`, `propose`, `apply`), and the curves share each + (query, neighbour) hit between the two sides, half each, against the fixed partners. - **J_T's join term:** follows `join.mode`, so the teacher trains on the kind of join it is scored on. `Self`: both sides of a pair compressed, the self join. `HeldOut`: every database trajectory also acts as an original query against its compressed partners, so each pair is @@ -308,7 +320,7 @@ not. They matter because the join metric is F1: simplification can make negative - **Computation:** in parallel, for each trajectory e: 1. find its k+2 nearest neighbours (`query::knn`) and remove e itself; 2. keep the first k as `top`; - 3. compute `cutoff`, the distance to the (k+1)th neighbour, with `distance()`. For continuous + 3. compute `cutoff`, the distance to the (k+1)th neighbour, with `query::frechet_distance`. For continuous Fréchet, the discrete distance is used as the upper bound of the search. - **Output:** one `KnnEntry` per trajectory. @@ -408,7 +420,7 @@ trajectory can't have a point in a box the original had none in. throws, because that metric can't be normalised. 6. **Scales:** `m_scale[m] = 1 / (4 · (1 − floor_m))`. This applies the normalisation and the equal average in one factor. - 7. **`m_v0[i]`:** trajectory i's scaled separable terms with endpoints only. Subtracting it makes + 7. **`m_endpoint_separable[i]`:** trajectory i's scaled separable terms with endpoints only. Subtracting it makes `separable` exactly 0 at the floor. - **Output:** a ready `Objective`. It keeps references to `db` and `workloads`, so both must outlive it (`Teacher` guarantees this). @@ -429,7 +441,7 @@ trajectory can't have a point in a box the original had none in. ### `separable(i, kept)` — public - **Input:** trajectory index and kept set. -- **Computation:** `Σ m_scale[m] · raw_separable[m] − m_v0[i]` over kNN, range and where-at. +- **Computation:** `Σ m_scale[m] · raw_separable[m] − m_endpoint_separable[i]` over kNN, range and where-at. - **Output:** i's part of J_T from the three separable terms, 0 at endpoints only. Summed over all trajectories, this equals those terms' share of J_T exactly. @@ -662,12 +674,9 @@ simplified. 4. Stops after a pass with no switches. - **Output:** the improved `Methods`. Stops after a pass with no switches or the 10-pass limit. -### `to_simplification`, `wins`, `seconds_since`, `is_arm` — helpers +### `wins`, `is_arm` — helpers -- `to_simplification`: `KeptSets` → `core::Simplification`, the format the pipeline evaluates and - exports. - `wins`: per-candidate counts for the log. -- `seconds_since`: wall time for the log. - `is_arm`: whether a name is in `kArms`. It is also used by the config parser. --- diff --git a/engine/src/teacher/objective.cpp b/engine/src/teacher/objective.cpp index f93de0f..8dc32c0 100644 --- a/engine/src/teacher/objective.cpp +++ b/engine/src/teacher/objective.cpp @@ -94,6 +94,8 @@ Objective::Objective(const core::TrajectoryDb &db, const TeacherWorkloads &workl if (!(config.min_tolerance_m < config.max_tolerance_m)) throw std::invalid_argument("objective: min_tolerance_m must be below max_tolerance_m"); const double weight_sum = check_term_weights(config.term_weights); + if (config.compressed_knn_query && !config.ranking_knn) + throw std::invalid_argument("objective: compressed kNN queries need the ranking kNN term"); // Extra thresholds use every pair, which would train on the pairs held back as unseen if (!config.all_join_pairs && !workloads.extra_joins.empty()) throw std::invalid_argument( @@ -266,6 +268,7 @@ Objective::prepare_independent(std::size_t i, const core::TrajectoryView &candid IndependentScore score; score.trajectory = i; score.candidate = candidate; + score.partners = partners; score.terms = raw_separable(i, candidate, false); std::array hits{}, found{}; for (const auto &link : m_links[i]) { @@ -286,7 +289,7 @@ Objective::prepare_independent(std::size_t i, const core::TrajectoryView &candid score.join = m_with_truth != 0 ? m_scale[Join] * f1_sum / static_cast(m_with_truth) : 0.0; score.bound = independent_sum(score, score.terms[Knn]) + - m_scale[Knn] * m_knn_weight * static_cast(m_knn[i].size()); + m_scale[Knn] * knn_check_weight() * static_cast(knn_checks(i)); return score; } @@ -302,19 +305,48 @@ double Objective::independent_value(const IndependentScore &score, double beat) double knn = score.terms[Knn]; auto value = [&] { return independent_sum(score, knn); }; // Stop once even all remaining hits cannot beat beat - const double per_hit = m_scale[Knn] * m_knn_weight; - std::size_t left = m_knn[i].size(); - for (const auto &m : m_knn[i]) { + const double weight = knn_check_weight(); + const double per_hit = m_scale[Knn] * weight; + std::size_t left = knn_checks(i); + auto check = [&](const core::TrajectoryView &a, const core::TrajectoryView &b, double cutoff) { if (value() + per_hit * static_cast(left) < beat - kBoundSlack) - return value(); + return false; --left; - if (query::frechet_within(m_db.trajectory(m.query), score.candidate, m.cutoff, - m_workloads.frechet)) - knn += m_knn_weight; + if (query::frechet_within(a, b, cutoff, m_workloads.frechet)) + knn += weight; + return true; + }; + if (!m_config.compressed_knn_query) { + for (const auto &m : m_knn[i]) + if (!check(m_db.trajectory(m.query), score.candidate, m.cutoff)) + return value(); + return value(); } + // Compressed queries: i is found by the queries it belongs to, and finds its own neighbours + auto partner = [&](std::size_t j) { + return score.partners.empty() ? m_db.trajectory(j) : score.partners[j]; + }; + for (const auto &m : m_knn[i]) + if (!check(partner(m.query), score.candidate, m.cutoff)) + return value(); + const auto &own = m_workloads.knn[i]; + if (own.valid) + for (const auto j : own.top) + if (!check(score.candidate, partner(j), own.cutoff)) + return value(); return value(); } +std::size_t Objective::knn_checks(std::size_t i) const { + const auto &own = m_workloads.knn[i]; + return m_knn[i].size() + (m_config.compressed_knn_query && own.valid ? own.top.size() : 0); +} + +double Objective::knn_check_weight() const { + // Compressed queries check each (query, neighbour) hit from both sides, so each counts half + return m_config.compressed_knn_query ? m_knn_weight / 2.0 : m_knn_weight; +} + double Objective::independent_bound(std::size_t i, const core::TrajectoryView &candidate, std::span partners) const { return prepare_independent(i, candidate, partners).bound; @@ -384,8 +416,10 @@ double Objective::knn_recall(const core::TrajectoryDb &compressed) const { const auto &entry = m_workloads.knn[e]; if (!entry.valid) return; - auto found = query::knn(compressed, m_db.trajectory(e), std::min(k + 1, m_db.size()), - m_workloads.frechet); + const auto query = + m_config.compressed_knn_query ? compressed.trajectory(e) : m_db.trajectory(e); + auto found = + query::knn(compressed, query, std::min(k + 1, m_db.size()), m_workloads.frechet); std::erase(found, e); found.resize(std::min(found.size(), k)); for (const auto i : entry.top) @@ -548,8 +582,11 @@ KnnState::KnnState(const Objective &objective, const KeptSets &state, auto &list = m_list[e]; list.assign(entry.top.begin(), entry.top.end()); for (const auto &compressed_db : compressed) - for (const auto j : query::knn(compressed_db, db.trajectory(e), std::min(watch + 1, n), - workloads.frechet)) + for (const auto j : + query::knn(compressed_db, + objective.config().compressed_knn_query ? compressed_db.trajectory(e) + : db.trajectory(e), + std::min(watch + 1, n), workloads.frechet)) if (j != e && std::ranges::find(list, j) == list.end()) list.push_back(static_cast(j)); }); @@ -562,12 +599,16 @@ KnnState::KnnState(const Objective &objective, const KeptSets &state, } const PreparedTrajectories prepared(db, state); const auto views = prepared.views(); - std::vector> own(n); - core::parallel_for(n, [&](std::size_t i) { own[i] = distances(i, views[i]); }); - for (std::size_t i = 0; i < n; ++i) - for (std::size_t watch_state = 0; watch_state < m_watchers[i].size(); ++watch_state) - m_distance[m_watchers[i][watch_state].query][m_watchers[i][watch_state].position] = - own[i][watch_state]; + if (objective.config().compressed_knn_query) { + core::parallel_for(n, [&](std::size_t e) { m_distance[e] = own_row(e, views[e], views); }); + } else { + std::vector> own(n); + core::parallel_for(n, [&](std::size_t i) { own[i] = distances(i, views[i]); }); + for (std::size_t i = 0; i < n; ++i) + for (std::size_t watch_state = 0; watch_state < m_watchers[i].size(); ++watch_state) + m_distance[m_watchers[i][watch_state].query][m_watchers[i][watch_state].position] = + own[i][watch_state]; + } core::parallel_for(n, [&](std::size_t e) { if (!m_list[e].empty()) m_hits[e] = hits(e, m_list[e].size(), 0.0); @@ -586,6 +627,73 @@ std::vector KnnState::distances(std::size_t i, return result; } +bool KnnState::active(std::size_t i) const { + return watched(i) || (m_objective.config().compressed_knn_query && !m_list[i].empty()); +} + +std::vector KnnState::own_row(std::size_t e, const core::TrajectoryView &query, + std::span state) const { + std::vector row; + row.reserve(m_list[e].size()); + for (const auto j : m_list[e]) + row.push_back(query::frechet_distance(query, state[j], m_objective.workloads().frechet)); + return row; +} + +KnnState::Move KnnState::move(std::size_t i, const core::TrajectoryView &candidate, + std::span state) const { + if (!m_objective.config().compressed_knn_query) + return {i, distances(i, candidate), {}}; + Move result{i, {}, own_row(i, candidate, state)}; + result.watched.reserve(m_watchers[i].size()); + for (const auto &watcher : m_watchers[i]) + result.watched.push_back(query::frechet_distance(state[watcher.query], candidate, + m_objective.workloads().frechet)); + return result; +} + +double KnnState::propose(const Move &move) const { + long change = 0; + for (std::size_t s = 0; s < m_watchers[move.trajectory].size(); ++s) { + const auto &watcher = m_watchers[move.trajectory][s]; + change += static_cast(hits(watcher.query, watcher.position, move.watched[s])) - + static_cast(m_hits[watcher.query]); + } + if (!move.own.empty() && !m_list[move.trajectory].empty()) + change += static_cast(hits(move.trajectory, move.own)) - + static_cast(m_hits[move.trajectory]); + return m_objective.scale(Objective::Knn) * static_cast(change) / + static_cast(m_objective.workloads().k * m_valid); +} + +void KnnState::apply(const Move &move) { + apply(move.trajectory, move.watched); + if (move.own.empty() || m_list[move.trajectory].empty()) + return; + const auto i = move.trajectory; + m_distance[i] = move.own; + m_hit_sum -= m_hits[i]; + m_hits[i] = hits(i, m_list[i].size(), 0.0); + m_hit_sum += m_hits[i]; +} + +std::size_t KnnState::hits(std::size_t e, std::span row) const { + const auto &list = m_list[e]; + const std::size_t k = std::min(m_objective.workloads().k, list.size()); + const std::size_t members = m_objective.workloads().knn[e].top.size(); + std::vector> order(list.size()); + for (std::size_t p = 0; p < list.size(); ++p) + order[p] = {row[p], static_cast(p)}; + std::ranges::partial_sort( + order, order.begin() + static_cast(k), [&](const auto &a, const auto &b) { + return a.first < b.first || (a.first == b.first && list[a.second] < list[b.second]); + }); + std::size_t count = 0; + for (std::size_t p = 0; p < k; ++p) + count += order[p].second < members; + return count; +} + std::size_t KnnState::hits(std::size_t e, std::size_t position, double distance) const { const auto &list = m_list[e]; const std::size_t k = std::min(m_objective.workloads().k, list.size()); diff --git a/engine/src/teacher/teacher.cpp b/engine/src/teacher/teacher.cpp index 569c1cd..a155c01 100644 --- a/engine/src/teacher/teacher.cpp +++ b/engine/src/teacher/teacher.cpp @@ -115,14 +115,8 @@ class CandidateMaker { AlgorithmType seed_algorithm) { if (!m_greedy) { m_greedy.emplace(m_objective, m_index); - // SED's and RDP's points in their own order of importance, then UNIFORM's; greedy - // skips points it already keeps + // SED's order of importance covers every point up to the cap m_fill = selection::sed::sequence(m_original, m_cap); - const auto rdp_order = selection::rdp::sequence(m_original, m_cap); - m_fill.insert(m_fill.end(), rdp_order.begin(), rdp_order.end()); - const auto uniform = - selection::simplify_trajectory(m_original, AlgorithmType::UNIFORM, m_cap); - m_fill.insert(m_fill.end(), uniform.begin(), uniform.end()); } const auto seed_type = seed_size ? seed_algorithm : AlgorithmType::SED; auto &cached = m_greedy_sequences[{seed_type, seed_size}]; @@ -292,12 +286,12 @@ class Search { public: Search(const core::TrajectoryDb &db, const Objective &objective, const Candidates &candidates, const Methods &method) - : m_start(method) { + : m_start(method), m_compressed_queries(objective.config().compressed_knn_query) { if (objective.config().ranking_knn) m_knn.emplace(objective, assemble(candidates, method), candidates, kKnnWatchFactor * objective.workloads().k); const auto active = [&](std::size_t i) { - return !objective.links(i).empty() || (m_knn && m_knn->watched(i)); + return !objective.links(i).empty() || (m_knn && m_knn->active(i)); }; for (std::size_t i = 0; i < db.size(); ++i) if (active(i)) @@ -326,7 +320,8 @@ class Search { owned.emplace_back(original, candidates[c][i]); m_views[c][i] = owned.back().view(); m_separable[i][c] = objective.separable(i, m_views[c][i]); - if (m_knn) + // Compressed queries depend on the partners' state, so are computed later + if (m_knn && !objective.config().compressed_knn_query) m_knn_distances[i][c] = m_knn->distances(i, m_views[c][i]); } else { m_views[c][i] = m_views[first][i]; @@ -358,6 +353,7 @@ class Search { const double here = m_separable[i][method[i]]; std::size_t best = method[i]; JoinState::Proposal best_proposal; + KnnState::Move best_move; double best_gain = kMinImprovement; // strict gains only, so ties cannot cycle for (std::size_t c = 0; c < kCandidates.size(); ++c) { // Each distinct kept set is tried once, and never the one i already has @@ -366,12 +362,20 @@ class Search { if (duplicate || current) continue; auto proposal = join.propose(i, m_views[c][i], state); - const double knn_gain = knn ? knn->propose(i, m_knn_distances[i][c]) : 0.0; + KnnState::Move move; + double knn_gain = 0.0; + if (knn && m_compressed_queries) { + move = knn->move(i, m_views[c][i], state); + knn_gain = knn->propose(move); + } else if (knn) { + knn_gain = knn->propose(i, m_knn_distances[i][c]); + } const double gain = m_separable[i][c] - here + proposal.gain + knn_gain; if (gain > best_gain) { best = c; best_gain = gain; best_proposal = std::move(proposal); + best_move = std::move(move); } } if (best == method[i]) @@ -382,7 +386,9 @@ class Search { continue; state[i] = m_views[best][i]; join.apply(best_proposal); - if (knn) + if (knn && m_compressed_queries) + knn->apply(best_move); + else if (knn) knn->apply(i, m_knn_distances[i][best]); } std::cout << std::format("teacher: pass {}: {} switches, join F1 {:.4f}{}\n", pass, @@ -396,6 +402,7 @@ class Search { private: Methods m_start; + bool m_compressed_queries; std::vector m_order; std::vector> m_prepared; std::array, kCandidates.size()> m_views; diff --git a/engine/src/teacher/workloads.cpp b/engine/src/teacher/workloads.cpp index c5603cf..aa0b85f 100644 --- a/engine/src/teacher/workloads.cpp +++ b/engine/src/teacher/workloads.cpp @@ -242,4 +242,18 @@ TeacherWorkloads load_or_build_workloads(const core::TrajectoryDb &db, const Wor return workloads; } +query::KnnWorkload self_knn_workload(const TeacherWorkloads &workloads, double p) { + query::KnnWorkload result{.queries = core::TrajectoryDb({0}, {}, {}, {}, {}), + .entries = {}, + .k = workloads.k, + .p = p, + .frechet = workloads.frechet, + .self = true}; + for (std::size_t e = 0; e < workloads.knn.size(); ++e) + if (workloads.knn[e].valid) + result.entries.push_back({e, std::vector(workloads.knn[e].top.begin(), + workloads.knn[e].top.end())}); + return result; +} + } // namespace teacher diff --git a/engine/tests/combined_test.cpp b/engine/tests/combined_test.cpp index 9aa877a..4b76f1c 100644 --- a/engine/tests/combined_test.cpp +++ b/engine/tests/combined_test.cpp @@ -160,3 +160,20 @@ TEST(NormalizeScores, WhereAtUsesAverageHitAndItsOwnEndpointFloor) { EXPECT_DOUBLE_EQ(normalized.where_at_within_radius, 0.5); EXPECT_DOUBLE_EQ(normalized.combined, 0.25); } + +TEST(NormalizeScores, SelfCombinedNeedsAllFourInternalScores) { + auto scores = all_scores(0.6); + auto floor = all_scores(0.2); + EXPECT_TRUE(std::isnan(evaluator::normalize_scores(scores, floor).combined_self)); + scores.self_knn.recall_at_k = 0.6; + floor.self_knn.recall_at_k = 0.2; + scores.self_join.f1 = 1.0; + floor.self_join.f1 = 0.2; + scores.self_join_grid = {{.f1 = 0.2}}; + floor.self_join_grid = {{.f1 = 0.2}}; + const auto normalized = evaluator::normalize_scores(scores, floor); + EXPECT_DOUBLE_EQ(normalized.self_knn, 0.5); + EXPECT_DOUBLE_EQ(normalized.self_join_mean, 0.5); // 1 at the main threshold, 0 on the grid + EXPECT_DOUBLE_EQ(normalized.combined_self, 0.5); + EXPECT_DOUBLE_EQ(normalized.combined, 0.5); +} diff --git a/engine/tests/config_test.cpp b/engine/tests/config_test.cpp index 433273d..59e9891 100644 --- a/engine/tests/config_test.cpp +++ b/engine/tests/config_test.cpp @@ -257,6 +257,17 @@ TEST(Config, TeacherIsOffByDefault) { const auto self = pipeline::load_config( write_config("teacher_self.json", R"({"data_path": "d", "join": {"mode": "Self"}})")); EXPECT_EQ(pipeline::teacher_objective_config(self).join_mode, query::JoinMode::Self); + // The kNN query is compressed in the in-database setting only, unless set + EXPECT_FALSE(pipeline::teacher_objective_config(config).compressed_knn_query); + EXPECT_TRUE(pipeline::teacher_objective_config(self).compressed_knn_query); + const auto original = pipeline::load_config(write_config( + "teacher_original.json", + R"({"data_path": "d", "join": {"mode": "Self"}, "teacher": {"knn_query": "Original"}})")); + EXPECT_FALSE(pipeline::teacher_objective_config(original).compressed_knn_query); + EXPECT_THROW(pipeline::load_config( + write_config("teacher_bad_query.json", + R"({"data_path": "d", "teacher": {"knn_query": "Sometimes"}})")), + std::invalid_argument); } TEST(Config, ReadsTheTeacherSection) { diff --git a/engine/tests/evaluator_test.cpp b/engine/tests/evaluator_test.cpp index 58fbca1..bece37c 100644 --- a/engine/tests/evaluator_test.cpp +++ b/engine/tests/evaluator_test.cpp @@ -215,3 +215,18 @@ TEST(Evaluator, SharedJoinMatchesIndependentClusteringAndFallsBackForDifferentWo EXPECT_EQ(join.entries[0].reference, (std::vector{2})); } } + +TEST(KnnScores, SelfQueriesUseTheCompressedQueryAndExcludeIt) { + const core::TrajectoryDb db({0, 2, 4, 6, 8}, {10, 20, 30, 40}, {0, 10, 0, 10, 0, 10, 0, 10}, + std::vector(8, 39.95), + {116.35, 116.35, 116.36, 116.36, 116.37, 116.37, 116.38, 116.38}); + // Each trajectory's nearest other is its neighbour in longitude + query::KnnWorkload workload{ + core::TrajectoryDb({0}, {}, {}, {}, {}), {{0, {1}}, {1, {0}}, {3, {2}}}, 1}; + workload.self = true; + const auto scores = evaluator::evaluate_knn(db, workload); + EXPECT_DOUBLE_EQ(scores.scores.recall_at_k, 1.0); + ASSERT_EQ(scores.answers.size(), 3u); + EXPECT_EQ(scores.answers[0].query_trajectory_id, 10); + EXPECT_EQ(scores.answers[0].neighbors, std::vector{20}); +} diff --git a/engine/tests/teacher/teacher_test.cpp b/engine/tests/teacher/teacher_test.cpp index 5025141..c9a056d 100644 --- a/engine/tests/teacher/teacher_test.cpp +++ b/engine/tests/teacher/teacher_test.cpp @@ -349,6 +349,77 @@ TEST_F(TeacherTest, RankingKnnIsTrackedExactly) { } } +TEST_F(TeacherTest, CompressedKnnQueriesAreTrackedExactly) { + const auto db = random_db(2, 24, 20); + auto config = small_config(); + config.frechet = query::FrechetVariant::Discrete; + const auto workloads = teacher::load_or_build_workloads(db, config, 9, m_dir); + const teacher::Objective objective( + db, workloads, + {.join_mode = query::JoinMode::Self, .ranking_knn = true, .compressed_knn_query = true}); + const auto floor = endpoints(db), full = every_point(db); + EXPECT_NEAR(objective.total(floor), 0.0, 1e-12); + EXPECT_DOUBLE_EQ(objective.metrics(full)[teacher::Objective::Knn], 1.0); + + // A switch changes i as a neighbour and as a query, both against the others' current state + auto state = floor; + const std::vector watch{full}; + teacher::KnnState knn(objective, state, watch, db.size()); + EXPECT_NEAR(knn.metric(), objective.metrics(state)[teacher::Objective::Knn], 1e-12); + const teacher::PreparedTrajectories prepared_floor(db, floor), prepared_full(db, full); + const auto floor_views = prepared_floor.views(), full_views = prepared_full.views(); + auto views = floor_views; + teacher::JoinState join(objective, views); + std::mt19937_64 rng(5); + for (int step = 0; step < 40; ++step) { + const std::size_t i = rng() % db.size(); + const bool grow = state[i].size() == 2; + const auto &next = grow ? full[i] : floor[i]; + const auto next_view = grow ? full_views[i] : floor_views[i]; + const double before = objective.total(state); + const auto move = knn.move(i, next_view, views); + const double knn_gain = knn.propose(move); + EXPECT_DOUBLE_EQ(knn.propose(knn.move(i, views[i], views)), 0.0); + const auto proposal = join.propose(i, next_view, views); + const double separable = objective.separable(i, next) - objective.separable(i, state[i]); + state[i] = next; + views[i] = next_view; + join.apply(proposal); + knn.apply(move); + EXPECT_NEAR(knn.metric(), objective.metrics(state)[teacher::Objective::Knn], 1e-12); + EXPECT_NEAR(objective.total(state) - before, knn_gain + proposal.gain + separable, 1e-9); + } +} + +TEST_F(TeacherTest, CompressedKnnQueriesNeedTheRankingTerm) { + const auto db = random_db(2, 24, 20); + const auto workloads = teacher::load_or_build_workloads(db, small_config(), 9, m_dir); + EXPECT_THROW( + teacher::Objective(db, workloads, {.ranking_knn = false, .compressed_knn_query = true}), + std::invalid_argument); +} + +TEST_F(TeacherTest, CompressedKnnArmsKeepTheBudgetAndLoopOnlyImproves) { + const auto db = random_db(3, 30, 25); + auto workloads = teacher::load_or_build_workloads(db, small_config(), 11, m_dir); + const teacher::Teacher model( + db, std::move(workloads), + {.join_mode = query::JoinMode::Self, .ranking_knn = true, .compressed_knn_query = true}, + 11); + const std::size_t budget = 2 * db.size() + 40; + const std::vector arms{"EQUAL_CHOICE", "EQUAL_CHOICE_RDP_PARTNERS", + "EQUAL_LOOP", "BUDGET_RDP", + "BUDGET_CHOICE", "BUDGET_LOOP"}; + const auto result = model.run(budget, arms); + for (const auto &arm : result) { + SCOPED_TRACE(arm.name); + EXPECT_EQ(arm.simplification.num_points(), budget); + } + // Watch lists hold every trajectory here, so the loop's tracked gains are exact + EXPECT_GE(result[2].objective, result[1].objective - 1e-12); + EXPECT_GE(result[5].objective, result[4].objective - 1e-12); +} + TEST_F(TeacherTest, RankingKnnArmsKeepTheBudget) { const auto db = random_db(3, 30, 25); auto workloads = teacher::load_or_build_workloads(db, small_config(), 11, m_dir);