diff --git a/README.md b/README.md index 76fc55b..9175b76 100644 --- a/README.md +++ b/README.md @@ -73,9 +73,9 @@ and it then falls back to its default. [engine/configs/geolife.json](engine/conf | --- | --- | | `data_path` | Parquet file produced by `convert` | | `seed` | Seed for all random sampling | -| `budget_shares` | Point budgets to test, as a share of all points (`0.01` = 1%) | +| `budget_shares` | Share of interior points to retain after reserving every endpoint: `B = 2T + round(share × (P − 2T))`. Report `B/P` for total retention | | `split` | `type` (`Random` or `Time`) and `query_percentage` held out for queries | -| `algorithms` | Simplification algorithms to run: `RDP`, `SED`, `UNIFORM` | +| `algorithms` | Simplification algorithms to run: `RDP`, `SED`, `UNIFORM`, `RDP_GLOBAL`. The global variant allocates the total budget by PED and runs once per share, independently of `allocation_alphas` | | `knn` | `k` and `query_count` (`0` = every trajectory in the query split) | | `range` | Box size (m), time window (s) and sampling limits for range queries | | `where_at` | Probes per trajectory and max time gap (s) between points to probe | @@ -85,6 +85,13 @@ and it then falls back to its default. [engine/configs/geolife.json](engine/conf | `clustering` | Route clustering metric: `enabled`, neighbour threshold `eps_m` (m) and `min_size` per cluster. Reuses the self join when one is scored with the same eps | | `allocation_alphas` | Allocation exponents to test: budgets follow size^α (`1` = size-proportional, `0` = equal). Exports for α ≠ 1 get an `_a<α>` suffix | +`RDP_GLOBAL` is database-wide top-down PED: reserve endpoints, then repeatedly split the +segment with the largest perpendicular error anywhere in the database. It uses no query +workloads and receives exactly the same total point budget as the teacher. + +The teacher's range term uses macro F1 like J4. The loop tracks box coverage counts exactly; +independent choices and value curves use F1 marginal gains against fixed partners. + ## Tests ```sh diff --git a/TEACHER_NEXT_STEPS.md b/TEACHER_NEXT_STEPS.md index 7358841..631f4e6 100644 --- a/TEACHER_NEXT_STEPS.md +++ b/TEACHER_NEXT_STEPS.md @@ -1,5 +1,11 @@ # Teacher: status and next steps +Implementation update, 2026-11-20: the range objective and endpoint floor now use macro F1 like +J4, with exact incremental updates in the loop and F1 marginals for candidate and budget scoring. +`RDP_GLOBAL`, a database-wide top-down PED baseline, runs alongside the other baselines in +`teacher_eval` and is enabled in `geolife.json`. It is included in the best-cheap-baseline comparison. +The numerical findings below predate this change and need to be checked with fresh runs. + Updated 2026-11-17 on branch `teacher`. The teacher is described in [engine/src/teacher/README.md](engine/src/teacher/README.md); this file says what is settled and what is open. diff --git a/engine/apps/teacher_eval.cpp b/engine/apps/teacher_eval.cpp index 0546e26..55692ce 100644 --- a/engine/apps/teacher_eval.cpp +++ b/engine/apps/teacher_eval.cpp @@ -17,6 +17,7 @@ #include "backend/engine/query/join.hpp" #include "backend/engine/query/knn.hpp" #include "backend/engine/query/route_clustering.hpp" +#include "backend/engine/selection/rdp.hpp" #include "backend/engine/selection/selector.hpp" #include "backend/engine/split/split.hpp" #include "backend/engine/teacher/teacher.hpp" @@ -40,7 +41,8 @@ namespace { using core::KeptSets; constexpr double kNan = std::numeric_limits::quiet_NaN(); -constexpr std::array kEvalEps{100.0, 125.0, 150.0, 175.0, 200.0, 250.0, 300.0}; +// Between the training thresholds, so the teacher never saw them +constexpr std::array kEvalEps{125.0, 175.0, 250.0}; constexpr double kRdpAlpha = 0.25; // RDP_a025 constexpr std::array kPartnerLambdas{0.5, 1.0}; // Unset options keep the config's value; arms default to all teacher arms @@ -53,7 +55,6 @@ struct Args { std::optional> shares; // config: budget_shares std::vector arms{teacher::kArms.begin(), teacher::kArms.end()}; bool baselines = true; - std::optional all_join_pairs; std::optional teacher_join; // config: teacher.join, else join.mode std::optional> train_eps; // config: teacher.extra_join_eps_m std::optional held_out_term; // config: teacher.held_out_term @@ -125,15 +126,13 @@ Args parse_args(int argc, char **argv) { args.teacher_join = mode == "Self" ? query::JoinMode::Self : query::JoinMode::HeldOut; } else if (key == "--no-baselines") args.baselines = false; - else if (key == "--seen-pairs-only") - args.all_join_pairs = false; else throw std::invalid_argument("unknown argument " + key + " (--config --data --out --cache --seeds --shares --arms " "--teacher-join --train-eps --held-out-term " "--independent-knn --knn-query --term-weights --eval-eps " "--knn-check " - "--no-baselines --seen-pairs-only)"); + "--no-baselines)"); } teacher::check_arms(args.arms); return args; @@ -147,8 +146,6 @@ pipeline::Config effective_config(const Args &args) { config.teacher.cache_dir = *args.cache; if (args.shares) config.budget_shares = *args.shares; - if (args.all_join_pairs) - config.teacher.all_join_pairs = *args.all_join_pairs; if (args.teacher_join) config.teacher.join = *args.teacher_join; if (args.train_eps) @@ -246,13 +243,12 @@ struct Arm { // PARTNER: RDP with weight 1 + lambda for trajectories that have a join partner std::vector baseline_arms(const core::TrajectoryDb &db, - const std::vector &pairs, bool all_join_pairs, + const std::vector &pairs, std::span shares) { using selection::AlgorithmType; std::vector partnered(db.size(), 0); for (const auto &pair : pairs) - if (all_join_pairs || pair.seen) - partnered[pair.i] = partnered[pair.j] = 1; + partnered[pair.i] = partnered[pair.j] = 1; std::vector arms; for (const double share : shares) { const auto budget = allocation::total_budget(db, share); @@ -261,6 +257,8 @@ std::vector baseline_arms(const core::TrajectoryDb &db, db, allocation::AllocatorType::SizeProportional, budget, alpha); }; const auto equal = geometric(0.0); + arms.push_back( + {"RDP_GLOBAL", share, core::to_kept(selection::rdp::simplify_database(db, budget))}); arms.push_back({"RDP_a0", share, select(db, AlgorithmType::RDP, equal)}); arms.push_back({"RDP_a025", share, select(db, AlgorithmType::RDP, geometric(kRdpAlpha))}); arms.push_back({"SED_a0", share, select(db, AlgorithmType::SED, equal)}); @@ -295,47 +293,11 @@ void validate(const Arm &arm, const core::TrajectoryDb &db) { throw std::runtime_error(std::format("{}: {} points, budget {}", arm.name, total, budget)); } -struct PairRecall { - double seen = kNan, unseen = kNan; -}; - -// Mean per-trajectory recall on the seen and unseen halves of the teacher's join pairs -PairRecall pair_recall(const core::TrajectoryDb &compressed, - const std::vector &pairs, const query::JoinConfig &join) { - std::vector matched(pairs.size()); - core::parallel_for(pairs.size(), [&](std::size_t k) { - matched[k] = - query::frechet_within(compressed.trajectory(pairs[k].i), - compressed.trajectory(pairs[k].j), join.eps_m, join.frechet); - }); - PairRecall result; - for (const bool seen : {true, false}) { - std::vector total(compressed.size(), 0), hits(compressed.size(), 0); - for (std::size_t k = 0; k < pairs.size(); ++k) { - if (pairs[k].seen != seen) - continue; - for (const auto t : {pairs[k].i, pairs[k].j}) { - ++total[t]; - hits[t] += matched[k]; - } - } - double sum = 0; - std::size_t count = 0; - for (std::size_t t = 0; t < total.size(); ++t) - if (total[t] != 0) { - sum += static_cast(hits[t]) / static_cast(total[t]); - ++count; - } - (seen ? result.seen : result.unseen) = count ? sum / static_cast(count) : kNan; - } - return result; -} - // Whether the teacher trained on the join at eps_m in mode: one of its thresholds in its own join // mode, or its held-out term at join.eps_m bool trained_on(const pipeline::Config &config, const teacher::ObjectiveConfig &objective, double eps_m, query::JoinMode mode) { - const auto &extra = config.teacher.extra_join_eps_m; + const auto extra = pipeline::teacher_extra_join_eps_m(config); const bool teacher_eps = eps_m == config.join.eps_m || std::ranges::find(extra, eps_m) != extra.end(); const bool held_out_term = @@ -364,7 +326,7 @@ int main(int argc, char **argv) try { args.out / "results.csv", "seed,share,arm,budget,knn_recall,range_f1,where_at_avg_hit,join_f1,clustering," "where_at_within_100,join_micro_f1,knn_tau,norm_knn,norm_range,norm_where_at,norm_join," - "norm_clustering,J4,J_T,join_recall_seen,join_recall_unseen,predicted_clusters," + "norm_clustering,J4,J_T,predicted_clusters," "eval_s,join_mode,self_join_f1,norm_self_join,teacher_join_mode,norm_join_main," "norm_clustering_main,self_knn_recall,norm_self_knn,norm_self_join_mean,J4_self," "teacher_knn_query"); @@ -412,8 +374,7 @@ int main(int argc, char **argv) try { core::seconds_since(start)); } if (args.baselines) { - for (auto &arm : - baseline_arms(db, pairs, config.teacher.all_join_pairs, config.budget_shares)) + for (auto &arm : baseline_arms(db, pairs, config.budget_shares)) arms.push_back(std::move(arm)); for (const double share : config.budget_shares) for (auto &arm : model.single_recipes(allocation::total_budget(db, share))) @@ -476,16 +437,15 @@ int main(int argc, char **argv) try { const auto scores = evaluator::Evaluator(db, compressed, test, scoring).evaluate().scores; const auto normalized = evaluator::normalize_scores(scores, floor); - const auto recall = pair_recall(compressed, pairs, config.join); results << std::format( - "{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{:.2f}," + "{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{:.2f}," "{},{},{},{},{},{},{},{},{},{},{}\n", seed, arm.share, arm.name, allocation::total_budget(db, arm.share), scores.knn.recall_at_k, scores.range_f1, scores.where_at.average_hit, scores.join.f1, scores.clustering.score, scores.where_at.within_radius, scores.join.micro_f1, scores.knn.kendalls_tau_topk, normalized.knn, normalized.range, normalized.where_at, normalized.join, normalized.clustering, - normalized.combined, objective.total(arm.kept), recall.seen, recall.unseen, + normalized.combined, objective.total(arm.kept), scores.clustering.predicted_clusters, core::seconds_since(start), mode_name(config.join.mode), scores.self_join.f1, normalized.self_join, mode_name(objective.config().join_mode), normalized.join_f1, diff --git a/engine/apps/teacher_stats.py b/engine/apps/teacher_stats.py index f080b3c..03f169f 100644 --- a/engine/apps/teacher_stats.py +++ b/engine/apps/teacher_stats.py @@ -26,7 +26,7 @@ ("loop", "BUDGET_CHOICE", "BUDGET_LOOP"), ("looking once", "BUDGET_CHOICE", "BUDGET_LOOP_NO_UPDATE"), ("reacting", "BUDGET_LOOP_NO_UPDATE", "BUDGET_LOOP")] -CHEAP = ["RDP_a0", "RDP_a025", "PARTNER_l0.5", "PARTNER_l1"] +CHEAP = ["RDP_a0", "RDP_a025", "RDP_GLOBAL", "SED_a0", "UNI_a0", "PARTNER_l0.5", "PARTNER_l1"] def num(v): @@ -71,7 +71,7 @@ def main(): parts = [num(r.get(c, "")) for c in ["norm_knn", "norm_range", "norm_where_at", "norm_join"]] parts = [x for x in parts if x is not None] r["J4"] = str(statistics.fmean(parts)) if parts else "" - for m in METRICS + ["J_T", "join_recall_seen", "join_recall_unseen"]: + for m in METRICS + ["J_T"]: v = num(r.get(m, "")) if v is not None: data[r["share"]][r["arm"]][m][int(r["seed"])] = v @@ -87,14 +87,11 @@ def main(): for share in sorted(data, key=float): arms = data[share] out.append(f"## Share {float(share) * 100:g}%\n") - out.append("| Arm | " + " | ".join(METRICS) + " | seen / unseen join recall |") - out.append("|---|" + "---:|" * (len(METRICS) + 1)) + out.append("| Arm | " + " | ".join(METRICS) + " |") + out.append("|---|" + "---:|" * len(METRICS)) for arm in sorted(arms): cells = [mean_sd(list(arms[arm][m].values())) for m in METRICS] - seen = list(arms[arm]["join_recall_seen"].values()) - unseen = list(arms[arm]["join_recall_unseen"].values()) - su = f"{statistics.fmean(seen):.4f} / {statistics.fmean(unseen):.4f}" if seen and unseen else "–" - out.append(f"| {arm} | " + " | ".join(cells) + f" | {su} |") + out.append(f"| {arm} | " + " | ".join(cells) + " |") out.append("") cheap = [a for a in CHEAP if a in arms and arms[a]["J4"]] best = max(cheap, key=lambda a: statistics.fmean(arms[a]["J4"].values())) if cheap else None diff --git a/engine/configs/geolife.json b/engine/configs/geolife.json index 697ad46..0933424 100644 --- a/engine/configs/geolife.json +++ b/engine/configs/geolife.json @@ -9,7 +9,7 @@ "type": "Random", "query_percentage": 0.15 }, - "algorithms": ["RDP", "SED", "UNIFORM"], + "algorithms": ["RDP", "SED", "UNIFORM", "RDP_GLOBAL"], "knn": { "k": 10, "query_count": 0 @@ -48,8 +48,7 @@ "enabled": false, "cache_dir": "cache", "arms": ["BUDGET_CHOICE", "BUDGET_LOOP"], - "range_query_count": 64000, - "extra_join_eps_m": [100.0, 125.0, 200.0, 300.0], + "range_query_count": 128000, "held_out_term": false, "ranking_knn": true, "term_weights": [1.0, 1.0, 1.0, 1.0] diff --git a/engine/include/backend/engine/pipeline/config.hpp b/engine/include/backend/engine/pipeline/config.hpp index 9599397..1f25dec 100644 --- a/engine/include/backend/engine/pipeline/config.hpp +++ b/engine/include/backend/engine/pipeline/config.hpp @@ -43,6 +43,8 @@ struct Config { // Reads a JSON config. Only data_path is required, a missing key keeps its default. Config load_config(const std::filesystem::path &path); +// teacher.extra_join_eps_m, by default join.score_eps_m +std::vector teacher_extra_join_eps_m(const Config &config); teacher::WorkloadConfig teacher_workload_config(const Config &config); teacher::ObjectiveConfig teacher_objective_config(const Config &config); diff --git a/engine/include/backend/engine/selection/rdp.hpp b/engine/include/backend/engine/selection/rdp.hpp index ee0de39..4765c95 100644 --- a/engine/include/backend/engine/selection/rdp.hpp +++ b/engine/include/backend/engine/selection/rdp.hpp @@ -1,5 +1,6 @@ #pragma once +#include "backend/engine/core/simplification.hpp" #include "backend/engine/core/trajectory_db.hpp" #include @@ -15,4 +16,9 @@ std::vector simplify(const core::TrajectoryView &trajectory, // reproduces simplify(trajectory, b), for 2 <= b <= min(limit, size). std::vector sequence(const core::TrajectoryView &trajectory, std::size_t limit); +// Database-wide top-down PED (RDP_GLOBAL / TD(W, PED)). Reserve every endpoint, then split +// the segment with the largest perpendicular error anywhere in the database until total_budget. +// Rejects budgets below the endpoint count or above the original point count. +core::Simplification simplify_database(const core::TrajectoryDb &db, std::size_t total_budget); + } // namespace selection::rdp diff --git a/engine/include/backend/engine/selection/selector.hpp b/engine/include/backend/engine/selection/selector.hpp index 2a50919..b11414d 100644 --- a/engine/include/backend/engine/selection/selector.hpp +++ b/engine/include/backend/engine/selection/selector.hpp @@ -8,7 +8,7 @@ namespace selection { -enum class AlgorithmType { RDP, SED, UNIFORM }; +enum class AlgorithmType { RDP, SED, UNIFORM, RDP_GLOBAL }; std::vector simplify_trajectory(const core::TrajectoryView &, AlgorithmType, std::size_t); diff --git a/engine/include/backend/engine/teacher/curves.hpp b/engine/include/backend/engine/teacher/curves.hpp index 87d52ec..e6c1d39 100644 --- a/engine/include/backend/engine/teacher/curves.hpp +++ b/engine/include/backend/engine/teacher/curves.hpp @@ -16,8 +16,9 @@ struct Curve { double at(std::size_t budget) const { return value[budget - 2]; } }; -// Maximises sum curves[i].at(b_i) with sum b_i = total_budget, greedily on the best gain per point -// over the next 1 to 3 points, which steps over flat stretches. Throws if the budget cannot fit +// Maximises sum curves[i].at(b_i) with sum b_i = total_budget: greedy on the slopes of each curve's +// upper concave envelope, which steps over flat stretches of any length. Optimal when the budget +// ends on envelope vertices. Throws if the budget cannot fit std::vector allocate(const std::vector &curves, std::size_t total_budget, double *value = nullptr); diff --git a/engine/include/backend/engine/teacher/greedy.hpp b/engine/include/backend/engine/teacher/greedy.hpp index 929c8e0..9e15b7e 100644 --- a/engine/include/backend/engine/teacher/greedy.hpp +++ b/engine/include/backend/engine/teacher/greedy.hpp @@ -10,7 +10,8 @@ namespace teacher { // BKH's workload-driven selector: from the endpoints and a seed, repeatedly keep the point that -// raises i's range and where-at terms most, then take the rest from fill, skipping kept points +// raises range F1 (other trajectories original) and i's where-at term most, then take the rest +// from fill, skipping kept points class Greedy { public: Greedy(const Objective &objective, std::size_t i); diff --git a/engine/include/backend/engine/teacher/objective.hpp b/engine/include/backend/engine/teacher/objective.hpp index 385a8e3..e797269 100644 --- a/engine/include/backend/engine/teacher/objective.hpp +++ b/engine/include/backend/engine/teacher/objective.hpp @@ -49,7 +49,6 @@ using TermWeights = std::array; struct ObjectiveConfig { double min_tolerance_m = 25.0; // where-at ramp, as in the evaluator double max_tolerance_m = 500.0; - bool all_join_pairs = true; // false: only the seen half // Self: both sides compressed. HeldOut: each trajectory also queries its partners as original query::JoinMode join_mode = query::JoinMode::Self; bool held_out_term = false; // with Self: add the held-out form at join_eps_m as a term @@ -74,11 +73,12 @@ struct JoinLink { // J_T: weighted mean of normalised kNN, range, where-at and join on the teacher's workloads, // (metric - floor) / (1 - floor) with floor = endpoints only, like J4. -// range recall: some kept point still inside each box i is in +// range macro F1 over nonempty reference boxes, like J4 // where-at average-hit ramp over i's probes -// join macro F1 over candidate pairs, pooled over all join terms (thresholds, held-out form) -// Range, where-at and the per-trajectory kNN add up over trajectories; join and ranking kNN do -// not, and are tracked under one-at-a-time changes by JoinState and KnnState +// join macro F1 over candidate pairs per join term (threshold, held-out form), each +// normalised by its own floor, then averaged: J4's join grid mean +// Where-at and the per-trajectory kNN add up over trajectories. Range, join and ranking kNN +// are tracked under one-at-a-time changes by RangeState, JoinState and KnnState class Objective { public: enum Term : std::size_t { Knn, Range, WhereAt, Join, TermCount }; @@ -92,11 +92,13 @@ class Objective { double separable(std::size_t i, std::span kept) const; double separable(std::size_t i, const core::TrajectoryView &candidate) const; - // Approximate score of i alone: separable plus i's join F1 against fixed partners (empty = - // original; ignored when held out). Exact above beat, otherwise some value below beat + // Approximate score of i alone: separable, marginal range F1, and i's join F1 against fixed + // partners (empty = original; the join ignores partners when held out). Exact above beat, + // otherwise some value below beat double independent_value(std::size_t i, const core::TrajectoryView &candidate, std::span partners, - double beat = -std::numeric_limits::infinity()) const; + double beat = -std::numeric_limits::infinity(), + std::span range_weights = {}) const; // independent_value with every kNN check a hit double independent_bound(std::size_t i, const core::TrajectoryView &candidate, std::span partners) const; @@ -114,7 +116,8 @@ class Objective { double join = 0.0; }; IndependentScore prepare_independent(std::size_t i, const core::TrajectoryView &candidate, - std::span partners) const; + std::span partners, + std::span range_weights = {}) const; double independent_value(const IndependentScore &score, double beat = -std::numeric_limits::infinity()) const; @@ -133,13 +136,21 @@ class Objective { std::size_t positive_count() const { return m_positive_count; } std::size_t slot(std::size_t t, std::size_t term) const { return t + term * m_db.size(); } std::size_t slots() const { return m_truth.size(); } + std::size_t join_terms() const { return m_eps.size(); } + double join_floor(std::size_t term) const { return m_join_floor[term]; } + // The join metric from each term's macro F1 + double join_value(std::span f1s) const; std::size_t truth(std::size_t slot) const { return m_truth[slot]; } bool held_out(std::size_t term) const { return m_term_held_out[term]; } struct RangeMembership { std::uint32_t box; - double weight; // 1 / (|reference| * scored boxes) + double weight; // marginal macro F1 of covering i with every other reference covered }; + // Marginal macro F1 of covering i, holding all other trajectories fixed (empty = original). + // RangeState::weights reuses its counts when scoring many candidates against one state. + std::vector range_weights(std::size_t i, + std::span partners) const; const core::TrajectoryDb &db() const { return m_db; } const TeacherWorkloads &workloads() const { return m_workloads; } const ObjectiveConfig &config() const { return m_config; } @@ -163,6 +174,7 @@ class Objective { std::size_t knn_checks(std::size_t i) const; double knn_check_weight() const; double join_metric(std::span state) const; + std::vector join_f1s(std::span state) const; const core::TrajectoryDb &m_db; const TeacherWorkloads &m_workloads; @@ -181,13 +193,42 @@ class Objective { std::size_t m_positive_count = 0; std::vector> m_links; // per trajectory std::vector m_truth; // positives per slot - std::size_t m_with_truth = 0; // slots with any positive + std::vector m_with_truth; // per join term, slots with any positive + std::vector m_join_floor; // per join term, macro F1 with endpoints only + std::vector m_join_weight; // per join term, 1 / (terms * (1 - floor)) Terms m_floor{}; Terms m_scale{}; // weight / (weight sum * (1 - floor)) std::vector m_endpoint_separable; // scaled separable terms, endpoints only }; +// Exact range macro F1 under one-at-a-time changes. A subset of original samples cannot add +// false positives, so each box's F1 is 2 * found / (reference size + found). +class RangeState { + public: + // Empty state means original trajectories + RangeState(const Objective &objective, std::span state); + + struct Proposal { + std::size_t trajectory; + double gain = 0.0; // change in J_T + std::vector covered; // in boxes(trajectory) order + }; + Proposal propose(std::size_t i, const core::TrajectoryView &candidate) const; + // A proposal is valid until the next apply() + void apply(const Proposal &proposal); + double metric() const; + std::vector weights(std::size_t i) const; + + private: + double f1(std::size_t box, std::size_t found) const; + const Objective &m_objective; + std::vector m_found; + std::vector> m_covered; + std::size_t m_scored = 0; + double m_sum = 0.0; +}; + // The exact join metric under one-at-a-time changes; changing i only rescores i's pairs class JoinState { public: @@ -204,7 +245,7 @@ class JoinState { std::span state) const; void apply(const Proposal &proposal); - double metric() const; // macro F1 over the pairs used + double metric() const; // as Objective's join metric private: bool counted(std::size_t t, std::size_t found) const; @@ -224,8 +265,8 @@ class JoinState { const Objective &m_objective; std::vector m_matched; // per pair and direction std::vector m_hits, m_found; - double m_sum = 0; // sum of F1 over counted trajectories - std::size_t m_counted = 0; + std::vector m_sum; // per join term, sum of F1 over counted slots + std::vector m_counted; }; // The ranking kNN metric under one-at-a-time changes. Each query watches its true top k and its diff --git a/engine/include/backend/engine/teacher/teacher.hpp b/engine/include/backend/engine/teacher/teacher.hpp index a30f938..ad7a0b3 100644 --- a/engine/include/backend/engine/teacher/teacher.hpp +++ b/engine/include/backend/engine/teacher/teacher.hpp @@ -48,10 +48,10 @@ struct TeacherConfig { std::filesystem::path cache_dir = "cache"; std::vector arms{"BUDGET_CHOICE", "BUDGET_LOOP"}; std::size_t range_query_count = kRangeQueryCount; - bool all_join_pairs = true; // false: only the seen half of the join pairs // Unset: join.mode std::optional join; - std::vector extra_join_eps_m; + // Unset: join.score_eps_m, the thresholds J4 averages over + std::optional> extra_join_eps_m; bool held_out_term = false; bool ranking_knn = true; // ObjectiveConfig::compressed_knn_query. Unset: compressed with a Self join, the in-database diff --git a/engine/include/backend/engine/teacher/workloads.hpp b/engine/include/backend/engine/teacher/workloads.hpp index fa88afa..2a87bdf 100644 --- a/engine/include/backend/engine/teacher/workloads.hpp +++ b/engine/include/backend/engine/teacher/workloads.hpp @@ -16,12 +16,10 @@ namespace teacher { // Offsets added to the run seed, one per random draw, so no two draws share a stream inline constexpr std::uint64_t kRangeSeedOffset = 7001; inline constexpr std::uint64_t kWhereAtSeedOffset = 7002; -inline constexpr std::uint64_t kPairSplitSeedOffset = 7003; inline constexpr std::uint64_t kOrderSeedOffset = 7004; -inline constexpr std::uint64_t kNegativeSeedOffset = 7005; inline constexpr std::uint64_t kShuffleSeedOffset = 7006; -inline constexpr std::size_t kRangeQueryCount = 64000; +inline constexpr std::size_t kRangeQueryCount = 128000; // Training workloads from the database alone, with their own seeds, never the held-out pool struct WorkloadConfig { @@ -42,7 +40,6 @@ struct KnnEntry { struct JoinPair { std::uint32_t i, j; // i < j - bool seen; // in the half used with all_join_pairs = false }; // Self-join pairs at a threshold other than join_eps_m, all of them used @@ -56,8 +53,8 @@ struct TeacherWorkloads { double join_eps_m = 0.0; query::FrechetVariant frechet = query::FrechetVariant::Continuous; std::vector knn; // one per database trajectory - std::vector positives; // self-join pairs, half seen - std::vector negatives; // endpoint candidates that are not pairs, half seen + std::vector positives; // self-join pairs + std::vector negatives; // endpoint candidates that are not pairs std::vector extra_joins; query::RangeWorkload range; query::WhereAtWorkload where_at; diff --git a/engine/src/pipeline/config.cpp b/engine/src/pipeline/config.cpp index 8912b25..731bcdc 100644 --- a/engine/src/pipeline/config.cpp +++ b/engine/src/pipeline/config.cpp @@ -116,18 +116,18 @@ void parse_clustering_config(Config &config, const json &clustering) { void parse_teacher_config(Config &config, const json §ion) { validate_keys(section, - {"enabled", "cache_dir", "arms", "range_query_count", "all_join_pairs", "join", - "extra_join_eps_m", "held_out_term", "ranking_knn", "knn_query", "term_weights"}, + {"enabled", "cache_dir", "arms", "range_query_count", "join", "extra_join_eps_m", + "held_out_term", "ranking_knn", "knn_query", "term_weights"}, "teacher"); auto &t = config.teacher; t.enabled = section.value("enabled", t.enabled); t.cache_dir = section.value("cache_dir", t.cache_dir.string()); t.arms = section.value("arms", t.arms); t.range_query_count = section.value("range_query_count", t.range_query_count); - t.all_join_pairs = section.value("all_join_pairs", t.all_join_pairs); if (section.contains("join")) t.join = parse_enum(section["join"]); - t.extra_join_eps_m = section.value("extra_join_eps_m", t.extra_join_eps_m); + if (section.contains("extra_join_eps_m")) + t.extra_join_eps_m = section["extra_join_eps_m"].get>(); t.held_out_term = section.value("held_out_term", t.held_out_term); t.ranking_knn = section.value("ranking_knn", t.ranking_knn); if (section.contains("knn_query")) { @@ -138,8 +138,9 @@ void parse_teacher_config(Config &config, const json §ion) { } t.term_weights = section.value("term_weights", t.term_weights); teacher::check_term_weights(t.term_weights); - query::check_extra_thresholds(t.extra_join_eps_m, config.join.eps_m, - "config: teacher.extra_join_eps_m"); + if (t.extra_join_eps_m) + query::check_extra_thresholds(*t.extra_join_eps_m, config.join.eps_m, + "config: teacher.extra_join_eps_m"); if (t.cache_dir.empty()) throw std::invalid_argument("config: teacher.cache_dir must not be empty"); if (t.enabled && t.arms.empty()) @@ -215,12 +216,16 @@ Config load_config(const std::filesystem::path &path) { } } +std::vector teacher_extra_join_eps_m(const Config &config) { + return config.teacher.extra_join_eps_m.value_or(config.join_score_eps_m); +} + teacher::WorkloadConfig teacher_workload_config(const Config &config) { auto range = config.range; range.query_count = config.teacher.range_query_count; return {.k = config.knn.k, .join_eps_m = config.join.eps_m, - .extra_join_eps_m = config.teacher.extra_join_eps_m, + .extra_join_eps_m = teacher_extra_join_eps_m(config), .frechet = config.join.frechet, .range = range, .where_at = config.where_at}; @@ -231,7 +236,6 @@ teacher::ObjectiveConfig teacher_objective_config(const Config &config) { const auto join_mode = config.teacher.join.value_or(config.join.mode); return {.min_tolerance_m = config.where_at_scoring.min_tolerance_m, .max_tolerance_m = config.where_at_scoring.max_tolerance_m, - .all_join_pairs = config.teacher.all_join_pairs, .join_mode = join_mode, .held_out_term = config.teacher.held_out_term, .ranking_knn = config.teacher.ranking_knn, diff --git a/engine/src/pipeline/pipeline.cpp b/engine/src/pipeline/pipeline.cpp index c21e8fd..8750f3a 100644 --- a/engine/src/pipeline/pipeline.cpp +++ b/engine/src/pipeline/pipeline.cpp @@ -5,6 +5,7 @@ #include "backend/engine/io/workload_export.hpp" #include "backend/engine/pipeline/scorer.hpp" #include "backend/engine/pipeline/test_workloads.hpp" +#include "backend/engine/selection/rdp.hpp" #include "backend/engine/selection/selector.hpp" #include "backend/engine/split/split.hpp" #include "backend/engine/teacher/teacher.hpp" @@ -12,6 +13,7 @@ #include +#include #include #include #include @@ -81,14 +83,23 @@ void run(const Config &config) { for (const double share : config.budget_shares) { const std::size_t budget = allocation::total_budget(database, share); + if (std::ranges::find(config.algorithms, selection::AlgorithmType::RDP_GLOBAL) != + config.algorithms.end()) + scorer.score(std::format("{}% RDP_GLOBAL", share * 100), + std::format("RDP_GLOBAL_{:g}pct", share * 100), + selection::rdp::simplify_database(database, budget)); + for (const double alpha : config.allocation_alphas) { const auto budgets = allocation::create_trajectory_budgets( database, allocation::AllocatorType::SizeProportional, budget, alpha); - for (const auto algorithm : config.algorithms) + for (const auto algorithm : config.algorithms) { + if (algorithm == selection::AlgorithmType::RDP_GLOBAL) + continue; scorer.score(std::format("{}% {} alpha={}", share * 100, magic_enum::enum_name(algorithm), alpha), baseline_name(algorithm, share, alpha), selection::simplify_database(database, algorithm, budgets)); + } } if (teacher) { diff --git a/engine/src/selection/rdp.cpp b/engine/src/selection/rdp.cpp index 5ef0407..88ce249 100644 --- a/engine/src/selection/rdp.cpp +++ b/engine/src/selection/rdp.cpp @@ -3,6 +3,7 @@ #include #include #include +#include namespace selection::rdp { @@ -34,10 +35,8 @@ struct Split { bool operator<(const Split &other) const { return error < other.error; } }; -void push_split(std::priority_queue &heap, std::span x, - std::span y, std::size_t first, std::size_t last) { - if (last - first < 2) - return; +Split split_at(std::span x, std::span y, std::size_t first, + std::size_t last) { Split split{-1.0, first + 1, first, last}; for (std::size_t k = first + 1; k < last; ++k) { const double d = squared_segment_distance(x[k], y[k], x[first], y[first], x[last], y[last]); @@ -46,7 +45,13 @@ void push_split(std::priority_queue &heap, std::span x, split.index = k; } } - heap.push(split); + return split; +} + +void push_split(std::priority_queue &heap, std::span x, + std::span y, std::size_t first, std::size_t last) { + if (last - first >= 2) + heap.push(split_at(x, y, first, last)); } } // namespace @@ -85,4 +90,46 @@ std::vector simplify(const core::TrajectoryView &trajectory, return indices; } +core::Simplification simplify_database(const core::TrajectoryDb &db, std::size_t total_budget) { + auto kept = core::endpoints_only(db); + std::size_t count = 0; + for (const auto &indices : kept) + count += indices.size(); + if (total_budget < count || total_budget > db.num_points()) + throw std::invalid_argument("global RDP: budget must be between endpoints and all points"); + + struct GlobalSplit { + Split split; + std::size_t trajectory; + bool operator<(const GlobalSplit &other) const { + if (split.error != other.split.error) + return split.error < other.split.error; + if (trajectory != other.trajectory) + return trajectory > other.trajectory; + return split.first > other.split.first; + } + }; + std::priority_queue heap; + auto push = [&](std::size_t i, std::size_t first, std::size_t last) { + if (last - first >= 2) { + const auto trajectory = db.trajectory(i); + heap.push({split_at(trajectory.x, trajectory.y, first, last), i}); + } + }; + for (std::size_t i = 0; i < db.size(); ++i) + if (db.trajectory(i).size() >= 3) + push(i, 0, db.trajectory(i).size() - 1); + while (count < total_budget) { + const auto [split, i] = heap.top(); + heap.pop(); + kept[i].push_back(split.index); + ++count; + push(i, split.first, split.index); + push(i, split.index, split.last); + } + for (auto &indices : kept) + std::ranges::sort(indices); + return core::to_simplification(kept); +} + } // namespace selection::rdp diff --git a/engine/src/selection/selector.cpp b/engine/src/selection/selector.cpp index 10d080d..f0b2e01 100644 --- a/engine/src/selection/selector.cpp +++ b/engine/src/selection/selector.cpp @@ -20,6 +20,8 @@ std::vector simplify_trajectory(const core::TrajectoryView &traject return sed::simplify(trajectory, points_to_keep); case AlgorithmType::UNIFORM: return uniform::simplify(trajectory, points_to_keep); + case AlgorithmType::RDP_GLOBAL: + throw std::invalid_argument("RDP_GLOBAL needs a database-wide total budget"); } throw std::invalid_argument("simplify_trajectory: unknown algorithm"); diff --git a/engine/src/teacher/README.md b/engine/src/teacher/README.md index 08f0a1a..6ee0ecf 100644 --- a/engine/src/teacher/README.md +++ b/engine/src/teacher/README.md @@ -8,6 +8,10 @@ It sees **only the database** (the 85% split) and workloads built from it with t never sees the held-out query pool that the pipeline scores on. The fixed-budget arms use equal shares (α = 0); the allocation arms redistribute the same total point budget. +Range scoring changed on 2026-11-20: J_T now uses macro F1, including its endpoint floor, +matching J4. Historical teacher selections and J_T values need a fresh run. `RDP_GLOBAL` is the +query-independent database-wide top-down PED baseline in both experiment runners. + ## How it fits together ``` @@ -34,7 +38,7 @@ Files: | File | Contents | |---|---| | `workloads.hpp/.cpp` | The teacher's training workloads, and the cache that saves the slow part | -| `objective.hpp/.cpp` | J_T, the score the teacher maximises, and `JoinState` and `KnnState` for fast exact join and kNN updates | +| `objective.hpp/.cpp` | J_T, the score the teacher maximises, and range, join and kNN state trackers | | `curves.hpp/.cpp` | Value curves and the budget allocator | | `greedy.hpp/.cpp` | Query-aware point selection, with optional classic-method seeds | | `teacher.hpp/.cpp` | The `Teacher` class and the selection steps | @@ -53,8 +57,8 @@ Files: it is built from the same Fréchet neighbour test as the join and would count it twice. With `join.mode` `HeldOut` (the Geolife config) the join term compares original trajectories from the query pool with the simplified database, so all four terms are held out. The database - self join, whose pairs the teacher trains on, is then reported separately (`norm_self_join`, - the seen / unseen recall and the 100 / 300 m thresholds) and is not part of J4. Results quoted + self join, whose pairs the teacher trains on, is then reported separately (`norm_self_join` + and the thresholds in `thresholds.csv`) and is not part of J4. Results quoted below predate this and have the self join inside the score. - **J4-self** (`J4_self`): the in-database setting, where the database's own trajectories are the kNN and join queries. Its terms are self kNN (each database trajectory, compressed, searched @@ -66,7 +70,7 @@ Files: - **J5:** the earlier headline, J4 plus clustering. Results quoted below from earlier runs use it; `teacher_eval` no longer writes it. - **J_T:** the teacher's own score on its training workloads. It has the same shape as J4. kNN - and range use training proxies. + uses a training proxy during value-curve construction; range uses macro F1 like J4. - **J_T's kNN term:** with `teacher.ranking_knn` (the default) it is recall@k of the teacher's database queries answered on the compressed database (`knn_recall`), so competitors are compressed too, like the test. The loop tracks it with `KnnState`. Value curves still use the per-trajectory check @@ -85,19 +89,20 @@ Files: This is the database-only stand-in for the held-out join, as the kNN proxy is for held-out kNN; the teacher still never sees the query pool. `teacher.join` (or `teacher_eval --teacher-join Self|HeldOut`) overrides it, e.g. to train on the self join for the self-join - experiment. Held out, `EQUAL_CHOICE_RDP_PARTNERS` and `EQUAL_CHOICE` are the same arm, and best response is - coupled only through each query's F1. Everything below describing the join term as "both + experiment. With held-out join terms the join scoring is independent of compressed partners, + but range F1 still couples trajectories that answer the same box. Everything below describing the join term as "both sides compressed" is the `Self` mode. -- **Join terms:** the join term can pool several terms. `teacher.extra_join_eps_m` (or - `teacher_eval --train-eps`) adds self-join thresholds next to `join.eps_m`, so the join gain is - not fitted to one threshold. `teacher.held_out_term` (or `--held-out-term`) adds the held-out +- **Join terms:** the join term can pool several terms. By default the teacher trains on + `join.eps_m` and `join.score_eps_m`, the thresholds J4 averages over, so the join gain is not + fitted to one threshold. `teacher.extra_join_eps_m` (or `teacher_eval --train-eps`) overrides + the extra thresholds. `teacher.held_out_term` (or `--held-out-term`) adds the held-out form of the join at `join.eps_m` next to a `Self` join term, so one teacher keeps both the self - join and the join from outside queries. Each trajectory's F1 is counted once per term and the - metric is the mean over all of them, so a term with more matching trajectories weighs more. -- **Join pairs:** the teacher uses every 150 m pair, because whoever compresses the database knows - all of it. The pairs are also split in half (seen / unseen); with `all_join_pairs = false` the - teacher only gets the seen half, which shows how much of its join gain is fitting those - specific pairs. Whether the join gain generalises is tested at other thresholds (100 m, 300 m). + join and the join from outside queries. Like J4's join grid, each term's macro F1 is normalised + by its own endpoints-only floor and the terms are averaged equally. A term without true pairs is + left out, as J4 skips NaN entries. +- **Join pairs:** the teacher uses every pair at each of its thresholds, because whoever + compresses the database knows all of it. Whether the join gain generalises to thresholds the + teacher never saw is tested in `teacher_eval` at 125, 175 and 250 m, between the trained ones. - **Arm:** one named output of the teacher, e.g. `EQUAL_CHOICE_RDP_PARTNERS`. --- @@ -123,10 +128,10 @@ Files: Candidates at the chosen budgets are built by `make_candidates` and reused during best response. ### Why a trajectory's choice depends on other trajectories -kNN, range and where-at only involve trajectory i itself, because everyone else is compared in -original form. The join is different: a pair (i, j) is scored with **both** sides simplified. Two -trajectories can gain or lose a join match when either kept set changes. So the best method for i -depends on what its join partners chose. +Range macro F1 depends on how many trajectories still answer each box, so the marginal reward +for recovering one trajectory depends on the others. Ranking kNN also depends on competing +trajectories. In a self join, both sides of each pair are simplified. These dependencies make the +best method for i depend on the current choices elsewhere in the database. ### Two stages The fixed-budget arms illustrate two stages, which differ in how the join partners are seen: @@ -140,7 +145,7 @@ Stage 2, EQUAL_LOOP: starts from stage 1, partners are their ACTUAL current c | | Stage 1 (EQUAL_CHOICE_RDP_PARTNERS) | Stage 2 (BR) | |---|---|---| | Points per trajectory | fixed budget | fixed budget | -| Methods tried for i | all 7 recipes | other distinct kept sets | +| Methods tried for i | all 6 recipes | other distinct kept sets | | Terms scored | all 4 (kNN, range, where-at, join) | all 4, as the change from i's current method | | i's join partners seen as | RDP (a guess) | their actual current methods | | Join change counted for | i only | i and every partner whose pair flips | @@ -162,7 +167,7 @@ actual J_T once both sides of every join pair use their chosen candidates. ### Allocation arms `build_curves` scores budgets from 2 up to a per-trajectory cap. `allocate` fills the total budget -with greedy additions of up to three points at a time. `BUDGET_RDP` uses RDP-only curves; `BUDGET_CHOICE` uses the best of all candidates. +with greedy steps along each curve's upper concave envelope. `BUDGET_RDP` uses RDP-only curves; `BUDGET_CHOICE` uses the best of all candidates. `BUDGET_LOOP` applies best response at those allocated budgets. `BUDGET_LOOP_SHUFFLED` is a control that searches with relabelled partners and kNN neighbours, then scores the result against the real workloads. `BUDGET_LOOP_NO_UPDATE` is the second control: one pass from the same start in which @@ -177,32 +182,34 @@ beat. `BUDGET_LOOP_REALLOCATE` starts from `BUDGET_LOOP`. Each round rebuilds curves against the current partner state, reallocates and runs best response. A round is accepted only if the exact J_T improves by -more than 1e-12; at most three rounds are tried. -With only held-out join terms (the outside-query teacher) the curves do not depend on how partners are compressed, so rebuilding them gives the same curves and budgets, and each round only reruns the loop from the same start in a new visit order. Re-budgeting only reacts to neighbours when the teacher has a self join term. +more than 1e-12. Rounds continue until J_T stops rising, at most `kMaxAllocationRounds` (10). +Even with only held-out join terms, rebuilding curves can change budgets: marginal range F1 +reacts to the current coverage of each box. ### Stage 2: BR (`best_response`) **In one sentence:** go through the trajectories one at a time, and let each switch to whichever of its other methods makes the *real* J_T highest, given what everyone else currently uses; repeat until nobody wants to switch. -1. **Who is visited:** only trajectories with at least one join pair. For the others, the - score doesn't depend on anyone else, so their stage-1 choice is already their best. +1. **Who is visited:** trajectories with a range membership, a join link or an active kNN query. 2. **Starting state:** stage 1's method for every trajectory, plus a `JoinState` holding every - pair's outcome (match or not) with both sides on their stage-1 methods. + pair's outcome, a `RangeState` holding box coverage counts, and the ranking kNN tracker. 3. **One pass:** visit those trajectories in random order. For trajectory i and each of its other distinct candidate kept sets c: ``` - gain(c) = [separable(i, c) − separable(i, current)] change in i's kNN, range, where-at + gain(c) = [separable(i, c) − separable(i, current)] where-at and independent kNN, if enabled + + range.propose(i, c).gain exact change in range macro F1 + join.propose(i, c, state).gain exact change in the join F1 + + knn.propose(...) ranking kNN change, if enabled ``` `join.propose` rescores **all** of i's pairs against the partners' current kept points, and updates the F1 of i and of every partner whose pair flips. - **It switches when the best gain is positive** (above 1e-12). The two parts can pull in + **It switches when the best gain is positive** (above 1e-12). The terms can pull in opposite directions: a switch that costs i a little where-at can still win if it fixes a join - pair. After a switch, `join.apply` stores the new pair outcomes, and later visits see i's new + pair. After a switch, the trackers store the new outcomes, and later visits see i's new method. 4. **Repeat passes** until a pass has no switches (at most 10). @@ -254,8 +261,7 @@ pairs, and the result is exact: the tests check it against a full recompute. ### What it shows In an earlier three-candidate run, seed 42 at 0.25%: EQUAL_CHOICE_RDP_PARTNERS 0.506 → EQUAL_LOOP 0.544 J5, with the same methods and budgets. The gain is in join (0.522 → 0.621) and clustering. The only difference is using the -partners' actual choices. Part of the join gain is in-sample, though: recall on unseen pairs falls -below seen pairs (0.78 against 0.89). +partners' actual choices. --- @@ -275,9 +281,9 @@ on disk. | Struct | Fields | Meaning | |---|---|---| -| `WorkloadConfig` | `k`, `join_eps_m`, `extra_join_eps_m`, `frechet`, `range`, `where_at` | Settings, taken from the pipeline config (`knn.k`, `join.eps_m`, `teacher.extra_join_eps_m`, `frechet`, `range` with the teacher's own query count, `where_at`) | +| `WorkloadConfig` | `k`, `join_eps_m`, `extra_join_eps_m`, `frechet`, `range`, `where_at` | Settings, taken from the pipeline config (`knn.k`, `join.eps_m`, `teacher.extra_join_eps_m` or else `join.score_eps_m`, `frechet`, `range` with the teacher's own query count, `where_at`) | | `KnnEntry` | `valid`, `top`, `cutoff` | Database trajectory e as a kNN query: its k nearest other trajectories, and `cutoff`, the Fréchet distance to the (k+1)th. `valid = false` if it has fewer than k+1 neighbours. | -| `JoinPair` | `i < j`, `seen` | A pair of database trajectories | +| `JoinPair` | `i < j` | A pair of database trajectories | | `ExtraJoin` | `eps_m`, `positives`, `negatives` | The self-join pairs at one extra threshold, all of them used | | `TeacherWorkloads` | `knn`, `positives`, `negatives`, `extra_joins`, `range`, `where_at`, plus the settings | Everything the objective needs | @@ -293,7 +299,7 @@ not. They matter because the join metric is F1: simplification can make negative 2. Calls `load`. If the file matches the current workload version, fingerprint, seed, `k`, `join_eps_m` and `frechet`, it uses the cached kNN entries and pairs. Older cache versions are rebuilt once because their metadata-only fingerprints cannot verify the point contents. - 3. Otherwise, it builds everything with `build_knn` and `build_pairs`, then `save`s it. + 3. Otherwise, it builds everything with `build_knn` and `collect_pairs`, then `save`s it. 4. Always regenerates the range boxes and where-at probes. They are fast, and use their own seeds (`seed + 7001` and `seed + 7002`). 5. Always regenerates the extra joins, one self join per extra threshold. They are not cached, so @@ -324,28 +330,22 @@ not. They matter because the join metric is F1: simplification can make negative Fréchet, the discrete distance is used as the upper bound of the search. - **Output:** one `KnnEntry` per trajectory. -### `build_pairs(db, config, seed, positives, negatives)` — internal +### `collect_pairs(db, eps_m, frechet, positives, negatives)` — internal - **Why:** this gives the join term its training pairs: what should match (positives) and what shouldn't (negatives). -- **Input:** the database, the settings, the seed, and output vectors. +- **Input:** the database, the threshold, the Fréchet variant, and output vectors. - **Computation:** runs the engine's `query::generate_self_join`. For every candidate pair of each - entry, a pair in `reference` is a positive and any other pair is a negative. Each list then goes - through `split_seen`. + entry, a pair in `reference` is a positive and any other pair is a negative. The pairs at + `join_eps_m` are cached; the extra thresholds are collected on every load. - **Output:** the filled vectors. -### `split_seen(pairs, seed)` — internal - -- **Input:** a pair list and a seed. -- **Computation:** a seeded shuffle; the first half is marked `seen`; then sorted by (i, j). - Positives use `seed + 7003`, negatives `seed + 7005`. -- **Output:** the same pairs, half of them seen. The split is deterministic for a given seed. - ### `save` / `load` / `write_pairs` / `read_pairs` / `put` / `get` — internal - **Why:** the cache file format. -- **Version 4** (read and written): a header (magic number, seed, full-content fingerprint, k, - eps, variant), then the kNN entries, positives and negatives. The version covers both file +- **Version 5** (read and written): a header (magic number, seed, full-content fingerprint, k, + eps, variant), then the kNN entries, positives and negatives. Version 4 also stored a seen / + unseen flag per pair and is rejected. The version covers both file layout and cached workload generation and must be bumped when either changes. - **Versions 2 and 3** are rejected because they used metadata-only fingerprints. - `save` writes to a `.tmp` file and renames it, so a crash never leaves a half-written cache. @@ -357,22 +357,21 @@ not. They matter because the join metric is F1: simplification can make negative ### Why -This is the score the teacher maximises. It has the same shape as J4, so that improving it -improves J4. It is computed only on the teacher's training workloads. Over the arms tested so far, -J_T correlated with the earlier J5 at r ≈ 0.95. +This is the score the teacher maximises on its training workloads. Range, where-at and join +use J4's metric definitions; the value curves retain an independent kNN proxy. Improvements +on training workloads must still be checked against held-out J4. The four terms, each normalised `(metric − floor) / (1 − floor)` and then averaged: | Term | Per trajectory i, simplified | Mirrors | |---|---|---| | kNN | for each query e that has i in its top k: Fréchet(original e, simplified i) ≤ cutoff(e) | recall@k | -| Range | for each box i is in: does a kept point still lie inside the box? | range F1, as recall (there are no false positives) | +| Range | average `2 * found / (reference size + found)` over boxes with nonempty references | range macro F1 | | Where-at | average-hit ramp over i's probes: `clamp((max − err) / (max − min), 0, 1)` | where-at average-hit | -| Join | macro F1 over the pairs used, both sides simplified | join F1, computed the same way | +| Join | macro F1 over the pairs per threshold, both sides simplified, each normalised by its floor, then averaged | J4's join grid mean, computed the same way | -The first three are **separable**: they add up over trajectories, because each check only involves -trajectory i (the others stay original). Join is **not separable**: whether a pair matches depends -on both sides' simplifications. +Where-at and independent kNN are **separable**. Range macro F1, join and ranking kNN depend +on other trajectories and use incremental state trackers during best response. Why range has no false positives: kept points are a subset of the original points, so a simplified trajectory can't have a point in a box the original had none in. @@ -390,11 +389,12 @@ trajectory can't have a point in a box the original had none in. retained points and rebuilds bounds while reusing coordinate storage between budgets. `PreparedTrajectories` owns a whole set and provides borrowed views. Fixed partners are prepared once for curve scoring; best response prepares each unique candidate once and shares duplicate - views. Trajectories without join links only need their selected candidate prepared. + views. Trajectories without range memberships, join links or active kNN queries only need + their selected candidate prepared. -### `inside(box, view, p)` and `f1_of(hits, found, expected)` — internal +### Range and join helpers — internal -- `inside`: is point p of the view inside the box's space and time window? +- `range_f1` and `range_marginal`: per-box F1 and the gain from recovering one more trajectory. - `f1_of`: per-trajectory join F1, `2·hits / (found + expected)`. This is exactly the formula in the engine's join evaluator. @@ -408,16 +408,19 @@ trajectory can't have a point in a box the original had none in. 1. **Inverts kNN.** The workloads say "query e has top k {i, …}". The constructor turns this into `m_knn[i]` = every (e, cutoff) where i is in e's top k, so scoring trajectory i only touches its own memberships. `m_knn_weight = 1 / (k · valid queries)`, so the term sums to recall@k. - 2. **Inverts range.** `m_range[i]` = every box i is in, each with weight - `1 / (|box reference| · boxes with any reference)`. Summed, this is recall averaged per box. + 2. **Inverts range.** `m_range[i]` contains every box i is in. Its default weight is the + marginal macro F1 of recovering i when all other reference trajectories are found. Greedy + candidate construction uses these weights; curve scoring recomputes them against fixed partners. 3. **Groups where-at probes** per trajectory in `m_probes[i]`, with `m_where_at_weight = 1 / trajectories with probes` (averaged per trajectory, like the evaluator). - 4. **Builds the join links.** Positives first, then negatives (all, or only the seen half), stored in + 4. **Builds the join links.** Positives first, then negatives, stored in `m_pair_i`/`m_pair_j`. Each pair is linked from both sides in `m_links`. `m_truth[t]` counts t's positives. - 5. **Floors:** `metrics(endpoints)` gives each metric's floor. If a floor is already 1, it - throws, because that metric can't be normalised. + 5. **Floors:** each join term's macro F1 with endpoints only gives `m_join_floor`, and + `m_join_weight[k] = 1 / (terms with true pairs · (1 − floor_k))`, so the join metric is already + normalised and 0 at the floor. Then `metrics(endpoints)` gives each metric's floor. If a floor + is already 1, it throws, because that metric can't be normalised. 6. **Scales:** `m_scale[m] = 1 / (4 · (1 − floor_m))`. This applies the normalisation and the equal average in one factor. 7. **`m_endpoint_separable[i]`:** trajectory i's scaled separable terms with endpoints only. Subtracting it makes @@ -427,22 +430,21 @@ trajectory can't have a point in a box the original had none in. ### `raw_separable(i, kept)` — private -- **Why:** the actual kNN, range and where-at checks for one trajectory. +- **Why:** the independent kNN and where-at checks for one trajectory. - **Input:** trajectory index and kept set. - **Computation:** - **kNN:** for each membership (e, cutoff), test `frechet_within(original e, simplified i, cutoff)` and add `m_knn_weight` if it holds. Competitors stay original. - - **Range:** for each box, add the box's weight if any kept point is `inside`. - **Where-at:** for each probe, interpolate the simplified position at the probe's time (`core::position_at`), take the distance to the true position, apply the ramp, and average over i's probes times `m_where_at_weight`. -- **Output:** `Terms`, with i's raw contribution to each metric (join slot 0). +- **Output:** `Terms`, with i's raw separable contributions (range and join slots 0). ### `separable(i, kept)` — public - **Input:** trajectory index and kept set. -- **Computation:** `Σ m_scale[m] · raw_separable[m] − m_endpoint_separable[i]` over kNN, range and where-at. -- **Output:** i's part of J_T from the three separable terms, 0 at endpoints only. Summed over all +- **Computation:** `Σ m_scale[m] · raw_separable[m] − m_endpoint_separable[i]` over independent kNN and where-at. +- **Output:** i's separable part of J_T, 0 at endpoints only. Summed over all trajectories, this equals those terms' share of J_T exactly. ### `independent_value(i, candidate, partners)` — public @@ -452,9 +454,10 @@ trajectory can't have a point in a box the original had none in. - **Input:** trajectory index, a prepared candidate view, and fixed partner views. An empty partner span means original partners (the `EQUAL_CHOICE` arm); otherwise the views contain RDP at equal budgets for `EQUAL_CHOICE_RDP_PARTNERS`. -- **Computation:** `separable(i, candidate)` plus i's own join F1 against those fixed partners: count +- **Computation:** `separable(i, candidate)`, plus the marginal range F1 of covering i against + fixed partners, plus i's own join F1 against those partners: count `found` and `hits` over i's links using `match`, then add - `m_scale[Join] · F1_i / (trajectories with truth)`. + `m_scale[Join] · Σ_k m_join_weight[k] · F1_ik / (trajectories with truth in term k)`. - **Output:** an approximate per-trajectory score. It's approximate because the real join depends on partners' actual choices and counts each pair on both sides. That is exactly what `best_response` corrects. @@ -484,7 +487,7 @@ trajectory can't have a point in a box the original had none in. - **Input:** a whole simplified database. - **Computation:** prepares the trajectories once, sums `raw_separable` over their views (in - parallel), and reuses those views in `join_metric`. + parallel), and reuses those views in `RangeState::metric` and `join_metric`. - **Output:** the four raw metric values. Used for the floors, and in tests. ### `total(kept)` — public @@ -494,6 +497,14 @@ trajectory can't have a point in a box the original had none in. - **Output:** the exact J_T. It is 0 with endpoints only, and higher is better. The pipeline prints it next to each arm. +### `RangeState` — public class + +Tracks the found trajectory count in every box and each trajectory's coverage. `propose` computes +an exact scaled macro-F1 change without mutation; `apply` updates the counts. `weights(i)` gives +F1's marginal reward for recovering i with everyone else fixed, reused across candidates and +budgets. Greedy candidates use the original-partner marginals; selection and value curves use +their specified partner state. The loop updates range F1 against the actual current state. + ### `JoinState` — public class #### Why @@ -507,8 +518,8 @@ The result is exact, not approximate: the tests check it against a full recomput - **Input:** the objective and a starting simplified database. - **Computation:** matches every pair used once (from its smaller side, in parallel). It fills - `m_matched`, `m_hits[t]` and `m_found[t]`, then sums F1 over counted trajectories into `m_sum` - and `m_counted`. + `m_matched`, `m_hits[t]` and `m_found[t]`, then sums F1 over counted slots into `m_sum` and + `m_counted`, per join term. - **Output:** a state whose `metric()` equals `join_metric(state)`. #### `propose(i, candidate, state)` @@ -523,8 +534,8 @@ The result is exact, not approximate: the tests check it against a full recomput 3. For each affected trajectory, remove its old F1 from the sum and add its new one, adjusting the counted trajectories when one starts or stops being counted. - **Output:** a `Proposal` with the trajectory index, pair outcomes, and - `gain = m_scale[Join] · (new macro F1 − current macro F1)`. Added to the cached `separable` - difference, this is the exact change in J_T. Proposals remain valid until the next `apply`. + `gain = m_scale[Join] · (new join metric − current join metric)`. Added to the cached `separable` + difference and range/kNN gains, this is the change in J_T. Proposals remain valid until the next `apply`. #### `apply(proposal)` @@ -535,7 +546,7 @@ The result is exact, not approximate: the tests check it against a full recomput #### `metric()`, `counted(t, found)`, `f1(t, hits, found)` -Helpers: the current macro F1; whether trajectory t counts (it has truth or matches); and t's F1. +Helpers: the current join metric; whether slot t counts (it has truth or matches); and t's F1. --- @@ -552,7 +563,10 @@ Helpers: the current macro F1; whether trajectory t counts (it has truth or matc every step. `allocate(curves, total_budget, value)` starts with endpoints and fills the remaining budget with -greedy steps of up to three points, taking the best gain per point each time. A donor/receiver +steps along each curve's upper concave envelope: each time it takes the step, of any length, with +the best gain per point. This is greedy allocation on the concave envelope, exactly optimal for the +relaxed problem and whenever the budget ends on envelope vertices, with no step-length limit to +choose. It replaced greedy steps of up to three points. A donor/receiver exchange pass used to follow; across the runs of November 2026 it never changed the curve sum by more than 0.0001, so it was removed. It preserves the exact total budget and respects every curve's maximum budget. It is a heuristic search of the curve sum, which is distinct from the exact J_T with both partners @@ -568,18 +582,18 @@ simplified. |---|---| | `CandidateIndex`, `kCandidates` | Named indices and labels for the six candidate recipes. The index is the "method". | | `kArms` | the nine valid arm names, listed in `teacher.hpp` | -| `TeacherConfig` | `enabled`, `cache_dir`, `arms`, `range_query_count`, `all_join_pairs`, `join` (optional, default: the pipeline's `join.mode`), `extra_join_eps_m`, `held_out_term`, `ranking_knn` (default true), `term_weights` (kNN, range, where-at, join; equal by default) | +| `TeacherConfig` | `enabled`, `cache_dir`, `arms`, `range_query_count` (default 128000), `join` (optional, default: the pipeline's `join.mode`), `extra_join_eps_m` (default: `join.score_eps_m`), `held_out_term`, `ranking_knn` (default true), `term_weights` (kNN, range, where-at, join; equal by default) | | `Arm` | one result: `name`, `simplification` (for the pipeline), `objective` (its J_T) | | `Candidates` | `candidates[c][i]` = trajectory i simplified with method c | | `Methods` | one method index per trajectory, i.e. a selection | | `kMaxResponsePasses` | at most 10 best-response passes | | `kKnnWatchFactor` | each kNN query watches its 3k nearest trajectories under each candidate state | -| `kMaxAllocationRounds` | at most 3 iterative reallocation rounds | +| `kMaxAllocationRounds` | at most 10 iterative reallocation rounds; they stop once J_T stops rising | | `kMinImprovement` | strict gain threshold, 1e-12 | ### `Teacher(db, workloads, config, seed)` — constructor -- **Input:** the database, the workloads (moved in), the where-at tolerances, and the seed. +- **Input:** the database, the workloads (moved in), the objective settings, and the seed. - **Computation:** stores the workloads, then builds the `Objective` on them. `m_workloads` is declared before `m_objective`, so it is constructed first. That ordering matters because the objective keeps a reference to it. Prints the pair counts and floors. @@ -662,15 +676,14 @@ simplified. single change helps. - **Input:** candidates, a starting selection (`EQUAL_CHOICE_RDP_PARTNERS` or `BUDGET_CHOICE`), and the seed. - **Computation:** - 1. `order` = the trajectories with any join link or, with the ranking kNN term, a watching - query. Only they can gain, since the other terms don't depend on anyone else. - 2. Prepares candidate views, caches their separable scores, and builds the starting `JoinState`. + 1. `order` = trajectories with a range membership, a join link or an active ranking kNN query. + 2. Prepares candidate views, caches separable scores, and builds range, join and kNN trackers. 3. For up to 10 passes, in a seeded random order: - for each trajectory i, and each other method c: gain = `separable(c) − separable(current)` - + `join.propose(c).gain` + `knn.propose(c)`. That's the exact change in J_T, up to + + `range.propose(c).gain` + `join.propose(c).gain` + `knn.propose(c)`. That's the exact change in J_T, up to trajectories outside a query's watch list entering its top k; - switches to the best c only if its gain beats 1e-12 (strictly positive, so ties can't - cycle), and calls `join.apply`. + cycle), and applies the move to all trackers. 4. Stops after a pass with no switches. - **Output:** the improved `Methods`. Stops after a pass with no switches or the 10-pass limit. @@ -714,16 +727,18 @@ These timings predate the seven-candidate teacher, allocation search and cache c - `CacheIsWrittenThenReadBack`: a cache round trip gives identical workloads. - `CacheForOtherSettingsIsRebuilt`: a cache built with a different `k` isn't reused. - `ObjectiveIsZeroAtTheFloorAndTracksJoinExactly`: J_T = 0 at endpoints only. After random - switches, `JoinState`'s tracked F1 equals a full recompute, and `delta + separable` predicts the + switches, the tracked range and join F1 equal full recomputations, and their gains plus separable predict the real change in J_T. - `SeveralJoinTermsAreTrackedExactly`: the same exactness with two extra thresholds, with and without the held-out term. - `RankingKnnIsTrackedExactly`: when every query watches every trajectory, `KnnState`'s tracked - recall equals a full recompute after random switches, and the kNN, join and separable changes + recall equals a full recompute after random switches, and the kNN, range, join and separable changes together predict the real change in J_T. - `RankingKnnArmsKeepTheBudget`: the arms keep exactly the budget with the ranking kNN term. - `SingleRecipesKeepTheBudget`: the six single-recipe baselines each keep exactly the budget. - `ArmsKeepTheBudgetAndBestResponseOnlyImproves`: every arm keeps exactly the budget, endpoints kept, indices sorted; and `EQUAL_LOOP`'s J_T ≥ `EQUAL_CHOICE_RDP_PARTNERS`'s. +- `JoinTermsAreNormalisedApartAndAveragedLikeJ4`: the join metric equals the mean over thresholds + of each threshold's macro F1, normalised by its own endpoints-only floor. `engine/tests/config_test.cpp` has the three teacher config tests. diff --git a/engine/src/teacher/curves.cpp b/engine/src/teacher/curves.cpp index 6402bd2..a36e7a3 100644 --- a/engine/src/teacher/curves.cpp +++ b/engine/src/teacher/curves.cpp @@ -8,8 +8,6 @@ namespace teacher { namespace { -constexpr std::size_t kMaxStep = 3; - struct Offer { double ratio; // gain per point std::size_t i, k, version; @@ -18,11 +16,12 @@ struct Offer { } }; -// The best step of 1 to kMaxStep points for trajectory i, at most left points. k = 0 if none +// The step of any length up to left with the best gain per point: the next vertex of the curve's +// upper concave envelope within the budget left. k = 0 if none Offer best_offer(const Curve &curve, std::size_t i, std::size_t budget, std::size_t left, std::size_t version) { Offer best{-std::numeric_limits::infinity(), i, 0, version}; - for (std::size_t k = 1; k <= kMaxStep && k <= left && budget + k <= curve.max_budget(); ++k) { + for (std::size_t k = 1; k <= left && budget + k <= curve.max_budget(); ++k) { const double ratio = (curve.at(budget + k) - curve.at(budget)) / static_cast(k); if (ratio > best.ratio) { best.ratio = ratio; diff --git a/engine/src/teacher/objective.cpp b/engine/src/teacher/objective.cpp index 8dc32c0..61f71d6 100644 --- a/engine/src/teacher/objective.cpp +++ b/engine/src/teacher/objective.cpp @@ -5,6 +5,7 @@ #include "backend/engine/core/simplification.hpp" #include "backend/engine/query/frechet_helper.hpp" #include "backend/engine/query/knn.hpp" +#include "backend/engine/query/range_query.hpp" #include #include @@ -59,10 +60,15 @@ std::vector PreparedTrajectories::views() const { namespace { -bool inside(const query::RangeBox &box, const core::TrajectoryView &trajectory, std::size_t p) { - return trajectory.t[p] >= box.t_min && trajectory.t[p] <= box.t_max && - trajectory.x[p] >= box.x_min && trajectory.x[p] <= box.x_max && - trajectory.y[p] >= box.y_min && trajectory.y[p] <= box.y_max; +double range_f1(std::size_t found, std::size_t expected) { + return 2.0 * static_cast(found) / static_cast(expected + found); +} + +double range_marginal(std::size_t others, std::size_t expected, std::size_t scored) { + // F1(others + 1) - F1(others), avoiding subtraction of nearly equal values + const double denominator = static_cast(expected + others); + return 2.0 * static_cast(expected) / + (denominator * (denominator + 1.0) * static_cast(scored)); } constexpr std::size_t kMaxJoinTerms = 8; @@ -71,6 +77,15 @@ double f1_of(std::size_t hits, std::size_t found, std::size_t expected) { return 2.0 * static_cast(hits) / static_cast(found + expected); } +// The join metric from each term's sum of F1 over its counted slots +double mean_f1(const Objective &objective, std::span sum, + std::span counted) { + std::array f1s{}; + for (std::size_t k = 0; k < sum.size(); ++k) + f1s[k] = counted[k] ? sum[k] / static_cast(counted[k]) : 0.0; + return objective.join_value(std::span(f1s).first(sum.size())); +} + } // namespace double check_term_weights(const TermWeights &weights) { @@ -96,10 +111,6 @@ Objective::Objective(const core::TrajectoryDb &db, const TeacherWorkloads &workl const double weight_sum = check_term_weights(config.term_weights); if (config.compressed_knn_query && !config.ranking_knn) throw std::invalid_argument("objective: compressed kNN queries need the ranking kNN term"); - // Extra thresholds use every pair, which would train on the pairs held back as unseen - if (!config.all_join_pairs && !workloads.extra_joins.empty()) - throw std::invalid_argument( - "objective: seen-pairs-only training cannot use extra join thresholds"); // kNN: invert each query's top k m_knn.resize(n); @@ -123,9 +134,8 @@ Objective::Objective(const core::TrajectoryDb &db, const TeacherWorkloads &workl for (std::size_t b = 0; b < workloads.range.queries.size(); ++b) { const auto &ref = workloads.range.queries[b].reference; for (const auto i : ref) - m_range[i].push_back( - {static_cast(b), - 1.0 / (static_cast(ref.size()) * static_cast(scored))}); + m_range[i].push_back({static_cast(b), + range_marginal(ref.size() - 1, ref.size(), scored)}); } // Where-at: probes per trajectory @@ -161,8 +171,6 @@ Objective::Objective(const core::TrajectoryDb &db, const TeacherWorkloads &workl m_links.resize(n); m_truth.assign(n * terms.size(), 0); auto add = [&](const JoinPair &join_pair, bool positive, std::size_t term) { - if (!config.all_join_pairs && !join_pair.seen) - return; const auto pair = static_cast(m_pair_i.size()); const auto k = static_cast(term); m_pair_i.push_back(join_pair.i); @@ -182,11 +190,24 @@ Objective::Objective(const core::TrajectoryDb &db, const TeacherWorkloads &workl for (std::size_t k = 0; k < terms.size(); ++k) for (const auto &join_pair : *terms[k].negatives) add(join_pair, false, k); - m_with_truth = static_cast( - std::ranges::count_if(m_truth, [](auto count) { return count > 0; })); + m_with_truth.assign(terms.size(), 0); + for (std::size_t t = 0; t < m_truth.size(); ++t) + m_with_truth[t / n] += m_truth[t] > 0; - // Floors: endpoints only, partners included, like the J4 floor + // Floors: endpoints only, partners included, like the J4 floor. Each join term has its own, + // so the join metric is 0 there. Terms without true pairs are left out, as J4 skips NaNs const auto endpoints = core::endpoints_only(db); + m_join_floor = join_f1s(PreparedTrajectories(db, endpoints).views()); + const auto active = std::ranges::count_if(m_with_truth, [](auto count) { return count > 0; }); + m_join_weight.assign(terms.size(), 0.0); + for (std::size_t k = 0; k < terms.size(); ++k) { + if (m_with_truth[k] == 0) + continue; + if (m_join_floor[k] >= 1.0) + throw std::runtime_error( + "objective: a join term is already perfect with endpoints only"); + m_join_weight[k] = 1.0 / (static_cast(active) * (1.0 - m_join_floor[k])); + } m_floor = metrics(endpoints); for (std::size_t m = 0; m < TermCount; ++m) { if (m_floor[m] >= 1.0) @@ -197,7 +218,7 @@ Objective::Objective(const core::TrajectoryDb &db, const TeacherWorkloads &workl core::parallel_for(n, [&](std::size_t i) { const auto endpoint_terms = raw_separable( i, PreparedTrajectory(db.trajectory(i), endpoints[i]).view(), !config.ranking_knn); - for (const auto m : {Knn, Range, WhereAt}) + for (const auto m : {Knn, WhereAt}) m_endpoint_separable[i] += m_scale[m] * endpoint_terms[m]; }); } @@ -222,15 +243,6 @@ Objective::Terms Objective::raw_separable(std::size_t i, const core::TrajectoryV terms[Knn] += m_knn_weight; } - for (const auto &[b, weight] : m_range[i]) { - const auto &box = m_workloads.range.queries[b].box; - for (std::size_t k = 0; k < candidate.size(); ++k) - if (inside(box, candidate, k)) { - terms[Range] += weight; - break; - } - } - if (!m_probes[i].empty()) { double sum = 0; for (const auto k : m_probes[i]) { @@ -252,24 +264,52 @@ double Objective::separable(std::size_t i, std::span kept) co double Objective::separable(std::size_t i, const core::TrajectoryView &candidate) const { const auto t = raw_separable(i, candidate, !m_config.ranking_knn); - return m_scale[Knn] * t[Knn] + m_scale[Range] * t[Range] + m_scale[WhereAt] * t[WhereAt] - - m_endpoint_separable[i]; + return m_scale[Knn] * t[Knn] + m_scale[WhereAt] * t[WhereAt] - m_endpoint_separable[i]; +} + +std::vector Objective::range_weights(std::size_t i, + std::span partners) const { + if (m_range[i].empty()) + return {}; + const auto scored = std::ranges::count_if(m_workloads.range.queries, + [](const auto &q) { return !q.reference.empty(); }); + std::vector weights; + weights.reserve(m_range[i].size()); + for (const auto &membership : m_range[i]) { + const auto &q = m_workloads.range.queries[membership.box]; + std::size_t others = 0; + for (const auto j : q.reference) + if (j != i && (partners.empty() || query::intersects(partners[j], q.box))) + ++others; + weights.push_back(range_marginal(others, q.reference.size(), scored)); + } + return weights; } double Objective::independent_value(std::size_t i, const core::TrajectoryView &candidate, - std::span partners, - double beat) const { - return independent_value(prepare_independent(i, candidate, partners), beat); + std::span partners, double beat, + std::span range_weights) const { + return independent_value(prepare_independent(i, candidate, partners, range_weights), beat); } Objective::IndependentScore Objective::prepare_independent(std::size_t i, const core::TrajectoryView &candidate, - std::span partners) const { + std::span partners, + std::span range_weights) const { IndependentScore score; score.trajectory = i; score.candidate = candidate; score.partners = partners; score.terms = raw_separable(i, candidate, false); + const auto owned_weights = + range_weights.empty() ? this->range_weights(i, partners) : std::vector{}; + if (range_weights.empty()) + range_weights = owned_weights; + if (range_weights.size() != m_range[i].size()) + throw std::invalid_argument("objective: one range weight per membership needed"); + for (std::size_t k = 0; k < m_range[i].size(); ++k) + if (query::intersects(candidate, m_workloads.range.queries[m_range[i][k].box].box)) + score.terms[Range] += range_weights[k]; std::array hits{}, found{}; for (const auto &link : m_links[i]) { // Held out: the partner is the original query and i the database side @@ -282,19 +322,19 @@ Objective::prepare_independent(std::size_t i, const core::TrajectoryView &candid found[link.term] += matched; hits[link.term] += matched && link.positive; } - double f1_sum = 0; + double join = 0; for (std::size_t k = 0; k < m_eps.size(); ++k) - if (found[k] + m_truth[slot(i, k)] != 0) - f1_sum += f1_of(hits[k], found[k], m_truth[slot(i, k)]); - score.join = - m_with_truth != 0 ? m_scale[Join] * f1_sum / static_cast(m_with_truth) : 0.0; + if (found[k] + m_truth[slot(i, k)] != 0 && m_with_truth[k] != 0) + join += m_join_weight[k] * f1_of(hits[k], found[k], m_truth[slot(i, k)]) / + static_cast(m_with_truth[k]); + score.join = m_scale[Join] * join; score.bound = independent_sum(score, score.terms[Knn]) + m_scale[Knn] * knn_check_weight() * static_cast(knn_checks(i)); return score; } double Objective::independent_sum(const IndependentScore &score, double knn) const { - // Same sums in the same order as separable, so the value is identical to the last bit + // Range is i's marginal F1 against fixed partners; the loop tracks the actual joint F1 return m_scale[Knn] * knn + m_scale[Range] * score.terms[Range] + m_scale[WhereAt] * score.terms[WhereAt] - m_endpoint_separable[score.trajectory] + score.join; @@ -352,7 +392,18 @@ double Objective::independent_bound(std::size_t i, const core::TrajectoryView &c return prepare_independent(i, candidate, partners).bound; } +double Objective::join_value(std::span f1s) const { + double value = 0; + for (std::size_t k = 0; k < f1s.size(); ++k) + value += m_join_weight[k] * (f1s[k] - m_join_floor[k]); + return value; +} + double Objective::join_metric(std::span state) const { + return join_value(join_f1s(state)); +} + +std::vector Objective::join_f1s(std::span state) const { // matched[p]: the pair's outcome, counted for i. reverse[p]: counted for j, the same outcome // unless held out, where each side is in turn the original query std::vector matched(m_pair_i.size()), reverse(m_pair_i.size()); @@ -380,14 +431,16 @@ double Objective::join_metric(std::span state) const hits[slot_j] += positive; } } - double sum = 0; - std::size_t counted = 0; + std::vector f1s(m_eps.size(), 0.0); + std::vector counted(m_eps.size(), 0); for (std::size_t t = 0; t < m_truth.size(); ++t) if (found[t] + m_truth[t] != 0) { - sum += f1_of(hits[t], found[t], m_truth[t]); - ++counted; + f1s[t / m_db.size()] += f1_of(hits[t], found[t], m_truth[t]); + ++counted[t / m_db.size()]; } - return counted ? sum / static_cast(counted) : 0.0; + for (std::size_t k = 0; k < f1s.size(); ++k) + f1s[k] = counted[k] ? f1s[k] / static_cast(counted[k]) : 0.0; + return f1s; } Objective::Terms Objective::metrics(const KeptSets &kept) const { @@ -401,8 +454,9 @@ Objective::Terms Objective::metrics(const KeptSets &kept) const { }); Terms total{}; for (const auto &terms : per) - for (const auto m : {Knn, Range, WhereAt}) + for (const auto m : {Knn, WhereAt}) total[m] += terms[m]; + total[Range] = RangeState(*this, views).metric(); total[Join] = join_metric(views); if (m_config.ranking_knn) total[Knn] = knn_recall(core::materialize(m_db, kept)); @@ -436,12 +490,91 @@ double Objective::total(const KeptSets &kept) const { return sum; } +RangeState::RangeState(const Objective &objective, std::span state) + : m_objective(objective), m_found(objective.workloads().range.queries.size(), 0), + m_covered(objective.db().size()) { + if (!state.empty() && state.size() != objective.db().size()) + throw std::invalid_argument("range state: one view per trajectory needed"); + core::parallel_for(objective.db().size(), [&](std::size_t i) { + const auto trajectory = state.empty() ? objective.db().trajectory(i) : state[i]; + m_covered[i].reserve(objective.boxes(i).size()); + for (const auto &membership : objective.boxes(i)) { + const auto &q = objective.workloads().range.queries[membership.box]; + const bool covered = state.empty() || query::intersects(trajectory, q.box); + m_covered[i].push_back(covered); + } + }); + for (std::size_t i = 0; i < objective.db().size(); ++i) { + const auto memberships = objective.boxes(i); + for (std::size_t k = 0; k < memberships.size(); ++k) + m_found[memberships[k].box] += m_covered[i][k]; + } + for (std::size_t b = 0; b < m_found.size(); ++b) + if (!objective.workloads().range.queries[b].reference.empty()) { + ++m_scored; + m_sum += f1(b, m_found[b]); + } +} + +double RangeState::f1(std::size_t box, std::size_t found) const { + return range_f1(found, m_objective.workloads().range.queries[box].reference.size()); +} + +RangeState::Proposal RangeState::propose(std::size_t i, + const core::TrajectoryView &candidate) const { + Proposal proposal{i, 0.0, {}}; + const auto memberships = m_objective.boxes(i); + proposal.covered.reserve(memberships.size()); + double change = 0.0; + for (std::size_t k = 0; k < memberships.size(); ++k) { + const auto b = memberships[k].box; + const bool covered = + query::intersects(candidate, m_objective.workloads().range.queries[b].box); + proposal.covered.push_back(covered); + const auto next = m_found[b] - m_covered[i][k] + covered; + change += f1(b, next) - f1(b, m_found[b]); + } + if (m_scored) + proposal.gain = + m_objective.scale(Objective::Range) * change / static_cast(m_scored); + return proposal; +} + +void RangeState::apply(const Proposal &proposal) { + const auto memberships = m_objective.boxes(proposal.trajectory); + auto &covered = m_covered[proposal.trajectory]; + for (std::size_t k = 0; k < memberships.size(); ++k) { + const auto b = memberships[k].box; + const auto next = m_found[b] - covered[k] + proposal.covered[k]; + m_sum += f1(b, next) - f1(b, m_found[b]); + m_found[b] = next; + covered[k] = proposal.covered[k]; + } +} + +double RangeState::metric() const { return m_scored ? m_sum / static_cast(m_scored) : 0.0; } + +std::vector RangeState::weights(std::size_t i) const { + std::vector weights; + const auto memberships = m_objective.boxes(i); + weights.reserve(memberships.size()); + for (std::size_t k = 0; k < memberships.size(); ++k) { + const auto b = memberships[k].box; + weights.push_back(range_marginal(m_found[b] - m_covered[i][k], + m_objective.workloads().range.queries[b].reference.size(), + m_scored)); + } + return weights; +} + JoinState::JoinState(const Objective &objective, std::span state) : m_objective(objective) { const std::size_t n = state.size(); m_matched.assign(2 * objective.pair_count(), 0); m_hits.assign(objective.slots(), 0); m_found.assign(objective.slots(), 0); + m_sum.assign(objective.join_terms(), 0.0); + m_counted.assign(objective.join_terms(), 0); // Each pair once, from its smaller side. Held out: each direction from its compressed side std::vector>> outcome(n); core::parallel_for(n, [&](std::size_t i) { @@ -468,8 +601,8 @@ JoinState::JoinState(const Objective &objective, std::span(m_counted) : 0.0; -} +double JoinState::metric() const { return mean_f1(m_objective, m_sum, m_counted); } JoinState::Proposal JoinState::propose(std::size_t i, const core::TrajectoryView &candidate, std::span state) const { @@ -510,23 +641,28 @@ JoinState::Proposal JoinState::propose(std::size_t i, const core::TrajectoryView } if (change.empty()) return proposal; - double sum = m_sum; - long n_counted = static_cast(m_counted); + const std::size_t n = m_objective.db().size(); + std::array sum{}; + std::array n_counted{}; + std::ranges::copy(m_sum, sum.begin()); + std::ranges::copy(m_counted, n_counted.begin()); for (const auto &[t, counts] : change) { const auto old_found = m_found[t], old_hits = m_hits[t]; const auto new_found = static_cast(static_cast(old_found) + counts.second); const auto new_hits = static_cast(static_cast(old_hits) + counts.first); if (counted(t, old_found)) { - sum -= f1(t, old_hits, old_found); - --n_counted; + sum[t / n] -= f1(t, old_hits, old_found); + --n_counted[t / n]; } if (counted(t, new_found)) { - sum += f1(t, new_hits, new_found); - ++n_counted; + sum[t / n] += f1(t, new_hits, new_found); + ++n_counted[t / n]; } } - const double after = n_counted ? sum / static_cast(n_counted) : 0.0; + const auto terms = m_objective.join_terms(); + const double after = + mean_f1(m_objective, std::span(sum).first(terms), std::span(n_counted).first(terms)); proposal.gain = m_objective.scale(Objective::Join) * (after - metric()); return proposal; } @@ -546,16 +682,17 @@ void JoinState::apply(const Proposal &proposal) { const long delta = then ? 1 : -1; const std::array touched{count(link.other, link), count(i, link)}; for (const auto t : std::span(touched).first(sides(link))) { + const auto term = t / m_objective.db().size(); if (counted(t, m_found[t])) { - m_sum -= f1(t, m_hits[t], m_found[t]); - --m_counted; + m_sum[term] -= f1(t, m_hits[t], m_found[t]); + --m_counted[term]; } m_found[t] = static_cast(static_cast(m_found[t]) + delta); if (link.positive) m_hits[t] = static_cast(static_cast(m_hits[t]) + delta); if (counted(t, m_found[t])) { - m_sum += f1(t, m_hits[t], m_found[t]); - ++m_counted; + m_sum[term] += f1(t, m_hits[t], m_found[t]); + ++m_counted[term]; } } } diff --git a/engine/src/teacher/teacher.cpp b/engine/src/teacher/teacher.cpp index a155c01..5e39440 100644 --- a/engine/src/teacher/teacher.cpp +++ b/engine/src/teacher/teacher.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -40,7 +41,7 @@ constexpr std::array kCandidates{ "RDP", "SED", "UNIFORM", "GREEDY", "GREEDY_SED2", "GREEDY_RDP2"}; constexpr std::size_t kMaxResponsePasses = 10; constexpr std::size_t kKnnWatchFactor = 3; -constexpr std::size_t kMaxAllocationRounds = 3; +constexpr std::size_t kMaxAllocationRounds = 10; constexpr double kMinImprovement = 1e-12; // candidates[c][i]: trajectory i simplified by kCandidates[c] at budgets[i] @@ -176,15 +177,17 @@ struct Pick { // best bound first so losers skip most kNN checks Pick pick(const Objective &objective, std::size_t i, std::span> options, - std::span partners) { - const double rdp = objective.independent_value(i, options[0].second, partners); + std::span partners, std::span range_weights) { + const double rdp = objective.independent_value( + i, options[0].second, partners, -std::numeric_limits::infinity(), range_weights); Pick best{options[0].first, rdp, rdp}; std::vector scores; scores.reserve(options.size() - 1); std::vector> order; // bound, position in options order.reserve(options.size() - 1); for (std::size_t k = 1; k < options.size(); ++k) { - scores.push_back(objective.prepare_independent(i, options[k].second, partners)); + scores.push_back( + objective.prepare_independent(i, options[k].second, partners, range_weights)); order.push_back({scores.back().bound, k}); } std::ranges::sort(order, std::greater{}); @@ -210,6 +213,7 @@ Methods independent_choice(const core::TrajectoryDb &db, const Objective &object prepared.emplace(db, *partners); partner_views = prepared->views(); } + const RangeState range(objective, partner_views); core::parallel_for(db.size(), [&](std::size_t i) { std::vector owned; std::vector> options; @@ -218,7 +222,7 @@ Methods independent_choice(const core::TrajectoryDb &db, const Objective &object owned.emplace_back(db.trajectory(i), candidates[c][i]); options.push_back({c, owned.back().view()}); } - method[i] = pick(objective, i, options, partner_views).method; + method[i] = pick(objective, i, options, partner_views, range.weights(i)).method; }); return method; } @@ -233,7 +237,9 @@ Curves build_curves(const core::TrajectoryDb &db, const Objective &objective, Curves curves{std::vector(db.size()), std::vector(db.size())}; const PreparedTrajectories prepared(db, partners); const auto partner_views = prepared.views(); + const RangeState range(objective, partner_views); core::parallel_for(db.size(), [&](std::size_t i) { + const auto range_weights = range.weights(i); CandidateMaker maker(objective, i, caps[i], true); const std::size_t max_b = std::min(caps[i], db.trajectory(i).size()); auto &best = curves.best[i]; @@ -260,7 +266,7 @@ Curves build_curves(const core::TrajectoryDb &db, const Objective &objective, owned[slot].assign(db.trajectory(i), kept[c]); options.push_back({c, owned[slot].view()}); } - const auto chosen = pick(objective, i, options, partner_views); + const auto chosen = pick(objective, i, options, partner_views, range_weights); rdp.value.push_back(chosen.rdp); rdp.method.push_back(kRdp); best.value.push_back(chosen.value); @@ -291,7 +297,8 @@ class Search { m_knn.emplace(objective, assemble(candidates, method), candidates, kKnnWatchFactor * objective.workloads().k); const auto active = [&](std::size_t i) { - return !objective.links(i).empty() || (m_knn && m_knn->active(i)); + return !objective.boxes(i).empty() || !objective.links(i).empty() || + (m_knn && m_knn->active(i)); }; for (std::size_t i = 0; i < db.size(); ++i) if (active(i)) @@ -333,6 +340,7 @@ class Search { for (std::size_t i = 0; i < db.size(); ++i) m_state[i] = m_views[method[i]][i]; m_join.emplace(objective, m_state); + m_range.emplace(objective, m_state); } Search(const Search &) = delete; Search &operator=(const Search &) = delete; @@ -342,6 +350,7 @@ class Search { auto method = m_start; auto state = m_state; auto join = *m_join; + auto range = *m_range; auto knn = m_knn; auto order = m_order; std::mt19937_64 rng(seed + kOrderSeedOffset); @@ -353,6 +362,7 @@ class Search { const double here = m_separable[i][method[i]]; std::size_t best = method[i]; JoinState::Proposal best_proposal; + RangeState::Proposal best_range; KnnState::Move best_move; double best_gain = kMinImprovement; // strict gains only, so ties cannot cycle for (std::size_t c = 0; c < kCandidates.size(); ++c) { @@ -362,6 +372,7 @@ class Search { if (duplicate || current) continue; auto proposal = join.propose(i, m_views[c][i], state); + auto range_proposal = range.propose(i, m_views[c][i]); KnnState::Move move; double knn_gain = 0.0; if (knn && m_compressed_queries) { @@ -370,11 +381,13 @@ class Search { } else if (knn) { knn_gain = knn->propose(i, m_knn_distances[i][c]); } - const double gain = m_separable[i][c] - here + proposal.gain + knn_gain; + const double gain = + m_separable[i][c] - here + proposal.gain + range_proposal.gain + knn_gain; if (gain > best_gain) { best = c; best_gain = gain; best_proposal = std::move(proposal); + best_range = std::move(range_proposal); best_move = std::move(move); } } @@ -386,6 +399,7 @@ class Search { continue; state[i] = m_views[best][i]; join.apply(best_proposal); + range.apply(best_range); if (knn && m_compressed_queries) knn->apply(best_move); else if (knn) @@ -412,6 +426,7 @@ class Search { std::optional m_knn; std::vector m_state; std::optional m_join; + std::optional m_range; }; Methods best_response(const core::TrajectoryDb &db, const Objective &objective, diff --git a/engine/src/teacher/workloads.cpp b/engine/src/teacher/workloads.cpp index aa0b85f..e3b47c7 100644 --- a/engine/src/teacher/workloads.cpp +++ b/engine/src/teacher/workloads.cpp @@ -12,15 +12,14 @@ #include #include #include -#include #include namespace teacher { namespace { -// Bump when the layout or workload generation changes. V4 fingerprints every point -constexpr std::uint64_t kMagicV4 = 0x5431'4341'4348'4534ull; +// Bump when the layout or workload generation changes. V5 drops the seen / unseen split +constexpr std::uint64_t kMagicV5 = 0x5431'4341'4348'4535ull; constexpr std::uint64_t kFnvOffsetBasis = 14695981039346656037ull; constexpr std::uint64_t kFnvPrime = 1099511628211ull; @@ -80,26 +79,16 @@ std::vector build_knn(const core::TrajectoryDb &db, const WorkloadConf return result; } -void split_seen(std::vector &pairs, std::uint64_t seed) { - std::mt19937_64 rng(seed); - std::shuffle(pairs.begin(), pairs.end(), rng); - for (std::size_t k = 0; k < pairs.size(); ++k) - pairs[k].seen = k < pairs.size() / 2; - std::ranges::sort(pairs, [](const JoinPair &a, const JoinPair &b) { - return a.i < b.i || (a.i == b.i && a.j < b.j); - }); -} - // Positives and negatives from one self-join void collect_pairs(const core::TrajectoryDb &db, double eps_m, query::FrechetVariant frechet, - bool seen, std::vector &positives, std::vector &negatives) { + std::vector &positives, std::vector &negatives) { const auto join = query::generate_self_join(db, eps_m, frechet); positives.clear(); negatives.clear(); for (const auto &entry : join.entries) for (const auto j : entry.candidates) { const JoinPair pair{static_cast(entry.query_index), - static_cast(j), seen}; + static_cast(j)}; if (std::ranges::binary_search(entry.reference, j)) positives.push_back(pair); else @@ -107,20 +96,11 @@ void collect_pairs(const core::TrajectoryDb &db, double eps_m, query::FrechetVar } } -// The pairs at join_eps_m, each list split into seen and unseen halves -void build_pairs(const core::TrajectoryDb &db, const WorkloadConfig &config, std::uint64_t seed, - std::vector &positives, std::vector &negatives) { - collect_pairs(db, config.join_eps_m, config.frechet, false, positives, negatives); - split_seen(positives, seed + kPairSplitSeedOffset); - split_seen(negatives, seed + kNegativeSeedOffset); -} - void write_pairs(std::ofstream &out, const std::vector &pairs) { put(out, pairs.size()); for (const auto &pair : pairs) { put(out, pair.i); put(out, pair.j); - put(out, pair.seen); } } @@ -129,7 +109,6 @@ void read_pairs(std::ifstream &in, std::vector &pairs) { for (auto &pair : pairs) { pair.i = get(in); pair.j = get(in); - pair.seen = get(in) != 0; } } @@ -140,7 +119,7 @@ void save(const std::filesystem::path &file, std::uint64_t seed, std::uint64_t d const auto tmp = file.string() + ".tmp"; { std::ofstream out(tmp, std::ios::binary); - put(out, kMagicV4); + put(out, kMagicV5); put(out, seed); put(out, db_fingerprint); put(out, workloads.k); @@ -166,7 +145,7 @@ bool load(const std::filesystem::path &file, std::uint64_t seed, std::uint64_t d std::ifstream in(file, std::ios::binary); if (!in) return false; - if (get(in) != kMagicV4 || get(in) != seed || + if (get(in) != kMagicV5 || get(in) != seed || get(in) != db_fingerprint) return false; const auto k = get(in); @@ -214,7 +193,8 @@ TeacherWorkloads load_or_build_workloads(const core::TrajectoryDb &db, const Wor workloads.knn = build_knn(db, config); std::cout << std::format("teacher: kNN {:.1f} s\n", core::seconds_since(start)); start = std::chrono::steady_clock::now(); - build_pairs(db, config, seed, workloads.positives, workloads.negatives); + collect_pairs(db, config.join_eps_m, config.frechet, workloads.positives, + workloads.negatives); std::cout << std::format("teacher: self-join pairs {:.1f} s\n", core::seconds_since(start)); save(file, seed, db_fingerprint, workloads); std::cout << "teacher: cache saved to " << file.string() << '\n'; @@ -225,7 +205,7 @@ TeacherWorkloads load_or_build_workloads(const core::TrajectoryDb &db, const Wor query::generate_where_at_queries(db, config.where_at, seed + kWhereAtSeedOffset); for (const double eps : config.extra_join_eps_m) { ExtraJoin extra{.eps_m = eps, .positives = {}, .negatives = {}}; - collect_pairs(db, eps, config.frechet, true, extra.positives, extra.negatives); + collect_pairs(db, eps, config.frechet, extra.positives, extra.negatives); std::cout << std::format("teacher: extra join at {} m, {} pairs, {} negatives\n", eps, extra.positives.size(), extra.negatives.size()); workloads.extra_joins.push_back(std::move(extra)); diff --git a/engine/tests/config_test.cpp b/engine/tests/config_test.cpp index 59e9891..59e8bfc 100644 --- a/engine/tests/config_test.cpp +++ b/engine/tests/config_test.cpp @@ -48,6 +48,12 @@ TEST(Config, ReadsJoinKeys) { EXPECT_DOUBLE_EQ(config.join.eps_m, 250); } +TEST(Config, ReadsGlobalRdpAlgorithm) { + const auto config = pipeline::load_config(write_config( + "global_rdp.json", R"({"data_path": "data.parquet", "algorithms": ["RDP_GLOBAL"]})")); + EXPECT_EQ(config.algorithms, std::vector{selection::AlgorithmType::RDP_GLOBAL}); +} + TEST(Config, ReadsTopLevelAndNestedKeys) { const auto config = pipeline::load_config(write_config("overrides.json", R"({ "data_path": "data.parquet", @@ -247,8 +253,12 @@ TEST(Config, TeacherIsOffByDefault) { EXPECT_FALSE(config.teacher.enabled); EXPECT_EQ(config.teacher.cache_dir, "cache"); EXPECT_EQ(config.teacher.arms, (std::vector{"BUDGET_CHOICE", "BUDGET_LOOP"})); - EXPECT_TRUE(config.teacher.all_join_pairs); - EXPECT_TRUE(config.teacher.extra_join_eps_m.empty()); + EXPECT_FALSE(config.teacher.extra_join_eps_m); + // The teacher trains on the thresholds J4 averages over, unless told otherwise + const auto grid = pipeline::load_config(write_config( + "teacher_grid.json", R"({"data_path": "d", "join": {"score_eps_m": [100, 300]}})")); + EXPECT_EQ(pipeline::teacher_workload_config(grid).extra_join_eps_m, + (std::vector{100.0, 300.0})); EXPECT_FALSE(pipeline::teacher_objective_config(config).held_out_term); EXPECT_TRUE(pipeline::teacher_objective_config(config).ranking_knn); // The teacher's join term follows the join it is scored on @@ -274,7 +284,7 @@ TEST(Config, ReadsTheTeacherSection) { const auto config = pipeline::load_config(write_config("teacher.json", R"({ "data_path": "d", "teacher": {"enabled": true, "cache_dir": "my_cache", "arms": ["EQUAL_CHOICE", "EQUAL_LOOP"], - "range_query_count": 1000, "all_join_pairs": false, + "range_query_count": 1000, "join": "Self", "extra_join_eps_m": [100, 300], "held_out_term": true, "ranking_knn": false, "term_weights": [2, 1, 1, 0]} })")); @@ -289,7 +299,6 @@ TEST(Config, ReadsTheTeacherSection) { EXPECT_EQ(config.teacher.cache_dir, "my_cache"); EXPECT_EQ(config.teacher.arms, (std::vector{"EQUAL_CHOICE", "EQUAL_LOOP"})); EXPECT_EQ(config.teacher.range_query_count, 1000u); - EXPECT_FALSE(config.teacher.all_join_pairs); } TEST(Config, RejectsBadTeacherSections) { diff --git a/engine/tests/selectors/simplify_database_test.cpp b/engine/tests/selectors/simplify_database_test.cpp index 7040719..3bb441f 100644 --- a/engine/tests/selectors/simplify_database_test.cpp +++ b/engine/tests/selectors/simplify_database_test.cpp @@ -1,6 +1,7 @@ #include #include +#include #include #include #include @@ -8,6 +9,7 @@ #include #include "backend/engine/core/trajectory_db.hpp" +#include "backend/engine/selection/rdp.hpp" #include "backend/engine/selection/selector.hpp" namespace { @@ -80,3 +82,77 @@ TEST(SimplifyDatabase, ThrowsWithoutOneBudgetPerTrajectory) { EXPECT_THROW(selection::simplify_database(db, selection::AlgorithmType::RDP, budgets), std::invalid_argument); } + +TEST(GlobalRdp, EachAddedPointHasTheLargestRemainingPedAcrossTheDatabase) { + const auto db = make_random_db({2, 5, 20, 50, 8}); + auto previous = core::endpoints_only(db); + for (std::size_t budget = 2 * db.size() + 1; budget <= db.num_points(); ++budget) { + const auto result = selection::rdp::simplify_database(db, budget); + ASSERT_EQ(result.num_points(), budget); + const auto kept = core::to_kept(result); + double largest = -1.0, added_error = -1.0; + std::size_t additions = 0; + for (std::size_t i = 0; i < db.size(); ++i) { + const auto trajectory = db.trajectory(i); + EXPECT_EQ(kept[i].front(), 0u); + EXPECT_EQ(kept[i].back(), trajectory.size() - 1); + EXPECT_TRUE(std::ranges::is_sorted(kept[i])); + EXPECT_EQ(std::ranges::adjacent_find(kept[i]), kept[i].end()); + EXPECT_TRUE(std::ranges::includes(kept[i], previous[i])); + // Independent reference: measure every omitted point against its enclosing segment. + for (std::size_t segment = 1; segment < previous[i].size(); ++segment) { + const auto a = previous[i][segment - 1], b = previous[i][segment]; + const double dx = trajectory.x[b] - trajectory.x[a]; + const double dy = trajectory.y[b] - trajectory.y[a]; + for (std::size_t p = a + 1; p < b; ++p) { + const double u = dx * dx + dy * dy == 0.0 + ? 0.0 + : std::clamp(((trajectory.x[p] - trajectory.x[a]) * dx + + (trajectory.y[p] - trajectory.y[a]) * dy) / + (dx * dx + dy * dy), + 0.0, 1.0); + const double error = std::hypot(trajectory.x[p] - trajectory.x[a] - u * dx, + trajectory.y[p] - trajectory.y[a] - u * dy); + largest = std::max(largest, error); + if (std::ranges::binary_search(kept[i], p)) { + added_error = error; + ++additions; + } + } + } + } + EXPECT_EQ(additions, 1u); + EXPECT_NEAR(added_error, largest, 1e-9); + previous = kept; + } +} + +TEST(GlobalRdp, ReservesEndpointsAndRejectsInfeasibleBudgets) { + const auto db = make_random_db({2, 5, 20}); + EXPECT_EQ(core::to_kept(selection::rdp::simplify_database(db, 6)), core::endpoints_only(db)); + const auto full = selection::rdp::simplify_database(db, db.num_points()); + EXPECT_EQ(full.num_points(), db.num_points()); + for (std::size_t i = 0; i < db.size(); ++i) + EXPECT_EQ(full.trajectory(i).size(), db.trajectory(i).size()); + EXPECT_THROW(selection::rdp::simplify_database(db, 5), std::invalid_argument); + EXPECT_THROW(selection::rdp::simplify_database(db, db.num_points() + 1), std::invalid_argument); + EXPECT_THROW( + selection::simplify_trajectory(db.trajectory(0), selection::AlgorithmType::RDP_GLOBAL, 2), + std::invalid_argument); +} + +TEST(GlobalRdp, HandlesTiesAndTrajectoriesShorterThanTwoPoints) { + const auto db = make_random_db({0, 1, 2, 8}); + const auto floor = core::endpoints_only(db); + EXPECT_EQ(core::to_kept(selection::rdp::simplify_database(db, 5)), floor); + EXPECT_EQ(selection::rdp::simplify_database(db, db.num_points()).num_points(), db.num_points()); + EXPECT_EQ(selection::rdp::simplify_database(make_random_db({}), 0).num_points(), 0u); + // Identical trajectories create exact cross-trajectory ties. + const auto one = make_random_db({8}); + const auto duplicated = core::subset(one, std::vector{0, 0}); + const auto first = selection::rdp::simplify_database(duplicated, 5); + EXPECT_EQ(first.trajectory(0).size(), 3u); + EXPECT_EQ(first.trajectory(1).size(), 2u); + EXPECT_EQ(core::to_kept(first), + core::to_kept(selection::rdp::simplify_database(duplicated, 5))); +} diff --git a/engine/tests/teacher/curves_test.cpp b/engine/tests/teacher/curves_test.cpp index ae3be8f..b7b4e50 100644 --- a/engine/tests/teacher/curves_test.cpp +++ b/engine/tests/teacher/curves_test.cpp @@ -57,6 +57,22 @@ TEST(TeacherAllocate, StepsOverFlatStretches) { EXPECT_EQ(b, (std::vector{4, 2})); } +TEST(TeacherAllocate, StepsOverFlatStretchesOfAnyLength) { + // A gains only at its fourth point, which a lookahead of 1 to 3 points would never see + const std::vector curves{curve({0.0, 0.0, 0.0, 0.0, 1.0}), curve({0.0, 0.1, 0.15, 0.2, 0.25})}; + const auto b = teacher::allocate(curves, 8); + EXPECT_EQ(b, (std::vector{6, 2})); +} + +TEST(TeacherAllocate, IsOptimalOnEnvelopeVertices) { + // Budgets that end on envelope vertices are optimal even on non-concave curves + const std::vector curves{curve({0.0, 0.0, 0.9, 0.9, 1.0}), curve({0.0, 0.5, 0.6})}; + for (const std::size_t budget : {std::size_t{5}, std::size_t{7}, std::size_t{8}}) { + const auto b = teacher::allocate(curves, budget); + EXPECT_NEAR(value_of(curves, b), brute_force(curves, budget), 1e-12) << budget; + } +} + TEST(TeacherAllocate, KeepsTheExactBudgetAndBounds) { std::mt19937_64 rng(3); std::uniform_real_distribution step(0.0, 1.0); diff --git a/engine/tests/teacher/teacher_test.cpp b/engine/tests/teacher/teacher_test.cpp index c9a056d..1493f1f 100644 --- a/engine/tests/teacher/teacher_test.cpp +++ b/engine/tests/teacher/teacher_test.cpp @@ -2,6 +2,7 @@ #include "backend/engine/core/projection.hpp" #include "backend/engine/core/trajectory_db.hpp" +#include "backend/engine/evaluator/range.hpp" #include "backend/engine/teacher/greedy.hpp" #include "backend/engine/teacher/objective.hpp" #include "backend/engine/teacher/teacher.hpp" @@ -105,9 +106,8 @@ void expect_same_cached_workloads(const teacher::TeacherWorkloads &a, EXPECT_DOUBLE_EQ(a.knn[i].cutoff, b.knn[i].cutoff); } auto same_pairs = [](const auto &x, const auto &y) { - return std::ranges::equal(x, y, [](const auto &p, const auto &q) { - return p.i == q.i && p.j == q.j && p.seen == q.seen; - }); + return std::ranges::equal( + x, y, [](const auto &p, const auto &q) { return p.i == q.i && p.j == q.j; }); }; EXPECT_TRUE(same_pairs(a.positives, b.positives)); EXPECT_TRUE(same_pairs(a.negatives, b.negatives)); @@ -230,6 +230,7 @@ TEST_F(TeacherTest, ObjectiveIsZeroAtTheFloorAndTracksJoinExactly) { const teacher::PreparedTrajectories prepared_floor(db, state); auto views = prepared_floor.views(); teacher::JoinState join(objective, views); + teacher::RangeState range(objective, views); const auto full = every_point(db); const teacher::PreparedTrajectories prepared_full(db, full); const auto full_views = prepared_full.views(); @@ -242,6 +243,7 @@ TEST_F(TeacherTest, ObjectiveIsZeroAtTheFloorAndTracksJoinExactly) { const auto next_view = state[i].size() == 2 ? full_views[i] : floor_views[i]; const double old_metric = join.metric(); const auto proposal = join.propose(i, next_view, views); + const auto range_proposal = range.propose(i, next_view); // Proposing must not change the state or this proposal const auto stay = join.propose(i, views[i], views); EXPECT_DOUBLE_EQ(stay.gain, 0.0); @@ -252,8 +254,105 @@ TEST_F(TeacherTest, ObjectiveIsZeroAtTheFloorAndTracksJoinExactly) { state[i] = next; views[i] = next_view; join.apply(proposal); + range.apply(range_proposal); EXPECT_NEAR(join.metric(), objective.metrics(state)[teacher::Objective::Join], 1e-12); - EXPECT_NEAR(objective.total(state) - before, predicted + separable, 1e-9); + EXPECT_NEAR(objective.total(state) - before, + predicted + range_proposal.gain + separable, 1e-9); + } + } +} + +TEST_F(TeacherTest, RangeObjectiveUsesMacroF1EvenWhenRecallPrefersAnotherSolution) { + std::vector offsets{0}; + std::vector ids; + std::vector times; + std::vector lat, lon; + for (std::size_t i = 0; i < 8; ++i) { + ids.push_back(static_cast(i)); + for (std::size_t k = 0; k < 3; ++k) { + const auto point = + core::to_lat_lon(k == 1 ? (i < 4 ? -100.0 : 100.0) : 0.0, k == 1 ? 0.0 : -100.0); + times.push_back(static_cast(k)); + lat.push_back(point.lat); + lon.push_back(point.lon); + } + offsets.push_back(times.size()); + } + const core::TrajectoryDb db(offsets, ids, times, lat, lon); + auto config = small_config(); + config.k = 1; + config.join_eps_m = 50.0; + auto workloads = teacher::load_or_build_workloads(db, config, 9, m_dir); + workloads.range.queries = { + {{-101, -99, -1, 1, 1, 1}, {0, 1, 2, 3}}, + {{99, 101, -1, 1, 1, 1}, {4, 5, 6, 7}}, + {{999, 1001, -1, 1, 1, 1}, {}}}; // Empty references do not enter the macro mean + const teacher::Objective objective(db, workloads, {.term_weights = {0, 1, 0, 0}}); + auto concentrated = endpoints(db), balanced = endpoints(db); + for (const auto i : {0u, 1u, 2u, 3u}) + concentrated[i] = {0, 1, 2}; + for (const auto i : {0u, 1u, 4u}) + balanced[i] = {0, 1, 2}; + const auto a = + evaluator::evaluate_range(core::materialize(db, concentrated), workloads.range.queries); + const auto b = + evaluator::evaluate_range(core::materialize(db, balanced), workloads.range.queries); + EXPECT_GT(a.recall, b.recall); + EXPECT_LT(a.f1, b.f1); + EXPECT_NEAR(objective.total(concentrated), a.f1, 1e-12); + EXPECT_NEAR(objective.total(balanced), b.f1, 1e-12); +} + +TEST_F(TeacherTest, RangeStateAndIndependentMarginalsMatchTheEvaluator) { + const auto db = random_db(2, 24, 20); + const auto workloads = teacher::load_or_build_workloads(db, small_config(), 9, m_dir); + const teacher::Objective objective(db, workloads, {.term_weights = {0, 1, 0, 0}}); + auto state = endpoints(db); + const auto floor = state, full = every_point(db); + const teacher::PreparedTrajectories prepared_floor(db, floor), prepared_full(db, full); + const auto floor_views = prepared_floor.views(), full_views = prepared_full.views(); + auto views = floor_views; + teacher::RangeState range(objective, views); + std::mt19937_64 rng(81); + for (int step = 0; step < 60; ++step) { + const auto i = rng() % db.size(); + const bool grow = state[i].size() == 2; + const auto candidate = grow ? full_views[i] : floor_views[i]; + const auto before = + evaluator::evaluate_range(core::materialize(db, state), workloads.range.queries).f1; + const auto proposal = range.propose(i, candidate); + EXPECT_DOUBLE_EQ(range.propose(i, views[i]).gain, 0.0); + EXPECT_NEAR(range.metric(), before, 1e-12); // propose does not mutate + + // Independent range scores are the gain over i missing every box, with partners fixed. + auto without_i = state; + without_i[i].clear(); + const double without = + evaluator::evaluate_range(core::materialize(db, without_i), workloads.range.queries).f1; + const auto weights = range.weights(i); + EXPECT_EQ(weights.size(), objective.boxes(i).size()); + const double independent = objective.independent_value( + i, candidate, views, -std::numeric_limits::infinity(), weights); + state[i] = grow ? full[i] : floor[i]; + views[i] = candidate; + range.apply(proposal); + const double after = + evaluator::evaluate_range(core::materialize(db, state), workloads.range.queries).f1; + EXPECT_NEAR(range.metric(), after, 1e-12); + EXPECT_NEAR(objective.metrics(state)[teacher::Objective::Range], after, 1e-12); + EXPECT_NEAR(proposal.gain, objective.scale(teacher::Objective::Range) * (after - before), + 1e-12); + EXPECT_NEAR(independent, objective.scale(teacher::Objective::Range) * (after - without), + 1e-12); + } + const teacher::RangeState original(objective, {}); + for (std::size_t i = 0; i < db.size(); ++i) { + const auto weights = original.weights(i); + const auto fallback = objective.range_weights(i, {}); + ASSERT_EQ(weights.size(), fallback.size()); + for (std::size_t k = 0; k < weights.size(); ++k) { + EXPECT_DOUBLE_EQ(weights[k], fallback[k]); + EXPECT_DOUBLE_EQ(weights[k], objective.boxes(i)[k].weight); } } } @@ -282,6 +381,7 @@ TEST_F(TeacherTest, SeveralJoinTermsAreTrackedExactly) { const auto floor_views = prepared_floor.views(), full_views = prepared_full.views(); auto views = floor_views; teacher::JoinState join(objective, views); + teacher::RangeState range(objective, views); std::mt19937_64 rng(4); for (int step = 0; step < 40; ++step) { const std::size_t i = rng() % db.size(); @@ -290,13 +390,16 @@ TEST_F(TeacherTest, SeveralJoinTermsAreTrackedExactly) { const auto next_view = grow ? full_views[i] : floor_views[i]; const double before = objective.total(state); const auto proposal = join.propose(i, next_view, views); + const auto range_proposal = range.propose(i, next_view); const double separable = objective.separable(i, next) - objective.separable(i, state[i]); state[i] = next; views[i] = next_view; join.apply(proposal); + range.apply(range_proposal); EXPECT_NEAR(join.metric(), objective.metrics(state)[teacher::Objective::Join], 1e-12); - EXPECT_NEAR(objective.total(state) - before, proposal.gain + separable, 1e-9); + EXPECT_NEAR(objective.total(state) - before, + proposal.gain + range_proposal.gain + separable, 1e-9); } } // Without extra terms the workloads' extra joins are the only difference @@ -307,6 +410,57 @@ TEST_F(TeacherTest, SeveralJoinTermsAreTrackedExactly) { EXPECT_GT(single.pair_count(), base.pair_count()); } +TEST_F(TeacherTest, JoinTermsAreNormalisedApartAndAveragedLikeJ4) { + const auto db = random_db(2, 24, 20); + auto config = small_config(); + config.extra_join_eps_m = {config.join_eps_m * 1.5, config.join_eps_m * 2.0}; + const auto workloads = teacher::load_or_build_workloads(db, config, 9, m_dir); + ASSERT_FALSE(workloads.positives.empty()); + for (const auto &extra : workloads.extra_joins) + ASSERT_FALSE(extra.positives.empty()); + const teacher::Objective objective(db, workloads, {.join_mode = query::JoinMode::Self}); + ASSERT_EQ(objective.join_terms(), 3u); + + auto macro_f1 = [&](const teacher::KeptSets &kept, std::size_t term) { + const teacher::PreparedTrajectories prepared(db, kept); + const auto views = prepared.views(); + double sum = 0; + std::size_t counted = 0; + for (std::size_t i = 0; i < db.size(); ++i) { + std::size_t hits = 0, found = 0; + for (const auto &link : objective.links(i)) { + if (link.term != term) + continue; + const bool matched = objective.match(views[i], views[link.other], term); + found += matched; + hits += matched && link.positive; + } + const auto truth = objective.truth(objective.slot(i, term)); + if (found + truth == 0) + continue; + sum += 2.0 * static_cast(hits) / static_cast(found + truth); + ++counted; + } + return counted ? sum / static_cast(counted) : 0.0; + }; + + std::mt19937_64 rng(31); + auto kept = endpoints(db); + for (std::size_t i = 0; i < db.size(); ++i) { + for (std::size_t k = 1; k + 1 < db.trajectory(i).size(); ++k) + if (rng() % 3 == 0) + kept[i].push_back(k); + std::ranges::sort(kept[i]); + } + double expected = 0; + for (std::size_t term = 0; term < 3; ++term) { + const double floor = macro_f1(endpoints(db), term); + EXPECT_NEAR(objective.join_floor(term), floor, 1e-12); + expected += (macro_f1(kept, term) - floor) / (1.0 - floor) / 3.0; + } + EXPECT_NEAR(objective.metrics(kept)[teacher::Objective::Join], expected, 1e-12); +} + TEST_F(TeacherTest, RankingKnnIsTrackedExactly) { const auto db = random_db(2, 24, 20); auto config = small_config(); @@ -328,6 +482,7 @@ TEST_F(TeacherTest, RankingKnnIsTrackedExactly) { const auto floor_views = prepared_floor.views(), full_views = prepared_full.views(); auto views = floor_views; teacher::JoinState join(objective, views); + teacher::RangeState range(objective, views); std::mt19937_64 rng(5); for (int step = 0; step < 40; ++step) { const std::size_t i = rng() % db.size(); @@ -339,13 +494,16 @@ TEST_F(TeacherTest, RankingKnnIsTrackedExactly) { const double knn_gain = knn.propose(i, distances); EXPECT_DOUBLE_EQ(knn.propose(i, knn.distances(i, views[i])), 0.0); const auto proposal = join.propose(i, next_view, views); + const auto range_proposal = range.propose(i, next_view); const double separable = objective.separable(i, next) - objective.separable(i, state[i]); state[i] = next; views[i] = next_view; join.apply(proposal); + range.apply(range_proposal); knn.apply(i, distances); EXPECT_NEAR(knn.metric(), objective.metrics(state)[teacher::Objective::Knn], 1e-12); - EXPECT_NEAR(objective.total(state) - before, knn_gain + proposal.gain + separable, 1e-9); + EXPECT_NEAR(objective.total(state) - before, + knn_gain + proposal.gain + range_proposal.gain + separable, 1e-9); } } @@ -370,6 +528,7 @@ TEST_F(TeacherTest, CompressedKnnQueriesAreTrackedExactly) { const auto floor_views = prepared_floor.views(), full_views = prepared_full.views(); auto views = floor_views; teacher::JoinState join(objective, views); + teacher::RangeState range(objective, views); std::mt19937_64 rng(5); for (int step = 0; step < 40; ++step) { const std::size_t i = rng() % db.size(); @@ -381,13 +540,16 @@ TEST_F(TeacherTest, CompressedKnnQueriesAreTrackedExactly) { const double knn_gain = knn.propose(move); EXPECT_DOUBLE_EQ(knn.propose(knn.move(i, views[i], views)), 0.0); const auto proposal = join.propose(i, next_view, views); + const auto range_proposal = range.propose(i, next_view); const double separable = objective.separable(i, next) - objective.separable(i, state[i]); state[i] = next; views[i] = next_view; join.apply(proposal); + range.apply(range_proposal); knn.apply(move); EXPECT_NEAR(knn.metric(), objective.metrics(state)[teacher::Objective::Knn], 1e-12); - EXPECT_NEAR(objective.total(state) - before, knn_gain + proposal.gain + separable, 1e-9); + EXPECT_NEAR(objective.total(state) - before, + knn_gain + proposal.gain + range_proposal.gain + separable, 1e-9); } } @@ -456,16 +618,6 @@ TEST_F(TeacherTest, TermWeightsScaleTheTerms) { std::invalid_argument); } -TEST_F(TeacherTest, SeenPairsOnlyRejectsExtraThresholds) { - const auto db = random_db(2, 24, 20); - auto config = small_config(); - config.extra_join_eps_m = {config.join_eps_m * 2.0}; - const auto workloads = teacher::load_or_build_workloads(db, config, 9, m_dir); - EXPECT_THROW(teacher::Objective(db, workloads, {.all_join_pairs = false}), - std::invalid_argument); - EXPECT_NO_THROW(teacher::Objective(db, workloads, {.all_join_pairs = true})); -} - TEST_F(TeacherTest, ExtraJoinThresholdsMustBeDistinctAndPositive) { const auto db = random_db(2, 24, 20); for (const auto &extra : @@ -509,8 +661,10 @@ TEST_F(TeacherTest, HeldOutJoinScoresOriginalQueriesAgainstCompressedPartners) { ++counted; } ASSERT_GT(counted, 0u); + // One term, normalised by its own floor + const double floor = objective.join_floor(0); EXPECT_NEAR(objective.metrics(kept)[teacher::Objective::Join], - sum / static_cast(counted), 1e-12); + (sum / static_cast(counted) - floor) / (1.0 - floor), 1e-12); } TEST_F(TeacherTest, PreparedIndependentScoresMatchFullScoringAndSurvivePruning) { @@ -557,20 +711,31 @@ TEST_F(TeacherTest, PreparedIndependentScoresMatchFullScoringAndSurvivePruning) ? 2.0 * static_cast(hits) / static_cast(found + objective.truth(i)) : 0.0; - const double join = with_truth ? objective.scale(teacher::Objective::Join) * - f1 / static_cast(with_truth) - : 0.0; - const double expected = objective.separable(i, views[i]) + join; + const double join = with_truth + ? objective.scale(teacher::Objective::Join) * f1 / + (static_cast(with_truth) * + (1.0 - objective.join_floor(0))) + : 0.0; + double range_gain = 0.0; + const auto weights = objective.range_weights(i, partners); + const auto boxes = objective.boxes(i); + for (std::size_t b = 0; b < boxes.size(); ++b) + if (query::intersects(views[i], + workloads.range.queries[boxes[b].box].box)) + range_gain += weights[b]; + const double expected = + objective.separable(i, views[i]) + + objective.scale(teacher::Objective::Range) * range_gain + join; const auto score = objective.prepare_independent(i, views[i], partners); EXPECT_GE(score.bound + 1e-12, expected); - EXPECT_DOUBLE_EQ(objective.independent_value(score, expected - 1e-6), - expected); + EXPECT_NEAR(objective.independent_value(score, expected - 1e-6), expected, + 1e-12); EXPECT_LE(objective.independent_value( score, std::numeric_limits::infinity()), expected + 1e-12); - EXPECT_DOUBLE_EQ(objective.independent_value(score), expected); - EXPECT_DOUBLE_EQ(objective.independent_value(i, views[i], partners), - expected); + EXPECT_NEAR(objective.independent_value(score), expected, 1e-12); + EXPECT_NEAR(objective.independent_value(i, views[i], partners), expected, + 1e-12); } } } @@ -589,8 +754,8 @@ TEST_F(TeacherTest, ArmsKeepTheBudgetAndBestResponseOnlyImproves) { const auto result = model.run(budget, arms); ASSERT_EQ(result.size(), arms.size()); - double equal_choice_jt = 0, rdp_partners_jt = 0, equal_loop_jt = 0, budget_choice_jt = 0, - budget_loop_jt = 0, reallocate_jt = 0; + double rdp_partners_jt = 0, equal_loop_jt = 0, budget_choice_jt = 0, budget_loop_jt = 0, + reallocate_jt = 0; for (const auto &arm : result) { SCOPED_TRACE(arm.name); ASSERT_EQ(arm.simplification.size(), db.size()); @@ -603,8 +768,6 @@ TEST_F(TeacherTest, ArmsKeepTheBudgetAndBestResponseOnlyImproves) { EXPECT_TRUE(std::ranges::is_sorted(kept)); EXPECT_EQ(std::ranges::adjacent_find(kept), kept.end()); } - if (arm.name == "EQUAL_CHOICE") - equal_choice_jt = arm.objective; if (arm.name == "EQUAL_CHOICE_RDP_PARTNERS") rdp_partners_jt = arm.objective; if (arm.name == "EQUAL_LOOP") @@ -619,10 +782,6 @@ TEST_F(TeacherTest, ArmsKeepTheBudgetAndBestResponseOnlyImproves) { EXPECT_GE(equal_loop_jt, rdp_partners_jt - 1e-12); EXPECT_GE(budget_loop_jt, budget_choice_jt - 1e-12); EXPECT_GE(reallocate_jt, budget_loop_jt - 1e-12); - // Held out, the partners are queries and stay original, so the two choices are the same - if (mode == query::JoinMode::HeldOut) { - EXPECT_DOUBLE_EQ(rdp_partners_jt, equal_choice_jt); - } } }