Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
43 changes: 30 additions & 13 deletions engine/apps/teacher_eval.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -58,6 +58,7 @@ struct Args {
std::optional<std::vector<double>> train_eps; // config: teacher.extra_join_eps_m
std::optional<bool> held_out_term; // config: teacher.held_out_term
std::optional<bool> ranking_knn; // config: teacher.ranking_knn
std::optional<bool> compressed_knn_query; // config: teacher.knn_query
std::optional<std::vector<double>> term_weights; // config: teacher.term_weights
std::vector<double> eval_eps{kEvalEps.begin(), kEvalEps.end()};
bool knn_check = false;
Expand Down Expand Up @@ -110,7 +111,12 @@ Args parse_args(int argc, char **argv) {
args.ranking_knn = false;
else if (key == "--held-out-term")
args.held_out_term = true;
else if (key == "--knn-check")
else if (key == "--knn-query") {
const auto query = value();
if (query != "Original" && query != "Compressed")
throw std::invalid_argument("--knn-query must be Original or Compressed");
args.compressed_knn_query = query == "Compressed";
} else if (key == "--knn-check")
args.knn_check = true;
else if (key == "--teacher-join") {
const auto mode = value();
Expand All @@ -125,7 +131,8 @@ Args parse_args(int argc, char **argv) {
throw std::invalid_argument("unknown argument " + key +
" (--config --data --out --cache --seeds --shares --arms "
"--teacher-join --train-eps --held-out-term "
"--independent-knn --term-weights --eval-eps --knn-check "
"--independent-knn --knn-query --term-weights --eval-eps "
"--knn-check "
"--no-baselines --seen-pairs-only)");
}
teacher::check_arms(args.arms);
Expand All @@ -150,6 +157,8 @@ pipeline::Config effective_config(const Args &args) {
config.teacher.held_out_term = *args.held_out_term;
if (args.ranking_knn)
config.teacher.ranking_knn = *args.ranking_knn;
if (args.compressed_knn_query)
config.teacher.compressed_knn_query = *args.compressed_knn_query;
if (args.term_weights)
std::ranges::copy(*args.term_weights, config.teacher.term_weights.begin());
config.self_join = true;
Expand Down Expand Up @@ -357,10 +366,12 @@ int main(int argc, char **argv) try {
"where_at_within_100,join_micro_f1,knn_tau,norm_knn,norm_range,norm_where_at,norm_join,"
"norm_clustering,J4,J_T,join_recall_seen,join_recall_unseen,predicted_clusters,"
"eval_s,join_mode,self_join_f1,norm_self_join,teacher_join_mode,norm_join_main,"
"norm_clustering_main");
"norm_clustering_main,self_knn_recall,norm_self_knn,norm_self_join_mean,J4_self,"
"teacher_knn_query");
auto floors =
open_csv(args.out / "floors.csv",
"seed,knn_recall,range_f1,where_at_avg_hit,join_f1,clustering,self_join_f1");
"seed,knn_recall,range_f1,where_at_avg_hit,join_f1,clustering,self_join_f1,"
"self_knn_recall");
auto thresholds =
open_csv(args.out / "thresholds.csv",
"seed,share,arm,join,eps_m,join_f1,norm_join,recall,teacher_threshold");
Expand Down Expand Up @@ -414,7 +425,9 @@ int main(int argc, char **argv) try {
// Held-out data from here on
start = std::chrono::steady_clock::now();
const auto pool = core::subset(all, parts.queries);
const pipeline::TestWorkloads test_workloads(db, pool, config);
const pipeline::TestWorkloads test_workloads(
db, pool, config,
teacher::self_knn_workload(model.objective().workloads(), config.knn.p));
// Both joins at other thresholds: does the join gain hold beyond the ones trained on?
std::vector<query::JoinWorkload> other_joins;
for (const double eps : args.eval_eps) {
Expand All @@ -433,9 +446,10 @@ int main(int argc, char **argv) try {
const auto &scoring = config.where_at_scoring;
const auto floor_db = core::materialize(db, core::endpoints_only(db));
const auto floor = evaluator::Evaluator(db, floor_db, test, scoring).evaluate().scores;
floors << std::format("{},{},{},{},{},{},{}\n", seed, floor.knn.recall_at_k, floor.range_f1,
floor.where_at.average_hit, floor.join.f1, floor.clustering.score,
floor.self_join.f1);
floors << std::format("{},{},{},{},{},{},{},{}\n", seed, floor.knn.recall_at_k,
floor.range_f1, floor.where_at.average_hit, floor.join.f1,
floor.clustering.score, floor.self_join.f1,
floor.self_knn.recall_at_k);
std::vector<double> other_floor;
for (const auto &workload : other_joins)
other_floor.push_back(evaluator::evaluate_join(floor_db, workload).f1);
Expand Down Expand Up @@ -465,7 +479,7 @@ int main(int argc, char **argv) try {
const auto recall = pair_recall(compressed, pairs, config.join);
results << std::format(
"{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{},{:.2f},"
"{},{},{},{},{},{}\n",
"{},{},{},{},{},{},{},{},{},{},{}\n",
seed, arm.share, arm.name, allocation::total_budget(db, arm.share),
scores.knn.recall_at_k, scores.range_f1, scores.where_at.average_hit,
scores.join.f1, scores.clustering.score, scores.where_at.within_radius,
Expand All @@ -475,7 +489,9 @@ int main(int argc, char **argv) try {
scores.clustering.predicted_clusters, core::seconds_since(start),
mode_name(config.join.mode), scores.self_join.f1, normalized.self_join,
mode_name(objective.config().join_mode), normalized.join_f1,
normalized.clustering_score);
normalized.clustering_score, scores.self_knn.recall_at_k, normalized.self_knn,
normalized.self_join_mean, normalized.combined_self,
objective.config().compressed_knn_query ? "Compressed" : "Original");

for (std::size_t k = 0; k < other_joins.size(); ++k) {
const auto &workload = other_joins[k];
Expand Down Expand Up @@ -526,9 +542,10 @@ int main(int argc, char **argv) try {
db.trajectory(i).size(), arm.kept[i].size());
for (auto *out : {&results, &thresholds, &terms, &budgets})
out->flush();
std::cerr << std::format("seed {} {} {:g}: J4 {:.4f}, join {:.4f}, self join {:.4f}\n",
seed, arm.name, arm.share, normalized.combined,
normalized.join, normalized.self_join);
std::cerr << std::format(
"seed {} {} {:g}: J4 {:.4f}, J4-self {:.4f}, join {:.4f}, self join {:.4f}\n", seed,
arm.name, arm.share, normalized.combined, normalized.combined_self, normalized.join,
normalized.self_join);
}
timings.flush();
floors.flush();
Expand Down
40 changes: 32 additions & 8 deletions engine/apps/teacher_stats.py
Original file line number Diff line number Diff line change
@@ -1,10 +1,12 @@
"""Summarises a teacher_eval run: python3 teacher_stats.py <out dir> [summary.md]

Per share: means and SDs over seeds, paired J4 differences with t intervals, the gate against
the best cheap baseline, the coordination checks, and the self join at 100, 150 and 300 m.
the best cheap baseline, the coordination checks, each step's gain in J_T (training) against J4
and J4-self (test), and the self join at 100, 150 and 300 m.

norm_join is J4's join term (held-out when join.mode is HeldOut). norm_self_join is the database
self join, which the teacher trains on: an in-sample diagnostic that is not part of J4.
norm_join is J4's join term (held-out when join.mode is HeldOut). J4_self scores the in-database
setting: the database's own kNN and self join (compressed on both sides), with J4's range and
where-at. Its kNN and join queries are the ones the teacher trains on, by definition of the setting.
"""
import csv
import math
Expand All @@ -14,7 +16,16 @@
from collections import defaultdict

T95 = {1: 12.706, 2: 4.303, 3: 3.182, 4: 2.776, 5: 2.571, 6: 2.447, 7: 2.365, 8: 2.306, 9: 2.262, 10: 2.228}
METRICS = ["J4", "norm_knn", "norm_range", "norm_where_at", "norm_join", "norm_clustering", "norm_self_join"]
METRICS = ["J4", "norm_knn", "norm_range", "norm_where_at", "norm_join", "J4_self", "norm_self_knn",
"norm_self_join_mean", "norm_clustering"]
# Each step of the teacher, from the arm before it to the arm after it
STEPS = [("budget search", "RDP_a0", "BUDGET_RDP"),
("recipe choice", "RDP_a0", "EQUAL_CHOICE"),
("recipe choice at searched budgets", "BUDGET_RDP", "BUDGET_CHOICE"),
("loop, equal budgets", "EQUAL_CHOICE_RDP_PARTNERS", "EQUAL_LOOP"),
("loop", "BUDGET_CHOICE", "BUDGET_LOOP"),
("looking once", "BUDGET_CHOICE", "BUDGET_LOOP_NO_UPDATE"),
("reacting", "BUDGET_LOOP_NO_UPDATE", "BUDGET_LOOP")]
CHEAP = ["RDP_a0", "RDP_a025", "PARTNER_l0.5", "PARTNER_l1"]


Expand Down Expand Up @@ -106,6 +117,23 @@ def main():
if a in arms and b in arms:
out.append(f"| {a} − {b} | {fmt(paired(arms[a]['J4'], arms[b]['J4']))} |")
out.append("")
steps = [st for st in STEPS if st[1] in arms and st[2] in arms and arms[st[1]]["J_T"]]
if steps:
out.append("Gain of each step in training (J_T) against test (J4, J4-self), "
"mean over seeds. Kept = test gain / training gain:\n")
out.append("| Step | ΔJ_T | ΔJ4 | kept | ΔJ4_self | kept |")
out.append("|---|---:|---:|---:|---:|---:|")
for name, before, after in steps:
row = [f"{name} ({before} → {after})"]
train = paired(arms[after]["J_T"], arms[before]["J_T"])
row.append(fmt(train))
for m in ["J4", "J4_self"]:
test = paired(arms[after][m], arms[before][m])
row.append(fmt(test))
kept = test[0] / train[0] if test and train and abs(train[0]) > 1e-9 else None
row.append("–" if kept is None else f"{kept:.0%}")
out.append("| " + " | ".join(row) + " |")
out.append("")
if share in other:
out.append("Normalised self-join F1 at the teacher's 150 m and at thresholds it does not optimise:\n")
out.append("| Arm | 100 m | 150 m | 300 m |")
Expand All @@ -118,10 +146,6 @@ def main():
mean_sd(list(other[share][arm].get(300.0, {}).values()))]
out.append(f"| {arm} | " + " | ".join(cells) + " |")
out.append("")
pairs = [(arms[a]["J_T"][s], arms[a]["J4"][s]) for a in arms for s in arms[a]["J_T"] if s in arms[a]["J4"]]
if len(pairs) >= 3:
xs, ys = zip(*pairs)
out.append(f"Surrogate check: Pearson r(J_T, J4) = {statistics.correlation(xs, ys):.3f} over {len(pairs)} arm-seeds\n")
text = "\n".join(out)
if len(sys.argv) > 2:
open(sys.argv[2], "w").write("# Teacher summary\n\n" + text + "\n")
Expand Down
5 changes: 5 additions & 0 deletions engine/include/backend/engine/evaluator/combined.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ namespace evaluator {
// database) are diagnostics only.
// join and clustering average over their threshold grids, each against its own floor;
// join_f1 and clustering_score keep the main threshold alone.
// combined_self (J4-self) scores the in-database setting: self kNN, the same range and where-at,
// and the self join over its grid (self_join_mean). NaN unless all four are present.
struct NormalizedScores {
static constexpr double missing = std::numeric_limits<double>::quiet_NaN();

Expand All @@ -30,8 +32,11 @@ struct NormalizedScores {
double join = missing;
double clustering = missing;
double self_join = missing;
double self_knn = missing;
double self_join_mean = missing;

double combined = missing;
double combined_self = missing;
};

// NaN if an input is NaN or the floor is already 1
Expand Down
5 changes: 5 additions & 0 deletions engine/include/backend/engine/evaluator/evaluator.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,9 @@ struct Workloads {
// Extra thresholds, averaged into the reported scores
std::span<const query::JoinWorkload> join_grid = {};
std::span<const query::RouteClusteringWorkload> clustering_grid = {};
// The database's own queries, for J4-self
const query::KnnWorkload *self_knn = nullptr;
std::span<const query::JoinWorkload> self_join_grid = {};
};

// Kendall's tau is in [-1, 1] and the others in [0, 1], or NaN if it could not be computed
Expand All @@ -42,6 +45,8 @@ struct Scoretable {
JoinScores self_join;
std::vector<JoinScores> join_grid; // in Workloads::join_grid order
std::vector<RouteClusteringScores> clustering_grid; // in Workloads::clustering_grid order
KnnScores self_knn;
std::vector<JoinScores> self_join_grid; // in Workloads::self_join_grid order
};

struct EvaluationResult {
Expand Down
5 changes: 4 additions & 1 deletion engine/include/backend/engine/pipeline/test_workloads.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -23,9 +23,12 @@ struct TestWorkloads {
std::optional<query::JoinWorkload> self_join;
std::optional<query::RouteClusteringWorkload> clustering;
std::vector<query::RouteClusteringWorkload> clustering_grid; // at config.join_score_eps_m
// For J4-self. The self join at config.join_score_eps_m, empty when join_grid already is
std::vector<query::JoinWorkload> self_join_grid;
std::optional<query::KnnWorkload> self_knn; // supplied, as its reference is slow to build

TestWorkloads(const core::TrajectoryDb &database, const core::TrajectoryDb &queries,
const Config &config);
const Config &config, std::optional<query::KnnWorkload> self_knn = std::nullopt);
TestWorkloads(const TestWorkloads &) = delete;
TestWorkloads &operator=(const TestWorkloads &) = delete;

Expand Down
3 changes: 3 additions & 0 deletions engine/include/backend/engine/query/knn.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,9 @@ struct KnnWorkload {
std::size_t k;
double p = 0.5;
FrechetVariant frechet = FrechetVariant::Continuous;
// query_index is a database trajectory, searched in its compressed form and excluded from its
// own answer and reference. queries stays empty
bool self = false;
};

// Samples query_count trajectories from query_pool (the held-out split) and stores their k
Expand Down
27 changes: 26 additions & 1 deletion engine/include/backend/engine/teacher/objective.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,9 @@ struct ObjectiveConfig {
// kNN ranked on the compressed database; false: per-trajectory check. Curves always use the
// latter
bool ranking_knn = false;
// The kNN query compressed too, the in-database setting; needs ranking_knn. Curves then share
// each hit between the query and its neighbour, both against the fixed partners
bool compressed_knn_query = false;
TermWeights term_weights{1.0, 1.0, 1.0, 1.0};
};

Expand Down Expand Up @@ -106,6 +109,7 @@ class Objective {
friend class Objective;
std::size_t trajectory = 0;
core::TrajectoryView candidate{};
std::span<const core::TrajectoryView> partners{};
Terms terms{};
double join = 0.0;
};
Expand Down Expand Up @@ -156,6 +160,8 @@ class Objective {
Terms raw_separable(std::size_t i, const core::TrajectoryView &candidate,
bool knn = true) const;
double independent_sum(const IndependentScore &score, double knn) const;
std::size_t knn_checks(std::size_t i) const;
double knn_check_weight() const;
double join_metric(std::span<const core::TrajectoryView> state) const;

const core::TrajectoryDb &m_db;
Expand Down Expand Up @@ -230,12 +236,26 @@ class KnnState {
std::span<const KeptSets> watch_states, std::size_t watch);

bool watched(std::size_t i) const { return !m_watchers[i].empty(); }
// From every query watching i, in the order propose and apply expect
// Whether i's choice can change the metric: watched, or with compressed queries a query
bool active(std::size_t i) const;
// Original queries only. From every query watching i, in the order propose and apply expect
std::vector<double> distances(std::size_t i, const core::TrajectoryView &candidate) const;
// Change in J_T
double propose(std::size_t i, std::span<const double> distances) const;
void apply(std::size_t i, std::span<const double> distances);

// Either query kind. With compressed queries a distance depends on both sides, so a move also
// rescores i's own list, against the current state
struct Move {
std::size_t trajectory = 0;
std::vector<double> watched; // as distances()
std::vector<double> own; // i's own list; empty with original queries
};
Move move(std::size_t i, const core::TrajectoryView &candidate,
std::span<const core::TrajectoryView> state) const;
double propose(const Move &move) const;
void apply(const Move &move);

double metric() const; // recall@k over the valid queries

private:
Expand All @@ -244,6 +264,11 @@ class KnnState {
};
// True neighbours in e's top k, with the distance at position replaced
std::size_t hits(std::size_t e, std::size_t position, double distance) const;
// The same with e's whole row replaced
std::size_t hits(std::size_t e, std::span<const double> row) const;
// Distances from query, e in some state, to e's list in state
std::vector<double> own_row(std::size_t e, const core::TrajectoryView &query,
std::span<const core::TrajectoryView> state) const;

const Objective &m_objective;
std::size_t m_valid = 0;
Expand Down
3 changes: 3 additions & 0 deletions engine/include/backend/engine/teacher/teacher.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ struct TeacherConfig {
std::vector<double> extra_join_eps_m;
bool held_out_term = false;
bool ranking_knn = true;
// ObjectiveConfig::compressed_knn_query. Unset: compressed with a Self join, the in-database
// setting
std::optional<bool> compressed_knn_query;
TermWeights term_weights{1.0, 1.0, 1.0, 1.0};
};

Expand Down
5 changes: 5 additions & 0 deletions engine/include/backend/engine/teacher/workloads.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@

#include "backend/engine/core/trajectory_db.hpp"
#include "backend/engine/query/frechet_helper.hpp"
#include "backend/engine/query/knn.hpp"
#include "backend/engine/query/range_query.hpp"
#include "backend/engine/query/where_at_query.hpp"

Expand Down Expand Up @@ -68,4 +69,8 @@ TeacherWorkloads load_or_build_workloads(const core::TrajectoryDb &db, const Wor
std::uint64_t seed,
const std::filesystem::path &cache_dir);

// The self-kNN test workload from the cached lists: every valid database query with its k nearest
// others. They depend on the database alone, so they are what a fresh build would give
query::KnnWorkload self_knn_workload(const TeacherWorkloads &workloads, double p);

} // namespace teacher
9 changes: 9 additions & 0 deletions engine/src/evaluator/combined.cpp
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
#include "backend/engine/evaluator/combined.hpp"

#include <algorithm>
#include <array>
#include <cmath>
#include <cstddef>
Expand Down Expand Up @@ -72,10 +73,18 @@ NormalizedScores normalize_scores(const Scoretable &scores, const Scoretable &fl
result.clustering_score, scores.clustering_grid, floor.clustering_grid,
[](const RouteClusteringScores &clustering) { return clustering.score; }, "clustering");
result.self_join = normalize(scores.self_join.f1, floor.self_join.f1);
result.self_knn = normalize(scores.self_knn.recall_at_k, floor.self_knn.recall_at_k);
result.self_join_mean = grid_mean(
result.self_join, scores.self_join_grid, floor.self_join_grid,
[](const JoinScores &join) { return join.f1; }, "self join");

// Clustering and the self join stay out (see combined.hpp)
result.combined =
mean_ignoring_nan(std::array{result.knn, result.range, result.where_at, result.join});
const std::array internal{result.self_knn, result.range, result.where_at,
result.self_join_mean};
if (std::ranges::none_of(internal, [](double v) { return std::isnan(v); }))
result.combined_self = mean_ignoring_nan(internal);
return result;
}

Expand Down
Loading
Loading