diff --git a/cache/teacher_cache_seed42.bin b/cache/teacher_cache_seed42.bin index ee4683a..fd78a75 100644 Binary files a/cache/teacher_cache_seed42.bin and b/cache/teacher_cache_seed42.bin differ diff --git a/engine/CMakeLists.txt b/engine/CMakeLists.txt index a7b869c..cfb15da 100644 --- a/engine/CMakeLists.txt +++ b/engine/CMakeLists.txt @@ -88,6 +88,7 @@ add_executable(engine_tests tests/route_clustering_test.cpp tests/continuous_frechet_test.cpp tests/teacher/curves_test.cpp + tests/teacher/certificate_test.cpp tests/teacher/teacher_test.cpp ) target_link_libraries(engine_tests PRIVATE engine GTest::gtest_main Arrow::arrow_shared Parquet::parquet_shared) diff --git a/engine/apps/teacher_eval.cpp b/engine/apps/teacher_eval.cpp index 55692ce..209ffaf 100644 --- a/engine/apps/teacher_eval.cpp +++ b/engine/apps/teacher_eval.cpp @@ -293,6 +293,24 @@ void validate(const Arm &arm, const core::TrajectoryDb &db) { throw std::runtime_error(std::format("{}: {} points, budget {}", arm.name, total, budget)); } +// FNV-1a over every kept index, so two runs can be checked for identical selections +std::uint64_t kept_hash(const KeptSets &kept) { + std::uint64_t hash = 14695981039346656037ull; + auto mix = [&](std::uint64_t value) { + for (unsigned int byte = 0; byte < 8; ++byte) { + hash ^= value & 0xff; + hash *= 1099511628211ull; + value >>= 8; + } + }; + for (const auto &trajectory : kept) { + mix(trajectory.size()); + for (const auto index : trajectory) + mix(index); + } + return hash; +} + // Whether the teacher trained on the join at eps_m in mode: one of its thresholds in its own join // mode, or its held-out term at join.eps_m bool trained_on(const pipeline::Config &config, const teacher::ObjectiveConfig &objective, @@ -342,6 +360,7 @@ int main(int argc, char **argv) try { "norm_knn,norm_range,norm_where_at,norm_join,norm_self_join"); auto budgets = open_csv(args.out / "budgets.csv", "seed,share,arm,points,budget"); auto timings = open_csv(args.out / "timings.csv", "seed,step,seconds"); + auto hashes = open_csv(args.out / "kept_hashes.csv", "seed,share,arm,hash"); std::ofstream knn_check; if (args.knn_check) knn_check = open_csv(args.out / "knn_check.csv", @@ -380,8 +399,12 @@ int main(int argc, char **argv) try { for (auto &arm : model.single_recipes(allocation::total_budget(db, share))) arms.push_back({arm.name, share, core::to_kept(arm.simplification)}); } - for (const auto &arm : arms) + for (const auto &arm : arms) { validate(arm, db); + hashes << std::format("{},{},{},{:016x}\n", seed, arm.share, arm.name, + kept_hash(arm.kept)); + } + hashes.flush(); // Held-out data from here on start = std::chrono::steady_clock::now(); diff --git a/engine/include/backend/engine/teacher/objective.hpp b/engine/include/backend/engine/teacher/objective.hpp index c8a8c2f..9cdd710 100644 --- a/engine/include/backend/engine/teacher/objective.hpp +++ b/engine/include/backend/engine/teacher/objective.hpp @@ -43,6 +43,10 @@ class PreparedTrajectories { std::vector m_trajectories; }; +// Upper bound on the continuous Fréchet distance between a trajectory and a simplification +// keeping a subsequence of its points. Infinity if kept is not such a subsequence +double simplification_bound(const core::TrajectoryView &original, const core::TrajectoryView &kept); + inline constexpr std::size_t kTermCount = 4; // kNN, range, where-at, join using TermWeights = std::array; @@ -160,6 +164,10 @@ class Objective { bool match(const core::TrajectoryView &a, const core::TrajectoryView &b, std::size_t term = 0) const; + // match(original of link.other, candidate), skipping the test when the originals' distance + // plus the candidate's own error already proves it. own_error < 0: not yet computed + bool held_out_match(std::size_t i, const JoinLink &link, const core::TrajectoryView &candidate, + double &own_error) const; private: struct KnnMembership { @@ -188,7 +196,8 @@ class Objective { std::vector m_pair_i, m_pair_j; // pairs used, positives first std::vector m_pair_term; - std::vector m_eps; // threshold of each join term + std::vector m_pair_upper; // held-out positives: bound on the originals' distance + std::vector m_eps; // threshold of each join term std::vector m_term_held_out; std::size_t m_positive_count = 0; std::vector> m_links; // per trajectory @@ -215,6 +224,9 @@ class RangeState { std::vector covered; // in boxes(trajectory) order }; Proposal propose(std::size_t i, const core::TrajectoryView &candidate) const; + // Which of i's boxes the candidate covers; independent of the state + std::vector coverage(std::size_t i, const core::TrajectoryView &candidate) const; + Proposal propose(std::size_t i, std::vector covered) const; // A proposal is valid until the next apply() void apply(const Proposal &proposal); double metric() const; @@ -243,6 +255,12 @@ class JoinState { // Valid until the next apply() Proposal propose(std::size_t i, const core::TrajectoryView &candidate, std::span state) const; + // Outcomes of i's links, in links(i) order. Held-out links ignore the state; with + // held_out_known their outcomes are taken from it instead of retested + std::vector outcomes(std::size_t i, const core::TrajectoryView &candidate, + std::span state, + std::span held_out_known = {}) const; + Proposal propose(std::size_t i, std::vector matched) const; void apply(const Proposal &proposal); double metric() const; // as Objective's join metric @@ -311,8 +329,12 @@ class KnnState { struct Watcher { std::uint32_t query, position; // position in the query's list }; - // True neighbours in e's top k, with the distance at position replaced + // True neighbours in e's top k, with the distance at position replaced. Reads e's ranking std::size_t hits(std::size_t e, std::size_t position, double distance) const; + // Re-ranks e after its row changed: m_rank[e] and m_hits[e] + void refresh(std::size_t e); + bool ranks_before(std::size_t e, double distance, std::size_t position, + std::size_t other) const; // The same with e's whole row replaced std::size_t hits(std::size_t e, std::span row) const; // True neighbours in the top k of order (distance, position) @@ -328,6 +350,7 @@ class KnnState { std::vector> m_list; // per query, its true top k first std::vector> m_distance; // per query, in list order std::vector m_hits; // per query + std::vector> m_rank; // per query, positions of its top k + 1 std::size_t m_hit_sum = 0; std::vector> m_watchers; // per trajectory }; diff --git a/engine/src/teacher/objective.cpp b/engine/src/teacher/objective.cpp index ca6ce62..476cbe6 100644 --- a/engine/src/teacher/objective.cpp +++ b/engine/src/teacher/objective.cpp @@ -72,6 +72,53 @@ double range_marginal(std::size_t others, std::size_t expected, std::size_t scor } constexpr std::size_t kMaxJoinTerms = 8; +// Relative margin for certificates, far above the decision predicate's 1e-12 rounding slack +constexpr double kCertificateMargin = 1e-9; + +} // namespace + +// Upper bound on the continuous Fréchet distance between original and a simplification keeping +// a subsequence of its points: each kept segment walks its original stretch at the same share +// of arc length. Along each original edge the gap is convex, so its maximum is at a vertex +double simplification_bound(const core::TrajectoryView &original, + const core::TrajectoryView &kept) { + const std::size_t n = original.size(); + if (kept.size() < 2 || n < 2) + return std::numeric_limits::infinity(); + double bound = 0.0; + std::size_t at = 0; + for (std::size_t k = 0; k + 1 < kept.size(); ++k) { + // Kept points are copies of original points, in order + while (at < n && !(original.t[at] == kept.t[k] && original.x[at] == kept.x[k] && + original.y[at] == kept.y[k])) + ++at; + std::size_t end = at + 1; + while (end < n && !(original.t[end] == kept.t[k + 1] && original.x[end] == kept.x[k + 1] && + original.y[end] == kept.y[k + 1])) + ++end; + if (end >= n) + return std::numeric_limits::infinity(); + double length = 0.0; + for (std::size_t j = at; j < end; ++j) + length += + std::hypot(original.x[j + 1] - original.x[j], original.y[j + 1] - original.y[j]); + double walked = 0.0; + for (std::size_t j = at; j <= end; ++j) { + if (j > at) + walked += std::hypot(original.x[j] - original.x[j - 1], + original.y[j] - original.y[j - 1]); + const double u = length > 0 ? walked / length : 0.0; + const double x = kept.x[k] + u * (kept.x[k + 1] - kept.x[k]); + const double y = kept.y[k] + u * (kept.y[k + 1] - kept.y[k]); + bound = std::max(bound, std::hypot(original.x[j] - x, original.y[j] - y)); + } + at = end; + } + return bound; +} + +namespace { + // Cached kNN distances are upper bounds on a fixed millimetre grid. constexpr double kKnnDistanceError = query::kKnnResolutionM; @@ -196,6 +243,20 @@ Objective::Objective(const core::TrajectoryDb &db, const TeacherWorkloads &workl for (std::size_t t = 0; t < m_truth.size(); ++t) m_with_truth[t / n] += m_truth[t] > 0; + // Held-out positives: the originals' discrete Fréchet distance bounds the continuous one + m_pair_upper.assign(m_pair_i.size(), std::numeric_limits::infinity()); + core::parallel_for(m_positive_count, [&](std::size_t p) { + const auto k = m_pair_term[p]; + if (!m_term_held_out[k] || m_workloads.frechet != query::FrechetVariant::Continuous) + return; + std::vector prev, curr; + const double cutoff = m_eps[k] * m_eps[k]; + const double squared = query::get_squared_frechet_distance( + db.trajectory(m_pair_i[p]), db.trajectory(m_pair_j[p]), cutoff, prev, curr); + if (squared < cutoff) + m_pair_upper[p] = std::sqrt(squared); + }); + // Floors: endpoints only, partners included, like the J4 floor. Each join term has its own, // so the join metric is 0 there. Terms without true pairs are left out, as J4 skips NaNs const auto endpoints = core::endpoints_only(db); @@ -225,6 +286,18 @@ Objective::Objective(const core::TrajectoryDb &db, const TeacherWorkloads &workl }); } +bool Objective::held_out_match(std::size_t i, const JoinLink &link, + const core::TrajectoryView &candidate, double &own_error) const { + // d(original partner, candidate) <= d(originals) + d(original i, candidate) + if (std::isfinite(m_pair_upper[link.pair])) { + if (own_error < 0) + own_error = simplification_bound(m_db.trajectory(i), candidate); + if (m_pair_upper[link.pair] + own_error <= m_eps[link.term] * (1.0 - kCertificateMargin)) + return true; + } + return match(m_db.trajectory(link.other), candidate, link.term); +} + bool Objective::match(const core::TrajectoryView &a, const core::TrajectoryView &b, std::size_t term) const { return query::frechet_within(a, b, m_eps[term], m_workloads.frechet); @@ -313,11 +386,12 @@ Objective::prepare_independent(std::size_t i, const core::TrajectoryView &candid if (query::intersects(candidate, m_workloads.range.queries[m_range[i][k].box].box)) score.terms[Range] += range_weights[k]; std::array hits{}, found{}; + double own_error = -1.0; for (const auto &link : m_links[i]) { // Held out: the partner is the original query and i the database side const bool matched = m_term_held_out[link.term] - ? match(m_db.trajectory(link.other), candidate, link.term) + ? held_out_match(i, link, candidate, own_error) : match(candidate, partners.empty() ? m_db.trajectory(link.other) : partners[link.other], link.term); @@ -524,16 +598,28 @@ double RangeState::f1(std::size_t box, std::size_t found) const { RangeState::Proposal RangeState::propose(std::size_t i, const core::TrajectoryView &candidate) const { - Proposal proposal{i, 0.0, {}}; + return propose(i, coverage(i, candidate)); +} + +std::vector RangeState::coverage(std::size_t i, const core::TrajectoryView &candidate) const { const auto memberships = m_objective.boxes(i); - proposal.covered.reserve(memberships.size()); + std::vector covered; + covered.reserve(memberships.size()); + for (const auto &membership : memberships) + covered.push_back(query::intersects( + candidate, m_objective.workloads().range.queries[membership.box].box)); + return covered; +} + +RangeState::Proposal RangeState::propose(std::size_t i, std::vector covered) const { + Proposal proposal{i, 0.0, std::move(covered)}; + const auto memberships = m_objective.boxes(i); + if (proposal.covered.size() != memberships.size()) + throw std::invalid_argument("range proposal: one outcome per box needed"); double change = 0.0; for (std::size_t k = 0; k < memberships.size(); ++k) { const auto b = memberships[k].box; - const bool covered = - query::intersects(candidate, m_objective.workloads().range.queries[b].box); - proposal.covered.push_back(covered); - const auto next = m_found[b] - m_covered[i][k] + covered; + const auto next = m_found[b] - m_covered[i][k] + proposal.covered[k]; change += f1(b, next) - f1(b, m_found[b]); } if (m_scored) @@ -580,11 +666,11 @@ JoinState::JoinState(const Objective &objective, std::span>> outcome(n); core::parallel_for(n, [&](std::size_t i) { + double own_error = -1.0; for (const auto &link : objective.links(i)) { if (objective.held_out(link.term)) outcome[i].push_back( - {slot(i, link), - objective.match(objective.db().trajectory(link.other), state[i], link.term)}); + {slot(i, link), objective.held_out_match(i, link, state[i], own_error)}); else if (i < link.other) outcome[i].push_back( {slot(i, link), objective.match(state[i], state[link.other], link.term)}); @@ -620,17 +706,41 @@ double JoinState::metric() const { return mean_f1(m_objective, m_sum, m_counted) JoinState::Proposal JoinState::propose(std::size_t i, const core::TrajectoryView &candidate, std::span state) const { - Proposal proposal{i, 0.0, {}}; - proposal.matched.reserve(m_objective.links(i).size()); - // New outcomes of i's pairs, and the hits/found changes they cause on each side + return propose(i, outcomes(i, candidate, state)); +} + +std::vector JoinState::outcomes(std::size_t i, const core::TrajectoryView &candidate, + std::span state, + std::span held_out_known) const { + const auto links = m_objective.links(i); + if (!held_out_known.empty() && held_out_known.size() != links.size()) + throw std::invalid_argument("join outcomes: one known outcome per link needed"); + std::vector matched; + matched.reserve(links.size()); + double own_error = -1.0; + for (std::size_t k = 0; k < links.size(); ++k) { + const auto &link = links[k]; + if (!m_objective.held_out(link.term)) + matched.push_back(m_objective.match(candidate, state[link.other], link.term)); + else if (!held_out_known.empty()) + matched.push_back(held_out_known[k]); + else + matched.push_back(m_objective.held_out_match(i, link, candidate, own_error)); + } + return matched; +} + +JoinState::Proposal JoinState::propose(std::size_t i, std::vector matched) const { + Proposal proposal{i, 0.0, std::move(matched)}; + const auto links = m_objective.links(i); + if (proposal.matched.size() != links.size()) + throw std::invalid_argument("join proposal: outcome count does not match links"); + // The hits/found changes the new outcomes cause on each side std::unordered_map> change; // t -> (hits, found) - for (const auto &link : m_objective.links(i)) { + for (std::size_t k = 0; k < links.size(); ++k) { + const auto &link = links[k]; const bool now = m_matched[slot(i, link)]; - const bool then = - m_objective.held_out(link.term) - ? m_objective.match(m_objective.db().trajectory(link.other), candidate, link.term) - : m_objective.match(candidate, state[link.other], link.term); - proposal.matched.push_back(then); + const bool then = proposal.matched[k]; if (now == then) continue; const long delta = then ? 1 : -1; @@ -748,9 +858,10 @@ KnnState::KnnState(const Objective &objective, const KeptSets &state, m_distance[m_watchers[i][watch_state].query][m_watchers[i][watch_state].position] = own[i][watch_state]; } + m_rank.resize(n); core::parallel_for(n, [&](std::size_t e) { if (!m_list[e].empty()) - m_hits[e] = hits(e, m_list[e].size(), 0.0); + refresh(e); }); m_hit_sum = std::accumulate(m_hits.begin(), m_hits.end(), 0ul); } @@ -818,7 +929,7 @@ void KnnState::apply(const Move &move) { const auto i = move.trajectory; m_distance[i] = move.own; m_hit_sum -= m_hits[i]; - m_hits[i] = hits(i, m_list[i].size(), 0.0); + refresh(i); m_hit_sum += m_hits[i]; } @@ -829,11 +940,52 @@ std::size_t KnnState::hits(std::size_t e, std::span row) const { return ranked_hits(e, order); } +bool KnnState::ranks_before(std::size_t e, double distance, std::size_t position, + std::size_t other) const { + return query::knn_rank_less(distance, m_list[e][position], m_distance[e][other], + m_list[e][other]); +} + +// Only position's entry changes, so the top k changes by at most one swap with the k-th or the +// (k + 1)-th entry of the current ranking std::size_t KnnState::hits(std::size_t e, std::size_t position, double distance) const { - std::vector> order(m_list[e].size()); - for (std::size_t p = 0; p < order.size(); ++p) - order[p] = {p == position ? distance : m_distance[e][p], static_cast(p)}; - return ranked_hits(e, order); + const std::size_t k = m_objective.workloads().k; + const auto &rank = m_rank[e]; + if (m_list[e].size() <= k) + return m_hits[e]; + const std::size_t members = m_objective.workloads().knn[e].top.size(); + const auto member = [&](std::size_t p) -> long { return p < members; }; + const auto current = static_cast(m_hits[e]); + const auto in_top = std::find(rank.begin(), rank.begin() + static_cast(k), + static_cast(position)); + if (in_top != rank.begin() + static_cast(k)) { + const std::size_t next = rank[k]; + const std::size_t winner = ranks_before(e, distance, position, next) ? position : next; + return static_cast(current - member(position) + member(winner)); + } + const std::size_t last = rank[k - 1]; + if (ranks_before(e, distance, position, last)) + return static_cast(current - member(last) + member(position)); + return m_hits[e]; +} + +void KnnState::refresh(std::size_t e) { + const std::size_t size = m_list[e].size(); + const std::size_t keep = std::min(m_objective.workloads().k + 1, size); + std::vector order(size); + std::iota(order.begin(), order.end(), 0u); + std::ranges::partial_sort(order, order.begin() + static_cast(keep), + [&](std::uint32_t a, std::uint32_t b) { + return query::knn_rank_less(m_distance[e][a], m_list[e][a], + m_distance[e][b], m_list[e][b]); + }); + order.resize(keep); + m_rank[e] = std::move(order); + const std::size_t members = m_objective.workloads().knn[e].top.size(); + const std::size_t top = std::min(m_objective.workloads().k, size); + m_hits[e] = static_cast( + std::count_if(m_rank[e].begin(), m_rank[e].begin() + static_cast(top), + [&](std::uint32_t p) { return p < members; })); } std::size_t KnnState::ranked_hits(std::size_t e, @@ -899,7 +1051,7 @@ void KnnState::watch(std::span> versions for (std::size_t e = 0; e < n; ++e) if (touched[e]) { m_hit_sum -= m_hits[e]; - m_hits[e] = hits(e, m_list[e].size(), 0.0); + refresh(e); m_hit_sum += m_hits[e]; } } @@ -958,7 +1110,7 @@ void KnnState::apply(std::size_t i, std::span distances) { const auto &watcher = m_watchers[i][s]; m_distance[watcher.query][watcher.position] = distances[s]; m_hit_sum -= m_hits[watcher.query]; - m_hits[watcher.query] = hits(watcher.query, m_list[watcher.query].size(), 0.0); + refresh(watcher.query); m_hit_sum += m_hits[watcher.query]; } } diff --git a/engine/src/teacher/teacher.cpp b/engine/src/teacher/teacher.cpp index 93781ec..8c74e2f 100644 --- a/engine/src/teacher/teacher.cpp +++ b/engine/src/teacher/teacher.cpp @@ -297,9 +297,49 @@ class Search { const Methods &method) : m_start(method), m_compressed_queries(objective.config().compressed_knn_query) { auto start = std::chrono::steady_clock::now(); - if (objective.config().ranking_knn) - m_knn.emplace(objective, assemble(candidates, method), candidates, - kKnnWatchFactor * objective.workloads().k); + // Every distinct candidate prepared, since any of them can enter a query's watch list + m_prepared.resize(db.size()); + for (auto &views : m_views) + views.resize(db.size()); + m_representative.resize(db.size()); + core::parallel_for(db.size(), [&](std::size_t i) { + auto &owned = m_prepared[i]; + // Reserve first: views borrow from these + owned.reserve(kCandidates.size()); + for (std::size_t c = 0; c < kCandidates.size(); ++c) { + const auto first = + first_equal(c, [&](std::size_t k) -> const auto & { return candidates[k][i]; }); + m_representative[i][c] = first; + if (first == c) { + owned.emplace_back(db.trajectory(i), candidates[c][i]); + m_views[c][i] = owned.back().view(); + } else { + m_views[c][i] = m_views[first][i]; + } + } + }); + m_state.resize(db.size()); + for (std::size_t i = 0; i < db.size(); ++i) + m_state[i] = m_views[method[i]][i]; + const double prepare_s = core::seconds_since(start); + start = std::chrono::steady_clock::now(); + if (objective.config().ranking_knn) { + const std::size_t watch = kKnnWatchFactor * objective.workloads().k; + if (m_compressed_queries) { + m_knn.emplace(objective, assemble(candidates, method), candidates, watch); + } else { + // Lists from the start, then every query also watches any candidate within the + // distance of its watch-th nearest there + m_knn.emplace(objective, assemble(candidates, method), std::span{}, + watch); + std::vector> versions(db.size()); + for (std::size_t i = 0; i < db.size(); ++i) + for (std::size_t c = 0; c < kCandidates.size(); ++c) + if (m_representative[i][c] == c) + versions[i].push_back(m_views[c][i]); + m_knn->watch(versions, m_state); + } + } const double knn_s = core::seconds_since(start); start = std::chrono::steady_clock::now(); const auto active = [&](std::size_t i) { @@ -309,44 +349,45 @@ class Search { for (std::size_t i = 0; i < db.size(); ++i) if (active(i)) m_order.push_back(i); - m_prepared.resize(db.size()); - for (auto &views : m_views) - views.resize(db.size()); m_separable.resize(db.size()); - m_representative.resize(db.size()); m_knn_distances.resize(db.size()); core::parallel_for(db.size(), [&](std::size_t i) { - auto &owned = m_prepared[i]; - const auto original = db.trajectory(i); - if (!active(i)) { - owned.emplace_back(original, candidates[method[i]][i]); - m_views[method[i]][i] = owned.back().view(); + if (!active(i)) return; - } - // Reserve first: views borrow from these - owned.reserve(kCandidates.size()); for (std::size_t c = 0; c < kCandidates.size(); ++c) { - const auto first = - first_equal(c, [&](std::size_t k) -> const auto & { return candidates[k][i]; }); - m_representative[i][c] = first; + const auto first = m_representative[i][c]; if (first == c) { - owned.emplace_back(original, candidates[c][i]); - m_views[c][i] = owned.back().view(); m_separable[i][c] = objective.separable(i, m_views[c][i]); // Compressed queries depend on the partners' state, so are computed later if (m_knn && !m_compressed_queries) m_knn_distances[i][c] = m_knn->distances(i, m_views[c][i]); } else { - m_views[c][i] = m_views[first][i]; m_separable[i][c] = m_separable[i][first]; } } }); - m_state.resize(db.size()); - for (std::size_t i = 0; i < db.size(); ++i) - m_state[i] = m_views[method[i]][i]; + std::cout << std::format("teacher: loop setup: prepare {:.1f} s\n", prepare_s); m_join.emplace(objective, m_state); m_range.emplace(objective, m_state); + // Box coverage and held-out join outcomes do not depend on the others, so are tested once + m_coverage.resize(db.size()); + m_held_out.resize(db.size()); + core::parallel_for(db.size(), [&](std::size_t i) { + if (!active(i)) + return; + const auto links = objective.links(i); + for (std::size_t c = 0; c < kCandidates.size(); ++c) { + if (m_representative[i][c] != c) + continue; + m_coverage[i][c] = m_range->coverage(i, m_views[c][i]); + auto &known = m_held_out[i][c]; + known.assign(links.size(), 0); + double own_error = -1.0; + for (std::size_t k = 0; k < links.size(); ++k) + if (objective.held_out(links[k].term)) + known[k] = objective.held_out_match(i, links[k], m_views[c][i], own_error); + } + }); std::cout << std::format("teacher: loop setup: kNN lists {:.1f} s, candidates {:.1f} s\n", knn_s, core::seconds_since(start)); } @@ -380,8 +421,9 @@ class Search { const bool current = m_representative[i][c] == m_representative[i][method[i]]; if (duplicate || current) continue; - auto proposal = join.propose(i, m_views[c][i], state); - auto range_proposal = range.propose(i, m_views[c][i]); + auto proposal = + join.propose(i, join.outcomes(i, m_views[c][i], state, m_held_out[i][c])); + auto range_proposal = range.propose(i, m_coverage[i][c]); KnnState::Move move; double knn_gain = 0.0; if (knn && m_compressed_queries) { @@ -433,6 +475,7 @@ class Search { std::vector> m_separable; std::vector> m_representative; std::vector, kCandidates.size()>> m_knn_distances; + std::vector, kCandidates.size()>> m_coverage, m_held_out; std::optional m_knn; std::vector m_state; std::optional m_join; @@ -476,7 +519,7 @@ struct Traded { // buyers with the cheapest sellers, and keeps a pair only if its exact gain, rescored when applied, // is positive. Each round the kNN lists are extended to watch the offered versions Traded trade_budgets(const core::TrajectoryDb &db, const Objective &objective, - const std::vector &caps, KeptSets kept) { + const std::vector &caps, KeptSets kept, double start_value) { const std::size_t n = db.size(); const bool ranking = objective.config().ranking_knn; const bool compressed = objective.config().compressed_knn_query; @@ -542,12 +585,13 @@ Traded trade_budgets(const core::TrajectoryDb &db, const Objective &objective, }; const std::size_t watch = kKnnWatchFactor * objective.workloads().k; const auto start_kept = kept; - const double start_value = objective.total(kept); double value = start_value; std::cout << std::format("teacher: trading from J_T {:.6f}\n", value); // Built once: the trades keep it current if (ranking) knn.emplace(objective, kept, std::span{}, watch); + // Sequences depend only on the trajectory and its cap, so are built once for every round + std::vector> makers(n); for (std::size_t round = 1; round <= kMaxTradeRounds; ++round) { auto start = std::chrono::steady_clock::now(); // buy[d][i], sell[d][i]: i's best exact change at b_i + delta and b_i - delta @@ -559,7 +603,9 @@ Traded trade_budgets(const core::TrajectoryDb &db, const Objective &objective, core::parallel_for(n, [&](std::size_t i) { const std::size_t b = kept[i].size(); const std::size_t top = std::min(caps[i], db.trajectory(i).size()); - CandidateMaker maker(objective, i, caps[i], true); + if (!makers[i]) + makers[i].emplace(objective, i, caps[i], true); + auto &maker = *makers[i]; auto best_at = [&](std::size_t budget) { Offer best; std::vector> seen; @@ -911,7 +957,8 @@ std::vector Teacher::run(std::size_t total_budget, std::span + +#include "backend/engine/core/projection.hpp" +#include "backend/engine/core/trajectory_db.hpp" +#include "backend/engine/query/frechet_helper.hpp" +#include "backend/engine/teacher/objective.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using Line = std::vector>; // (x, y) in local metres + +core::TrajectoryDb make_db(const std::vector &lines) { + std::vector offsets{0}; + std::vector ids; + std::vector t; + std::vector lat; + std::vector lon; + for (std::size_t i = 0; i < lines.size(); ++i) { + ids.push_back(static_cast(i)); + for (std::size_t k = 0; k < lines[i].size(); ++k) { + const auto ll = core::to_lat_lon(lines[i][k].first, lines[i][k].second); + t.push_back(static_cast(k)); + lat.push_back(ll.lat); + lon.push_back(ll.lon); + } + offsets.push_back(t.size()); + } + return {std::move(offsets), std::move(ids), std::move(t), std::move(lat), std::move(lon)}; +} + +// Random walks with turns, stops (repeated points) and backtracking +std::vector random_lines(std::mt19937_64 &rng, std::size_t count, std::size_t max_len) { + std::uniform_real_distribution start(-200.0, 200.0); + std::normal_distribution step(0.0, 50.0); + std::uniform_int_distribution length(2, max_len); + std::bernoulli_distribution stop(0.1); + std::vector lines(count); + for (auto &line : lines) { + double x = start(rng), y = start(rng); + for (std::size_t k = 0, n = length(rng); k < n; ++k) { + line.emplace_back(x, y); + if (!stop(rng)) { + x += step(rng) + 20.0; + y += step(rng); + } + } + } + return lines; +} + +// Sorted random subset of [0, n) with both endpoints +std::vector random_kept(std::mt19937_64 &rng, std::size_t n) { + std::bernoulli_distribution keep(std::uniform_real_distribution(0.0, 0.6)(rng)); + std::vector kept{0}; + for (std::size_t j = 1; j + 1 < n; ++j) + if (keep(rng)) + kept.push_back(j); + kept.push_back(n - 1); + return kept; +} + +TEST(SimplificationBound, IsZeroForTheTrajectoryItself) { + const auto db = make_db({{{0, 0}, {100, 0}, {100, 100}, {0, 100}}}); + const std::vector all{0, 1, 2, 3}; + const teacher::PreparedTrajectory kept(db.trajectory(0), all); + EXPECT_NEAR(teacher::simplification_bound(db.trajectory(0), kept.view()), 0.0, 1e-9); +} + +TEST(SimplificationBound, IsInfiniteWithoutASegment) { + const auto db = make_db({{{0, 0}, {100, 0}, {200, 0}}}); + const std::vector one{0}; + const teacher::PreparedTrajectory kept(db.trajectory(0), one); + EXPECT_TRUE(std::isinf(teacher::simplification_bound(db.trajectory(0), kept.view()))); +} + +TEST(SimplificationBound, IsInfiniteForPointsNotInTheOriginal) { + const auto db = make_db({{{0, 0}, {100, 0}, {200, 0}}, {{0, 0}, {50, 50}, {200, 0}}}); + const std::vector all{0, 1, 2}; + const teacher::PreparedTrajectory other(db.trajectory(1), all); + EXPECT_TRUE(std::isinf(teacher::simplification_bound(db.trajectory(0), other.view()))); +} + +TEST(SimplificationBound, MatchesTheDetourOfAnExcursion) { + // Dropping the peak of a 50 m spike leaves 50 m as the continuous Fréchet distance + const auto db = make_db({{{0, 0}, {100, 0}, {100, 50}, {100, 0}, {200, 0}}}); + const std::vector ends{0, 4}; + const teacher::PreparedTrajectory kept(db.trajectory(0), ends); + const double bound = teacher::simplification_bound(db.trajectory(0), kept.view()); + EXPECT_GE(bound, 50.0 - 1e-6); + EXPECT_TRUE(query::continuous_frechet_within(db.trajectory(0), kept.view(), bound)); +} + +TEST(SimplificationBound, NeverBelowTheContinuousFrechetDistance) { + std::mt19937_64 rng(7); + const auto db = make_db(random_lines(rng, 200, 40)); + for (std::size_t i = 0; i < db.size(); ++i) { + const auto original = db.trajectory(i); + for (int trial = 0; trial < 5; ++trial) { + const auto indices = random_kept(rng, original.size()); + const teacher::PreparedTrajectory kept(original, indices); + const double bound = teacher::simplification_bound(original, kept.view()); + ASSERT_TRUE(std::isfinite(bound)); + EXPECT_TRUE(query::continuous_frechet_within(original, kept.view(), bound)) + << "trajectory " << i << " trial " << trial << " bound " << bound; + } + } +} + +// The certificate in Objective::held_out_match: d(P, C) <= discrete d(P, T) + bound(T, C) +TEST(SimplificationBound, CertificateNeverBelowTheContinuousFrechetDistance) { + std::mt19937_64 rng(11); + const auto db = make_db(random_lines(rng, 60, 30)); + std::vector prev, curr; + for (std::size_t a = 0; a < db.size(); ++a) { + for (std::size_t b = 0; b < db.size(); ++b) { + if (a == b) + continue; + const auto partner = db.trajectory(a); + const auto original = db.trajectory(b); + const double pair = std::sqrt(query::get_squared_frechet_distance( + partner, original, std::numeric_limits::infinity(), prev, curr)); + EXPECT_TRUE(query::continuous_frechet_within(partner, original, pair)); + const auto indices = random_kept(rng, original.size()); + const teacher::PreparedTrajectory kept(original, indices); + const double certificate = pair + teacher::simplification_bound(original, kept.view()); + EXPECT_TRUE(query::continuous_frechet_within(partner, kept.view(), certificate)) + << "pair " << a << ", " << b << " certificate " << certificate; + } + } +} + +} // namespace