Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 4 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -89,6 +89,10 @@ and it then falls back to its default. [engine/configs/geolife.json](engine/conf
segment with the largest perpendicular error anywhere in the database. It uses no query
workloads and receives exactly the same total point budget as the teacher.

Continuous kNN uses the same fixed millimetre-grid ranking in the evaluator and teacher tracker,
with ties resolved by database index. Teacher workload caches from earlier ranking versions are
rebuilt automatically. Recompute older kNN/J4 results when comparing runs across this change.

The teacher's range term uses macro F1 like J4. The loop tracks box coverage counts exactly;
independent choices and value curves use F1 marginal gains against fixed partners.

Expand Down
4 changes: 3 additions & 1 deletion TEACHER_NEXT_STEPS.md
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,9 @@ Implementation update, 2026-11-20: the range objective and endpoint floor now us
J4, with exact incremental updates in the loop and F1 marginals for candidate and budget scoring.
`RDP_GLOBAL`, a database-wide top-down PED baseline, runs alongside the other baselines in
`teacher_eval` and is enabled in `geolife.json`. It is included in the best-cheap-baseline comparison.
The numerical findings below predate this change and need to be checked with fresh runs.
Continuous kNN now shares a deterministic millimetre-grid ranking between the evaluator and
tracker; V6 caches rebuild old reference rankings. Limited watch lists remain approximate.
The numerical findings below predate these changes and need to be checked with fresh runs.

Updated 2026-11-17 on branch `teacher`. The teacher is described in
[engine/src/teacher/README.md](engine/src/teacher/README.md); this file says what is settled and
Expand Down
17 changes: 15 additions & 2 deletions engine/include/backend/engine/query/knn.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,19 @@

namespace query {

inline constexpr double kKnnResolutionM = 1e-3;

// Continuous distances are rounded up to a fixed millimetre grid using the decision predicate.
// The grid makes ranking independent of the search bounds, candidate order and requested k.
// Discrete distances are unchanged. Use this distance for cached kNN rankings too.
double knn_distance(const core::TrajectoryView &a, const core::TrajectoryView &b,
FrechetVariant variant = FrechetVariant::Continuous);

inline bool knn_rank_less(double distance_a, std::size_t index_a, double distance_b,
std::size_t index_b) {
return distance_a < distance_b || (distance_a == distance_b && index_a < index_b);
}

struct KnnQuery {
std::size_t query_index; // index into KnnWorkload::queries
std::vector<std::size_t> reference; // knn in the uncompressed database, closest first
Expand Down Expand Up @@ -39,8 +52,8 @@ KnnWorkload generate_knn_queries(const core::TrajectoryDb &original,
std::uint64_t seed = 42);

// Indices into trajectories of the k trajectories closest to query by Fréchet distance, closest
// first. Equal distances are ordered by database index; continuous distances within 1 mm count
// as equal.
// first, using knn_distance and knn_rank_less. Continuous distances in the same millimetre bin
// are tied by database index; discrete distance ties also use the database index.
std::vector<std::size_t> knn(const core::TrajectoryDb &trajectories,
const core::TrajectoryView &query, std::size_t k,
FrechetVariant variant = FrechetVariant::Continuous);
Expand Down
14 changes: 13 additions & 1 deletion engine/include/backend/engine/teacher/objective.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -280,9 +280,17 @@ class KnnState {
// Whether i's choice can change the metric: watched, or with compressed queries a query
bool active(std::size_t i) const;
// Original queries only. From every query watching i, in the order propose and apply expect
std::vector<double> distances(std::size_t i, const core::TrajectoryView &candidate) const;
std::vector<double> distances(std::size_t i, const core::TrajectoryView &candidate,
bool parallel = false) const;
// Change in J_T
double propose(std::size_t i, std::span<const double> distances) const;
// The same change from i becoming candidate, computing a distance only near a query's k-th
// place
double propose_bounded(std::size_t i, const core::TrajectoryView &candidate) const;
// Original queries only. Adds trajectory i to the list of every query that one of versions[i]
// comes within the query's watch radius of, the distance of its watch-th nearest in state
void watch(std::span<const std::vector<core::TrajectoryView>> versions,
std::span<const core::TrajectoryView> state);
void apply(std::size_t i, std::span<const double> distances);

// Either query kind. With compressed queries a distance depends on both sides, so a move also
Expand All @@ -307,12 +315,16 @@ class KnnState {
std::size_t hits(std::size_t e, std::size_t position, double distance) const;
// The same with e's whole row replaced
std::size_t hits(std::size_t e, std::span<const double> row) const;
// True neighbours in the top k of order (distance, position)
std::size_t ranked_hits(std::size_t e,
std::vector<std::pair<double, std::uint32_t>> &order) const;
// Distances from query, e in some state, to e's list in state
std::vector<double> own_row(std::size_t e, const core::TrajectoryView &query,
std::span<const core::TrajectoryView> state) const;

const Objective &m_objective;
std::size_t m_valid = 0;
std::size_t m_watch = 0;
std::vector<std::vector<std::uint32_t>> m_list; // per query, its true top k first
std::vector<std::vector<double>> m_distance; // per query, in list order
std::vector<std::size_t> m_hits; // per query
Expand Down
20 changes: 11 additions & 9 deletions engine/include/backend/engine/teacher/teacher.hpp
Original file line number Diff line number Diff line change
Expand Up @@ -27,17 +27,19 @@ namespace teacher {
// BUDGET_LOOP_REALLOCATE then curves, budgets and loop again while J_T rises. With only
// held-out join terms the curves ignore partners, so this only reruns
// the loop in a new visit order
// BUDGET_LOOP_TRADE then moves points between trajectories while exact J_T rises
// BUDGET_LOOP_SHUFFLED control: loop on relabelled partners and neighbours
// BUDGET_LOOP_NO_UPDATE control: one pass against the starting state
inline constexpr std::array<std::string_view, 9> kArms{"EQUAL_CHOICE",
"EQUAL_CHOICE_RDP_PARTNERS",
"EQUAL_LOOP",
"BUDGET_RDP",
"BUDGET_CHOICE",
"BUDGET_LOOP",
"BUDGET_LOOP_REALLOCATE",
"BUDGET_LOOP_SHUFFLED",
"BUDGET_LOOP_NO_UPDATE"};
inline constexpr std::array<std::string_view, 10> kArms{"EQUAL_CHOICE",
"EQUAL_CHOICE_RDP_PARTNERS",
"EQUAL_LOOP",
"BUDGET_RDP",
"BUDGET_CHOICE",
"BUDGET_LOOP",
"BUDGET_LOOP_REALLOCATE",
"BUDGET_LOOP_TRADE",
"BUDGET_LOOP_SHUFFLED",
"BUDGET_LOOP_NO_UPDATE"};

bool is_arm(std::string_view name);
// Throws std::invalid_argument on an unknown or repeated arm
Expand Down
25 changes: 25 additions & 0 deletions engine/include/backend/engine/teacher/trade_matching.hpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#pragma once

#include <cstddef>
#include <span>
#include <utility>

namespace teacher::detail {

// Sellers are sorted by descending gain. Only used sellers can permanently advance first;
// a seller excluded for being this buyer remains available to subsequent buyers.
inline std::size_t find_profitable_seller(std::span<const std::pair<double, std::size_t>> sellers,
std::span<const char> used, std::size_t &first,
std::size_t buyer, double buy_gain, double min_gain) {
while (first < sellers.size() && used[sellers[first].second])
++first;
auto candidate = first;
while (candidate < sellers.size() &&
(used[sellers[candidate].second] || sellers[candidate].second == buyer))
++candidate;
if (candidate == sellers.size() || buy_gain + sellers[candidate].first <= min_gain)
return sellers.size();
return candidate;
}

} // namespace teacher::detail
176 changes: 77 additions & 99 deletions engine/src/query/knn.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -229,92 +229,56 @@ std::vector<std::size_t> discrete_knn(const core::TrajectoryDb &trajectory_db,
return neighbor_ids;
}

// A candidate whose continuous distance is known to lie in [lo, hi]
struct Bracket {
// The first millimetre radius accepted by the continuous decision predicate. Unlike an
// arbitrary bisection midpoint, this key is identical for different valid initial bounds.
double continuous_rank_distance(const core::TrajectoryView &a, const core::TrajectoryView &b,
double lower, double upper) {
if (!std::isfinite(upper))
return std::numeric_limits<double>::infinity();
// Geometric lower bounds and the decision predicate have rounding slack. Start below it,
// so an accepted grid point is never skipped just because a bound rounded upward.
const double slack = 2e-12 * std::max(1.0, lower);
const double first = std::ceil(std::max(0.0, lower - slack) / kKnnResolutionM);
const double last = std::ceil(upper / kKnnResolutionM);
if (last >= static_cast<double>(std::numeric_limits<std::uint64_t>::max()))
throw std::overflow_error("knn: distance exceeds millimetre ranking range");
auto lo = static_cast<std::uint64_t>(first);
auto hi = static_cast<std::uint64_t>(last);
// Most distances equal their Hausdorff bound. Resolve those with one decision instead of
// bisecting the entire interval up to the discrete upper bound.
if (lo < hi && continuous_frechet_within(a, b, static_cast<double>(lo) * kKnnResolutionM))
return static_cast<double>(lo) * kKnnResolutionM;
if (lo < hi)
++lo;
while (lo < hi) {
const auto mid = lo + (hi - lo) / 2;
if (continuous_frechet_within(a, b, static_cast<double>(mid) * kKnnResolutionM))
hi = mid;
else
lo = mid + 1;
}
return static_cast<double>(lo) * kKnnResolutionM;
}

struct RankedNeighbor {
double distance;
std::size_t idx;
double lo;
double hi;
bool operator<(const RankedNeighbor &other) const {
return knn_rank_less(distance, idx, other.distance, other.idx);
}
};

// Each distance is narrowed only as far as top-k membership and order need. Distances within
// tolerance of each other count as equal and are ordered by database index, and the same rule
// decides membership at the k-th place. Every prune allows the tolerance as well.
std::vector<std::size_t> continuous_knn(const core::TrajectoryDb &trajectory_db,
const core::TrajectoryView &query, std::size_t k) {
constexpr double tolerance = 1e-3;
CandidateQueue queue(trajectory_db, query, k);
std::vector<Bracket> kept;
std::vector<double> scratch;
const auto halve = [&](Bracket &b) {
const double mid = 0.5 * (b.lo + b.hi);
if (continuous_frechet_within(query, trajectory_db.trajectory(b.idx), mid))
b.hi = mid;
else
b.lo = mid;
};
const auto kth_upper = [&] {
scratch.clear();
for (const auto &b : kept)
scratch.push_back(b.hi);
std::nth_element(scratch.begin(), scratch.begin() + static_cast<std::ptrdiff_t>(k - 1),
scratch.end());
return scratch[k - 1];
};
// Narrows overlapping brackets until they are disjoint or within tolerance, then orders runs
// of overlapping brackets, which are ties, by index
const auto rank = [&] {
const auto by_lower = [](const Bracket &b) { return std::pair(b.lo, b.idx); };
for (bool overlapping = true; overlapping;) {
overlapping = false;
std::ranges::sort(kept, {}, by_lower);
for (std::size_t i = 0; i + 1 < kept.size(); ++i) {
Bracket &a = kept[i], &b = kept[i + 1];
if (a.hi < b.lo)
continue;
Bracket &wider = a.hi - a.lo >= b.hi - b.lo ? a : b;
if (wider.hi - wider.lo <= tolerance)
continue;
halve(wider);
overlapping = true;
}
}
for (std::size_t start = 0; start < kept.size();) {
std::size_t end = start + 1;
double reach = kept[start].hi;
while (end < kept.size() && kept[end].lo <= reach)
reach = std::max(reach, kept[end++].hi);
std::ranges::sort(kept.begin() + static_cast<std::ptrdiff_t>(start),
kept.begin() + static_cast<std::ptrdiff_t>(end), {}, &Bracket::idx);
start = end;
}
};
// Drops brackets that are certainly beyond the k-th, refining the widest uncertain one.
// Once only ties remain at the boundary, the ranking decides who stays.
const auto trim = [&] {
while (kept.size() > k) {
const double cutoff = kth_upper();
std::erase_if(kept, [&](const Bracket &b) { return b.lo > cutoff + tolerance; });
if (kept.size() <= k)
return;
Bracket *widest = nullptr;
for (auto &b : kept)
if (b.hi >= cutoff && b.hi - b.lo > tolerance &&
(!widest || b.hi - b.lo > widest->hi - widest->lo))
widest = &b;
if (!widest) {
rank();
kept.resize(k);
return;
}
halve(*widest);
}
};

std::priority_queue<RankedNeighbor> nearest;
std::vector<double> prev, curr;
while (!queue.empty()) {
const Candidate candidate = queue.next();
const double cutoff = kept.size() < k ? std::sqrt(queue.initial_cutoff()) : kth_upper();
const double limit = cutoff + tolerance;
// One extra bin absorbs floating-point rounding in bounds and includes all distance ties.
const double limit =
(nearest.size() < k ? std::sqrt(queue.initial_cutoff()) : nearest.top().distance) +
kKnnResolutionM;
const double squared_limit = limit * limit;
if (candidate.squared_lower_bound > squared_limit)
break;
Expand All @@ -328,36 +292,50 @@ std::vector<std::size_t> continuous_knn(const core::TrajectoryDb &trajectory_db,
std::max(forward, squared_directed_hausdorff(reference, query, squared_limit));
if (squared_hausdorff > squared_limit)
continue;
// The discrete distance bounds the continuous one from above
double hi = limit;
const double squared_discrete = get_squared_frechet_distance(
query, reference, std::nextafter(squared_limit, squared_limit + 1.0), prev, curr);
if (squared_discrete <= squared_limit)
hi = std::min(hi, std::sqrt(squared_discrete));
double upper = limit;
const double discrete = get_squared_frechet_distance(
query, reference,
std::nextafter(squared_limit, std::numeric_limits<double>::infinity()), prev, curr);
if (discrete <= squared_limit)
upper = std::min(upper, std::sqrt(discrete));
else if (!continuous_frechet_within(query, reference, limit))
continue;
double lo = std::min(
hi, std::max(std::sqrt(candidate.squared_lower_bound), std::sqrt(squared_hausdorff)));
// The distance often equals the Hausdorff bound
if (hi - lo > tolerance) {
if (continuous_frechet_within(query, reference, lo + tolerance))
hi = lo + tolerance;
else
lo += tolerance;
const double lower =
std::min(upper, std::sqrt(std::max(candidate.squared_lower_bound, squared_hausdorff)));
const RankedNeighbor next{continuous_rank_distance(query, reference, lower, upper),
candidate.idx};
if (!std::isfinite(next.distance))
continue;
if (nearest.size() < k)
nearest.push(next);
else if (next < nearest.top()) {
nearest.pop();
nearest.push(next);
}
kept.push_back({candidate.idx, lo, hi});
trim();
}

rank();
std::vector<std::size_t> neighbor_ids;
for (const auto &b : kept)
neighbor_ids.push_back(b.idx);
return neighbor_ids;
std::vector<std::size_t> result(nearest.size());
for (std::size_t i = result.size(); i > 0; --i) {
result[i - 1] = nearest.top().idx;
nearest.pop();
}
return result;
}

} // namespace

double knn_distance(const core::TrajectoryView &a, const core::TrajectoryView &b,
FrechetVariant variant) {
const double upper = frechet_distance(a, b);
if (variant == FrechetVariant::Discrete || !std::isfinite(upper))
return upper;
const double squared_upper = upper * upper;
const double lower = std::sqrt(
std::max({squared_endpoint_lower_bound(a, b), squared_box_extent_lower_bound(a, b),
squared_directed_hausdorff(a, b, squared_upper),
squared_directed_hausdorff(b, a, squared_upper)}));
return continuous_rank_distance(a, b, std::min(lower, upper), upper);
}

std::vector<std::size_t> knn(const core::TrajectoryDb &trajectory_db,
const core::TrajectoryView &query_trajectory, std::size_t k,
FrechetVariant variant) {
Expand Down
Loading
Loading