Add single-query search using the optimised model

README.md

@@ -109,6 +109,13 @@ print(linker.last_cluster.quality)
 ```
 
 `cpplink-viewer` (`pip install "cpplink[viewer]"`) then serves the clusters as a page: each cluster's members side by side with every disagreeing cell highlighted, and every prediction behind it as the ledger the scorer produced.
+A model is also a search index. `cpplink search` takes a query record and returns the records that score highest against it, top-k over the same weight `predict` writes, and it answers in tens of milliseconds at four million records because interning makes a string level a question about a *value* rather than about a row.
+
+```sh
+cpplink search --schema schema.json --model model.json sample.parquet \
+    --field last_name=zolnerowich --field dob=1979-08-06 --expected-matches 1
+```
+
 Between `init` and `estimate` sit the diagnostics that cost seconds and decide the quality of the result: `profile` says what each column can be worth before any model exists, `levels` places the fuzzy thresholds from the data, `explain-blocking` prices every source without enumerating a pair, and `recall` measures what blocking reaches.
 See [Getting started](https://4ment.github.io/cpplink/getting-started/) for the whole pipeline with its output explained, [Commands](https://4ment.github.io/cpplink/commands/) for every command at a glance, the [schema reference](https://4ment.github.io/cpplink/reference/schema/) for every field, and [From Python](https://4ment.github.io/cpplink/python/) for the package.
 

bench/scale/search_latency.json

@@ -0,0 +1,338 @@
+[
+  {
+    "rows": 250000,
+    "query": "exact",
+    "threads": 1,
+    "walk_ms": 0.19808299999999998,
+    "gather_ms": 10.897167,
+    "values_walked": 9000,
+    "rescored": 377,
+    "tabulated": 1,
+    "evaluated": 1,
+    "constant": 7,
+    "top": "r0",
+    "top_weight": 11.717544504910038
+  },
+  {
+    "rows": 250000,
+    "query": "exact",
+    "threads": 8,
+    "walk_ms": 0.31670899999999996,
+    "gather_ms": 2.720917,
+    "values_walked": 9000,
+    "rescored": 2957,
+    "tabulated": 1,
+    "evaluated": 1,
+    "constant": 7,
+    "top": "r0",
+    "top_weight": 11.717544504910038
+  },
+  {
+    "rows": 250000,
+    "query": "full",
+    "threads": 1,
+    "walk_ms": 58.307625,
+    "gather_ms": 14.936166,
+    "values_walked": 1274236,
+    "rescored": 234,
+    "tabulated": 6,
+    "evaluated": 1,
+    "constant": 2,
+    "top": "r0",
+    "top_weight": 70.46587527914643
+  },
+  {
+    "rows": 250000,
+    "query": "full",
+    "threads": 8,
+    "walk_ms": 13.3905,
+    "gather_ms": 3.3235,
+    "values_walked": 1274236,
+    "rescored": 1122,
+    "tabulated": 6,
+    "evaluated": 1,
+    "constant": 2,
+    "top": "r0",
+    "top_weight": 70.46587527914643
+  },
+  {
+    "rows": 250000,
+    "query": "fuzzy",
+    "threads": 1,
+    "walk_ms": 4.767792,
+    "gather_ms": 7.853333,
+    "values_walked": 100787,
+    "rescored": 53869,
+    "tabulated": 1,
+    "evaluated": 0,
+    "constant": 8,
+    "top": "r0",
+    "top_weight": 5.36650263108208
+  },
+  {
+    "rows": 250000,
+    "query": "fuzzy",
+    "threads": 8,
+    "walk_ms": 1.330916,
+    "gather_ms": 5.376417,
+    "values_walked": 100787,
+    "rescored": 245007,
+    "tabulated": 1,
+    "evaluated": 0,
+    "constant": 8,
+    "top": "r0",
+    "top_weight": 5.36650263108208
+  },
+  {
+    "rows": 1000000,
+    "query": "exact",
+    "threads": 1,
+    "walk_ms": 0.42812500000000003,
+    "gather_ms": 44.227959,
+    "values_walked": 9000,
+    "rescored": 747,
+    "tabulated": 1,
+    "evaluated": 1,
+    "constant": 7,
+    "top": "r0",
+    "top_weight": 10.207822943323324
+  },
+  {
+    "rows": 1000000,
+    "query": "exact",
+    "threads": 8,
+    "walk_ms": 0.558833,
+    "gather_ms": 10.704084000000002,
+    "values_walked": 9000,
+    "rescored": 3339,
+    "tabulated": 1,
+    "evaluated": 1,
+    "constant": 7,
+    "top": "r0",
+    "top_weight": 10.207822943323324
+  },
+  {
+    "rows": 1000000,
+    "query": "full",
+    "threads": 1,
+    "walk_ms": 215.632667,
+    "gather_ms": 58.280333,
+    "values_walked": 4796686,
+    "rescored": 272,
+    "tabulated": 6,
+    "evaluated": 1,
+    "constant": 2,
+    "top": "r0",
+    "top_weight": 72.45608783602324
+  },
+  {
+    "rows": 1000000,
+    "query": "full",
+    "threads": 8,
+    "walk_ms": 46.683583,
+    "gather_ms": 13.764042,
+    "values_walked": 4796686,
+    "rescored": 1414,
+    "tabulated": 6,
+    "evaluated": 1,
+    "constant": 2,
+    "top": "r0",
+    "top_weight": 72.45608783602324
+  },
+  {
+    "rows": 1000000,
+    "query": "fuzzy",
+    "threads": 1,
+    "walk_ms": 7.484375,
+    "gather_ms": 28.290792,
+    "values_walked": 149192,
+    "rescored": 54029,
+    "tabulated": 1,
+    "evaluated": 0,
+    "constant": 8,
+    "top": "r0",
+    "top_weight": 2.4799602682808395
+  },
+  {
+    "rows": 1000000,
+    "query": "fuzzy",
+    "threads": 8,
+    "walk_ms": 2.4970830000000004,
+    "gather_ms": 13.482959,
+    "values_walked": 149192,
+    "rescored": 396874,
+    "tabulated": 1,
+    "evaluated": 0,
+    "constant": 8,
+    "top": "r0",
+    "top_weight": 2.4799602682808395
+  },
+  {
+    "rows": 4000000,
+    "query": "exact",
+    "threads": 1,
+    "walk_ms": 1.325875,
+    "gather_ms": 174.675042,
+    "values_walked": 9000,
+    "rescored": 861,
+    "tabulated": 1,
+    "evaluated": 1,
+    "constant": 7,
+    "top": "r0",
+    "top_weight": 8.068591826542518
+  },
+  {
+    "rows": 4000000,
+    "query": "exact",
+    "threads": 8,
+    "walk_ms": 1.632416,
+    "gather_ms": 38.276083,
+    "values_walked": 9000,
+    "rescored": 4795,
+    "tabulated": 1,
+    "evaluated": 1,
+    "constant": 7,
+    "top": "r0",
+    "top_weight": 8.068591826542518
+  },
+  {
+    "rows": 4000000,
+    "query": "full",
+    "threads": 1,
+    "walk_ms": 861.101667,
+    "gather_ms": 242.876541,
+    "values_walked": 18718486,
+    "rescored": 324,
+    "tabulated": 6,
+    "evaluated": 1,
+    "constant": 2,
+    "top": "r0",
+    "top_weight": 73.44513500940663
+  },
+  {
+    "rows": 4000000,
+    "query": "full",
+    "threads": 8,
+    "walk_ms": 200.91,
+    "gather_ms": 58.472625,
+    "values_walked": 18718486,
+    "rescored": 1961,
+    "tabulated": 6,
+    "evaluated": 1,
+    "constant": 2,
+    "top": "r0",
+    "top_weight": 73.44513500940663
+  },
+  {
+    "rows": 4000000,
+    "query": "fuzzy",
+    "threads": 1,
+    "walk_ms": 12.12425,
+    "gather_ms": 109.033125,
+    "values_walked": 237310,
+    "rescored": 54046,
+    "tabulated": 1,
+    "evaluated": 0,
+    "constant": 8,
+    "top": "r0",
+    "top_weight": 0.6526652113186122
+  },
+  {
+    "rows": 4000000,
+    "query": "fuzzy",
+    "threads": 8,
+    "walk_ms": 4.594708000000001,
+    "gather_ms": 33.168375,
+    "values_walked": 237310,
+    "rescored": 307665,
+    "tabulated": 1,
+    "evaluated": 0,
+    "constant": 8,
+    "top": "r0",
+    "top_weight": 0.6526652113186122
+  },
+  {
+    "rows": 20000000,
+    "query": "exact",
+    "threads": 1,
+    "walk_ms": 13.007166,
+    "gather_ms": 848.898125,
+    "values_walked": 9000,
+    "rescored": 886,
+    "tabulated": 1,
+    "evaluated": 1,
+    "constant": 7,
+    "top": "r0",
+    "top_weight": 5.719266274137542
+  },
+  {
+    "rows": 20000000,
+    "query": "exact",
+    "threads": 8,
+    "walk_ms": 8.869084,
+    "gather_ms": 186.34825,
+    "values_walked": 9000,
+    "rescored": 8753,
+    "tabulated": 1,
+    "evaluated": 1,
+    "constant": 7,
+    "top": "r0",
+    "top_weight": 5.719266274137542
+  },
+  {
+    "rows": 20000000,
+    "query": "full",
+    "threads": 1,
+    "walk_ms": 4194.0,
+    "gather_ms": 1308.0,
+    "values_walked": 91645846,
+    "rescored": 636,
+    "tabulated": 6,
+    "evaluated": 1,
+    "constant": 2,
+    "top": "r0",
+    "top_weight": 75.98
+  },
+  {
+    "rows": 20000000,
+    "query": "full",
+    "threads": 8,
+    "walk_ms": 901.0,
+    "gather_ms": 293.0,
+    "values_walked": 91645846,
+    "rescored": 2249,
+    "tabulated": 6,
+    "evaluated": 1,
+    "constant": 2,
+    "top": "r0",
+    "top_weight": 75.98
+  },
+  {
+    "rows": 20000000,
+    "query": "fuzzy",
+    "threads": 1,
+    "walk_ms": 42.3645,
+    "gather_ms": 555.802167,
+    "values_walked": 657528,
+    "rescored": 54123,
+    "tabulated": 1,
+    "evaluated": 0,
+    "constant": 8,
+    "top": "r0",
+    "top_weight": -1.5300124376597202
+  },
+  {
+    "rows": 20000000,
+    "query": "fuzzy",
+    "threads": 8,
+    "walk_ms": 28.6,
+    "gather_ms": 137.6,
+    "values_walked": 657528,
+    "rescored": 356775,
+    "tabulated": 1,
+    "evaluated": 0,
+    "constant": 8,
+    "top": "r0",
+    "top_weight": -1.5300124376597202
+  }
+]