diff --git a/.claude/board/AGENT_LOG.md b/.claude/board/AGENT_LOG.md index 707065e8b..fda047b85 100644 --- a/.claude/board/AGENT_LOG.md +++ b/.claude/board/AGENT_LOG.md @@ -1,3 +1,17 @@ +## 2026-10-16 — 5+3 council: WCOJ closing atoms (premise + 5 savants + 3 reviewers) + +- Premise auditor: PREMISE-SPLIT (Part B is an EXISTS, not a comparison). +- Savants on Sonnet: prior art, iron rules, code truth, cascade impact, different views. Spot checks on the main thread: `reference.rs:372-405` (validate's `if let` and `_ => {}`), `quack/lib.rs:575-615` (`if let Filter::Semijoin`, one-mask `leaf_validity`), `value.rs:178-188`, the two non-AVX-512 `gather` sites, `ISSUES.md` accumulator-gate entry. All held. +- Reviewers: overclaim, dilution/collapse, firewall. Three BLOCKs (inverted G5 polarity, visited-only order check, partial sink on refusal), all resolved in v3. +- Misses: three savants could not write their tag-files (no write tool); reports were taken from their hand-backs. S3 found that gate fixtures F2/F3a/F3b named in v1 did not exist. +- Result: `.claude/plans/wcoj-closing-atoms-v1.md`, entry `2026-10-16-wcoj-closing-atoms-council.md`. Written by the orchestrating main thread only. + +## 2026-10-16 — WCOJ items 1–3: DuckDB and bind harvest (three Haiku runs) + +- harvest-scout (as general-purpose on Haiku) over a DuckDB source clone at `e1ecb5b8`: 10 patterns, per-pattern counts. Spot checks: `physical_asof_join.cpp:1182`, `delim_join_cte_rewriter.cpp:1172` and `query_graph_manager.cpp:399` re-read. All three matched. No conclusion drawn. +- Web scout (general-purpose on Haiku): 12 items with read grades; most were ABSTRACT-ONLY or SECONDHAND and are recorded as leads only. +- substrate-mapper over `lance-graph-sap` (7 files, 96 rows). Spot checks: `binder.rs` and `bind.rs:1-140` re-read in full. The `Binder` trait was read at `lance-graph-quack/src/bind.rs:92`. No misses. +- Result: entry `2026-10-16-wcoj-probes-1-3.md`. Written by the orchestrating main thread only. ## 2026-10-16 — /coresearch: worst-case-optimal joins for the join path (D-ASM-11) - Phase 0: brief by the orchestrator; premise-auditor → PREMISE-SPLIT (semijoin chain = existence mask, leapfrog = bindings); re-asked as Q-E / Q-A. diff --git a/.claude/board/INTEGRATION_PLANS.md b/.claude/board/INTEGRATION_PLANS.md index 0c7201e24..920511b14 100644 --- a/.claude/board/INTEGRATION_PLANS.md +++ b/.claude/board/INTEGRATION_PLANS.md @@ -1,3 +1,19 @@ +## 2026-10-16 (3) — wcoj-closing-atoms-v1 — the exact cyclic existence mask: two closing atoms → `.claude/plans/wcoj-closing-atoms-v1.md` + +**Status:** RATIFIED SPEC (5+3 council); not authorized. D-ASM-11A, D-ASM-11B. + +- Part A `Pred::EqU32ViaLane`: comparison through an address lane, three-valued, O(1) per row; functional relations. +- Part B `MaskOp::RunsMeet`: EXISTS over two sorted runs, gated beside `Gather`, Semijoin's NULL rule, order checked at validate; n:m relations. +- Council record: entry `2026-10-16-wcoj-closing-atoms-council.md`. + +## 2026-10-16 (2) — quack-multitable-bind-v1 — one bind path for joins, shared by MeTTa bodies and SAP → `.claude/plans/quack-multitable-bind-v1.md` + +**Status:** PROPOSAL; nothing built. D-QMB-0..5. + +- `Binder` gains a default `addresses(table, field)`; `Draft::via(field)` scopes predicates to the addressed table; `bind_joined` returns the `Query` plus the `Foreign` slot layout it assumes. +- Every hop conjoins the target's live plane as a `Semijoin`, so a deleted or dangling target is a known FALSE. +- v1 binds `=` on coded foreign fields only; `<>`, `I32` and optional foreign fields are refused until pinned; cyclic and n:m bodies wait for the council-p7 predicates. + ## 2026-10-16 — d-holo-h14d-logic-mask-read-v1 — C's own operation: logic → address mask → masked read → terminal → `.claude/plans/d-holo-h14d-logic-mask-read-v1.md` **Status:** PRE-REGISTERED; not run. diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index 7384767fe..c47e9c1f7 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,7 +1,11 @@ -## 2026-10-16 — main after #1584 +## 2026-10-16 — WCOJ map items 1–3 run (branch `ccr-9f3b8075-74u47l-wcoj-probes`, in PR) + +Contract inventory delta: `ordered_lane::gallop_partition_point` (cursor seek, galloping), `ordered_lane::seek_partition_point` + `SEEK_LINEAR_GAP` (hybrid by gap hint, thresholds from v4 timings) and `SealedFacetLane::seek_from` (witnessed). Outside the contract: `lance-graph-relalg::join_body::is_alpha_acyclic` (GYO) and `lance-graph-metta/examples/wcoj_probe.rs` (F4 fixture, bag oracle, P-7 check). Results: entry `2026-10-16-wcoj-probes-1-3.md`. + +## 2026-10-16 — main after #1584, #1585, #1586 - **On `main` (#1584):** D-ASM-1b and D-ASM-8..11 queued; entry `2026-10-16-asm-accepted-proposals.md`. Nothing built. -- **In this PR:** D-ASM-12 (the `lance-graph-cog` content → NodeGuid address that `atomspace-lance` and the D-ASM-1b binder share); D-ASM-13 (the `lance-graph-cog` thinking API as a facade over NAL / ONA-shaped tables / R2IL / relalg); the forks `matrix` and `atomspace-storage` recorded; the MORK stable branch recorded. +- **#1586 (merged):** D-ASM-12 (the `lance-graph-cog` content → NodeGuid address that `atomspace-lance` and the D-ASM-1b binder share); D-ASM-13 (the `lance-graph-cog` thinking API as a facade over NAL / ONA-shaped tables / R2IL / relalg); the forks `matrix` and `atomspace-storage` recorded; the MORK stable branch recorded. ## 2026-10-16 — main after #1582 (merged `e08e5659`) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 1e52873f5..58df10a3a 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -1,3 +1,17 @@ +## 2026-10-16 — lance-graph PR #1586 (merged `f2892f98`, branch `ccr-9f3b8075-74u47l-coresearch-wcoj`, head `2284376a`, 1 commit) — board: co-research map for worst-case-optimal joins (D-ASM-11) + +- **Added:** entry `2026-10-16-coresearch-wcoj-leapfrog.md` (leapfrog family mapped onto our surfaces, ADOPT-NOW / PROBE / PARK / SKIP per idea, fixtures F2/F3a/F3b/F4, probes P-1..P-12). +- **Locked:** none; the map ratifies nothing. +- **Deferred / open:** items 1-3 run in #1587; closing atoms specified in `.claude/plans/wcoj-closing-atoms-v1.md`. +- **Confidence:** n/a (board). + +## 2026-10-16 — lance-graph PR #1585 (merged `9252e5cf`, branch `ccr-9f3b8075-74u47l-asm-queue-2`, head `4399cb72`, 3 commits) — board: `lance-graph-cog` address + thinking API (D-ASM-12/13), leapfrog arms + +- **Added:** STATUS D-ASM-12 (content → NodeGuid atom address in `lance-graph-cog`), D-ASM-13 (thinking API as a facade over NAL / ONA-shaped tables / R2IL / relalg); D-ASM-11 split into Q-E / Q-A with arms A and B; the #1584 arc entry. +- **Locked:** the thinking API adds no opcode, interpreter or executor; leapfrog is prior art to learn from, decided by measurement. +- **Deferred / open:** all rows Queued; `lance-graph-cog` not created. +- **Confidence:** n/a (board). + ## 2026-10-16 — lance-graph PR #1584 (merged, branch `ccr-9f3b8075-74u47l-asm-queue`, head `018be049`, 1 commit) — board: queue the accepted D-ASM follow-ups (D-ASM-1b, 8–11) - **Added:** STATUS D-ASM-1b (Atomese front end in the binder), D-ASM-8 (learn measures), D-ASM-9 (`lance-graph-das` probe), D-ASM-10 (MORK oracle), D-ASM-11 (leapfrog probe); entry `2026-10-16-asm-accepted-proposals.md`. diff --git a/.claude/board/STATUS_BOARD.md b/.claude/board/STATUS_BOARD.md index 1af344cb8..de1cb3fe8 100644 --- a/.claude/board/STATUS_BOARD.md +++ b/.claude/board/STATUS_BOARD.md @@ -1,3 +1,16 @@ +## D-QMB — multi-table bind for Quack (2026-10-16) + +Plan: `.claude/plans/quack-multitable-bind-v1.md`. Shared by D-ASM-1/11 and SAP joins. + +| D-id | scope | status | gate / falsifier | +|---|---|---|---| +| **D-QMB-0** | tests first: two-table draft == expert query + slot layout; binder call count; nested-loop oracle incl. dangling and deleted targets | Queued (not authorized) | disable: drop the live `Semijoin` → deleted target kept (red); skip slot dedup (red) | +| **D-QMB-1** | `Binder::addresses` (default), `Draft::via`, `Bound`, `bind_joined`, `BindError::{NotAnAddress, Unsupported}` | Queued (not authorized) | the four existing `Binder` implementers compile unchanged | +| **D-QMB-2** | SAP: CATS hours through a functional employee assignment | Queued (not authorized) | answer == the SAP fixture's expected rows | +| **D-QMB-3** | MeTTa: two-atom acyclic body through `via` | Queued (not authorized) | answer == datafrog SET | +| **D-QMB-4** | nested `via` chains in one program | Queued (not authorized) | check the #1521/#1522 withdrawal first | +| **D-QMB-5** | cyclic bodies and n:m relations | Queued (gated on the council-p7 predicates) | answer == datafrog SET on F2/F3a/F3b/F4 | + ## D-WAL — `LanceCycleWriter` vs the lancedb 0.40 MemWAL write path (2026-10-14) Plan: `.claude/plans/wal-shootout-v1.md`. Feeds D-LNC-7; decides nothing. @@ -31,7 +44,9 @@ Council 2026-10-16: `metta-datalog-assimilation-v1-council-v3.md` is the ratifie | **D-ASM-8** | `learn`-shaped measures (marginals, MI/PMI, cosine, Jaccard) as a reading of pair counts in `lance-graph-arm-discovery`; counts from D-ASM-3, never a loop; no `lance-graph-learn` crate (entry `2026-10-16-asm-accepted-proposals.md`) | Queued (not authorized; gated on D-ASM-3) | measures == an oracle run of upstream `learn` on one corpus; `(opencog matrix)` definitions sourced first | | **D-ASM-9** | `lance-graph-das` attention-broker probe: `w = edge.count / node.count`, rent, spread over the D-ASM-2 hop | Queued (not authorized; gated on D-ASM-2; crate not created) | attention values == DAS broker on one fixture; crate created only when the probe runs | | **D-ASM-10** | MORK as fourth oracle (Hyperon · PeTTa · datafrog · MORK), built as a separate binary outside the workspace, results only | Queued (not authorized) | MORK answers recorded beside the other three, divergences recorded, never averaged | -| **D-ASM-11** | Leapfrog probe on a cyclic 3-way join, split into Q-E (exact existence mask where the semijoin chain over-approximates) and Q-A (binding enumeration emitting keyed counts, D-ASM-3's missing carrier), written from the paper (Veldhuizen, *Leapfrog Triejoin*, arXiv 1210.0481v5, ICDT '14); the zipper shape is the probe's question, not a premise — arm A: leapfrog triejoin over the existing sealed ordered lanes, with aperture depth as `open`/`up` and a galloping `seek` from the cursor (the paper's amortized O(1 + log(N/m)); today's `bound_aperture` is a fresh `partition_point`); cleanest, already integrated; arm B: leapfrog triejoin over a prefix-shared trie zipper (PathMap-like), studied as prior art to learn from (worst-case optimal on cyclic joins, which a semijoin chain is not); the arm that wins on cost per element and integration is adopted, and what B teaches (variable order, prefix sharing, seek pattern) is carried into A either way | Queued (not authorized; gated on D-ASM-2) | Q-E: mask == datafrog's SET of head rows; Q-A: == datafrog as a BAG; per-element cost on the v4 arm only; margin and lessons pre-registered before the run; findings from B recorded even when A wins | +| **D-ASM-11** | Leapfrog probe on a cyclic 3-way join, split into Q-E (exact existence mask where the semijoin chain over-approximates) and Q-A (binding enumeration emitting keyed counts, D-ASM-3's missing carrier), written from the paper (Veldhuizen, *Leapfrog Triejoin*, arXiv 1210.0481v5, ICDT '14); the zipper shape is the probe's question, not a premise — arm A: leapfrog triejoin over the existing sealed ordered lanes, with aperture depth as `open`/`up` and a galloping `seek` from the cursor (the paper's amortized O(1 + log(N/m)); today's `bound_aperture` is a fresh `partition_point`); cleanest, already integrated; arm B: leapfrog triejoin over a prefix-shared trie zipper (PathMap-like), studied as prior art to learn from (worst-case optimal on cyclic joins, which a semijoin chain is not); the arm that wins on cost per element and integration is adopted, and what B teaches (variable order, prefix sharing, seek pattern) is carried into A either way | Queued (not authorized; gated on D-ASM-2). Pre-work in #1587: F4 fixture, bag oracle, `seek_from` (P-1), GYO (P-3), P-7 insufficiency shown (entry `2026-10-16-wcoj-probes-1-3.md`); closing atoms split into D-ASM-11A/B | Q-E: mask == datafrog's SET of head rows; Q-A: == datafrog as a BAG; per-element cost on the v4 arm only; margin and lessons pre-registered before the run; findings from B recorded even when A wins | +| **D-ASM-11A** | Closing atom for functional relations: `Pred::EqU32ViaLane` + ndarray `eq_u32_via_lane_to_mask` (spec `.claude/plans/wcoj-closing-atoms-v1.md` §2) | Spec ratified 2026-10-16; authorized 2026-10-16 (ungated from D-ASM-2) | G1 exact mask == datafrog; G2 op count fixed and < the disjunction; G5-A disables | +| **D-ASM-11B** | Closing atom for n:m relations: `MaskOp::RunsMeet` (gated, beside `Gather`) + ndarray `runs_meet_under_to_mask` + lgj-abi status arm (spec §3) | Spec ratified 2026-10-16; authorized 2026-10-16, after D-ASM-11A | G0 fixtures; G3 edge oracle + projection; G4 elements-touched slope 1.5 ± 0.1; G5-B disables | | **D-ASM-12** | `lance-graph-cog`: deterministic content → NodeGuid atom address, shared by the D-ASM-1b binder and the `atomspace-lance` StorageNode adapter (which calls it); replaces the Rocks `n@`/`l@`/`a@`/`h@` indexes | Queued (not authorized; crate not created) | same content → same address from both callers; no string or hash index reachable from a row; adapter carries no layout of its own | | **D-ASM-13** | `lance-graph-cog` thinking API for OpenCog callers: a facade over existing algebra only — NAL (`lance-graph-nal`, global core methods), ONA-shaped term table + rule catalogue (D-ASM-5/6), R2IL procedures, relalg IR, all lowering to mask-risc | Queued (not authorized; gated on D-ASM-5/6 and D-ASM-12) | answers == OpenCog (PLN / ONA) on the same inputs, run outside the workspace; no new opcode, interpreter or executor (guard: every entry point resolves to an existing NAL/R2IL/relalg call) | diff --git a/.claude/board/TECH_DEBT.md b/.claude/board/TECH_DEBT.md index bcc1a0008..40fd9f340 100644 --- a/.claude/board/TECH_DEBT.md +++ b/.claude/board/TECH_DEBT.md @@ -1,3 +1,13 @@ +## TD-NO-BOUNDED-U32-GATHER-1 (2026-10-16) — OPEN + +`eq_u32_via_to_mask` and the planned `eq_u32_via_lane_to_mask` are scalar. An +`unsafe`, unbounded f32 gather exists on several ndarray backends +(`simd_avx512.rs:236-246`, `simd_avx2.rs:703`, `simd_scalar.rs:1099`), but no +bounds-masked u32 gather-and-compare wrapper does, and the out-of-range-is-false +contract needs one. Whether it pays on random access is unmeasured. Payoff: a +W1a wrapper plus a v4 per-element measurement against the scalar kernel. +Source: `.claude/plans/wcoj-closing-atoms-v1.md` §2. + ## TD-KEYED-SINK-MERGE-IDENTITY-1 (2026-10-05) — OPEN **Equal length is not semantic compatibility.** `Terminal::merge_group_sink` diff --git a/.claude/board/entries/2026-10-16-wcoj-closing-atoms-council.md b/.claude/board/entries/2026-10-16-wcoj-closing-atoms-council.md new file mode 100644 index 000000000..c7e176b42 --- /dev/null +++ b/.claude/board/entries/2026-10-16-wcoj-closing-atoms-council.md @@ -0,0 +1,39 @@ +# 2026-10-16 — WCOJ closing atoms: 5+3 council record + +**Status:** RATIFIED SPEC (`.claude/plans/wcoj-closing-atoms-v1.md`); not +authorized to build. Follows the P-7 insufficiency in +`2026-10-16-wcoj-probes-1-3.md`. + +## Outcome + +- **Part A, `Pred::EqU32ViaLane { fk, key, lane }`**: `EqU32Via` with the + constant replaced by the row's own lane. Comparison family, three-valued, + O(1) per row. Closes a cyclic body over functional relations. +- **Part B, `MaskOp::RunsMeet { .. }`**: for each row of R-edges, do two sorted + runs share a value. EXISTS family, beside `Gather{under}`, Semijoin's NULL + rule. Closes a cyclic body over n:m relations; elements touched ≤ N^1.5 [H]. +- The two answer different questions over different head populations (node + rows vs R-edge rows); they coincide only when R is functional. + +## What changed between rounds + +| round | finding | consequence | +|---|---|---| +| premise | PREMISE-SPLIT: Part B is an EXISTS over a set, not a comparison | split into Q-A / Q-B; Part B gets Semijoin's NULL rule and a gated execution | +| S1 | no prior ruling; planner `AdjacencyBatch::intersect` and `WcoJoin` are stale parallels | not reused | +| S2 | `if let Filter::Semijoin` (`quack/lib.rs:584`) would misroute a new EXISTS leaf under `NOT` | becomes `matches!` over both | +| S3 | F2/F3a/F3b fixtures do not exist yet; the ndarray "no gather" doc is overstated | G0 writes them; sibling doc corrected | +| S4 | exhaustive `ExecError` match in lgj-abi; Pred `_under` absent in the Via family | Part B is a `MaskOp`; new error variant paired with lgj-abi | +| S5 | re-anchoring does not remove Part B; k-way is not AND of 2-way | decisions 10, 11 | +| R1, R2 | G5 NULL-under-`NOT` polarity was inverted | fixed: a NULL key is KEPT under `NOT` | +| R1 | visited-only order check made Ok/Err depend on the gate | complete check at validate | +| R1 | quack gates `RunsMeet` on the first resident plane, not all earlier conjuncts | stated; no "rejected rows do no work" claim | +| R1 | G3 compared R-edge rows with head rows `a`; G2 could not fail; G4 metric changed | edge-level oracle + projection; G2 vs the disjunction; G4 on elements touched | +| R3 | a refusal inside the tile loop leaves a partly written sink | order invariant checked before any tile; `ForeignLaneNotOrdered` | + +## OPEN + +- Authorization of D-ASM-11 (still gated on D-ASM-2). +- Part B's insufficiency is argued, not measured; G0 measures it. +- The sealed CSR witness that removes the per-execution O(|S| + |T|) check. +- A bounds-masked u32 gather backend for Part A (TECH_DEBT). diff --git a/.claude/board/entries/2026-10-16-wcoj-probes-1-3.md b/.claude/board/entries/2026-10-16-wcoj-probes-1-3.md new file mode 100644 index 000000000..85d13c656 --- /dev/null +++ b/.claude/board/entries/2026-10-16-wcoj-probes-1-3.md @@ -0,0 +1,147 @@ +# 2026-10-16 — WCOJ map, items 1–3 run: fixture, bag oracle, seek, GYO, P-7 + +**Status:** MEASURED (tests in this PR). Feeds D-ASM-11; nothing routed or wired. + +The co-research map (`2026-10-16-coresearch-wcoj-leapfrog.md`, PR #1586) listed +ADOPT-NOW test items, probes P-1 and P-3, and an insufficiency proof before P-7. +This entry records what running them showed. + +## What landed + +| item | where | tests | disable run (went red) | +|---|---|---|---| +| X6: F4 cyclic fixture (2+2+2 × k) | `lance-graph-metta/examples/wcoj_probe.rs` | semijoin chain keeps all 2k rows; datafrog finds no triangle | — (the fixture is the falsifier of A0) | +| X17: bag oracle | same file | datafrog with counts == nested loop on a bag (382 keys, multiplicity 650) and on a set | counts collapsed to 1: the bag goes red, the set stays green (kept as a test) | +| P-1: galloping seek | `lance-graph-contract::ordered_lane::{gallop_partition_point, SealedFacetLane::seek_from}` | bound, regime table, edges, witnessed seek | start every seek at 0: the bound and the dense regime go red | +| P-3: GYO α-acyclicity | `lance-graph-relalg::join_body::is_alpha_acyclic` | 7 acyclic, 4 cyclic bodies (triangle, 4-cycle, barbell, triangle with a tail) | always `true`: triangle red; skip the ear-variable step: a single atom reads as cyclic | +| P-7: exact cyclic mask with today's ops | `wcoj_probe.rs` | equals datafrog on F4 and on a mixed fixture with real triangles | drop the `tc == v` conjunct: it collapses to the semijoin chain | + +## P-1 result: galloping is a middle band, not a win everywhere + +Comparisons per seek over N = 2²⁰ keys (counted, not timed): + +| N/m | galloping | fresh binary search | linear step | +|---:|---:|---:|---:| +| 2 | 4.00 | 21.00 | 3.00 | +| 16 | 11.00 | 21.00 | 17.00 | +| 256 | 19.00 | 21.00 | 256.94 | +| 4,096 | 26.90 | 21.00 | 4,081 | +| 65,536 | 32.88 | 21.00 | 61,441 | + +- The bound holds with constant 4: `m·(2·log₂(N/m) + 4)`. The pre-registered + constant was 2, and N/m = 4,096 measured 26.9 against 26, from the ceilings + in the bracket search and the cursor check. The slope matches; the test pins + the measured constant. +- The pre-registered kill ("no speedup over a fresh `partition_point` at + N/m ≥ 10³") was stated the wrong way round. Galloping beats a fresh binary + search only for N/m below about 1,000, and beats a linear step only above + about 16. A leapfrog seek therefore wants a step-then-gallop-then-search + hybrid; that is a design input for D-ASM-11, not a kill. +- Timed below; see the next section. + +## P-1 timed on v4: the hybrid seek + +`examples/seek_probe.rs`, `avx512f=true`, ns per seek. Each row is the second of +two runs; the other run agrees within about 20%. + +| N | N/m | gallop | fresh search | linear | hybrid | +|---|---:|---:|---:|---:|---:| +| 2²⁰ | 2 | 3.6 | 22.8 | 2.3 | 1.7 | +| 2²⁰ | 16 | 21.8 | 26.5 | 11.5 | 8.0 | +| 2²⁰ | 256 | 62.9 | 102.4 | 167.2 | 63.3 | +| 2²⁰ | 4,096 | 159.8 | 100.4 | 2,624 | 112.2 | +| 2²⁰ | 65,536 | 102.1 | 52.7 | — | 49.4 | +| 2²⁴ | 2 | 5.3 | 53.3 | 3.4 | 3.6 | +| 2²⁴ | 16 | 41.0 | 53.6 | 24.7 | 19.6 | +| 2²⁴ | 256 | 170.4 | 260.3 | 336.8 | 151.0 | +| 2²⁴ | 4,096 | 258.4 | 246.8 | 4,605 | 291.5 | +| 2²⁴ | 65,536 | 526.9 | 271.6 | — | 261.1 | + +`seek_partition_point(keys, from, gap_hint, pred)` routes as follows: +- a gap of at most 16: linear step; +- `gap² ≥ N`: a whole-lane binary search; +- otherwise: gallop. + +It is fastest or within noise of the fastest at every ratio. Two findings +moved the thresholds away from the comparison counts: +- **A linear step beats galloping up to a gap of about 16 in time**, although + galloping makes fewer comparisons there (11 vs 17). Branch prediction + reverses the count. +- **A fresh search of the whole lane beats a search of the remaining tail by + about 2×.** The first hybrid searched the tail. A whole-lane search revisits + the same top midpoints, which stay in cache; a tail search does not. The + whole-lane search gives the same answer, because `pred` holds below the + cursor. + +The test `hybrid_seek_routes_by_gap_hint` pins the routing. Disabling the +whole-lane branch turns it red. + +## P-7 result: expressible, not at the bound + +mask-risc compares a lane with a constant, never with another lane. The exact +triangle mask needs `s_dst[rb[a]] == tc[a]`, so today it can only be written as +a disjunction over the value domain, `∨_v (EqU32Via(rb, s_dst, v) ∧ tc == v)`. +That is correct, but the program grows by about 4 ops per value (66 ops at +|D| = 16, 1,026 at |D| = 256), so the row work is Θ(rows · |D|). This misses +N^1.5 once |D| > √N. Two things are missing: a lane-to-lane comparison through +the fk, and, for non-functional relations, a sorted-run intersection. The +functional-lane layout cannot represent the n:m case at all. Rule 25 is met: the +insufficiency is shown, so one fused op may be proposed. It is not built here. + +## DuckDB (two Haiku scouts; source clone at `e1ecb5b8`, plus web) + +- Core: no leapfrog, trie join or WCOJ in `src/` or `extension/`. The + patterns `leapfrog`, `triejoin`, `wcoj` and `multiway` had 0 hits, which is + a search result, not proof of absence. +- Leapfrog-adjacent in core, read here: the ASOF join gallops forward from a + cursor ("exponential search … to avoid thrashing the block manager", + `physical_asof_join.cpp:1182`). That is the P-1 kernel. IEJoin and the + piecewise merge join handle range predicates (`plan_comparison_join.cpp:223-271`). + Join filters (min/max, Bloom) are pushed to the probe side. A semijoin + reduction is preserved in the delim-join rewriter (`delim_join_cte_rewriter.cpp:1172`). + The hypergraph code is for join ordering (`query_graph_manager.cpp:399`). +- Research on DuckDB, read from abstracts and summaries only: + - WCOJ through adaptive factorization (CIDR 2025); + - predicate transfer as the `robust` community extension; + - RPT+ (VLDB 2026, on DuckDB 1.3.0); + - Parachute. + These are leads, not mechanism claims. + +## Bind DTO synergies (for SAP and others) + +The bind surface is quack's `Binder` trait (`lance-graph-quack/src/bind.rs:92`): +`table`, `live`, `field → BoundField{col, kind, validity}`, `code`. It never +mints. Implementers today: +- `lance-graph-sap::CatsBinder`; +- `lance-graph-dir-sim`; +- `lance-graph-report` (its test); +- `lance-graph-glove-parity`. + +- **D-ASM-1/1b should implement `Binder`, not add a binder.** Atom and type + names resolve through `code`, and the label codebook sits behind it. The + content address (D-ASM-12) is issued at ingest; binding never mints it, + which matches the trait's contract. +- **Keep codebook storage separate.** CATS codes are batch-local (the batch + dictionary). Atom addresses must be stable across batches. Share the trait, + not the dictionary. +- **SAP optional fields.** `CatsBinder::field` refuses optional fields because + CATS stores NULL as a sentinel. `BoundField.validity` and the three-valued + `Draft::bind` already exist (`bind.rs:77-89`). A validity plane built at + `CatsBatch::bind` would unlock them. This is a candidate, not built. +- **Joins do not bind yet.** `Draft` is one table (`bind.rs:195`), so a + multi-atom body (D-ASM-11, or SAP CATS ⋈ employee assignment) has no bind + path. This gap is shared by MeTTa bodies and SAP joins. +- **SAP aggregates across n:m joins need the X17 oracle.** For example, hours + per cost center through an n:m assignment. A mask cannot carry the count, so + this is the fan-out undercount D-ASM-3 names. +- **`Registrar` has no production implementation** (`bind.rs` module doc). + An atomspace-lance type registration would be the first real one; it is not + proposed here. + +## OPEN + +- The hybrid seek on a `FacetCascade` lane (timed here on `u32` keys only). +- The fused lane-vs-lane-through-fk op and a sorted-run intersection: their + names and shape, after the insufficiency proof above. +- A multi-table `Draft` for joins: shared by D-ASM-11 and SAP. +- The DuckDB research items above were read at abstract or secondhand level only. diff --git a/.claude/board/entries/README.md b/.claude/board/entries/README.md index 412f31797..949112aa0 100644 --- a/.claude/board/entries/README.md +++ b/.claude/board/entries/README.md @@ -25,10 +25,12 @@ index row, (3) no duplicate entry id. Checks 1 and 2 are deliberately opposite directions; the stranding this convention prevents shows up in exactly one of them, never both. -356 entries, 2026-08-06 .. 2026-10-16. +358 entries, 2026-08-06 .. 2026-10-16. | date | entry id | finding | file | |---|---|---|---| +| 2026-10-16 | `wcoj-probes-1-3` | | [2026-10-16-wcoj-probes-1-3.md](2026-10-16-wcoj-probes-1-3.md) | +| 2026-10-16 | `wcoj-closing-atoms-council` | | [2026-10-16-wcoj-closing-atoms-council.md](2026-10-16-wcoj-closing-atoms-council.md) | | 2026-10-16 | `synergy-map-since-1011` | | [2026-10-16-synergy-map-since-1011.md](2026-10-16-synergy-map-since-1011.md) | | 2026-10-16 | `opencog-oracle-scope` | | [2026-10-16-opencog-oracle-scope.md](2026-10-16-opencog-oracle-scope.md) | | 2026-10-16 | `D-ASM-1` | | [2026-10-16-metta-datalog-assimilation-council-v3.md](2026-10-16-metta-datalog-assimilation-council-v3.md) | diff --git a/.claude/plans/quack-multitable-bind-v1.md b/.claude/plans/quack-multitable-bind-v1.md new file mode 100644 index 000000000..a568d9638 --- /dev/null +++ b/.claude/plans/quack-multitable-bind-v1.md @@ -0,0 +1,159 @@ +# quack-multitable-bind-v1 — one bind path for joins, shared by MeTTa bodies and SAP + +> Status: PROPOSAL (2026-10-16). Nothing below is built. D-QMB-0..5. +> Feeds D-ASM-1 / D-ASM-11 (MeTTa bodies) and SAP CATS joins +> (`sap-glove-quack-v1.md`). Read against `origin/main` + PR #1587. + +## 0. Why + +`Draft` binds one table (`lance-graph-quack/src/bind.rs:192-199`). A predicate +naming another table is refused (`BindError::WrongTable`, `bind.rs:295-301`). +The executor already runs joins inside one program, through three leaves that +read a caller-supplied `Foreign { planes, lanes }`: + +- `Filter::EqU32Via { fk, key: ForeignLane, v }` — a foreign value compared + through the address lane, no foreign plane (`lib.rs:348-365`); +- `Filter::Semijoin { fk, foreign: ForeignPlane }` — existence through a + RESIDENT foreign plane, lowered to `MaskOp::Gather` (`lib.rs:366-392`); +- `Agg::GroupSumViaI32` — a group key read through the address lane. + +So execution is not the gap. Two things are: + +1. **No name reaches those leaves.** A caller hand-picks `ForeignLane(0)`, + `ForeignPlane(1)` and must assemble `Foreign` in the same order. No binder + says which field of which table sits in which slot. +2. **No binder says which lane is an address.** `Binder::field` returns a + `BoundField { col, kind, validity }`; nothing marks `lines.partner_id` as + holding row coordinates of `partner`. + +Both consumers hit the same gap (entry `2026-10-16-wcoj-probes-1-3.md` +§ "Bind DTO synergies"): a MeTTa body `(R a b) (S b c)` and an SAP +`CATS ⋈ assignment` both need one head table plus foreign reads. + +## 1. Frozen decisions + +1. Bind once, execute numeric: the bound result holds no name (`bind.rs:15-20`, + `tests/string_fence.rs`). +2. The binder never mints (`Binder::code`, `bind.rs:99-102`). +3. An address lane holds row coordinates of another table; it is not a foreign + key in the relational sense and the plan does not add one (rule: hops are + coordinate ranges). The existing IR field name `fk` stays as it is. +4. Chained hops fold into one program; nothing is materialized per hop + (`Semijoin` already forbids a plane another `Program` produced, `lib.rs:193-198`). +5. One trait: `Binder` grows by DEFAULT methods only, so the four implementers + (`lance-graph-sap::CatsBinder`, `lance-graph-dir-sim`, `lance-graph-report`'s + test, `lance-graph-glove-parity`) compile unchanged. +6. Codebooks stay per binder. CATS codes are batch-local; atom addresses are + stable. Share the trait, never the dictionary. + +## 2. Shape + +### 2.1 The binder declares addresses + +```rust +trait Binder { + // … existing four methods … + /// The table whose rows `field` addresses, if it is an address lane. + fn addresses(&self, _table: TableId, _field: Col) -> Option { None } +} +``` + +Default `None`: nothing changes for a binder that declares no address lane. + +### 2.2 The draft names the hop + +```rust +table("lines") + .via("partner_id") // the address lane, by name + .where_eq("country", "DE") // a field of the ADDRESSED table + .count() +``` + +`via(field)` switches the predicate scope to the addressed table until the +next `via` or `back()`. Nested `via` chains (`lines → partner → region`) are +D-QMB-4, not v1. + +### 2.3 The bound result carries its slot layout + +```rust +pub struct Bound { + pub query: Query, + /// Slot i of `Foreign::lanes` must be this (table, column). + pub lanes: Vec<(TableId, Col)>, + /// Slot i of `Foreign::planes` must be this (table, plane). + pub planes: Vec<(TableId, Mask)>, +} +``` + +Slots are assigned in first-use order and deduplicated, so the same foreign +field read twice takes one slot. The caller (or a later catalog) builds +`Foreign` from this list; a mismatch is a caller bug the executor already +rejects by lane kind (`reference.rs:383-388`). `Draft::bind` keeps returning +`Query` for one-table drafts; the new entry point is `Draft::bind_joined`. + +### 2.4 What a hop binds to + +For `via(a)` to table `T` with predicates `p₁ … pₙ`: + +- each `T.f = literal` binds to `EqU32Via { fk: a, key: slot(T, f), v }`; +- the hop always adds `Semijoin { fk: a, foreign: slot(T, live(T)) }`. + +The second leaf is load-bearing. `eq_u32_via_to_mask` drops an out-of-range +address (`ndarray/src/simd_masking_ops.rs:3156-3158`) but does not read the +foreign table's live plane, so a deleted target row still matches. And +`NOT EqU32Via` keeps every dangling address. Conjoining the live-plane +semijoin makes a dangling or deleted target a known FALSE. + +Three-valued rules follow `Filter::sql_where` (`lib.rs:506-535`): +`EqU32Via` is UNKNOWN when its address lane is nullable; `Semijoin` is never +UNKNOWN (NULL address = FALSE). + +### 2.5 What v1 does not bind + +- `<>` on a foreign field: `NeU32Via` does not exist. A `<>` through a hop is + `NOT EqU32Via ∧ Semijoin(live)`; v1 refuses it with a new + `BindError::Unsupported` until a test pins that rewrite. +- `I32` foreign fields: there is no `EqI32Via`. Refused. +- Optional foreign fields with their own validity plane: refused until the + foreign validity plane is a `ForeignPlane` slot conjoined like `live`. +- Comparing two lanes (a cyclic body's closing atom) and n:m runs: these need + the predicates under council in `council-p7` (Part A / Part B). The bind + path takes them when they exist; it does not invent them. + +## 3. Work items + +- [ ] **D-QMB-0** tests first (`lance-graph-quack/src/bind.rs` tests): + - a two-table draft binds to exactly the query an expert writes + (`EqU32Via` + live `Semijoin`) and the expected slot layout; + - binder call count: one `table`, `live`, `field`, `code`, `addresses` per + distinct name, as the one-table test pins (`bind.rs:536-557`); + - executed answer == a nested-loop oracle over the same rows, including a + dangling address and a deleted target row (both must drop); + - disable runs: drop the live `Semijoin` → the deleted-target row is kept + (red); skip slot dedup → two slots for one field (red). +- [ ] **D-QMB-1** `Binder::addresses` (default) + `Draft::via` + `Bound` + + `bind_joined`; `BindError::{NotAnAddress, Unsupported}`. +- [ ] **D-QMB-2** SAP first consumer: CATS hours through the employee + assignment where the assignment is functional (one row per employee): + `CatsBinder::addresses`. Oracle: the SAP fixture's own expected rows. +- [ ] **D-QMB-3** MeTTa first consumer: a two-atom acyclic body + `(R a b) (S b c)` binds through `via`; answer == datafrog SET + (`lance-graph-metta/examples/wcoj_probe.rs` oracle). +- [ ] **D-QMB-4** nested `via` chains (`a → b → c`): one program; needs a + read through two address lanes. Check first whether `GatherChain`'s + withdrawal (#1521/#1522) already settled this shape. +- [ ] **D-QMB-5** cyclic bodies and n:m relations: after the `council-p7` + predicates land; a cyclic body binds its closing atom to Part A or Part B. + +## 4. Non-goals + +- A table catalog or storage owner (`Registrar` stays unimplemented, `bind.rs:403-412`). +- Aggregates across an n:m join (fan-out counting is D-ASM-3). +- Join ordering, a planner, DataFusion (in grace). +- Any change to the executor beyond what D-QMB-5 inherits from the council. + +## 5. Kill conditions + +- The bound query differs from the expert query on any D-QMB-0 fixture. +- A deleted or dangling target is kept by `=` (the live semijoin is missing). +- An existing `Binder` implementer needs an edit to compile. diff --git a/.claude/plans/wcoj-closing-atoms-v1.md b/.claude/plans/wcoj-closing-atoms-v1.md new file mode 100644 index 000000000..d42fafede --- /dev/null +++ b/.claude/plans/wcoj-closing-atoms-v1.md @@ -0,0 +1,236 @@ +# wcoj-closing-atoms-v1 — the exact cyclic existence mask: two closing atoms + +> Status: RATIFIED SPEC (5+3 council, 2026-10-16); not authorized to build. +> D-ASM-11 is "Queued (not authorized)"; ratifying this spec does not change that. +> Council record: board entry `2026-10-16-wcoj-closing-atoms-council.md`. +> Evidence: entry `2026-10-16-wcoj-probes-1-3.md` (P-7 insufficiency); +> co-research map `2026-10-16-coresearch-wcoj-leapfrog.md` (PR #1586). +> Grades: [G] read in source; [H] design hypothesis awaiting its gate. + +## 0. Two questions, two populations + +The semijoin chain over-approximates a cyclic body (F4 keeps every row of an +empty query). The exact mask needs a closing atom. The premise audit split the +question in two, by signature: + +- **Q-A, comparison family.** Compare a local lane with a foreign lane read + through an address lane. Three-valued like `EqU32Via`. O(1) work per row. + Head population: the table holding both local lanes (triangle fixture: node + rows `a`, lanes `rb`, `tc`). +- **Q-B, EXISTS family.** For each row of a named head population (triangle: + R-edge rows), do two set-valued addresses (sorted runs) share a value? + Two-valued NULL rule like `Semijoin`. Data-dependent work per row. + +The two populations coincide only when R is functional. Part A's head rows are +node rows `a`; Part B's are R-edge rows; a Part A mask is never a Part B mask +on the same index space. + +## 1. Frozen decisions + +1. No opcode merely to pass an experiment; each opcode carries its own + insufficiency proof (decision 14). +2. No second algebra or interpreter beside mask-risc / Quack. An IR that + describes is fine; a layer that executes beside mask-risc is not. +3. Chained hops fold into one mask expression; nothing is materialized per hop. +4. Nothing is a foreign key in the relational sense; a hop is a coordinate + range (D-TELEPORT). The IR field name `fk` on `Pred::EqU32Via` is an address + lane; new fields keep it for continuity. +5. Performance is cited only from the v4 arm, per element, tier printed. +6. All SIMD comes from `ndarray::simd`; W1a contract + (`ndarray/.claude/knowledge/vertical-simd-consumer-contract.md`); a + scalar-only kernel states its exemption in its doc comment. +7. The frontier stays inside one mask-risc program and never crosses as a + `ForeignPlane`; integer semirings only; routing lives in relalg/R2IL, not the + planner; DataFusion is in grace. +8. No strings in rules, queries or IR. +9. Lance family upstream; AdaWorldAPI forks elsewhere; ndarray `master`. +10. [H] Part B is not expressible with today's ops: re-anchoring on an edge + table leaves a second fan-out, which the gremlin adapter refuses as + `Refusal::NonFunctionalChain` (`quack/tests/gremlin_parity.rs:515`, `:33-35`). +11. k-way is not AND of 2-way atoms: `∃c∈A∩B ∧ ∃c∈B∩C ∧ ∃c∈A∩C` is weaker than + `∃c∈A∩B∩C`, the semijoin-chain over-approximation one level up. k-way needs + its own fixed arity (`Pred` is `Copy + Eq`, `ir.rs:126`). +12. A new `ExecError` variant is accepted and paired: lgj-abi + `exports.rs:1737-1769` matches `ExecError` exhaustively, so the Part B + lance-graph PR carries the lgj-abi status mapping in a paired PR. + `LaneNotOrdered { lane }` is NOT reused: its `lane` indexes local lanes + (`value.rs:181`), and foreign/local errors are kept apart + (`ForeignLaneOutOfRange` vs `LaneOutOfRange`, `value.rs:185-187`). +13. Planner `AdjacencyBatch::intersect` (all-pairs two-pointer merge on u64) + and the logical-only `WcoJoin` node are not reused. +14. Insufficiency, per part: + - A: #1587 `p7_disjunction_costs_one_pass_per_domain_value`: the functional + disjunction `∨_v (EqU32Via(rb, s_dst, v) ∧ tc == v)` grows by about 4 ops + per value (66 at |D| = 16, 1,026 at 256), so rows · |D|. Measured. + - B: n:m cannot be represented in functional lanes (P-7), and decision 10. + Its per-value disjunction `∨_c (Semijoin(x, S∋c) ∧ Semijoin(y, T∋c))` + needs one resident plane per value. Argued, not measured: G0 measures it. + +## 2. Part A — `Pred::EqU32ViaLane { fk, key, lane }` + +`EqU32Via` with the constant replaced by this row's own lane value. Row i +matches iff `fk[i] < len(foreign.lanes[key])` and +`foreign.lanes[key][fk[i]] == lanes[lane][i]`. + +- **ndarray** `eq_u32_via_lane_to_mask(index, table, local, out_words)` next to + `eq_u32_via_to_mask` (`simd_masking_ops.rs:3141`); re-export beside + `simd.rs:844`; parity block, code 0xD41. + - Scalar, exemption stated. Precedent: `eq_u32_via_to_mask` and + `mask_gather_u32_under` (scalar because the address comes out of another + array). The sibling's stated reason ("no vector gather on any backend", + `:3114-3120`) is corrected in the same PR to: "an `unsafe`, unbounded f32 + gather exists on several backends (`simd_avx512.rs:236-246`, `simd_avx2.rs:703`, + `simd_scalar.rs:1099`); there is no bounds-masked u32 gather-and-compare + wrapper, and the out-of-range-is-false contract needs one; whether one pays + on random access is unmeasured." The gather backend is a TECH_DEBT row. + - `local.len() != index.len()` panics with the facade's assert message. + - W1a criterion 7: the PR body names the consumer site (`run_pred`). +- **mask-risc** + - `run_pred` arm: gate ANDed after, as `EqU32Via` (O(1) per row). + - `pred_lane_and_kind` (`reference.rs:64-93`, "the one place") returns a small + fixed list of local lanes, so `fk` and `lane` are both checked through it; + `check_foreign_lane(key, U32)` beside the `EqU32Via` check (`:386-388`). + - `eval_pred` oracle arm. +- **quack** `Filter::EqU32ViaLane { fk: Col, key: ForeignLane, lane: Col }`, + builder, lowering, `gate_walk`, `Fx::oracle`, `sql_where` doc row. + - Three-valued: UNKNOWN if `fk` OR `lane` is NULL. `leaf_validity` + (`lib.rs:606-614`) returns up to two masks and `rewrite` (`:580-592`) gates + on all of them. A NULL in the FOREIGN value lane is not modelled (`:537`); + Part A inherits this, stated in its doc. +- **Claim [H]:** `R.valid ∧ T.valid ∧ Semijoin(rb, S.valid) ∧ + EqU32ViaLane(rb, s_dst, tc)` is the exact triangle mask over functional R, S, + T, in one program whose op count does not grow with |D|. + +## 3. Part B — `MaskOp::RunsMeet { x, y, s_off, s_val, t_off, t_val, under, dst }` + +(v1.1 `Pred::RunsIntersectVia`, renamed and re-placed by the council.) + +Row i is set iff `under` keeps it (or there is no gate), `x[i]`, `y[i]` index +runs that exist, and `s_val[s_off[x[i]]..s_off[x[i]+1]]` and +`t_val[t_off[y[i]]..t_off[y[i]+1]]` share a value. A key with no run → clear. + +- **Placement: `MaskOp`, beside `Gather`.** `Gather{under}` is the template + (`exec.rs:1753-1782`, `mask_gather_u32_under`): the kernel visits only rows + set in its `under` operand; a zero gate word still costs one test, so the + work is O(words + Σ over gated rows). This matches `D-V4-FOLD-LAB.md:175-177` + (WCOJ as a physical intersect, not a semantic op). +- **What `under` is under quack lowering.** `gate_walk` (`lib.rs:2079-2097`), + `emit_gated` (`:2243`, `:2258`) and `assign_slots` (`:2340`, `:2356`) use the + AND's first resident plane (or the accumulator when no plane exists), not the + conjunction of all earlier conjuncts. A preceding `Semijoin` or `Cmp` does not + narrow the rows `RunsMeet` visits. In the triangle body the gate is `R.valid`. + Narrowing on the accumulator is the cost already filed in `ISSUES.md`. +- **Order invariant: checked completely, at validate, before any tile runs.** + For each of the two CSRs: `len(off) ≥ 1`, `off` non-decreasing, + `off[last] ≤ len(val)`, every run non-decreasing. A violation refuses with + the new `ExecError::ForeignLaneNotOrdered { key }` before any output is + written. Why at validate: a refusal raised inside the tile loop would leave + earlier tiles written in a sink terminal (`Keep`, `GroupSum*`, `ScatterOrU32`, + `BlendI32`; `exec.rs:1319-1337`), and a check on visited elements only would + make Ok/Err depend on `under`, the data and the gallop schedule, so the oracle + and exec could disagree. It also closes the `NOT` hazard: on an unsorted run + the leaf can miss a common value (never invent one), and `NOT` would turn + that into a wrongly kept row. Cost: O(|S| + |T|) per execution, stated in the + op's doc. A sealed layout witness (per Lance version) that removes this cost + is a named follow-up. +- **ndarray** `runs_meet_under_to_mask(x, y, s_off, s_val, t_off, t_val, under, + out_words)`; re-export; parity block, 0xD6x. Precondition: the order + invariant holds (checked by the caller at validate; `debug_assert` in the + kernel). Returns nothing. Out-of-range keys → clear bit. + - Per row: the shorter run proposes; the longer seeks forward from a cursor; + exit at the first common value. + - Seek routing follows `ordered_lane::seek_partition_point` (step if gap ≤ 16; + whole-range search if gap² ≥ len; else gallop). Both thresholds came from + v4 timing against the comparison counts (branch prediction; cached top + midpoints, measured on 2²⁰–2²⁴ lanes). Neither is assumed to transfer to + per-row runs: re-measure both inside ndarray, and record a regime where + the whole-range branch stops winning. + - Scalar, exemption stated. Precedent: `mask_gather_u32_under` and + `eq_u32_via_to_mask` (the address comes out of another array). Not + `mask_shift_morton`, whose pass 1 is vectorized. + - A `#[cfg(test)]` element counter in the kernel feeds G4; never a probe-local + copy of the kernel. +- **mask-risc**: `MaskOp` variant. Exhaustive matches that grow an arm: + exec tile loop, `reference.rs` `dst_of` / validate / `run` (×2), `ir.rs` touch + loop, the two fuse recognizers (explicit `=> return None`), `ir.rs` + `op_histogram` (`:1279-1341`, a new `runs_meet` field), `program_collapse_probe` + `op_dst`. Validate checks 2 local + 4 foreign `U32` lanes, the order + invariant, and `under != Scratch(dst)`. +- **lgj-abi** (paired PR): `exec_error_to_status` arm for + `ForeignLaneNotOrdered`. +- **quack** `Filter::RunsMeet { .. }` leaf, its own `Node` variant, 2 emitter + arms, `gate_walk`, `gate_shape`, the test helper at `lib.rs:3148`. + - Three-valued: Semijoin's row. Gate on `valid(x) ∧ valid(y)` (two masks, the + same `leaf_validity` change as Part A), then complement the GATED leaf. A + NULL key is a known FALSE, never UNKNOWN; under `NOT` it is KEPT. Part A's + NULL row is UNKNOWN and dropped under both polarities. + - The `if let Filter::Semijoin` at `lib.rs:584` becomes `matches!` over both + EXISTS leaves. +- **Layout.** An edge table sorted by `(key, val)` plus a run index is the same + carrier. Probe-local `Csr { off: Vec }` shapes exist only in probes + (`holo_coord_probe.rs:329-332`); no contract carrier is added here. +- **Not D-TELEPORT.** Each run is a coordinate range, but the closing step + intersects run CONTENTS in C's coordinate space (merge / gallop). That is not + D-TELEPORT's `[max lo, min hi)`; do not cite it as covering this step. +- **Claim [H]:** `R.valid ∧ RunsMeet(x = rb, y = ra, S, T)` is the exact mask + over R-edge rows; elements touched are Σ_{(a,b)∈R} min(|S_b|, |T_a|) ≤ N^1.5 + by Cauchy–Schwarz, and comparisons add a factor of at most 1 + log(max/min). + +## 4. Non-goals + +- Multi-table `Draft` binding (`quack-multitable-bind-v1`, D-QMB). +- Counting (P-11). It needs strictly increasing runs or multiplicity lanes; + its oracle is the X17 bag oracle in `wcoj_probe.rs`. G3 is SET-only by design + and cannot detect a multiplicity defect. +- `materialize_bindings`; the relalg router (A3); GYO wiring. +- k-way (decision 11); bodies beyond the triangle. +- A gather backend for Part A (TECH_DEBT); the sealed CSR witness. +- Hash tries; any change to `Gather` or `EqU32Via` semantics. + +## 5. Gates (pre-registered) + +- **G0** Write F2, F3a, F3b (map entry) in `wcoj_probe.rs` first, each with its + datafrog SET answer. Measure Part B's per-value disjunction (decision 14) on + one of them. +- **G1** `R.valid ∧ T.valid ∧ Semijoin(rb, S.valid) ∧ EqU32ViaLane(rb, s_dst, tc)` + == datafrog SET on F4 ×{1, 8, 50} and `mixed` {30, 64, 200}, plus one fixture + with `fk[i] ≥ len(foreign)`. +- **G2** Part A's program op count == a fixed constant AND < the disjunction + program's count at |D| = 16 (`p7_disjunction_costs_one_pass_per_domain_value` + counts `program.ops.len()`). +- **G3** Part B: the R-edge mask == the edge-level oracle + `{(a,b) ∈ R : S_b ∩ T_a ≠ ∅}`, and its projection onto `a` + (`Terminal::ScatterOrU32`, `ir.rs:338`) == datafrog SET, on F2, F3a, F3b, F4 + at 10²..10⁵. +- **G4** Elements touched (the kernel counter) on F3b: fitted slope in + 1.5 ± 0.1 over 10³..10⁶. Comparisons are reported, with the log term in the + prediction, but not gated. +- **G5** disable runs, each red: + - A: compare with `v = 0` instead of the local lane; + - A: drop `lane`'s validity in `leaf_validity`, on a NULL-`lane` row whose + stale payload equals `foreign[key][fk]` (sql_null_3vl fixture) → kept; + - B: fixed proposer AND no galloping (the map's A1′), F3a oriented hub-run + first → elements touched ≥ 10× at 10⁶; + - B: ignore `under` on a 1%-survivor gate → elements touched ≥ 10×; + - B: restore the Semijoin-only `if let` → the NULL-key row is DROPPED under + `NOT` (correct: kept); the paired correct run keeps it; + - B: an unsorted run → `ForeignLaneNotOrdered` before any write, with a sink + terminal and the bad run in a later tile; a sorted twin stays silent. +- **G6** Seek thresholds: an inertness test per threshold (raising the step + threshold to ∞ and lowering it to 0 each change the elements-touched count on + a fixture that exercises it). +- **G7** Portability: the one scalar body per kernel runs identically on each + target (`simd-masking-parity`, `wasm-simd-parity`, `neon-simd-parity`). This + catches width/portability defects only, not backend divergence. Clippy + `-D warnings`; mask-risc / quack / metta / sql_null_3vl / lgj-abi green. +- **G8** v4 ns per row printed with the tier; no v3 numbers. +- **G9** CI does not run `wcoj_probe`; add a CI line or state G1–G5 as + local-only in the PR body. + +## 6. Order of work + +1. Authorization of D-ASM-11. +2. Part A: ndarray PR (with its blackboard entry) → lance-graph PR (with its + STATUS_BOARD row and LATEST_STATE inventory line). +3. Part B: ndarray PR → lance-graph PR + paired lgj-abi PR, each with its board + rows in the same commit. diff --git a/crates/lance-graph-contract/examples/seek_probe.rs b/crates/lance-graph-contract/examples/seek_probe.rs new file mode 100644 index 000000000..87a0a79b8 --- /dev/null +++ b/crates/lance-graph-contract/examples/seek_probe.rs @@ -0,0 +1,98 @@ +//! D-ASM-11 pre-work, P-1 timed: four cursor seeks over a sorted `u32` lane. +//! +//! `m` increasing targets spread evenly over `N` keys; each seek starts at the +//! previous answer. Reports ns per seek and ns per key (`N`) for: +//! galloping, a fresh whole-lane binary search, a linear step, and the hybrid +//! `seek_partition_point` given the true gap as its hint. Every method's +//! answers are checked against `partition_point` before any timing counts. +//! Only a v4 run (`avx512f=true` printed below) may be cited. +//! +//! ```text +//! CARGO_PROFILE_RELEASE_DEBUG=0 RUSTFLAGS=-Ctarget-cpu=native \ +//! cargo run --release -p lance-graph-contract --example seek_probe +//! ``` + +use std::hint::black_box; +use std::time::Instant; + +use lance_graph_contract::ordered_lane::{gallop_partition_point, seek_partition_point}; + +fn best(reps: usize, mut f: impl FnMut() -> u64) -> (u64, f64) { + let mut t_best = f64::INFINITY; + let mut out = 0; + for _ in 0..reps { + let t = Instant::now(); + out = black_box(f()); + t_best = t_best.min(t.elapsed().as_secs_f64()); + } + (out, t_best) +} + +fn main() { + println!( + "tier: avx512f={} avx2={}", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx2") + ); + println!("N | N/m | gallop ns/seek | fresh ns/seek | linear ns/seek | hybrid ns/seek"); + for log_n in [20u32, 24] { + let n = 1usize << log_n; + let keys: Vec = (0..n as u32).map(|i| 2 * i).collect(); + for ratio in [2usize, 16, 256, 4096, 65536] { + let m = n / ratio; + let targets: Vec = (0..m).map(|j| ((2 * n * j) / m) as u32 + 1).collect(); + let want: u64 = targets + .iter() + .map(|&t| keys.partition_point(|&k| k < t) as u64) + .sum(); + let (g, tg) = best(5, || { + let mut c = 0; + let mut s = 0u64; + for &t in &targets { + c = gallop_partition_point(&keys, c, |&k| k < t); + s += c as u64; + } + s + }); + let (f, tf) = best(5, || { + targets + .iter() + .map(|&t| keys.partition_point(|&k| k < t) as u64) + .sum() + }); + let (l, tl) = if ratio <= 4096 { + best(3, || { + let mut c = 0usize; + let mut s = 0u64; + for &t in &targets { + while c < n && keys[c] < t { + c += 1; + } + s += c as u64; + } + s + }) + } else { + (want, f64::NAN) + }; + let (h, th) = best(5, || { + let mut c = 0; + let mut s = 0u64; + for &t in &targets { + c = seek_partition_point(&keys, c, ratio, |&k| k < t); + s += c as u64; + } + s + }); + assert_eq!((g, f, l, h), (want, want, want, want), "N={n} N/m={ratio}"); + let per = |t: f64| t * 1e9 / m as f64; + println!( + "2^{log_n} | {ratio} | {:.2} | {:.2} | {:.2} | {:.2}", + per(tg), + per(tf), + per(tl), + per(th) + ); + } + } +} diff --git a/crates/lance-graph-contract/src/ordered_lane.rs b/crates/lance-graph-contract/src/ordered_lane.rs index f2d1d83d8..80c9849e5 100644 --- a/crates/lance-graph-contract/src/ordered_lane.rs +++ b/crates/lance-graph-contract/src/ordered_lane.rs @@ -331,6 +331,27 @@ impl SealedFacetLane { Ok(bound_unwitnessed(&self.keys, prefix)) } + /// **The witnessed seek.** The first row at or after `from` whose key is + /// not below `key` (a lower bound that starts at a cursor), found by + /// [`gallop_partition_point`]. Successive seeks with increasing `key` and + /// the previous answer as `from` cost `O(1 + log(gap))` each, which is + /// what a leapfrog join's per-variable intersection needs. + /// + /// # Errors + /// + /// Any [`validate`](Self::validate) error — the seek does not run. + pub fn seek_from( + &self, + w: &OrderedLaneWitness, + from: u32, + key: &FacetCascade, + ) -> Result { + self.validate(w)?; + Ok(gallop_partition_point(&self.keys, from as usize, |k| { + k.cmp_numeric_projection(key) == Ordering::Less + }) as u32) + } + /// **The witnessed bound for an aperture.** A prefix aperture /// ([`SemanticAperture::interval`]) selects a closed key interval, so on a /// validated lane its matches are the row range `[lo, hi)` located by two @@ -365,6 +386,90 @@ impl SealedFacetLane { } } +/// `partition_point` that starts at a cursor and gallops forward: the first +/// index `i >= from` with `!pred(&keys[i])`. It probes `from`, `from + 1`, +/// `from + 3`, `from + 7`, … until one fails `pred`, then binary-searches the +/// last bracket, so a seek that lands `d` rows ahead costs about +/// `2·log2(d) + 2` comparisons instead of `log2(len)`. +/// +/// Precondition, as for `partition_point`: `pred` is true on a prefix of +/// `keys` and false after it, and true at every index below `from` (the +/// cursor never moves backward). Leapfrog triejoin's `seek` needs exactly +/// this (Veldhuizen 1210.0481 §3): a sequence of `m` increasing seeks over +/// `N` keys costs `O(m·(1 + log(N/m)))` in total. +#[must_use] +pub fn gallop_partition_point( + keys: &[T], + from: usize, + mut pred: impl FnMut(&T) -> bool, +) -> usize { + let n = keys.len(); + if from >= n || !pred(&keys[from]) { + return from.min(n); + } + // `lo` always satisfies `pred`; `hi` is the first probe that fails, or `n`. + let mut lo = from; + let mut step = 1usize; + let hi = loop { + let probe = lo.saturating_add(step); + if probe >= n { + break n; + } + if !pred(&keys[probe]) { + break probe; + } + lo = probe; + step = step.saturating_mul(2); + }; + lo + 1 + keys[lo + 1..hi].partition_point(pred) +} + +/// Gaps up to this many rows are stepped one row at a time before galloping. +pub const SEEK_LINEAR_GAP: usize = 16; + +/// The cursor seek that picks its method from the caller's expected gap +/// between successive seeks (in a leapfrog join, the other side's length +/// divided into this one's). The thresholds come from the v4 timings of +/// `examples/seek_probe.rs`, not from comparison counts, which disagree: +/// - a gap of at most [`SEEK_LINEAR_GAP`] rows: step, then gallop if the +/// steps run out (a predictable step beats galloping's branches even where +/// galloping makes fewer comparisons); +/// - a gap whose square reaches the lane length: binary search the WHOLE +/// lane. The answer is the same, since `pred` holds below `from`, and a +/// whole-lane search revisits the same top midpoints, which stay in cache; +/// a search over the shrinking tail measured about 2× slower; +/// - otherwise gallop ([`gallop_partition_point`]). +/// +/// Same contract as [`gallop_partition_point`]; the hint changes only the +/// cost, never the answer. +#[must_use] +pub fn seek_partition_point( + keys: &[T], + from: usize, + gap_hint: usize, + mut pred: impl FnMut(&T) -> bool, +) -> usize { + let n = keys.len(); + if from >= n { + return n; + } + if gap_hint <= SEEK_LINEAR_GAP { + let stop = (from + 2 * SEEK_LINEAR_GAP).min(n); + let mut i = from; + while i < stop { + if !pred(&keys[i]) { + return i; + } + i += 1; + } + return gallop_partition_point(keys, i, pred); + } + if gap_hint.saturating_mul(gap_hint) >= n { + return keys.partition_point(pred); + } + gallop_partition_point(keys, from, pred) +} + /// The raw bound with NO ordering gate: two `partition_point`s over `keys`. /// Correct only on a slice in numeric projection order; on anything else it /// returns a PLAUSIBLE WRONG RANGE — which is exactly the failure the witness @@ -707,4 +812,155 @@ mod tests { Err(WitnessError::VersionMismatch { .. }) )); } + + // ── P-1: galloping seek from a cursor (D-ASM-11, coresearch map 2026-10-16) ── + + /// The hybrid seek routes by its gap hint and agrees with + /// `partition_point`. Its comparison count equals that of the method its + /// rule picks: the linear step at gaps up to `SEEK_LINEAR_GAP`, the + /// whole-lane search once `gap² >= N`, galloping in between. (The + /// thresholds are v4 timings, `examples/seek_probe.rs`; the linear step + /// makes more comparisons than galloping at a gap of 16 and is still + /// faster.) Disable run: ignore the hint (always gallop) and the + /// whole-lane ratios go red. + #[test] + fn hybrid_seek_routes_by_gap_hint() { + let n = 1 << 20; + let keys: Vec = (0..n as u32).map(|i| 2 * i).collect(); + for m in [n / 2, n / 16, n / 256, n / 4096, n / 65536] { + let ratio = n / m; + let targets: Vec = (0..m).map(|j| ((2 * n * j) / m) as u32 + 1).collect(); + let mut hybrid = 0u64; + let mut cursor = 0usize; + for &t in &targets { + let h = seek_partition_point(&keys, cursor, ratio, |&k| { + hybrid += 1; + k < t + }); + assert_eq!(h, keys.partition_point(|&k| k < t), "t={t}"); + cursor = h; + } + let (g, f, l) = seek_costs(n, m); + let picked = if ratio <= SEEK_LINEAR_GAP { + l + } else if ratio * ratio >= n { + f + } else { + g + }; + assert_eq!( + hybrid, picked, + "N/m = {ratio}: gallop {g}, fresh {f}, linear {l}" + ); + } + } + + /// `m` increasing targets spread evenly over `0..2N`, sought over the + /// even keys `0, 2, …, 2N-2`. Returns total comparisons for the galloping + /// seek from the previous answer, the fresh whole-lane binary search, and + /// a forward linear scan from the previous answer. + fn seek_costs(n: usize, m: usize) -> (u64, u64, u64) { + let keys: Vec = (0..n as u32).map(|i| 2 * i).collect(); + let targets: Vec = (0..m).map(|j| ((2 * n * j) / m) as u32 + 1).collect(); + let (mut gallop, mut fresh, mut linear) = (0u64, 0u64, 0u64); + let mut cursor = 0usize; + let mut lin = 0usize; + for &t in &targets { + let g = gallop_partition_point(&keys, cursor, |&k| { + gallop += 1; + k < t + }); + let f = keys.partition_point(|&k| { + fresh += 1; + k < t + }); + while lin < n && { + linear += 1; + keys[lin] < t + } { + lin += 1; + } + assert_eq!(g, f, "galloping seek disagrees with partition_point at {t}"); + assert_eq!(lin, f, "linear scan disagrees at {t}"); + cursor = g; + } + (gallop, fresh, linear) + } + + /// FAILS IF the seek is not amortized `O(m·(1 + log(N/m)))`: total + /// comparisons must stay within `m·(2·log2(N/m) + 4)` at every ratio. + /// Disable run: ignore `from` (start every seek at 0) and the dense + /// ratios exceed the bound. + #[test] + fn p1_galloping_seek_meets_the_leapfrog_bound() { + let n = 1 << 20; + for m in [1usize << 4, 1 << 8, 1 << 12, 1 << 16, 1 << 19] { + let (g, f, l) = seek_costs(n, m); + let ratio = (n / m) as f64; + std::println!( + "N/m = {ratio:>7}: gallop {:.2}/seek fresh {:.2}/seek linear {:.2}/seek", + g as f64 / m as f64, + f as f64 / m as f64, + l as f64 / m as f64 + ); + // Pre-registered as `2·log2(N/m) + 2`; measured one comparison + // over it (ceilings in the bracket and the cursor check), so the + // constant is 4. The slope is the claim. + let bound = m as f64 * (2.0 * ratio.log2() + 4.0); + assert!( + (g as f64) <= bound, + "N/m = {ratio}: {g} comparisons > bound {bound:.0}" + ); + } + } + + /// Where galloping wins, measured in comparisons (the X21 i-cost metric). + /// Against a FRESH binary search it wins when seeks are dense (small + /// N/m); against a forward LINEAR merge it wins when they are sparse + /// (large N/m). The pre-registered kill condition named the fresh search + /// at N/m >= 10^3, where neither is expected to win; this pins both + /// regimes instead. + #[test] + fn p1_galloping_beats_binary_search_dense_and_linear_merge_sparse() { + let n = 1 << 20; + // Dense: N/m = 2. Galloping ~4 per seek, binary search ~21. + let (g, f, _) = seek_costs(n, n / 2); + assert!(g * 3 < f, "dense: gallop {g} vs fresh {f}"); + // Sparse: N/m = 2^14. Galloping ~30 per seek, linear ~16k. + let (g, _, l) = seek_costs(n, n >> 14); + assert!(g * 100 < l, "sparse: gallop {g} vs linear {l}"); + } + + #[test] + fn p1_gallop_edges() { + let k = [1u32, 3, 5, 7]; + assert_eq!(gallop_partition_point(&k, 0, |&x| x < 1), 0); + assert_eq!(gallop_partition_point(&k, 0, |&x| x < 100), 4); + assert_eq!(gallop_partition_point(&k, 2, |&x| x < 6), 3); + assert_eq!(gallop_partition_point(&k, 4, |&x| x < 100), 4); + assert_eq!(gallop_partition_point(&k, 9, |&x| x < 100), 4); + assert_eq!(gallop_partition_point::(&[], 0, |_| true), 0); + } + + /// The witnessed seek agrees with the witnessed aperture bound, refuses a + /// stale witness, and starts from its cursor. + #[test] + fn p1_seek_from_on_a_sealed_lane() { + let lane = SealedFacetLane::seal(skewed_keys(4000, 9), 3).unwrap(); + let w = lane.witness(); + let keys = lane.keys().to_vec(); + let mut cursor = 0u32; + for i in (0..keys.len()).step_by(97) { + let key = keys[i]; + let got = lane.seek_from(&w, cursor, &key).unwrap(); + let want = + keys.partition_point(|k| k.cmp_numeric_projection(&key) == Ordering::Less) as u32; + assert_eq!(got, want, "row {i}"); + cursor = got; + } + let stale = SealedFacetLane::seal(skewed_keys(4000, 9), 4) + .unwrap() + .witness(); + assert!(lane.seek_from(&stale, 0, &keys[0]).is_err()); + } } diff --git a/crates/lance-graph-metta/Cargo.toml b/crates/lance-graph-metta/Cargo.toml index c2aa62b00..4f4f92f8b 100644 --- a/crates/lance-graph-metta/Cargo.toml +++ b/crates/lance-graph-metta/Cargo.toml @@ -26,3 +26,7 @@ test = true [[example]] name = "join_range_probe" test = true + +[[example]] +name = "wcoj_probe" +test = true diff --git a/crates/lance-graph-metta/examples/wcoj_probe.rs b/crates/lance-graph-metta/examples/wcoj_probe.rs new file mode 100644 index 000000000..7d9e637bc --- /dev/null +++ b/crates/lance-graph-metta/examples/wcoj_probe.rs @@ -0,0 +1,388 @@ +//! D-ASM-11 pre-work — the cyclic fixture, the bag oracle, and the P-7 +//! expressibility check (`.claude/board/entries/2026-10-16-coresearch-wcoj-leapfrog.md`). +//! +//! The query is the triangle `Q(a) :- R(a,b), S(b,c), T(a,c)`, existence half: +//! which head rows `a` extend to an answer. R and T are functional over `a` +//! (lanes `rb`, `tc` of the head table), S is functional over `b` (a foreign +//! table: plane `s_valid`, lane `s_dst`). +//! +//! - **A0, the semijoin chain**: `R.valid ∧ T.valid ∧ Semijoin(rb, s_valid)`. +//! On a cyclic body it over-approximates: it never checks that the `c` +//! reached through `S` is the `c` that `T` names. +//! - **P-7, the exact mask with today's ops**: the missing check is a +//! comparison of two lanes (`s_dst[rb[a]] == tc[a]`), and mask-risc only +//! compares a lane with a constant. It can still be written, as a +//! disjunction over every value `v` of the `c` domain: +//! `∨_v (EqU32Via(rb, s_dst, v) ∧ tc == v)`. That is correct but costs one +//! pass per value, `Θ(rows · |D|)`, which misses the `N^1.5` bound once +//! `|D|` exceeds `√N`. This is the insufficiency proof the map asked for: +//! expressible, not at the bound. +//! - **X17, the bag oracle**: datafrog is a SET; answers with multiplicity +//! need a count carried through the join and summed after it. +//! +//! ```text +//! cargo test --manifest-path crates/lance-graph-metta/Cargo.toml --example wcoj_probe +//! ``` + +use std::collections::BTreeMap; + +use datafrog::Relation; +use lance_graph_mask_risc::exec::{execute_into, Scratch}; +use lance_graph_mask_risc::{words_for, Foreign, ForeignPlane, LaneRef, Out, Planes, Value}; +use lance_graph_quack::{ + lower, Agg, Cmp, Col, Filter, ForeignLane, ForeignPlane as QForeign, Mask, Query, +}; + +const RB: Col = Col(0); +const TC: Col = Col(1); +const R_VALID: Mask = Mask(0); +const T_VALID: Mask = Mask(1); +const S_VALID: QForeign = QForeign(0); +const S_DST: ForeignLane = ForeignLane(0); + +/// A functional relation over the address space `0..n`: `dst[a]` where +/// `valid` has bit `a`. +struct Func { + dst: Vec, + valid: Vec, +} + +impl Func { + fn new(n: usize, facts: &[(u32, u32)]) -> Self { + let mut dst = vec![0u32; n]; + let mut valid = vec![0u64; words_for(n)]; + for &(s, o) in facts { + assert!((s as usize) < n && (o as usize) < n, "fact outside 0..n"); + assert_eq!( + (valid[s as usize / 64] >> (s % 64)) & 1, + 0, + "not functional" + ); + valid[s as usize / 64] |= 1 << (s % 64); + dst[s as usize] = o; + } + Func { dst, valid } + } +} + +/// A triangle instance over one address space `0..n`. +struct Triangle { + n: usize, + r: Func, + s: Func, + t: Func, + facts: [Vec<(u32, u32)>; 3], +} + +impl Triangle { + fn new(n: usize, r: Vec<(u32, u32)>, s: Vec<(u32, u32)>, t: Vec<(u32, u32)>) -> Self { + Triangle { + n, + r: Func::new(n, &r), + s: Func::new(n, &s), + t: Func::new(n, &t), + facts: [r, s, t], + } + } + + /// Runs `filter` over the head table and returns the kept rows. + fn run(&self, filter: Filter) -> (Vec, usize) { + let program = lower(&Query { + filter, + agg: Agg::Rows, + }) + .expect("lowers"); + let ops = program.ops.len(); + let lanes = [LaneRef::U32(&self.r.dst), LaneRef::U32(&self.t.dst)]; + let masks = [self.r.valid.as_slice(), self.t.valid.as_slice()]; + let planes = Planes { + n_rows: self.n, + masks: &masks, + lanes: &lanes, + }; + let fplanes = [ForeignPlane { + words: &self.s.valid, + rows: self.n, + }]; + let flanes = [LaneRef::U32(&self.s.dst)]; + let foreign = Foreign { + planes: &fplanes, + lanes: &flanes, + }; + let mut scratch = Scratch::for_program(&program, self.n).expect("carves"); + let mut kept = vec![0u64; words_for(self.n)]; + match execute_into( + &program, + &planes, + &foreign, + &mut scratch, + Out::Mask(&mut kept), + ) + .expect("runs") + { + Value::Mask(_) => {} + other => panic!("not a kept mask: {other:?}"), + } + let rows = (0..self.n as u32) + .filter(|&a| (kept[a as usize / 64] >> (a % 64)) & 1 == 1) + .collect(); + (rows, ops) + } + + /// A0: the semijoin chain. + fn semijoin_chain(&self) -> Vec { + self.run(Filter::and([ + Filter::plane(R_VALID), + Filter::plane(T_VALID), + Filter::semijoin(RB, S_VALID), + ])) + .0 + } + + /// P-7: the exact mask as a disjunction over the `c` domain `0..d`. + /// Returns the rows and the program's op count. + fn exact_by_disjunction(&self, d: u32) -> (Vec, usize) { + self.run(Filter::and([ + Filter::plane(R_VALID), + Filter::plane(T_VALID), + Filter::semijoin(RB, S_VALID), + Filter::or((0..d).map(|v| { + Filter::and([ + Filter::eq_u32_via(RB, S_DST, v), + Filter::cmp(TC, Cmp::EqU32(v)), + ]) + })), + ])) + } + + /// The datafrog SET oracle: head rows `a` with a triangle. + fn oracle(&self) -> Vec { + let [r, s, t] = &self.facts; + let r_by_b: Relation<(u32, u32)> = r.iter().map(|&(a, b)| (b, a)).collect(); + let s: Relation<(u32, u32)> = s.iter().copied().collect(); + let ac: Relation<((u32, u32), ())> = + Relation::from_join(&r_by_b, &s, |_b, &a, &c| ((a, c), ())); + let t: Relation<((u32, u32), ())> = t.iter().map(|&p| (p, ())).collect(); + let heads: Relation<(u32, ())> = Relation::from_join(&ac, &t, |&(a, _c), _, _| (a, ())); + heads.elements.iter().map(|&(a, ())| a).collect() + } +} + +/// F4: `k` disjoint copies of the 2+2+2 counterexample. Every semijoin keeps +/// every row; the query is empty. +fn f4(k: u32) -> Triangle { + let (mut r, mut s, mut t) = (Vec::new(), Vec::new(), Vec::new()); + for i in 0..k { + let (x, y) = (2 * i, 2 * i + 1); + r.extend([(x, x), (y, y)]); + s.extend([(x, y), (y, x)]); + t.extend([(x, x), (y, y)]); + } + Triangle::new(2 * k as usize, r, s, t) +} + +/// A functional instance with real triangles: `T` agrees with `S ∘ R` on +/// every fourth row and disagrees on the rest; R and S have holes. +fn mixed(n: u32) -> Triangle { + let r: Vec<(u32, u32)> = (0..n) + .filter(|a| a % 3 != 0) + .map(|a| (a, (7 * a + 3) % n)) + .collect(); + let s: Vec<(u32, u32)> = (0..n) + .filter(|b| b % 5 != 0) + .map(|b| (b, (5 * b + 1) % n)) + .collect(); + let sf: BTreeMap = s.iter().copied().collect(); + let t: Vec<(u32, u32)> = r + .iter() + .map(|&(a, b)| { + let c = sf.get(&b).copied().unwrap_or(a); + (a, if a % 4 == 0 { c } else { (c + 1) % n }) + }) + .collect(); + Triangle::new(n as usize, r, s, t) +} + +// ───────────── X17: the bag oracle ───────────── + +/// Bag answer of `Q(x, z) :- R(x, y), S(y, z)` with duplicate facts, by +/// nested loops: the multiplicity of `(x, z)` is the number of pairs of +/// facts that produce it. +fn bag_brute(r: &[(u32, u32)], s: &[(u32, u32)]) -> BTreeMap<(u32, u32), u64> { + let mut out = BTreeMap::new(); + for &(x, y) in r { + for &(y2, z) in s { + if y == y2 { + *out.entry((x, z)).or_insert(0) += 1; + } + } + } + out +} + +/// The same bag through datafrog. Datafrog relations are sets, so each fact +/// is collapsed to `(fact, count)` first and the count rides the join; the +/// join tuple keeps `y` so two paths with equal counts are not merged. +fn bag_datafrog(r: &[(u32, u32)], s: &[(u32, u32)], dedup: bool) -> BTreeMap<(u32, u32), u64> { + fn counted(facts: &[(u32, u32)], dedup: bool) -> BTreeMap<(u32, u32), u64> { + let mut m = BTreeMap::new(); + for &f in facts { + let c = m.entry(f).or_insert(0); + *c = if dedup { 1 } else { *c + 1 }; + } + m + } + let r: Relation<(u32, (u32, u64))> = counted(r, dedup) + .into_iter() + .map(|((x, y), m)| (y, (x, m))) + .collect(); + let s: Relation<(u32, (u32, u64))> = counted(s, dedup) + .into_iter() + .map(|((y, z), m)| (y, (z, m))) + .collect(); + let paths: Relation<((u32, u32, u32), u64)> = + Relation::from_join(&r, &s, |&y, &(x, m1), &(z, m2)| ((x, z, y), m1 * m2)); + let mut out = BTreeMap::new(); + for &((x, z, _y), m) in paths.elements.iter() { + *out.entry((x, z)).or_insert(0) += m; + } + out +} + +/// n:m facts with duplicates (a bag), and an n:1 set with none. +/// `(R, S, has_duplicates)`. +type BagFixture = (Vec<(u32, u32)>, Vec<(u32, u32)>, bool); + +fn bag_fixtures() -> [BagFixture; 2] { + let mut r = Vec::new(); + let mut s = Vec::new(); + for x in 0..40u32 { + for j in 0..3 { + r.push((x, (x + j) % 13)); + } + if x % 4 == 0 { + r.push((x, x % 13)); // a duplicate fact + } + } + for y in 0..13u32 { + for j in 0..4 { + s.push((y, (y * 3 + j) % 17)); + } + s.push((y, (y * 3) % 17)); // a duplicate fact + } + let r1: Vec<(u32, u32)> = (0..40).map(|x| (x, x % 13)).collect(); + let s1: Vec<(u32, u32)> = (0..13).map(|y| (y, (y * 3) % 17)).collect(); + [(r, s, true), (r1, s1, false)] +} + +fn main() { + println!( + "tier: avx512f={} avx2={}", + cfg!(target_feature = "avx512f"), + cfg!(target_feature = "avx2") + ); + println!("fixture | rows | A0 kept | exact | oracle | ops(disjunction over |D|)"); + for (name, tri) in [ + ("F4 x8", f4(8)), + ("mixed 64", mixed(64)), + ("mixed 256", mixed(256)), + ] { + let a0 = tri.semijoin_chain(); + let (exact, ops) = tri.exact_by_disjunction(tri.n as u32); + let want = tri.oracle(); + println!( + "{name} | {} | {} | {} | {} | {ops} (|D| = {})", + tri.n, + a0.len(), + exact.len(), + want.len(), + tri.n + ); + } + for (r, s, dup) in bag_fixtures() { + let brute = bag_brute(&r, &s); + println!( + "bag (duplicates={dup}): {} keys, total multiplicity {}, datafrog+counts agrees: {}", + brute.len(), + brute.values().sum::(), + bag_datafrog(&r, &s, false) == brute + ); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// X6 / F4. FAILS IF the semijoin chain stops over-approximating on the + /// counterexample (then the fixture no longer tests anything), or if the + /// oracle finds a triangle in it. + #[test] + fn f4_semijoin_chain_keeps_every_row_and_the_query_is_empty() { + for k in [1, 2, 8, 50] { + let tri = f4(k); + assert_eq!(tri.semijoin_chain().len(), 2 * k as usize, "k={k}"); + assert!(tri.oracle().is_empty(), "k={k}"); + } + } + + /// P-7, correctness: the disjunction over the value domain is the exact + /// mask. Disable run: drop the `tc == v` conjunct and it collapses to A0. + #[test] + fn p7_disjunction_over_the_domain_equals_the_set_oracle() { + for tri in [f4(1), f4(8), mixed(30), mixed(64), mixed(200)] { + let want = tri.oracle(); + let (got, _) = tri.exact_by_disjunction(tri.n as u32); + assert_eq!(got, want, "n={}", tri.n); + } + // Anti-vacuity: on `mixed` the semijoin chain keeps rows the oracle + // rejects, and the oracle keeps some rows. + let tri = mixed(200); + let (a0, want) = (tri.semijoin_chain(), tri.oracle()); + assert!( + !want.is_empty() && want.len() * 2 < a0.len(), + "{} vs {}", + want.len(), + a0.len() + ); + } + + /// P-7, cost: the program grows linearly with the value domain, so the + /// row work is `Θ(rows · |D|)`. This is why the exact cyclic mask is not + /// at the `N^1.5` bound with today's ops. + #[test] + fn p7_disjunction_costs_one_pass_per_domain_value() { + let tri = mixed(256); + let (_, ops64) = tri.exact_by_disjunction(64); + let (_, ops128) = tri.exact_by_disjunction(128); + let (_, ops256) = tri.exact_by_disjunction(256); + assert!( + ops128 - ops64 >= 64 && ops256 - ops128 >= 128, + "{ops64} {ops128} {ops256}" + ); + assert_eq!(ops256 - ops128, 2 * (ops128 - ops64), "not linear in |D|"); + } + + /// X17. FAILS IF datafrog-with-counts disagrees with the nested loop on + /// the bag. Disable run: collapse counts to 1 (`dedup = true`) and the + /// n:m fixture with duplicates goes red while the n:1 one stays green. + #[test] + fn x17_bag_oracle_matches_the_nested_loop() { + for (r, s, _) in bag_fixtures() { + assert_eq!(bag_datafrog(&r, &s, false), bag_brute(&r, &s)); + } + } + + /// The disable run of X17, kept as a test: deduplicating inside the + /// oracle must be detectable on the bag and invisible on the set. + #[test] + fn x17_dedup_is_caught_on_the_bag_only() { + for (r, s, has_duplicates) in bag_fixtures() { + let same = bag_datafrog(&r, &s, true) == bag_brute(&r, &s); + assert_eq!(same, !has_duplicates); + } + // Multiplicities above one exist, so the bag is not a set. + let (r, s, _) = &bag_fixtures()[0]; + assert!(bag_brute(r, s).values().any(|&m| m > 1)); + } +} diff --git a/crates/lance-graph-relalg/src/join_body.rs b/crates/lance-graph-relalg/src/join_body.rs new file mode 100644 index 000000000..73509b534 --- /dev/null +++ b/crates/lance-graph-relalg/src/join_body.rs @@ -0,0 +1,136 @@ +//! A join body as a hypergraph, and its α-acyclicity test (GYO reduction). +//! +//! Probe P-3 of the worst-case-optimal-join map +//! (`.claude/board/entries/2026-10-16-coresearch-wcoj-leapfrog.md`): the +//! router that sends an α-acyclic body to the semijoin chain (Yannakakis, +//! exact) and a cyclic one to a leapfrog-style intersection needs this test +//! at compile time, before anything runs. +//! +//! Coordinates only: a variable is a `u16`, an atom is the set of variables +//! it binds. Which relation an atom reads does not matter for acyclicity, so +//! it is not part of the description. + +/// The variables one atom of the body binds. +pub type Edge = Vec; + +/// GYO (Graham; Yu–Özsoyoğlu) reduction. Repeat until nothing changes: +/// drop every variable that occurs in exactly one edge, and drop every edge +/// that is empty or contained in another edge. The body is α-acyclic iff +/// this leaves no edge. +#[must_use] +pub fn is_alpha_acyclic(body: &[Edge]) -> bool { + let mut edges: Vec = body + .iter() + .map(|e| { + let mut e = e.clone(); + e.sort_unstable(); + e.dedup(); + e + }) + .collect(); + loop { + let before: usize = edges.iter().map(Vec::len).sum::() + edges.len(); + // Variables in exactly one edge. + let mut count = std::collections::BTreeMap::::new(); + for e in &edges { + for &v in e { + *count.entry(v).or_default() += 1; + } + } + for e in &mut edges { + e.retain(|v| count[v] > 1); + } + // Edges that are empty or contained in another edge. Of two equal + // edges, keep the first. + let mut keep = vec![true; edges.len()]; + for i in 0..edges.len() { + if edges[i].is_empty() { + keep[i] = false; + continue; + } + for j in 0..edges.len() { + if i == j || !keep[j] { + continue; + } + let contained = edges[i].iter().all(|v| edges[j].binary_search(v).is_ok()); + if contained && (edges[i].len() < edges[j].len() || j < i) { + keep[i] = false; + break; + } + } + } + let mut k = keep.into_iter(); + edges.retain(|_| k.next().unwrap_or(false)); + let after: usize = edges.iter().map(Vec::len).sum::() + edges.len(); + if edges.is_empty() { + return true; + } + if after == before { + return false; + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const A: u16 = 0; + const B: u16 = 1; + const C: u16 = 2; + const D: u16 = 3; + const E: u16 = 4; + const F: u16 = 5; + + fn body(edges: &[&[u16]]) -> Vec { + edges.iter().map(|e| e.to_vec()).collect() + } + + /// Acyclic bodies the router must send to the semijoin chain. + #[test] + fn acyclic_bodies_are_recognised() { + for (name, b) in [ + ("one atom", body(&[&[A, B]])), + ("chain", body(&[&[A, B], &[B, C], &[C, D]])), + ("star", body(&[&[A, B], &[A, C], &[A, D]])), + ("tree", body(&[&[A, B], &[B, C], &[B, D], &[D, E]])), + // The 3-ary atom covers the triangle: α-acyclic, not cyclic. + ( + "covered triangle", + body(&[&[A, B, C], &[A, B], &[B, C], &[A, C]]), + ), + ("repeated atom", body(&[&[A, B], &[A, B], &[B, C]])), + ("cross product", body(&[&[A], &[B]])), + ] { + assert!(is_alpha_acyclic(&b), "{name} misread as cyclic"); + } + } + + /// Cyclic bodies the router must send to the leapfrog arm. Disable run: + /// make the test return `true` and every case here goes red. + #[test] + fn cyclic_bodies_are_recognised() { + for (name, b) in [ + ("triangle", body(&[&[A, B], &[B, C], &[A, C]])), + ("4-cycle", body(&[&[A, B], &[B, C], &[C, D], &[D, A]])), + ( + "barbell", + body(&[ + &[A, B], + &[B, C], + &[A, C], + &[C, D], + &[D, E], + &[E, F], + &[D, F], + ]), + ), + ( + "triangle with a tail", + body(&[&[A, B], &[B, C], &[A, C], &[C, D]]), + ), + ] { + assert!(!is_alpha_acyclic(&b), "{name} misread as acyclic"); + } + } +} diff --git a/crates/lance-graph-relalg/src/lib.rs b/crates/lance-graph-relalg/src/lib.rs index c36764941..9944d524d 100644 --- a/crates/lance-graph-relalg/src/lib.rs +++ b/crates/lance-graph-relalg/src/lib.rs @@ -29,6 +29,7 @@ //! built. pub mod ir; +pub mod join_body; pub mod lower; pub mod normalize;