diff --git a/.claude/board/LATEST_STATE.md b/.claude/board/LATEST_STATE.md index c47e9c1f7..070411e6d 100644 --- a/.claude/board/LATEST_STATE.md +++ b/.claude/board/LATEST_STATE.md @@ -1,11 +1,15 @@ -## 2026-10-16 — WCOJ map items 1–3 run (branch `ccr-9f3b8075-74u47l-wcoj-probes`, in PR) +## 2026-10-16 — CATS optional fields bind (branch `ccr-9f3b8075-74u47l-cats-validity`, in PR) + +`lance-graph-sap`: the six optional `U32` CATS fields bind through validity planes built at `CatsBatch::bind` (`OPTIONAL_CODE_FIELDS`, `validity_mask`, `CatsBatch::masks`), so `=` and `<>` drop NULL rows. Before this, `CatsBinder` refused them. U64 instants stay refused. Found in the bind-DTO synergy check (entry `2026-10-16-wcoj-probes-1-3.md`, #1587). + +## 2026-10-16 — WCOJ map items 1–3 run (#1587, merged) Contract inventory delta: `ordered_lane::gallop_partition_point` (cursor seek, galloping), `ordered_lane::seek_partition_point` + `SEEK_LINEAR_GAP` (hybrid by gap hint, thresholds from v4 timings) and `SealedFacetLane::seek_from` (witnessed). Outside the contract: `lance-graph-relalg::join_body::is_alpha_acyclic` (GYO) and `lance-graph-metta/examples/wcoj_probe.rs` (F4 fixture, bag oracle, P-7 check). Results: entry `2026-10-16-wcoj-probes-1-3.md`. -## 2026-10-16 — main after #1584, #1585, #1586 +## 2026-10-16 — main after #1584, #1585, #1586 (#1586: the co-research map, entry `2026-10-16-coresearch-wcoj-leapfrog.md`) - **On `main` (#1584):** D-ASM-1b and D-ASM-8..11 queued; entry `2026-10-16-asm-accepted-proposals.md`. Nothing built. -- **#1586 (merged):** D-ASM-12 (the `lance-graph-cog` content → NodeGuid address that `atomspace-lance` and the D-ASM-1b binder share); D-ASM-13 (the `lance-graph-cog` thinking API as a facade over NAL / ONA-shaped tables / R2IL / relalg); the forks `matrix` and `atomspace-storage` recorded; the MORK stable branch recorded. +- **#1585 (merged):** D-ASM-12 (the `lance-graph-cog` content → NodeGuid address that `atomspace-lance` and the D-ASM-1b binder share); D-ASM-13 (the `lance-graph-cog` thinking API as a facade over NAL / ONA-shaped tables / R2IL / relalg); the forks `matrix` and `atomspace-storage` recorded; the MORK stable branch recorded. ## 2026-10-16 — main after #1582 (merged `e08e5659`) diff --git a/.claude/board/PR_ARC_INVENTORY.md b/.claude/board/PR_ARC_INVENTORY.md index 58df10a3a..1f9834b32 100644 --- a/.claude/board/PR_ARC_INVENTORY.md +++ b/.claude/board/PR_ARC_INVENTORY.md @@ -1,3 +1,10 @@ +## 2026-10-16 — lance-graph PR #1587 (merged `215f9753`, branch `ccr-9f3b8075-74u47l-wcoj-probes`, head `ab5a1047`, 11 commits) — WCOJ map items 1–3, hybrid seek, closing-atom spec, multi-table bind plan + +- **Added:** contract `ordered_lane::{gallop_partition_point, seek_partition_point, SEEK_LINEAR_GAP}`, `SealedFacetLane::seek_from`; `lance-graph-relalg::join_body::is_alpha_acyclic` (GYO); `lance-graph-metta/examples/wcoj_probe.rs` (F4, bag oracle, P-7); `lance-graph-contract/examples/seek_probe.rs`; entries `2026-10-16-wcoj-probes-1-3.md`, `2026-10-16-wcoj-closing-atoms-council.md`; plans `wcoj-closing-atoms-v1.md`, `quack-multitable-bind-v1.md`; STATUS D-ASM-11A/B, D-QMB-0..5; TD-NO-BOUNDED-U32-GATHER-1; the #1585/#1586 arc entries. +- **Locked:** seek routing step ≤ 16 / whole-lane search at gap² ≥ N / gallop otherwise, from v4 timings; P-7: the exact cyclic mask needs a closing atom (a disjunction costs about 4 ops per value). +- **Deferred / open:** D-ASM-11A/B and D-QMB-0/1 authorized 2026-10-16, not built; DuckDB research items read at abstract level only. +- **Confidence:** tests + v4 timings (`avx512f=true`). + ## 2026-10-16 — lance-graph PR #1586 (merged `f2892f98`, branch `ccr-9f3b8075-74u47l-coresearch-wcoj`, head `2284376a`, 1 commit) — board: co-research map for worst-case-optimal joins (D-ASM-11) - **Added:** entry `2026-10-16-coresearch-wcoj-leapfrog.md` (leapfrog family mapped onto our surfaces, ADOPT-NOW / PROBE / PARK / SKIP per idea, fixtures F2/F3a/F3b/F4, probes P-1..P-12). diff --git a/crates/lance-graph-glove-parity/src/basis.rs b/crates/lance-graph-glove-parity/src/basis.rs index a40bcb219..16d4448b7 100644 --- a/crates/lance-graph-glove-parity/src/basis.rs +++ b/crates/lance-graph-glove-parity/src/basis.rs @@ -87,12 +87,12 @@ pub const SAP_FIELDS: &[FieldMap] = &[ hyp( "project_code", "project", - "a PS element identifies a Project; optional, CatsBinder refuses it", + "a PS element identifies a Project; optional, bound through its validity plane", ), hyp( "task_code", "about", - "an order identifies a ProjectWorkItem; optional, CatsBinder refuses it", + "an order identifies a ProjectWorkItem; optional, bound through its validity plane", ), hyp( "hours_logged", diff --git a/crates/lance-graph-sap/src/bind.rs b/crates/lance-graph-sap/src/bind.rs index fb67fb33f..7f46e1f02 100644 --- a/crates/lance-graph-sap/src/bind.rs +++ b/crates/lance-graph-sap/src/bind.rs @@ -23,6 +23,21 @@ pub const BILLING_VALUES: &[(&str, bool)] = &[ ("Non-Billable", false), ("Internal Cost", false), ]; +/// The optional dictionary/NUMC fields, in validity-plane order: the plane of +/// `OPTIONAL_CODE_FIELDS[k]` is mask `1 + k` (mask 0 is the live plane). These +/// are exactly the optional fields with a `U32` carrier; the U64 instant has +/// no `FieldKind` and gets no plane. Pinned against `FIELDS` by a test. +pub const OPTIONAL_CODE_FIELDS: [usize; 6] = [6, 7, 8, 16, 20, 21]; + +/// The validity mask index of an optional code field, `None` for any other. +#[must_use] +pub fn validity_mask(ordinal: usize) -> Option { + OPTIONAL_CODE_FIELDS + .iter() + .position(|&o| o == ordinal) + .map(|k| lance_graph_quack::Mask(1 + k as u16)) +} + /// A billing indicator the DTO does not document: neither billable nor not. /// A filter on either boolean never selects it. pub const BILLABLE_UNKNOWN: u32 = u32::MAX; @@ -70,6 +85,9 @@ pub struct CatsBatch { pub schema: CatsSchema, columns: Vec, dictionaries: [Vec; FIELD_COUNT], + /// One plane per [`OPTIONAL_CODE_FIELDS`] entry: bit set where the value + /// is present. NULL is the clear bit, never the sentinel in the lane. + validity: Vec>, len: usize, scale: u32, } @@ -88,6 +106,7 @@ impl CatsBatch { } let mut dictionaries: [Vec; FIELD_COUNT] = std::array::from_fn(|_| Vec::new()); let mut columns = Vec::with_capacity(LANE_COUNT); + let mut validity = Vec::with_capacity(OPTIONAL_CODE_FIELDS.len()); let mut scale = 0; for f in FIELDS { let index = usize::from(f.ordinal.0); @@ -108,6 +127,15 @@ impl CatsBatch { } } } + if OPTIONAL_CODE_FIELDS.contains(&index) { + let mut plane = vec![0u64; len.div_ceil(64)]; + for (i, value) in values.iter().enumerate() { + if value.is_some() { + plane[i / 64] |= 1 << (i % 64); + } + } + validity.push(plane); + } let column = if index == HOURS { let decimals: Vec<_> = values .iter() @@ -197,6 +225,7 @@ impl CatsBatch { schema, columns, dictionaries, + validity, len, scale, }) @@ -210,6 +239,13 @@ impl CatsBatch { pub fn scale(&self) -> u32 { self.scale } + /// The executor's mask planes: `live` first (mask 0), then one validity + /// plane per [`OPTIONAL_CODE_FIELDS`] entry (masks `1..`). + pub fn masks<'a>(&'a self, live: &'a [u64]) -> Vec<&'a [u64]> { + std::iter::once(live) + .chain(self.validity.iter().map(Vec::as_slice)) + .collect() + } pub fn lanes(&self) -> [LaneRef<'_>; LANE_COUNT] { std::array::from_fn(|i| self.columns[i].borrow()) } @@ -388,6 +424,17 @@ fn format_utc(n: u64) -> String { #[cfg(test)] mod tests { use super::*; + /// FAILS IF the validity-plane list drifts from the schema. + #[test] + fn optional_code_fields_are_exactly_the_optional_u32_fields() { + let want: Vec = FIELDS + .iter() + .filter(|f| f.optional && matches!(f.carrier, lance_graph_mask_risc::LaneKind::U32)) + .map(|f| usize::from(f.ordinal.0)) + .collect(); + assert_eq!(OPTIONAL_CODE_FIELDS.to_vec(), want); + } + #[test] fn lossless_adapters_reject_lossy_inputs() { assert_eq!(numc("00000042"), Ok(42)); diff --git a/crates/lance-graph-sap/src/binder.rs b/crates/lance-graph-sap/src/binder.rs index 577e1f927..cd79bc4b6 100644 --- a/crates/lance-graph-sap/src/binder.rs +++ b/crates/lance-graph-sap/src/binder.rs @@ -13,9 +13,11 @@ //! ABAP_BOOL, the batch dictionary otherwise. Nothing is minted; //! - `live`: plane [`LIVE`], supplied by the executor as [`live_words`]. //! -//! Refused, deliberately, as unknown fields: optional fields (CATS stores NULL -//! as a sentinel, not a validity plane, so `<>` would keep NULL rows) and U64 -//! instants (no `FieldKind`; `work_day` is their bindable lens). +//! Optional code fields bind with a validity plane built at +//! [`CatsBatch::bind`](crate::bind::CatsBatch::bind) (masks `1..`, see +//! [`CatsBatch::masks`](crate::bind::CatsBatch::masks)), so `=` and `<>` drop +//! NULL rows. Refused, deliberately, as unknown fields: U64 instants (no +//! `FieldKind`; `work_day` is their bindable lens). use crate::bind::{numc, CatsBatch, BILLABLE, WORK_DAY}; use crate::schema::FIELDS; use lance_graph_mask_risc::LaneKind; @@ -80,9 +82,14 @@ impl Binder for CatsBinder<'_> { } let col = self.batch.schema.resolve(name)?; let field = &FIELDS[usize::from(col.0)]; - if field.optional { - return None; - } + // An optional field binds only through its validity plane: CATS + // stores NULL as a sentinel, and `<>` read through the lane alone + // would keep NULL rows. + let validity = if field.optional { + Some(crate::bind::validity_mask(usize::from(col.0))?) + } else { + None + }; let kind = match field.carrier { LaneKind::I32 => FieldKind::I32, LaneKind::U32 => FieldKind::Code, @@ -91,7 +98,7 @@ impl Binder for CatsBinder<'_> { Some(BoundField { col, kind, - validity: None, + validity, }) } fn code(&self, _: TableId, col: Col, literal: &str) -> Option { diff --git a/crates/lance-graph-sap/tests/binder.rs b/crates/lance-graph-sap/tests/binder.rs index e0f5c2c05..4b9d213f4 100644 --- a/crates/lance-graph-sap/tests/binder.rs +++ b/crates/lance-graph-sap/tests/binder.rs @@ -200,25 +200,128 @@ fn unknown_names_and_values_are_bind_errors_never_defaults() { )); } +/// Runs a bound query over the batch with its validity planes and returns the +/// kept rows. +fn kept_rows(batch: &lance_graph_sap::bind::CatsBatch, q: &Query) -> Vec { + let n = batch.len(); + let program = lower(q).unwrap(); + let live = live_words(n); + let lanes = batch.lanes(); + let masks = batch.masks(&live); + let planes = Planes { + n_rows: n, + masks: &masks, + lanes: &lanes, + }; + let mut scratch = Scratch::for_program(&program, n).unwrap(); + let mut mask = vec![0; words_for(n)]; + execute_into( + &program, + &planes, + &Foreign::NONE, + &mut scratch, + Out::Mask(&mut mask), + ) + .unwrap(); + (0..n) + .filter(|i| (mask[i / 64] >> (i % 64)) & 1 != 0) + .collect() +} + +/// Optional fields bind through a validity plane built at `CatsBatch::bind`. +/// NULL is the plane's clear bit, so `=` and `<>` both drop NULL rows, as in +/// SQL. (Replaces the earlier refusal: CATS stores NULL as a sentinel, and a +/// field bound without a plane would let `<>` keep NULL rows.) #[test] -fn optional_and_instant_fields_are_refused_not_read_through_sentinels() { - // CATS stores NULL as a sentinel (code 0, u32::MAX, timestamp 0), not as a - // validity plane. Binding an optional field without one would let `<>` - // keep NULL rows, so the binder refuses it. - let mut input = mixed(3); - input[6][1] = None; // customer_number is optional +fn optional_fields_bind_three_valued_through_a_validity_plane() { + use lance_graph_quack::bind::FieldRef; + for n in [3, 64, 65, 200] { + let mut input = mixed(n); + for (i, v) in input[6].iter_mut().enumerate() { + *v = match i % 3 { + 0 => None, // NULL customer + 1 => Some("0000000123"), + _ => Some("0000000456"), + }; + } + let batch = bind(&input); + let b = CatsBinder::new(&batch); + let f = b.field(T, "customer_number").unwrap(); + assert_eq!((f.col, f.kind), (Col(6), FieldKind::Code)); + assert!(f.validity.is_some(), "an optional field carries its plane"); + + let eq = table(TABLE) + .where_eq("customer_number", "0000000123") + .rows() + .bind(&b) + .unwrap(); + assert_eq!( + kept_rows(&batch, &eq), + (0..n).filter(|i| i % 3 == 1).collect::>() + ); + let ne = table(TABLE) + .where_(FieldRef::new(TABLE, "customer_number").ne("0000000123")) + .rows() + .bind(&b) + .unwrap(); + let want: Vec = (0..n).filter(|i| i % 3 == 2).collect(); + assert_eq!( + kept_rows(&batch, &ne), + want, + "n={n}: <> must drop NULL rows" + ); + + // Can-fire: the same `<>` read through the sentinel lane alone keeps + // the NULL rows (code 0 differs from 123's code). + let raw = Query { + filter: Filter::and([ + Filter::plane(lance_graph_sap::binder::LIVE), + Filter::cmp(f.col, Cmp::NeU32(b.code(T, f.col, "0000000123").unwrap())), + ]), + agg: Agg::Rows, + }; + let raw_rows = kept_rows(&batch, &raw); + assert!(raw_rows.iter().any(|i| i % 3 == 0)); + assert_ne!(raw_rows, want); + } +} + +/// Every optional `U32` field gets its own plane; the PERNR one uses NUMC +/// literals; U64 instants still have no `FieldKind` and stay refused. +#[test] +fn every_optional_code_field_has_a_plane_and_instants_stay_refused() { + let mut input = mixed(5); + input[21][2] = Some("00000042"); let batch = bind(&input); let b = CatsBinder::new(&batch); - assert_eq!(b.field(T, "customer_number"), None); - assert_eq!(b.field(T, "approver_employee_num"), None); + let mut planes = Vec::new(); + for name in [ + "customer_number", + "project_code", + "task_code", + "sanitization_reason", + "notes", + "approver_employee_num", + ] { + let f = b.field(T, name).unwrap_or_else(|| panic!("{name} binds")); + planes.push(f.validity.expect("plane")); + } + planes.sort(); + planes.dedup(); + assert_eq!(planes.len(), 6, "one plane per optional field"); + assert!(!planes.contains(&lance_graph_sap::binder::LIVE)); + assert_eq!(batch.masks(&live_words(5)).len(), 7); + let f = b.field(T, "approver_employee_num").unwrap(); + assert_eq!(b.code(T, f.col, "00000042"), Some(42)); + let q = table(TABLE) + .where_eq("approver_employee_num", "00000042") + .rows() + .bind(&b) + .unwrap(); + assert_eq!(kept_rows(&batch, &q), vec![2]); // U64 instants have no `FieldKind`; the derived `work_day` is the bindable lens. + assert_eq!(b.field(T, "approval_timestamp_utc"), None); assert_eq!(b.field(T, "work_date_utc"), None); - assert!(matches!( - table(TABLE) - .where_eq("customer_number", "0000000123") - .bind(&b), - Err(BindError::UnknownField { .. }) - )); } /// The billable lens is a VERIFIED conversion (plan §C.6.2): on real rows with