diff --git a/docs/ai/design/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/design/2026-09-02-feature-memory-noise-reduction.md new file mode 100644 index 00000000..bae91c78 --- /dev/null +++ b/docs/ai/design/2026-09-02-feature-memory-noise-reduction.md @@ -0,0 +1,56 @@ +--- +phase: design +title: Memory Hybrid Noise Reduction Design +description: Apply confidence only to semantic-only entrants and tighten reciprocal-rank decay +--- + +# Memory Hybrid Noise Reduction Design + +## Architecture + +```mermaid +flowchart LR + L[Lexical top 20] --> F[RRF fusion k=10] + S[Semantic top 20] --> C{Lexical match?} + C -->|yes| F + C -->|no and cosine >= 0.50| F + C -->|no and cosine < 0.50| X[Suppress] + F --> O[Requested result limit] +``` + +The change stays inside `fuseSearchResults`. It adds no API, persisted data, model work, or caller-specific branch. + +## Ranking Rules + +1. Every lexical candidate enters fusion. +2. A semantic candidate already present lexically contributes its semantic reciprocal-rank score at any cosine value. Independent channel agreement remains useful evidence. +3. A semantic-only candidate enters fusion only at cosine similarity 0.50 or greater. +4. Both channels use reciprocal-rank fusion with `k=10`, increasing separation between high and deep ranks. +5. Existing deterministic tie-breaking remains score, lexical presence, lexical rank, semantic rank, then ID. + +The threshold is internal rather than configurable because no current caller needs a tuning surface. Removing either policy is a one-line change plus its unit test. + +## Alternatives Tested + +| Candidate | hit@1 | hit@3 | hit@5 | Irrelevant top-3 | Coverage | Zero | Decision | +|---|---:|---:|---:|---:|---:|---:|---| +| Semantic baseline, `k=60` | 88% | 97% | 98% | 4.7% (6/127) | 42.3% | 0% | Replace | +| Equal RRF `k=10` | 88% | 97% | 98% | 4.0% (5/126) | 42.0% | 0% | Useful but incomplete | +| Asymmetric RRF 60/90 | 86% | 97% | 98% | 4.0% (5/125) | 41.7% | 0% | Rejected: loses 2 pp hit@1 | +| Semantic-only cosine >=0.50 | 88% | 97% | 98% | 3.3% (4 judged irrelevant) | 55.5% | 1% | Retained | +| Combined cosine >=0.50 + `k=10` | 88% | 97% | 98% | 2.5% (3 judged irrelevant) | 55.0% | 1% | Selected | + +The rank-discount experiment remains visible in Git history and was reverted before testing the threshold. The selected combination targets both observed mechanisms and passes the hit@3 gate. + +## Trade-offs + +- The selected policy can return fewer than the requested limit when all semantic-only candidates are weak. This is intentional: the former lexical-zero query `169.254.169.254` received five unrelated semantic results with cosine 0.14–0.18 and no relevant hit. +- Coverage rises partly because weak unjudged results are no longer returned. Report raw counts and returned-slot denominators alongside conditional rates. +- A fixed threshold may not generalize perfectly beyond expanded-100. Pooled judgments are a follow-up, not a reason to retain demonstrated noise. + +## Non-functional Properties + +- Performance: no additional inference or database work; filtering reduces fusion work. +- Security/privacy: unchanged local inference and data handling. +- Reliability: semantic failure still degrades to lexical-only search. +- Compatibility: no public type, response, configuration, or schema change. diff --git a/docs/ai/implementation/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/implementation/2026-09-02-feature-memory-noise-reduction.md new file mode 100644 index 00000000..f95edb43 --- /dev/null +++ b/docs/ai/implementation/2026-09-02-feature-memory-noise-reduction.md @@ -0,0 +1,39 @@ +--- +phase: implementation +title: Memory Hybrid Noise Reduction Implementation +description: Record the selected fusion changes and empirical development workflow +--- + +# Memory Hybrid Noise Reduction Implementation + +## Changed Code + +- `packages/memory/src/semantic/embeddings.ts` + - Changes `RRF_K` from 60 to 10. + - Adds the internal `SEMANTIC_ONLY_MIN_SIMILARITY = 0.5` constant. + - Applies the threshold only after checking for an existing lexical candidate, preserving low-similarity semantic reinforcement of lexical results. +- `packages/memory/tests/unit/semantic/embeddings.test.ts` + - Verifies a 0.49 semantic-only candidate is suppressed, a 0.50 candidate is admitted, and a shared lexical candidate remains. + - Verifies lexical rank 1 outranks weak agreement at lexical/semantic rank 20 with tighter RRF decay. + +## Integration + +`fuseSearchResults` remains the single ranking seam used by `searchKnowledgeHybrid`. CLI and MCP callers receive the existing result structure and semantic explanation fields. The storage, embedding, fallback, and model-loading paths are untouched. + +## Benchmark Procedure + +The community benchmark was run in dev mode against the built CLI, not an npm release: + +```bash +cd /home/ubuntu/code/agent-memory-bench +AI_DEVKIT_BIN=/home/ubuntu/code/ai-devkit/.worktrees/feature-memory-noise/packages/cli/dist/cli.js \ + npm run bench -- --label --semantic --output /tmp/.json +``` + +Each candidate was applied, built, and measured against the same expanded-100 fixture definition. Rejected ranking candidates were reverted before the next standalone experiment. No experiment row was added to the released-version leaderboard. + +## Operational Considerations + +- No migration, deployment configuration, or rollback operation is required. +- Reverting the two selected commits restores the previous fusion policy. +- The threshold can reduce result count for weak semantic-only queries; the API already permits fewer results than the requested maximum. diff --git a/docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md new file mode 100644 index 00000000..1966433d --- /dev/null +++ b/docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md @@ -0,0 +1,34 @@ +--- +phase: planning +title: Memory Hybrid Noise Reduction Plan +description: Track diagnosis, isolated experiments, selected implementation, and release validation +--- + +# Memory Hybrid Noise Reduction Plan + +## Completed Work + +- [x] Reproduce the 0.59.0 lexical and semantic expanded-100 rows. +- [x] Run all queries with retrieval explanations against one shared seeded corpus. +- [x] Identify all six judged-irrelevant hybrid top-three appearances and classify channel membership. +- [x] Correct the coverage interpretation using raw judged and returned slot counts. +- [x] Test equal `k=10` in isolation and record 88/97/98 with 4.0% irrelevant. +- [x] Test asymmetric lexical/semantic 60/90 in isolation and record 86/97/98 with 4.0% irrelevant. +- [x] Revert the asymmetric experiment before the next candidate. +- [x] Test a semantic-only cosine floor of 0.50 and record 88/97/98 with 3.3% irrelevant. +- [x] Test the combined floor plus `k=10` and record 88/97/98 with 2.5% irrelevant. +- [x] Add deterministic unit tests for both selected ranking rules. +- [x] Prove the tests fail when the selected implementation is removed. +- [x] Run final build, full test, lint, and E2E gates after documentation changes. +- [x] Publish the branch and open a review request without merging. + +## Dependencies and Sequencing + +The real model and expanded-100 fixture were required only for empirical selection. Unit tests use explicit candidates and ranks, so routine validation does not download or execute the model. A released-version leaderboard row depends on the next ai-devkit release and is deliberately deferred. + +## Risks and Mitigations + +- **Sparse judgments:** include raw counts and coverage in review evidence; expand pooled judgments separately. +- **Threshold overfitting:** retain lexical matches regardless of cosine and preserve all measured hit metrics. +- **Ranking regression:** deterministic tests cover the threshold boundary and deep dual-channel agreement. +- **History noise:** rejected experiments remain as evidence, with explicit revert commits; reviewers can evaluate the final diff against `origin/main` independently. diff --git a/docs/ai/requirements/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/requirements/2026-09-02-feature-memory-noise-reduction.md new file mode 100644 index 00000000..d7d4a653 --- /dev/null +++ b/docs/ai/requirements/2026-09-02-feature-memory-noise-reduction.md @@ -0,0 +1,54 @@ +--- +phase: requirements +title: Memory Hybrid Noise Reduction Requirements +description: Reduce judged-irrelevant hybrid search results without giving back semantic recall +--- + +# Memory Hybrid Noise Reduction Requirements + +## Problem + +Opt-in hybrid search improved expanded-100 hit@1/3/5 from 81/91/96 to 88/97/98 and removed the one empty response, but judged-irrelevant top-three results rose from 2.9% to 4.7%. Agents and CLI users benefit from the recall gain only if plausible but answer-wrong memories do not crowd the first results. + +The percentage needs context. Lexical search produced 3 irrelevant results among 104 judged slots and 207 returned top-three slots. Hybrid produced 6 among 127 judged slots and 300 returned slots. Hybrid therefore had more judged slots in absolute terms, while its reported coverage fell from 50.2% to 42.3% because it filled more positions. The conditional 2.9% to 4.7% comparison is valid but sensitive to incomplete, non-random judgments. + +## Goals + +- Preserve the semantic-on hit@1/3/5 improvement. +- Reduce explicitly irrelevant top-three results using a small, deterministic fusion change. +- Keep semantic search opt-in, local, fail-open, and API-compatible. +- Keep lexical matches eligible even when their semantic similarity is low. + +## Non-goals + +- Changing the embedding model, storage schema, corpus cap, or public configuration. +- Adding a learned reranker or another inference pass. +- Publishing a benchmark leaderboard row before a released ai-devkit version exists. +- Treating unjudged results as irrelevant. + +## Success Criteria + +- Expanded-100 hit@3 must remain at least 96%, no more than 1 percentage point below the 97% semantic baseline. +- Judged-irrelevant top-three rate and raw irrelevant count must decrease. +- Identifier and paraphrase recall gains must remain represented in overall hit@1/3/5. +- Fusion remains deterministic and requires no model in unit tests. +- Existing semantic degradation behavior and external result shapes remain unchanged. + +## Diagnosis + +The six hybrid noisy appearances were: + +| Case | Variant | Irrelevant result | Hybrid evidence | +|---|---|---|---| +| `errors-boundary-exact` | identifier | `errors-boundary-message` | semantic-only rank 2, cosine 0.334 | +| `obs-trace-exact` | identifier | `obs-trace-pii` | semantic-only rank 3, cosine 0.475 | +| `obs-trace-natural` | natural language | `obs-trace-dev` | lexical 3 + semantic 1, cosine 0.606 | +| `perf-batch-natural` | natural language | `perf-batch-interactive` | lexical 2 + semantic 2, cosine 0.544 | +| `dto-paraphrase` | paraphrase | `api-pagination` | lexical 3 + semantic 16, cosine 0.239 | +| `logging-natural` | natural language | `log-request-id` | lexical 9 + semantic 2, cosine 0.403 | + +Two failures were weak semantic-only fillers. Four were lexical distractors reinforced by semantic rank. A global cosine floor was rejected because a noisy result scored 0.606 while a judged-relevant result scored as low as 0.238. + +## Assumptions and Follow-up + +The current fixture is sufficient for a release-gating comparison but not a final precision estimate. The benchmark should eventually judge the pooled lexical and hybrid top-three results: only 127 of 360 unique query/result pairs are currently judged, leaving 233 missing judgments. diff --git a/docs/ai/testing/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/testing/2026-09-02-feature-memory-noise-reduction.md new file mode 100644 index 00000000..6c78d6e2 --- /dev/null +++ b/docs/ai/testing/2026-09-02-feature-memory-noise-reduction.md @@ -0,0 +1,42 @@ +--- +phase: testing +title: Memory Hybrid Noise Reduction Testing +description: Validate deterministic ranking behavior, regression safety, and expanded-100 quality +--- + +# Memory Hybrid Noise Reduction Testing + +## Automated Coverage + +- [x] Suppress a semantic-only candidate below cosine 0.50. +- [x] Admit a semantic-only candidate at the 0.50 boundary. +- [x] Preserve a low-similarity candidate independently found by lexical retrieval. +- [x] Rank lexical rank 1 above a candidate appearing at rank 20 in both channels. +- [x] Preserve deterministic fusion and existing lexical tie protection. +- [x] Regression proof: restoring baseline `k=60` and removing the guard makes exactly the two new tests fail. +- [x] Final `npm run build`: all six projects built successfully. +- [x] Final `npm test`: 174 test files and 2,165 tests passed across six projects. +- [x] Final `npm run lint`: all six projects passed with zero errors and three pre-existing CLI warnings. +- [x] Final `npx vitest run --config e2e/vitest.config.ts`: 1 file and 41 tests passed. + +Unit tests construct candidates and ranks directly. They do not load MiniLM or depend on floating model output. + +## Expanded-100 Evidence + +| Configuration | hit@1 | hit@3 | hit@5 | Known bad | Judged irrelevant | Bad / returned slots | Coverage | Zero | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| Dev semantic baseline | 88% | 97% | 98% | 6 | 4.7% | 2.00% | 42.3% | 0% | +| Equal RRF `k=10` | 88% | 97% | 98% | 5 | 4.0% | 1.67% | 42.0% | 0% | +| Asymmetric RRF 60/90 | 86% | 97% | 98% | 5 | 4.0% | 1.67% | 41.7% | 0% | +| Semantic-only cosine >=0.50 | 88% | 97% | 98% | 4 | 3.3% | 1.82% | 55.5% | 1% | +| Combined selected policy | 88% | 97% | 98% | 3 | 2.5% | 1.36% | 55.0% | 1% | + +The selected policy has no hit@3 regression and reduces the conditional noise rate by 2.2 percentage points. The 1% zero rate is the `169.254.169.254` query, whose former semantic results contained no relevant top-five answer. + +## Coverage Caveat + +The benchmark judges only returned query/result pairs that have explicit fixture labels. Hybrid baseline coverage is 127/300 top-three slots, not evidence that the other 173 are relevant or irrelevant. Across the pooled lexical and hybrid top-three results, 233 of 360 unique pairs still need judgments. Review decisions therefore use raw known-bad counts, conditional rate, returned-slot rate, coverage, and recall together. + +## Release Follow-up + +After the next npm release, rerun expanded-100 using the released package and add that released-version result to the benchmark leaderboard. Dev-mode experiment files remain outside the repository leaderboard. diff --git a/packages/memory/src/semantic/embeddings.ts b/packages/memory/src/semantic/embeddings.ts index 001fb3bc..135a849e 100644 --- a/packages/memory/src/semantic/embeddings.ts +++ b/packages/memory/src/semantic/embeddings.ts @@ -1,7 +1,8 @@ import type { SearchResultItem, SearchRetrievalExplanation } from '../domain/knowledge/types.js'; export const EMBEDDING_DIMENSION = 384; -export const RRF_K = 60; +export const RRF_K = 10; +const SEMANTIC_ONLY_MIN_SIMILARITY = 0.5; interface EmbeddingDocument { title: string; @@ -92,6 +93,7 @@ export function fuseSearchResults( existing.similarity = candidate.similarity; return; } + if (candidate.similarity < SEMANTIC_ONLY_MIN_SIMILARITY) return; const { similarity, ...item } = candidate; fused.set(candidate.id, { item: { ...item, score: 0 }, diff --git a/packages/memory/tests/unit/semantic/embeddings.test.ts b/packages/memory/tests/unit/semantic/embeddings.test.ts index 71bf9edb..77c35ac9 100644 --- a/packages/memory/tests/unit/semantic/embeddings.test.ts +++ b/packages/memory/tests/unit/semantic/embeddings.test.ts @@ -43,4 +43,30 @@ describe('semantic retrieval primitives', () => { expect(first).toEqual(second); expect(first[1]?.retrieval).toMatchObject({ lexicalRank: 1, semanticRank: null }); }); + + it('suppresses weak semantic-only candidates without discarding lexical matches', () => { + const item = (id: string) => ({ id, title: id, content: 'x', tags: [], scope: 'global', score: 1 }); + const results = fuseSearchResults( + [item('lexical-low-similarity')], + [ + { ...item('lexical-low-similarity'), similarity: 0.2 }, + { ...item('semantic-low-similarity'), similarity: 0.49 }, + { ...item('semantic-confident'), similarity: 0.5 }, + ], + 5, + ); + + expect(results.map(result => result.id)).toEqual(['lexical-low-similarity', 'semantic-confident']); + }); + + it('keeps a top lexical match above weak agreement deep in both channels', () => { + const item = (id: string) => ({ id, title: id, content: 'x', tags: [], scope: 'global', score: 1 }); + const lexical = [item('exact'), ...Array.from({ length: 18 }, (_, index) => item(`lexical-${index}`)), item('weak')]; + const semantic = [...Array.from({ length: 19 }, (_, index) => ({ + ...item(`semantic-${index}`), + similarity: 1 - index / 100, + })), { ...item('weak'), similarity: 0.5 }]; + + expect(fuseSearchResults(lexical, semantic, 2)[0]?.id).toBe('exact'); + }); });