From 3b705f523a5f01e862380df222e0863bab9d5273 Mon Sep 17 00:00:00 2001 From: Hoang Nguyen Date: Wed, 2 Sep 2026 16:16:17 +0000 Subject: [PATCH 1/9] fix(memory): emphasize top ranks in hybrid fusion Expanded-100 semantic benchmark: hit@1/3/5 88/97/98, judged irrelevant top-3 4.0% (baseline 4.7%), coverage 42.0%, zero results 0%. --- packages/memory/src/semantic/embeddings.ts | 2 +- .../memory/tests/unit/semantic/embeddings.test.ts | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/memory/src/semantic/embeddings.ts b/packages/memory/src/semantic/embeddings.ts index 001fb3bc..a9cb95aa 100644 --- a/packages/memory/src/semantic/embeddings.ts +++ b/packages/memory/src/semantic/embeddings.ts @@ -1,7 +1,7 @@ import type { SearchResultItem, SearchRetrievalExplanation } from '../domain/knowledge/types.js'; export const EMBEDDING_DIMENSION = 384; -export const RRF_K = 60; +export const RRF_K = 10; interface EmbeddingDocument { title: string; diff --git a/packages/memory/tests/unit/semantic/embeddings.test.ts b/packages/memory/tests/unit/semantic/embeddings.test.ts index 71bf9edb..e1cfe9b0 100644 --- a/packages/memory/tests/unit/semantic/embeddings.test.ts +++ b/packages/memory/tests/unit/semantic/embeddings.test.ts @@ -43,4 +43,15 @@ describe('semantic retrieval primitives', () => { expect(first).toEqual(second); expect(first[1]?.retrieval).toMatchObject({ lexicalRank: 1, semanticRank: null }); }); + + it('keeps a top lexical match above weak agreement deep in both channels', () => { + const item = (id: string) => ({ id, title: id, content: 'x', tags: [], scope: 'global', score: 1 }); + const lexical = [item('exact'), ...Array.from({ length: 18 }, (_, index) => item(`lexical-${index}`)), item('weak')]; + const semantic = [...Array.from({ length: 19 }, (_, index) => ({ + ...item(`semantic-${index}`), + similarity: 1 - index / 100, + })), { ...item('weak'), similarity: 0.5 }]; + + expect(fuseSearchResults(lexical, semantic, 2)[0]?.id).toBe('exact'); + }); }); From 7948e7848efe62292104b04b786492928a76f0b6 Mon Sep 17 00:00:00 2001 From: Hoang Nguyen Date: Wed, 2 Sep 2026 16:16:26 +0000 Subject: [PATCH 2/9] Revert "fix(memory): emphasize top ranks in hybrid fusion" This reverts commit 3b705f523a5f01e862380df222e0863bab9d5273. --- packages/memory/src/semantic/embeddings.ts | 2 +- .../memory/tests/unit/semantic/embeddings.test.ts | 11 ----------- 2 files changed, 1 insertion(+), 12 deletions(-) diff --git a/packages/memory/src/semantic/embeddings.ts b/packages/memory/src/semantic/embeddings.ts index a9cb95aa..001fb3bc 100644 --- a/packages/memory/src/semantic/embeddings.ts +++ b/packages/memory/src/semantic/embeddings.ts @@ -1,7 +1,7 @@ import type { SearchResultItem, SearchRetrievalExplanation } from '../domain/knowledge/types.js'; export const EMBEDDING_DIMENSION = 384; -export const RRF_K = 10; +export const RRF_K = 60; interface EmbeddingDocument { title: string; diff --git a/packages/memory/tests/unit/semantic/embeddings.test.ts b/packages/memory/tests/unit/semantic/embeddings.test.ts index e1cfe9b0..71bf9edb 100644 --- a/packages/memory/tests/unit/semantic/embeddings.test.ts +++ b/packages/memory/tests/unit/semantic/embeddings.test.ts @@ -43,15 +43,4 @@ describe('semantic retrieval primitives', () => { expect(first).toEqual(second); expect(first[1]?.retrieval).toMatchObject({ lexicalRank: 1, semanticRank: null }); }); - - it('keeps a top lexical match above weak agreement deep in both channels', () => { - const item = (id: string) => ({ id, title: id, content: 'x', tags: [], scope: 'global', score: 1 }); - const lexical = [item('exact'), ...Array.from({ length: 18 }, (_, index) => item(`lexical-${index}`)), item('weak')]; - const semantic = [...Array.from({ length: 19 }, (_, index) => ({ - ...item(`semantic-${index}`), - similarity: 1 - index / 100, - })), { ...item('weak'), similarity: 0.5 }]; - - expect(fuseSearchResults(lexical, semantic, 2)[0]?.id).toBe('exact'); - }); }); From 2b444e5b72bbb930926426d77b992a834c54b3c9 Mon Sep 17 00:00:00 2001 From: Hoang Nguyen Date: Wed, 2 Sep 2026 16:21:55 +0000 Subject: [PATCH 3/9] fix(memory): discount semantic-only fusion ranks Expanded-100 semantic benchmark: hit@1/3/5 86/97/98, judged irrelevant top-3 4.0% (baseline 4.7%), coverage 41.7%, zero results 0%. --- packages/memory/src/semantic/embeddings.ts | 7 ++++--- packages/memory/tests/unit/semantic/embeddings.test.ts | 8 ++++++++ 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/packages/memory/src/semantic/embeddings.ts b/packages/memory/src/semantic/embeddings.ts index 001fb3bc..dc5ddced 100644 --- a/packages/memory/src/semantic/embeddings.ts +++ b/packages/memory/src/semantic/embeddings.ts @@ -1,7 +1,8 @@ import type { SearchResultItem, SearchRetrievalExplanation } from '../domain/knowledge/types.js'; export const EMBEDDING_DIMENSION = 384; -export const RRF_K = 60; +export const LEXICAL_RRF_K = 60; +export const SEMANTIC_RRF_K = 90; interface EmbeddingDocument { title: string; @@ -76,7 +77,7 @@ export function fuseSearchResults( lexical.forEach((item, index) => { fused.set(item.id, { item, - score: 1 / (RRF_K + index + 1), + score: 1 / (LEXICAL_RRF_K + index + 1), lexicalRank: index + 1, semanticRank: null, similarity: null, @@ -85,7 +86,7 @@ export function fuseSearchResults( semantic.forEach((candidate, index) => { const existing = fused.get(candidate.id); - const semanticScore = 1 / (RRF_K + index + 1); + const semanticScore = 1 / (SEMANTIC_RRF_K + index + 1); if (existing) { existing.score += semanticScore; existing.semanticRank = index + 1; diff --git a/packages/memory/tests/unit/semantic/embeddings.test.ts b/packages/memory/tests/unit/semantic/embeddings.test.ts index 71bf9edb..0f875f9e 100644 --- a/packages/memory/tests/unit/semantic/embeddings.test.ts +++ b/packages/memory/tests/unit/semantic/embeddings.test.ts @@ -43,4 +43,12 @@ describe('semantic retrieval primitives', () => { expect(first).toEqual(second); expect(first[1]?.retrieval).toMatchObject({ lexicalRank: 1, semanticRank: null }); }); + + it('keeps lexical candidates above semantic-only candidates during fusion', () => { + const item = (id: string) => ({ id, title: id, content: 'x', tags: [], scope: 'global', score: 1 }); + const lexical = Array.from({ length: 20 }, (_, index) => item(`lexical-${index}`)); + const semantic = [{ ...item('semantic-only'), similarity: 0.9 }]; + + expect(fuseSearchResults(lexical, semantic, 21).at(-1)?.id).toBe('semantic-only'); + }); }); From d557a6ec76e8212ff05de546a1007f4b8839775f Mon Sep 17 00:00:00 2001 From: Hoang Nguyen Date: Wed, 2 Sep 2026 16:22:01 +0000 Subject: [PATCH 4/9] Revert "fix(memory): discount semantic-only fusion ranks" This reverts commit 2b444e5b72bbb930926426d77b992a834c54b3c9. --- packages/memory/src/semantic/embeddings.ts | 7 +++---- packages/memory/tests/unit/semantic/embeddings.test.ts | 8 -------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/packages/memory/src/semantic/embeddings.ts b/packages/memory/src/semantic/embeddings.ts index dc5ddced..001fb3bc 100644 --- a/packages/memory/src/semantic/embeddings.ts +++ b/packages/memory/src/semantic/embeddings.ts @@ -1,8 +1,7 @@ import type { SearchResultItem, SearchRetrievalExplanation } from '../domain/knowledge/types.js'; export const EMBEDDING_DIMENSION = 384; -export const LEXICAL_RRF_K = 60; -export const SEMANTIC_RRF_K = 90; +export const RRF_K = 60; interface EmbeddingDocument { title: string; @@ -77,7 +76,7 @@ export function fuseSearchResults( lexical.forEach((item, index) => { fused.set(item.id, { item, - score: 1 / (LEXICAL_RRF_K + index + 1), + score: 1 / (RRF_K + index + 1), lexicalRank: index + 1, semanticRank: null, similarity: null, @@ -86,7 +85,7 @@ export function fuseSearchResults( semantic.forEach((candidate, index) => { const existing = fused.get(candidate.id); - const semanticScore = 1 / (SEMANTIC_RRF_K + index + 1); + const semanticScore = 1 / (RRF_K + index + 1); if (existing) { existing.score += semanticScore; existing.semanticRank = index + 1; diff --git a/packages/memory/tests/unit/semantic/embeddings.test.ts b/packages/memory/tests/unit/semantic/embeddings.test.ts index 0f875f9e..71bf9edb 100644 --- a/packages/memory/tests/unit/semantic/embeddings.test.ts +++ b/packages/memory/tests/unit/semantic/embeddings.test.ts @@ -43,12 +43,4 @@ describe('semantic retrieval primitives', () => { expect(first).toEqual(second); expect(first[1]?.retrieval).toMatchObject({ lexicalRank: 1, semanticRank: null }); }); - - it('keeps lexical candidates above semantic-only candidates during fusion', () => { - const item = (id: string) => ({ id, title: id, content: 'x', tags: [], scope: 'global', score: 1 }); - const lexical = Array.from({ length: 20 }, (_, index) => item(`lexical-${index}`)); - const semantic = [{ ...item('semantic-only'), similarity: 0.9 }]; - - expect(fuseSearchResults(lexical, semantic, 21).at(-1)?.id).toBe('semantic-only'); - }); }); From 1213226cb5098cd766f0d5e1b0bfba582fecc8c5 Mon Sep 17 00:00:00 2001 From: Hoang Nguyen Date: Wed, 2 Sep 2026 16:27:44 +0000 Subject: [PATCH 5/9] fix(memory): filter weak semantic-only matches Expanded-100 semantic benchmark: hit@1/3/5 88/97/98, judged irrelevant top-3 3.3% (baseline 4.7%), coverage 55.5%, zero results 1%. --- packages/memory/src/semantic/embeddings.ts | 2 ++ .../memory/tests/unit/semantic/embeddings.test.ts | 15 +++++++++++++++ 2 files changed, 17 insertions(+) diff --git a/packages/memory/src/semantic/embeddings.ts b/packages/memory/src/semantic/embeddings.ts index 001fb3bc..d307d2ed 100644 --- a/packages/memory/src/semantic/embeddings.ts +++ b/packages/memory/src/semantic/embeddings.ts @@ -2,6 +2,7 @@ import type { SearchResultItem, SearchRetrievalExplanation } from '../domain/kno export const EMBEDDING_DIMENSION = 384; export const RRF_K = 60; +const SEMANTIC_ONLY_MIN_SIMILARITY = 0.5; interface EmbeddingDocument { title: string; @@ -92,6 +93,7 @@ export function fuseSearchResults( existing.similarity = candidate.similarity; return; } + if (candidate.similarity < SEMANTIC_ONLY_MIN_SIMILARITY) return; const { similarity, ...item } = candidate; fused.set(candidate.id, { item: { ...item, score: 0 }, diff --git a/packages/memory/tests/unit/semantic/embeddings.test.ts b/packages/memory/tests/unit/semantic/embeddings.test.ts index 71bf9edb..49b25cec 100644 --- a/packages/memory/tests/unit/semantic/embeddings.test.ts +++ b/packages/memory/tests/unit/semantic/embeddings.test.ts @@ -43,4 +43,19 @@ describe('semantic retrieval primitives', () => { expect(first).toEqual(second); expect(first[1]?.retrieval).toMatchObject({ lexicalRank: 1, semanticRank: null }); }); + + it('suppresses weak semantic-only candidates without discarding lexical matches', () => { + const item = (id: string) => ({ id, title: id, content: 'x', tags: [], scope: 'global', score: 1 }); + const results = fuseSearchResults( + [item('lexical-low-similarity')], + [ + { ...item('lexical-low-similarity'), similarity: 0.2 }, + { ...item('semantic-low-similarity'), similarity: 0.49 }, + { ...item('semantic-confident'), similarity: 0.5 }, + ], + 5, + ); + + expect(results.map(result => result.id)).toEqual(['lexical-low-similarity', 'semantic-confident']); + }); }); From 3b70e34f00ba0ac5339dc66466313e80bdcb55e5 Mon Sep 17 00:00:00 2001 From: Hoang Nguyen Date: Wed, 2 Sep 2026 16:33:29 +0000 Subject: [PATCH 6/9] fix(memory): combine semantic guard with top-rank RRF Expanded-100 semantic benchmark: hit@1/3/5 88/97/98, judged irrelevant top-3 2.5% (baseline 4.7%), coverage 55.0%, zero results 1%. --- packages/memory/src/semantic/embeddings.ts | 2 +- .../memory/tests/unit/semantic/embeddings.test.ts | 11 +++++++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/packages/memory/src/semantic/embeddings.ts b/packages/memory/src/semantic/embeddings.ts index d307d2ed..135a849e 100644 --- a/packages/memory/src/semantic/embeddings.ts +++ b/packages/memory/src/semantic/embeddings.ts @@ -1,7 +1,7 @@ import type { SearchResultItem, SearchRetrievalExplanation } from '../domain/knowledge/types.js'; export const EMBEDDING_DIMENSION = 384; -export const RRF_K = 60; +export const RRF_K = 10; const SEMANTIC_ONLY_MIN_SIMILARITY = 0.5; interface EmbeddingDocument { diff --git a/packages/memory/tests/unit/semantic/embeddings.test.ts b/packages/memory/tests/unit/semantic/embeddings.test.ts index 49b25cec..77c35ac9 100644 --- a/packages/memory/tests/unit/semantic/embeddings.test.ts +++ b/packages/memory/tests/unit/semantic/embeddings.test.ts @@ -58,4 +58,15 @@ describe('semantic retrieval primitives', () => { expect(results.map(result => result.id)).toEqual(['lexical-low-similarity', 'semantic-confident']); }); + + it('keeps a top lexical match above weak agreement deep in both channels', () => { + const item = (id: string) => ({ id, title: id, content: 'x', tags: [], scope: 'global', score: 1 }); + const lexical = [item('exact'), ...Array.from({ length: 18 }, (_, index) => item(`lexical-${index}`)), item('weak')]; + const semantic = [...Array.from({ length: 19 }, (_, index) => ({ + ...item(`semantic-${index}`), + similarity: 1 - index / 100, + })), { ...item('weak'), similarity: 0.5 }]; + + expect(fuseSearchResults(lexical, semantic, 2)[0]?.id).toBe('exact'); + }); }); From 40664636abf9ddb09a3333fbc7400b3a99ca69ad Mon Sep 17 00:00:00 2001 From: Hoang Nguyen Date: Wed, 2 Sep 2026 18:28:27 +0000 Subject: [PATCH 7/9] docs(memory): record hybrid noise reduction evidence --- ...26-09-02-feature-memory-noise-reduction.md | 57 +++++++++++++++++++ ...26-09-02-feature-memory-noise-reduction.md | 40 +++++++++++++ ...26-09-02-feature-memory-noise-reduction.md | 34 +++++++++++ ...26-09-02-feature-memory-noise-reduction.md | 55 ++++++++++++++++++ ...26-09-02-feature-memory-noise-reduction.md | 42 ++++++++++++++ 5 files changed, 228 insertions(+) create mode 100644 docs/ai/design/2026-09-02-feature-memory-noise-reduction.md create mode 100644 docs/ai/implementation/2026-09-02-feature-memory-noise-reduction.md create mode 100644 docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md create mode 100644 docs/ai/requirements/2026-09-02-feature-memory-noise-reduction.md create mode 100644 docs/ai/testing/2026-09-02-feature-memory-noise-reduction.md diff --git a/docs/ai/design/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/design/2026-09-02-feature-memory-noise-reduction.md new file mode 100644 index 00000000..0dfd2a62 --- /dev/null +++ b/docs/ai/design/2026-09-02-feature-memory-noise-reduction.md @@ -0,0 +1,57 @@ +--- +phase: design +title: Memory Hybrid Noise Reduction Design +description: Apply confidence only to semantic-only entrants and tighten reciprocal-rank decay +--- + +# Memory Hybrid Noise Reduction Design + +## Architecture + +```mermaid +flowchart LR + L[Lexical top 20] --> F[RRF fusion k=10] + S[Semantic top 20] --> C{Lexical match?} + C -->|yes| F + C -->|no and cosine >= 0.50| F + C -->|no and cosine < 0.50| X[Suppress] + F --> O[Requested result limit] +``` + +The change stays inside `fuseSearchResults`. It adds no API, persisted data, model work, or caller-specific branch. + +## Ranking Rules + +1. Every lexical candidate enters fusion. +2. A semantic candidate already present lexically contributes its semantic reciprocal-rank score at any cosine value. Independent channel agreement remains useful evidence. +3. A semantic-only candidate enters fusion only at cosine similarity 0.50 or greater. +4. Both channels use reciprocal-rank fusion with `k=10`, increasing separation between high and deep ranks. +5. Existing deterministic tie-breaking remains score, lexical presence, lexical rank, semantic rank, then ID. + +The threshold is internal rather than configurable because no current caller needs a tuning surface. Removing either policy is a one-line change plus its unit test. + +## Alternatives Tested + +| Candidate | hit@1 | hit@3 | hit@5 | Irrelevant top-3 | Coverage | Zero | Decision | +|---|---:|---:|---:|---:|---:|---:|---| +| Semantic baseline, `k=60` | 88% | 97% | 98% | 4.7% (6/127) | 42.3% | 0% | Replace | +| Equal RRF `k=10` | 88% | 97% | 98% | 4.0% (5/126) | 42.0% | 0% | Useful but incomplete | +| Asymmetric RRF 60/90 | 86% | 97% | 98% | 4.0% (5/125) | 41.7% | 0% | Rejected: loses 2 pp hit@1 | +| Semantic-only cosine >=0.50 | 88% | 97% | 98% | 3.3% (4 judged irrelevant) | 55.5% | 1% | Retained | +| Combined cosine >=0.50 + `k=10` | 88% | 97% | 98% | 2.5% (3 judged irrelevant) | 55.0% | 1% | Selected | + +The rank-discount experiment remains visible in Git history and was reverted before testing the threshold. The selected combination targets both observed mechanisms and passes the hit@3 gate. + +## Trade-offs + +- The selected policy can return fewer than the requested limit when all semantic-only candidates are weak. This is intentional: the former lexical-zero query `169.254.169.254` received five unrelated semantic results with cosine 0.14–0.18 and no relevant hit. +- Coverage rises partly because weak unjudged results are no longer returned. Report raw counts and returned-slot denominators alongside conditional rates. +- A fixed threshold may not generalize perfectly beyond expanded-100. Pooled judgments are a follow-up, not a reason to retain demonstrated noise. + +## Non-functional Properties + +- Performance: no additional inference or database work; filtering reduces fusion work. +- Security/privacy: unchanged local inference and data handling. +- Reliability: semantic failure still degrades to lexical-only search. +- Compatibility: no public type, response, configuration, or schema change. + diff --git a/docs/ai/implementation/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/implementation/2026-09-02-feature-memory-noise-reduction.md new file mode 100644 index 00000000..50709724 --- /dev/null +++ b/docs/ai/implementation/2026-09-02-feature-memory-noise-reduction.md @@ -0,0 +1,40 @@ +--- +phase: implementation +title: Memory Hybrid Noise Reduction Implementation +description: Record the selected fusion changes and empirical development workflow +--- + +# Memory Hybrid Noise Reduction Implementation + +## Changed Code + +- `packages/memory/src/semantic/embeddings.ts` + - Changes `RRF_K` from 60 to 10. + - Adds the internal `SEMANTIC_ONLY_MIN_SIMILARITY = 0.5` constant. + - Applies the threshold only after checking for an existing lexical candidate, preserving low-similarity semantic reinforcement of lexical results. +- `packages/memory/tests/unit/semantic/embeddings.test.ts` + - Verifies a 0.49 semantic-only candidate is suppressed, a 0.50 candidate is admitted, and a shared lexical candidate remains. + - Verifies lexical rank 1 outranks weak agreement at lexical/semantic rank 20 with tighter RRF decay. + +## Integration + +`fuseSearchResults` remains the single ranking seam used by `searchKnowledgeHybrid`. CLI and MCP callers receive the existing result structure and semantic explanation fields. The storage, embedding, fallback, and model-loading paths are untouched. + +## Benchmark Procedure + +The community benchmark was run in dev mode against the built CLI, not an npm release: + +```bash +cd /home/ubuntu/code/agent-memory-bench +AI_DEVKIT_BIN=/home/ubuntu/code/ai-devkit/.worktrees/feature-memory-noise/packages/cli/dist/cli.js \ + npm run bench -- --label --semantic --output /tmp/.json +``` + +Each candidate was applied, built, and measured against the same expanded-100 fixture definition. Rejected ranking candidates were reverted before the next standalone experiment. No experiment row was added to the released-version leaderboard. + +## Operational Considerations + +- No migration, deployment configuration, or rollback operation is required. +- Reverting the two selected commits restores the previous fusion policy. +- The threshold can reduce result count for weak semantic-only queries; the API already permits fewer results than the requested maximum. + diff --git a/docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md new file mode 100644 index 00000000..53bdd930 --- /dev/null +++ b/docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md @@ -0,0 +1,34 @@ +--- +phase: planning +title: Memory Hybrid Noise Reduction Plan +description: Track diagnosis, isolated experiments, selected implementation, and release validation +--- + +# Memory Hybrid Noise Reduction Plan + +## Completed Work + +- [x] Reproduce the 0.59.0 lexical and semantic expanded-100 rows. +- [x] Run all queries with retrieval explanations against one shared seeded corpus. +- [x] Identify all six judged-irrelevant hybrid top-three appearances and classify channel membership. +- [x] Correct the coverage interpretation using raw judged and returned slot counts. +- [x] Test equal `k=10` in isolation and record 88/97/98 with 4.0% irrelevant. +- [x] Test asymmetric lexical/semantic 60/90 in isolation and record 86/97/98 with 4.0% irrelevant. +- [x] Revert the asymmetric experiment before the next candidate. +- [x] Test a semantic-only cosine floor of 0.50 and record 88/97/98 with 3.3% irrelevant. +- [x] Test the combined floor plus `k=10` and record 88/97/98 with 2.5% irrelevant. +- [x] Add deterministic unit tests for both selected ranking rules. +- [x] Prove the tests fail when the selected implementation is removed. +- [x] Run final build, full test, lint, and E2E gates after documentation changes. +- [ ] Publish the branch and open a review request without merging. + +## Dependencies and Sequencing + +The real model and expanded-100 fixture were required only for empirical selection. Unit tests use explicit candidates and ranks, so routine validation does not download or execute the model. A released-version leaderboard row depends on the next ai-devkit release and is deliberately deferred. + +## Risks and Mitigations + +- **Sparse judgments:** include raw counts and coverage in review evidence; expand pooled judgments separately. +- **Threshold overfitting:** retain lexical matches regardless of cosine and preserve all measured hit metrics. +- **Ranking regression:** deterministic tests cover the threshold boundary and deep dual-channel agreement. +- **History noise:** rejected experiments remain as evidence, with explicit revert commits; reviewers can evaluate the final diff against `origin/main` independently. diff --git a/docs/ai/requirements/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/requirements/2026-09-02-feature-memory-noise-reduction.md new file mode 100644 index 00000000..4716cbda --- /dev/null +++ b/docs/ai/requirements/2026-09-02-feature-memory-noise-reduction.md @@ -0,0 +1,55 @@ +--- +phase: requirements +title: Memory Hybrid Noise Reduction Requirements +description: Reduce judged-irrelevant hybrid search results without giving back semantic recall +--- + +# Memory Hybrid Noise Reduction Requirements + +## Problem + +Opt-in hybrid search improved expanded-100 hit@1/3/5 from 81/91/96 to 88/97/98 and removed the one empty response, but judged-irrelevant top-three results rose from 2.9% to 4.7%. Agents and CLI users benefit from the recall gain only if plausible but answer-wrong memories do not crowd the first results. + +The percentage needs context. Lexical search produced 3 irrelevant results among 104 judged slots and 207 returned top-three slots. Hybrid produced 6 among 127 judged slots and 300 returned slots. Hybrid therefore had more judged slots in absolute terms, while its reported coverage fell from 50.2% to 42.3% because it filled more positions. The conditional 2.9% to 4.7% comparison is valid but sensitive to incomplete, non-random judgments. + +## Goals + +- Preserve the semantic-on hit@1/3/5 improvement. +- Reduce explicitly irrelevant top-three results using a small, deterministic fusion change. +- Keep semantic search opt-in, local, fail-open, and API-compatible. +- Keep lexical matches eligible even when their semantic similarity is low. + +## Non-goals + +- Changing the embedding model, storage schema, corpus cap, or public configuration. +- Adding a learned reranker or another inference pass. +- Publishing a benchmark leaderboard row before a released ai-devkit version exists. +- Treating unjudged results as irrelevant. + +## Success Criteria + +- Expanded-100 hit@3 must remain at least 96%, no more than 1 percentage point below the 97% semantic baseline. +- Judged-irrelevant top-three rate and raw irrelevant count must decrease. +- Identifier and paraphrase recall gains must remain represented in overall hit@1/3/5. +- Fusion remains deterministic and requires no model in unit tests. +- Existing semantic degradation behavior and external result shapes remain unchanged. + +## Diagnosis + +The six hybrid noisy appearances were: + +| Case | Variant | Irrelevant result | Hybrid evidence | +|---|---|---|---| +| `errors-boundary-exact` | identifier | `errors-boundary-message` | semantic-only rank 2, cosine 0.334 | +| `obs-trace-exact` | identifier | `obs-trace-pii` | semantic-only rank 3, cosine 0.475 | +| `obs-trace-natural` | natural language | `obs-trace-dev` | lexical 3 + semantic 1, cosine 0.606 | +| `perf-batch-natural` | natural language | `perf-batch-interactive` | lexical 2 + semantic 2, cosine 0.544 | +| `dto-paraphrase` | paraphrase | `api-pagination` | lexical 3 + semantic 16, cosine 0.239 | +| `logging-natural` | natural language | `log-request-id` | lexical 9 + semantic 2, cosine 0.403 | + +Two failures were weak semantic-only fillers. Four were lexical distractors reinforced by semantic rank. A global cosine floor was rejected because a noisy result scored 0.606 while a judged-relevant result scored as low as 0.238. + +## Assumptions and Follow-up + +The current fixture is sufficient for a release-gating comparison but not a final precision estimate. The benchmark should eventually judge the pooled lexical and hybrid top-three results: only 127 of 360 unique query/result pairs are currently judged, leaving 233 missing judgments. + diff --git a/docs/ai/testing/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/testing/2026-09-02-feature-memory-noise-reduction.md new file mode 100644 index 00000000..6c78d6e2 --- /dev/null +++ b/docs/ai/testing/2026-09-02-feature-memory-noise-reduction.md @@ -0,0 +1,42 @@ +--- +phase: testing +title: Memory Hybrid Noise Reduction Testing +description: Validate deterministic ranking behavior, regression safety, and expanded-100 quality +--- + +# Memory Hybrid Noise Reduction Testing + +## Automated Coverage + +- [x] Suppress a semantic-only candidate below cosine 0.50. +- [x] Admit a semantic-only candidate at the 0.50 boundary. +- [x] Preserve a low-similarity candidate independently found by lexical retrieval. +- [x] Rank lexical rank 1 above a candidate appearing at rank 20 in both channels. +- [x] Preserve deterministic fusion and existing lexical tie protection. +- [x] Regression proof: restoring baseline `k=60` and removing the guard makes exactly the two new tests fail. +- [x] Final `npm run build`: all six projects built successfully. +- [x] Final `npm test`: 174 test files and 2,165 tests passed across six projects. +- [x] Final `npm run lint`: all six projects passed with zero errors and three pre-existing CLI warnings. +- [x] Final `npx vitest run --config e2e/vitest.config.ts`: 1 file and 41 tests passed. + +Unit tests construct candidates and ranks directly. They do not load MiniLM or depend on floating model output. + +## Expanded-100 Evidence + +| Configuration | hit@1 | hit@3 | hit@5 | Known bad | Judged irrelevant | Bad / returned slots | Coverage | Zero | +|---|---:|---:|---:|---:|---:|---:|---:|---:| +| Dev semantic baseline | 88% | 97% | 98% | 6 | 4.7% | 2.00% | 42.3% | 0% | +| Equal RRF `k=10` | 88% | 97% | 98% | 5 | 4.0% | 1.67% | 42.0% | 0% | +| Asymmetric RRF 60/90 | 86% | 97% | 98% | 5 | 4.0% | 1.67% | 41.7% | 0% | +| Semantic-only cosine >=0.50 | 88% | 97% | 98% | 4 | 3.3% | 1.82% | 55.5% | 1% | +| Combined selected policy | 88% | 97% | 98% | 3 | 2.5% | 1.36% | 55.0% | 1% | + +The selected policy has no hit@3 regression and reduces the conditional noise rate by 2.2 percentage points. The 1% zero rate is the `169.254.169.254` query, whose former semantic results contained no relevant top-five answer. + +## Coverage Caveat + +The benchmark judges only returned query/result pairs that have explicit fixture labels. Hybrid baseline coverage is 127/300 top-three slots, not evidence that the other 173 are relevant or irrelevant. Across the pooled lexical and hybrid top-three results, 233 of 360 unique pairs still need judgments. Review decisions therefore use raw known-bad counts, conditional rate, returned-slot rate, coverage, and recall together. + +## Release Follow-up + +After the next npm release, rerun expanded-100 using the released package and add that released-version result to the benchmark leaderboard. Dev-mode experiment files remain outside the repository leaderboard. From 5def4dd7dfca2f41919d5f1492568bbec6e4cf5f Mon Sep 17 00:00:00 2001 From: Hoang Nguyen Date: Wed, 2 Sep 2026 18:29:04 +0000 Subject: [PATCH 8/9] docs(memory): normalize lifecycle markdown --- docs/ai/design/2026-09-02-feature-memory-noise-reduction.md | 1 - .../implementation/2026-09-02-feature-memory-noise-reduction.md | 1 - .../ai/requirements/2026-09-02-feature-memory-noise-reduction.md | 1 - 3 files changed, 3 deletions(-) diff --git a/docs/ai/design/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/design/2026-09-02-feature-memory-noise-reduction.md index 0dfd2a62..bae91c78 100644 --- a/docs/ai/design/2026-09-02-feature-memory-noise-reduction.md +++ b/docs/ai/design/2026-09-02-feature-memory-noise-reduction.md @@ -54,4 +54,3 @@ The rank-discount experiment remains visible in Git history and was reverted bef - Security/privacy: unchanged local inference and data handling. - Reliability: semantic failure still degrades to lexical-only search. - Compatibility: no public type, response, configuration, or schema change. - diff --git a/docs/ai/implementation/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/implementation/2026-09-02-feature-memory-noise-reduction.md index 50709724..f95edb43 100644 --- a/docs/ai/implementation/2026-09-02-feature-memory-noise-reduction.md +++ b/docs/ai/implementation/2026-09-02-feature-memory-noise-reduction.md @@ -37,4 +37,3 @@ Each candidate was applied, built, and measured against the same expanded-100 fi - No migration, deployment configuration, or rollback operation is required. - Reverting the two selected commits restores the previous fusion policy. - The threshold can reduce result count for weak semantic-only queries; the API already permits fewer results than the requested maximum. - diff --git a/docs/ai/requirements/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/requirements/2026-09-02-feature-memory-noise-reduction.md index 4716cbda..d7d4a653 100644 --- a/docs/ai/requirements/2026-09-02-feature-memory-noise-reduction.md +++ b/docs/ai/requirements/2026-09-02-feature-memory-noise-reduction.md @@ -52,4 +52,3 @@ Two failures were weak semantic-only fillers. Four were lexical distractors rein ## Assumptions and Follow-up The current fixture is sufficient for a release-gating comparison but not a final precision estimate. The benchmark should eventually judge the pooled lexical and hybrid top-three results: only 127 of 360 unique query/result pairs are currently judged, leaving 233 missing judgments. - From 6306d51d0e0af28ed9dfaf044691145a8da060f1 Mon Sep 17 00:00:00 2001 From: Hoang Nguyen Date: Wed, 2 Sep 2026 18:30:14 +0000 Subject: [PATCH 9/9] docs(memory): mark review publication complete --- docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md b/docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md index 53bdd930..1966433d 100644 --- a/docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md +++ b/docs/ai/planning/2026-09-02-feature-memory-noise-reduction.md @@ -20,7 +20,7 @@ description: Track diagnosis, isolated experiments, selected implementation, and - [x] Add deterministic unit tests for both selected ranking rules. - [x] Prove the tests fail when the selected implementation is removed. - [x] Run final build, full test, lint, and E2E gates after documentation changes. -- [ ] Publish the branch and open a review request without merging. +- [x] Publish the branch and open a review request without merging. ## Dependencies and Sequencing