diff --git a/.gitattributes b/.gitattributes index 8aedc091..26d4603b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,3 +1,4 @@ * text=auto eol=lf -# The standalone preview embeds generated JavaScript with significant whitespace. +# Vendored upstream reference source stays out of GitHub's language stats. +repos/** linguist-vendored diff --git a/.project-kit/manifest.json b/.project-kit/manifest.json index 9ac9b22f..1d40e3b0 100644 --- a/.project-kit/manifest.json +++ b/.project-kit/manifest.json @@ -26,13 +26,11 @@ }, "design": { "status": "production-workbench", - "selection": "design/selection.json", - "surface": "design/DESIGN.md" + "surface": "docs/design.md" }, "reviewRuntime": { "status": "implemented-local-and-cloud-review", "documentation": "docs/review-runtime.md", - "comparisonDocumentation": "docs/research/agent-harness-comparison.md", "harness": "@earendil-works/pi-durable", "applicationRuntime": "bun", "localExecutionDefault": true, @@ -46,7 +44,7 @@ "automaticReturn": "live-transport-and-recovery-with-uncertain-model-outcome", "drawingSetReview": "recorded-provider-and-real-package-roundtrip" }, - "validationDocumentation": "docs/research/cloud-validation.md", + "validationDocumentation": "docs/validation/cloud-validation.md", "documentation": "docs/cloud-reviews.md", "sdk": "modal@0.11.1", "language": "TypeScript on Bun", @@ -76,7 +74,7 @@ "https://modal.com/docs/guide/sandbox-snapshots", "https://modal.com/docs/guide/volumes" ], - "implementationDocumentation": "docs/research/pi-runtime.md", + "implementationDocumentation": "docs/architecture/pi-runtime.md", "versions": { "pi": "1.4.0", "pi-ai": "1.4.0", @@ -86,8 +84,8 @@ "heroCase": "docs/fixtures/hero-case.md", "primaryDemoPlatform": "linux-x64", "promptInstructions": { - "version": "moke-packet-review-v8", - "source": "https://github.com/sc-frederick/skills/tree/8a0e2648738018631eaa6d24ee0b6ebaa239afb1/moke-review", + "version": "moke-packet-review-v10", + "source": "Author's private review-instruction repository at revision 8a0e2648738018631eaa6d24ee0b6ebaa239afb1", "provenance": "backend/review/prompts/provenance.json", "scope": "Whole supplied packet or selected discipline with explicit coverage, governing sources, findings and offline HTML report", "legacyCountVersion": "moke-selected-coordination-v2", @@ -95,7 +93,6 @@ "findingStyle": { "instructions": "backend/review/prompts/finding-style.md", "contract": "src/lib/review/finding-summary.ts", - "examples": "docs/research/meta-vs-sonnet-unslop-findings.md", "purpose": "Validated short display summaries with complete technical findings retained" } }, @@ -113,14 +110,13 @@ ] }, "firmMemory": { - "decision": "docs/firm-memory-plan.md", "decidedOn": "2026-10-30", "implemented": ["finding decisions", "firm checklist", "jurisdiction leads"], "deferred": "firm detail library, visual precedent retrieval, learned playbook", "migration": "migrations/0011_firm_checklist.sql" }, "codeIndex": { - "version": "florida-code-index-v2", + "version": "florida-code-index-v3", "module": "backend/review/code-index/code-index.ts", "provenance": "backend/review/code-index/provenance.json", "tool": "search_code", @@ -138,9 +134,6 @@ } }, "hackathon": { - "bonusDocumentation": "docs/hackathon-bonuses.md", - "primaryBonusTarget": "Best Agentic Use-Case", - "secondaryBonusTarget": "Best Build on Meta Open Weight Models", "sourcesCheckedOn": "2026-10-27", "sources": [ "https://cerebralvalley.ai/e/meta-ai-developer-hackathon/details", @@ -150,19 +143,19 @@ "segmentationExperiment": { "entrypoint": "scripts/test-sam.ts", "cases": "experiments/sam/cases.json", - "documentation": "docs/segmentation-test.md", + "documentation": "docs/experiments/segmentation-test.md", "model": "sam-3.1", "parser": "@meta-sam/parser@0.1.0", "pdfRenderer": "Poppler pdftoppm, external test dependency", "fixtureManifest": "Moke-Hackathon-Drawings/drawing-manifest.json", "artifactDirectory": ".local/sam", "plumbingStudy": { - "documentation": "docs/plumbing-segmentation.md", + "documentation": "docs/experiments/plumbing-segmentation.md", "cropRecipe": "experiments/sam/plumbing-crops.json", "candidateCases": "experiments/sam/plumbing-cases.json", "refinementCases": "experiments/sam/plumbing-refinement-cases.json", "reconciliationEntrypoint": "scripts/reconcile-fixtures.ts", - "mokeInstructions": "https://github.com/sc-frederick/skills/tree/8a0e2648738018631eaa6d24ee0b6ebaa239afb1/moke-review" + "mokeInstructions": "backend/review/prompts/assets" }, "sourcesCheckedOn": "2026-10-27", "sources": [ @@ -171,12 +164,8 @@ "https://dev.meta.ai/docs/sam/client-libraries" ], "drawingExpansion": { - "documentation": "docs/sam-drawing-expansion.md", + "documentation": "docs/experiments/sam-drawing-expansion.md", "recipes": "experiments/sam/symbols", - "researchTools": [ - "Executor Exa", - "Executor Firecrawl" - ], "sourcesCheckedOn": "2026-10-27", "sources": [ "https://arxiv.org/html/2511.16719v2", @@ -189,7 +178,7 @@ }, "adaptiveReview": { "entrypoint": "scripts/test-review-agent.ts", - "documentation": "docs/adaptive-meta-review.md", + "documentation": "docs/experiments/adaptive-meta-review.md", "cases": "experiments/sam/adaptive-cases.json", "models": [ "muse-spark-1.3", @@ -205,7 +194,7 @@ ] }, "additionalDisciplineSets": { - "documentation": "docs/sam-expanded-sets.md", + "documentation": "docs/experiments/sam-expanded-sets.md", "recipes": "experiments/sam/expanded", "fixtureManifest": "Moke-Hackathon-Drawings/expanded/drawing-manifest.json", "artifactDirectory": ".local/sam/iteration-5", @@ -230,7 +219,7 @@ "hackathonOverride": { "provider": "baseten", "model": "meta-models/muse-spark-open-1026", - "source": "User-provided Baseten hackathon onboarding screenshot, image.png" + "source": "Baseten hackathon onboarding" } } } diff --git a/AGENTS.md b/AGENTS.md index 63b8874b..4ba3bebb 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -22,13 +22,10 @@ vendored in tools/oxlint/anti-slop; preserve local customizations when updating. Use SvelteKit 3 configuration in vite.config.ts, Svelte 5 runes, and #lib imports. Use the Svelte coding skills and run their autofixer when editing components. -The initial notes UI is a functional smoke example. Choose visual design during -app design from any entry in sc-frederick/uiverse-templates, the user's private -backup of downloaded Uiverse design systems. Forge and Aether are recent choices. -Run project-kit designs to list the archive and project-kit design to -import the chosen reference. Preserve its DESIGN.md, assets, and license. Keep -reference source outside public distributions and adapt the selected design to -the application rather than adding a second UI system. +The workbench adapts the Forge design system; docs/design.md records how. Use the +tokens in src/lib/forge.css rather than adding a second UI system. A licensed copy +of the Forge pack may sit in the ignored design/reference/ directory. Its license +forbids republishing it, so never commit it or ship it in a distribution. ## Runtime boundaries @@ -47,7 +44,7 @@ migration or recovery behavior. Read docs/deployment.md before release changes. Authentication and cloud capabilities are selected per application. Implement authorization at operation boundaries. Choose desktop updater delivery and access -per project; aecPermitMonitor is a reference to evaluate for R2-hosted updates. +per project. Scaffolding itself does not deploy resources or publish applications. Follow the user's requested scope when moving from local implementation to external changes. diff --git a/PRODUCT.md b/PRODUCT.md index 6d7c5cda..d4ef7427 100644 --- a/PRODUCT.md +++ b/PRODUCT.md @@ -37,8 +37,8 @@ are known, and its wording stays unconfirmed until the engineer opens the link. Its stored status remains unverified. Calculations retain formulas, input values and declared units; units are not automatically reconciled. Model findings are review results, not professional -approval. Sequential discipline passes are supported; the app does not claim to -launch parallel reviewers. +approval. A packet with four or more sheets runs as a coordinator plus one +reviewer per sheet. The conversation presents a short review summary with compact findings. Opening a finding shows readable details and source evidence in the central canvas. @@ -96,7 +96,8 @@ Until a revised set arrives, an **Initial Review** button sits above the compose One click posts the preset first-submittal request (`initial-review-v1`, in `src/lib/review/initial-review.ts`) as the reviewer's message and starts a review of every sheet in the first set, with all disciplines and the four passes, on -Muse Spark 1.3. It runs in the cloud when cloud review is set up and can take the +the reviewer's preferred model. The default is Muse Spark 1.3, or Muse Spark 1.3 +Contributor when OpenCode Go is connected. It runs in the cloud when cloud review is set up and can take the workspace, and on this computer otherwise; one line beside the button says which. The review records the preset's identity. The button is disabled while a review runs in the conversation and can run again during the initial phase, for example @@ -105,11 +106,9 @@ span several PDFs: the phase ends when a PDF is linked to earlier sheets through revision correspondence, or when a PDF is imported after the project's first drawing-set review started. After that, reviewers use typed requests. -The original desktop UI study established chat as the main work surface. -Projects group their conversations in navigation, and selecting a conversation -updates the project and conversation breadcrumb. Closing the source pane restores -the chat's width. The user referenced T3 Code's information menu and the existing -result-to-evidence behavior in the aec_pgrep codebase. +Chat is the main work surface. Projects group their conversations in navigation, +and selecting a conversation updates the project and conversation breadcrumb. +Closing the source pane restores the chat's width. The application uses persisted projects, conversations, review requests, findings and source evidence. Sheet confirmation and revision correspondence remain @@ -117,8 +116,6 @@ explicit for count checks. ## Brand Commitments -The user named OpenCode, T3 Code and Cloudflare OS as visual references, with -particular emphasis on T3 Code's desktop polish. The current workbench adapts the -Forge design system selected from the Uiverse catalog: a dark, industrial -interface with hairline panels, one ember accent, compact navigation and -evidence beside the conversation. +The workbench adapts the Forge design system: a dark, industrial interface with +hairline panels, one ember accent, compact navigation and evidence beside the +conversation. See [the design notes](docs/design.md). diff --git a/README.md b/README.md index 1db28b97..9fa0196a 100644 --- a/README.md +++ b/README.md @@ -49,6 +49,15 @@ Regulatory findings need applicable governing sources; unavailable sources and unknown editions remain Needs input. SAM geometry does not establish physical dimensions. +## Install + +Download the installer for Windows, macOS (Apple silicon) or Linux from +[GitHub Releases](https://github.com/sc-frederick/moke-meta-hackathon/releases/latest). +The installers are unsigned. macOS can block the first launch; allow it under +System Settings › Privacy & Security › Open Anyway. On Linux, make the AppImage +executable or install the `.deb`. Model reviews need a provider key; see +[inference setup](docs/inference.md). + ## How it works | Process | Owns | @@ -69,8 +78,8 @@ optional call budget. The default packet review has no application call ceiling; long conversations use Pi compaction. Complete packages export and restore the database, original PDFs, rendered evidence and execution journals together. -Projects run locally, but inference uses hosted Meta or Baseten APIs and sends -selected drawing evidence to the provider. **Push to cloud** transfers the complete +Projects run locally, but inference uses hosted Meta, Baseten or OpenCode Go APIs, +or a custom endpoint you add, and sends selected drawing evidence to that provider. **Push to cloud** transfers the complete workspace for a new bounded review, streams progress, and automatically syncs results with local ownership checks. See [cloud setup and limits](docs/cloud-reviews.md), [the runtime decision](docs/review-runtime.md), and @@ -88,7 +97,9 @@ bun run dev You can import drawings and use manual count checks without provider keys. For model review, copy [.env.example](.env.example) to `.env` and set `OPENCODE_API_KEY`, `MODEL_API_KEY` or `BASETEN_API_KEY`. With a Go key, reviews run -on OpenCode Go, and Meta is used only for SAM and count checks. Otherwise choose +on Muse Spark 1.3 Contributor through OpenCode Go, and Meta's API is used only for +SAM and count checks. Contributor trains on your drawings, and Go refuses it until +a workspace owner allows that. Otherwise choose Muse Spark, Muse Wonder or Baseten Muse Spark Open in the app. Restart after changing credentials. See [inference setup](docs/inference.md) for packaged-app credentials, model capabilities and optional review budgets. @@ -125,12 +136,11 @@ migrations/ Immutable SQLite migrations tests/ Smoke tests, Electron end-to-end flows, and synthetic fixtures scripts/ Development, build, packaged checks, and opt-in model experiments docs/ Setup, architecture, screenshots, and validation evidence -design/ Forge design selection, adaptation notes, and icon license experiments/ SAM recipes, captured proposals, and research evidence ``` The [documentation index](docs/README.md) separates setup and architecture from -hackathon research and drawing experiments. The [workbench design notes](design/DESIGN.md) +validation records and drawing experiments. The [workbench design notes](docs/design.md) record the Forge design system, its adaptations, and third-party asset attribution. ## Hackathon provenance @@ -138,11 +148,16 @@ record the Forge design system, its adaptations, and third-party asset attributi The Moke name and review prompts predate this entry. This repository implements a fresh application from the project-kit desktop starter, with drawing evidence, corrections, and revision tracking built for the hackathon. It does not reuse the -earlier Moke application. Prompt and dependency provenance live in +earlier Moke application. Pi Durable provides agent execution as an MIT-licensed +dependency; the new work is the drawing-review application, evidence handling, +corrections, revision behavior and portable review execution. Prompt and +dependency provenance live in [the project manifest](.project-kit/manifest.json) and [the review prompt sources](backend/review/prompts/README.md). ## License -Moke is released under the [MIT License](LICENSE). Vendored code and design -assets keep their own licenses, which sit beside them in the repository. +Moke is released under the [MIT License](LICENSE). Vendored code and third-party +assets keep their own licenses. The styles in `src/lib/forge.css` are adapted from +Uiverse Design's Forge and remain under its license; see +[the design notes](docs/design.md). diff --git a/backend/review/code-index/README.md b/backend/review/code-index/README.md index b002ed9e..f41fe805 100644 --- a/backend/review/code-index/README.md +++ b/backend/review/code-index/README.md @@ -46,7 +46,7 @@ for a section listed in a combined title. Every section the host links this way, the sections in the additions, was checked to exist with its title and Reserved status, against ICC's Eighth Edition tables of contents as returned by a search index and against UpCodes. The check is recorded in -[the research note](../../../docs/research/citation-links-real-set.md). An owner's +[the research note](../../../docs/validation/citation-links-real-set.md). An owner's single-section link was not checked this way, and whether a browser scrolls to an anchor can only be seen by opening one. A wrong anchor opens the top of the right chapter. diff --git a/backend/review/prompts/README.md b/backend/review/prompts/README.md index 18bb3605..33d2d685 100644 --- a/backend/review/prompts/README.md +++ b/backend/review/prompts/README.md @@ -1,9 +1,8 @@ # Moke review instructions -`assets/` contains unmodified files from `sc-frederick/skills/moke-review` at -revision `8a0e2648738018631eaa6d24ee0b6ebaa239afb1`. `provenance.json` records -the upstream paths and SHA-256 hashes. Read the original skill repository to -update them; this application does not edit that source repository. +`assets/` contains unmodified files from the author's private review-instruction +repository at revision `8a0e2648738018631eaa6d24ee0b6ebaa239afb1`. `provenance.json` records +the upstream paths and SHA-256 hashes. This application does not edit that source. `review-prompt.ts` imports the files with Bun's text loader, embedding them in the compiled backend. Packet review version `moke-packet-review-v10` supplies the full @@ -14,10 +13,9 @@ instruction revision. `finding-style.md` is application-owned guidance, embedded in every packet reviewer's instructions for both Meta and Baseten, including coordinator and -sheet roles. It uses the short Sonnet 5.5 rewrites in -`docs/research/meta-vs-sonnet-unslop-findings.md`, rows 28, 36 and 41, as wording -examples. The calculation example expands the circuit abbreviation; a fourth -example illustrates an uncertain reading. The comparison is a style reference, +sheet roles. Its wording examples are short Sonnet 5.5 rewrites of earlier Meta +findings. The calculation example expands the circuit abbreviation; a fourth +example illustrates an uncertain reading. They are a style reference, not a technical validation of those earlier findings. The authoritative contract is `src/lib/review/finding-summary.ts`. A display @@ -103,7 +101,7 @@ own project/conversation root. Both paths preserve reviewer acceptance of engineering count inputs; general model findings and prose do not become accepted count facts. -Audit 3 adds a page-by-page contradiction search before missing-content claims. +The method adds a page-by-page contradiction search before missing-content claims. The tool boundary requires current-run inspection of every source page, targeted inspection at the claimed location, an explicit source search and permit-stage basis. This is an evidence gate, not proof that model interpretation is correct; @@ -126,7 +124,7 @@ coordinator's guidance describes setup's sheet and request limits, and Packet review `moke-packet-review-v8` leaves quotes empty for unread ICC and Florida rule text, and notes empty except for project-specific facts. The host records the access limit required by REVIEW_METHOD.md. -Packet review `moke-packet-review-v9` tells every reviewer to log a citation when a finding rests on a code requirement or a sheet prints one, as REVIEW_METHOD.md requires. A real 46-sheet review on v8 recorded 3 citations in 698 calls, none from a sheet reviewer, because the sheet role never mentioned them. Search is now for when the section is unknown, since naming the document, edition and section gets the same link. See [the research note](../../../docs/research/citation-links-real-set.md). +Packet review `moke-packet-review-v9` tells every reviewer to log a citation when a finding rests on a code requirement or a sheet prints one, as REVIEW_METHOD.md requires. A real 46-sheet review on v8 recorded 3 citations in 698 calls, none from a sheet reviewer, because the sheet role never mentioned them. Search is now for when the section is unknown, since naming the document, edition and section gets the same link. See [the research note](../../../docs/validation/citation-links-real-set.md). Packet review `moke-packet-review-v10` has models mark finding locations and crops with `imageBox`, a pixel box in an image they received, which the host converts to PDF points. diff --git a/backend/review/prompts/provenance.json b/backend/review/prompts/provenance.json index dbbaa3fd..b0ec1013 100644 --- a/backend/review/prompts/provenance.json +++ b/backend/review/prompts/provenance.json @@ -1,5 +1,5 @@ { - "repository": "https://github.com/sc-frederick/skills", + "repository": "Author's private review-instruction repository", "revision": "8a0e2648738018631eaa6d24ee0b6ebaa239afb1", "source": "moke-review", "promptVersion": "moke-selected-coordination-v2", @@ -75,5 +75,5 @@ "sha256": "b30491a08e7fafbb9c75501c4bc8fc1c01d8f25401c4a6b0bf7baaf95e0d95e4" } ], - "packetPromptVersion": "moke-packet-review-v9" + "packetPromptVersion": "moke-packet-review-v10" } diff --git a/design/selection.json b/design/selection.json deleted file mode 100644 index 69762601..00000000 --- a/design/selection.json +++ /dev/null @@ -1,6 +0,0 @@ -{ - "repository": "sc-frederick/uiverse-templates", - "revision": "88f6fb4b913606085633292659ac60ec6918cf28", - "design": "forge", - "reference": "design/reference/forge" -} diff --git a/docs/README.md b/docs/README.md index ac92ad7d..473a32f6 100644 --- a/docs/README.md +++ b/docs/README.md @@ -11,19 +11,32 @@ Start with [Moke's README](../README.md) for the product, workflow, and screensh - [Desktop builds and releases](deployment.md): packaging, CI, credentials, and runtime boundaries. - [Data and recovery](data.md): SQLite migrations, consistent backups, complete packages, and restore behavior. - [Demonstration fixture](fixtures/hero-case.md): supported issue, correction, unchanged control, and revision recheck. -- [Workbench design](../design/DESIGN.md): the Forge design system, its desktop adaptations, and asset licenses. +- [Workbench design](design.md): the Forge design system, its desktop adaptations, and asset licenses. -## Architecture and validation +## Architecture -- [Beta workflow](research/beta-implementation.md): conversations, sheet setup, reviewer acceptance, and product boundaries. -- [Beta validation](research/beta-validation.md): recorded desktop and packaged checks, screenshots, fixes, and limits. -- [Core validation](research/core-validation.md): recorded Meta/SAM receipts, recovery checks, and limits. +- [Product scope](../PRODUCT.md): what the workbench does and who it serves. - [Review runtime](review-runtime.md): local ownership, Pi Durable, hosted inference, and cloud review scope. -- [Pi execution](research/pi-runtime.md): journals, request admission, controls, and crash recovery. -- [Review packages](research/review-packages.md): complete export/restore and interrupted replacement recovery. -- [SAM integration](research/sam-integration.md): image geometry, intent ledger, source fences, and proposals. -- [PDF packaging](research/pdf-packaging.md): renderer assets and historical packaging measurements. -- [Baseten rate limits and caching](research/baseten-rate-limits.md): measured key limits, 429 shape, session affinity and the request pacer. +- [Pi execution for count checks](architecture/pi-runtime.md): journals, request admission, controls, and crash recovery. +- [Review packages](architecture/review-packages.md): complete export/restore and interrupted replacement recovery. +- [SAM integration](architecture/sam-integration.md): image geometry, intent ledger, source fences, and proposals. +- [Review prompt provenance](../backend/review/prompts/README.md): where the review instructions come from and how they version. +- [Project choices and dependency provenance](../.project-kit/manifest.json) + +## Validation records + +These record specific live runs on the dates they state. They are not benchmarks +for unseen drawings. Raw request and response receipts stayed in the ignored +`.local/` directory of the machine that ran them; the `*-result.json` files here +are the committed records. + +- [Core validation](validation/core-validation.md): recorded Meta/SAM receipts, recovery checks, and limits. +- [Cloud validation](validation/cloud-validation.md): a live cloud review, its automatic return, and its recovery. +- [Fast initial review](validation/fast-initial-review.md): measured time and cost for the Initial Review button. +- [Parallel sheet review](validation/parallel-sheet-review.md): live results for one reviewer per sheet on Muse Spark 1.3. +- [Citation links on a real set](validation/citation-links-real-set.md): what a public 46-sheet set showed, and how far code links are checked. +- [Muse Spark Open reviews](validation/muse-spark-open.md): completed reviews on one of Meta's open-weight models, and their limits. +- [Baseten rate limits and caching](validation/baseten-rate-limits.md): measured key limits, 429 shape, session affinity and the request pacer. ## Drawing experiments @@ -31,27 +44,8 @@ These studies record bounded development experiments. Captured model results are not a benchmark for unseen drawings. Reproducing live requests requires provider access and the separately supplied drawing pack. -- [SAM drawing experiment](segmentation-test.md): crop rendering, prompts, overlays, and coordinates. -- [Plumbing coordination](plumbing-segmentation.md): count experiments, corrections, and the offline worksheet. -- [SAM beyond plumbing](sam-drawing-expansion.md): electrical, lighting, HVAC, doors, and observed limits. -- [Adaptive Meta review](adaptive-meta-review.md): independent inventories, targeted zooms, and fresh SAM guidance. -- [Additional discipline sets](sam-expanded-sets.md): electrical, fire alarm, structural, and civil probes. - -## Planning and provenance - -These documents retain the decisions and source research behind the implementation. -For current behavior, use the setup, architecture, and validation documents above. - -- [Product scope](../PRODUCT.md) -- [Core implementation plan](core-implementation-plan.md) -- [Implementation source research](research/core-implementation-sources.md) -- [Core council findings](research/core-council.md) -- [Agent harness comparison](research/agent-harness-comparison.md) -- [Hackathon bonuses](hackathon-bonuses.md) -- [Firm memory](firm-memory-plan.md): finding decisions, the firm checklist and - jurisdiction leads are implemented; the full memory system is deferred. With - [agent memory research](research/memory-sota-notes.md) and - [AEC firm knowledge research](research/aec-firm-knowledge-notes.md) -- [Hackathon product audit](research/hackathon-product-audit.md): requirements and revision gaps as of October 30 -- [Project choices and dependency provenance](../.project-kit/manifest.json) -- [Review prompt provenance](../backend/review/prompts/README.md) +- [SAM drawing experiment](experiments/segmentation-test.md): crop rendering, prompts, overlays, and coordinates. +- [Plumbing coordination](experiments/plumbing-segmentation.md): count experiments, corrections, and the offline worksheet. +- [SAM beyond plumbing](experiments/sam-drawing-expansion.md): electrical, lighting, HVAC, doors, and observed limits. +- [Adaptive Meta review](experiments/adaptive-meta-review.md): independent inventories, targeted zooms, and fresh SAM guidance. +- [Additional discipline sets](experiments/sam-expanded-sets.md): electrical, fire alarm, structural, and civil probes. diff --git a/docs/research/pi-runtime.md b/docs/architecture/pi-runtime.md similarity index 88% rename from docs/research/pi-runtime.md rename to docs/architecture/pi-runtime.md index d8ed3ea0..ca573d37 100644 --- a/docs/research/pi-runtime.md +++ b/docs/architecture/pi-runtime.md @@ -1,15 +1,19 @@ -# Selected Pi runtime slice +# Pi runtime for count checks Implemented against installed `@earendil-works/pi-durable`, `pi-ai`, `chord`, and `pi-telemetry` 1.4.0, Effect 4.0.4, and TypeBox 1.3.27. Production imports use -installed packages. The read-only preflight at `/tmp/moke-pi-preflight-RSsqwv` -established the published SQLite facade, telemetry identity propagation, -recovery resend behavior, and safe-tool receipt pattern. This slice uses the +installed packages. A read-only preflight established the published SQLite facade, telemetry identity propagation, +recovery resend behavior, and safe-tool receipt pattern. This adapter uses the published Harness and SQLite storage rather than replacing their internals. +This describes the execution adapter that runs selected count checks, +`backend/review/execution.ts`. Packet reviews use the same Pi Durable packages from +`backend/packet-review/service.ts`, with automatic compaction and parallel sheet +reviewers; see [the review runtime](../review-runtime.md). + ## Composition contract -Acquire `acquireDataDirectoryOwnership(dataDirectory)` in the parent's application +Acquire `acquireDataDirectoryOwnership(dataDirectory)` in the application scope **before migrations**, domain SQL, or journal work. Its realpath-canonical directory and generation go into: @@ -30,7 +34,7 @@ ReviewExecution.layer({ }); ``` -The parent owns one ManagedRuntime. The service creates none. During Layer +The application owns one ManagedRuntime. The service creates none. During Layer finalization, its callback entry uses `Effect.runPromiseWith` with that finalizer's Context because ManagedRuntime marks its public entry disposed before releasing Layers. Tool callbacks do not acquire another runtime. @@ -43,7 +47,7 @@ transport. Without provider configuration, start returns a typed unavailable fai and creates no phantom run. No execution RPC carries credentials, filesystem paths, journals, or transcripts. -`backend/review/execution-port.ts` defines the parent's required capabilities: +`backend/review/execution-port.ts` defines the capabilities it needs from the application: | Operation | Contract | | -------------------------- | ------------------------------------------------------------------------------------------- | @@ -51,16 +55,16 @@ transcripts. | `render(fence, detail)` | Bounded PDF region plus a named missing fact; returns an application Artifact. | | `readImage(artifact)` | Application-resized PNG pixels, actual delivered dimensions, and both PDF/pixel transforms. | | `propose(fence, proposal)` | Proposed count and evidence; never accepts or confirms coverage. | -| `evaluate(fence)` | Parent's deterministic check over accepted inputs. | +| `evaluate(fence)` | The application's deterministic check over accepted inputs. | | `message(fence, text)` | Application progress or a reviewer clarification request. | -The optional segmentation capability is application-owned and not exposed as a -paid tool by this slice. SAM integration remains separate. +When a Meta key is configured, the application supplies the segmentation capability +and this adapter adds a `segment` tool. SAM has [its own adapter](sam-integration.md). Every mutation fence includes project/finding/run/journal IDs, owner generation, captured source/fact versions, and a stable command UUID derived from journal ID and durable tool task ID. Overview render commands also include a stable sheet -discriminator. The parent must return an existing receipt **before** checking +discriminator. The application must return an existing receipt **before** checking current versions, then atomically check the fence and commit mutation plus receipt. Already admitted work may drain while state is `pausing`; `paused`, `cancelled`, `uncertain`, stale owner, and changed source/fact versions cannot @@ -75,7 +79,7 @@ delivered dimensions/transforms. Moke owns resizing and provenance. ## Persistence and admission `0003_review_execution.sql` stores runs, attempts, and cursor-based status events -in `app.sqlite`. It deliberately makes no assumption about the parent's project +in `app.sqlite`. It deliberately makes no assumption about the application's project table key; project authorization is through the port. Each short SQL operation opens a scoped Effect SQL connection and shares the recovery semaphore. No provider wait holds that permit. @@ -92,7 +96,7 @@ The separate `.execution-owner.sqlite` connection holds `BEGIN IMMEDIATE` with busy timeout zero for the process lifetime. Recovery/export must exclude this lock file. When uncooperative shutdown exceeds the bound, an exit listener retains the native lock handle through process exit, even after Layer disposal. -The parent must terminate its Bun process within the overall three-second +The application must terminate its Bun process within the overall three-second shutdown policy when `close` returns false. A local lock on a network filesystem or after machine power loss has not been validated. @@ -156,7 +160,7 @@ The final focused run passed 13 tests and 71 assertions on Bun 1.4.4, Linux x64. - Duplicate stable command receipts return one committed proposal even after the run becomes terminal. The preflight separately exercised crash immediately before and after domain commit; that crash-window probe has not been repeated - against the parent's final domain adapter. + against the final domain adapter. - Timeout, retry-bearing 503, malformed arguments, stale late proposal, call bound, and conservative text bound are exercised. - Cancel intent persists before abort and blocks a late proposed-count response. @@ -174,10 +178,8 @@ The final focused run passed 13 tests and 71 assertions on Bun 1.4.4, Linux x64. These establish local process-crash and admission behavior, not real provider acceptance, billing, retrieval, power-loss guarantees, or target-OS installed -Electron packaging. Parent integration owns the explicit budgeted live probe, -real domain/PDF acceptance, complete export/restore consistency, E2E, council, -and final repository verification. Gates 0C/0D are not declared fully passed by -this slice alone. +Electron packaging. The [core validation](../validation/core-validation.md) +records the live provider, packaged-app and restore checks. ## Lifecycle command receipts and explicit new attempts diff --git a/docs/research/review-packages.md b/docs/architecture/review-packages.md similarity index 96% rename from docs/research/review-packages.md rename to docs/architecture/review-packages.md index 772e6d14..e2aaf178 100644 --- a/docs/research/review-packages.md +++ b/docs/architecture/review-packages.md @@ -1,6 +1,6 @@ # Review package export, restore, and restart recovery -Status: local behavior tested on Linux x64 through WSL2. This is the whole-application package used before a parent wires it into the desktop shell. It is not a single-project subset, and it does not start Pi or dispatch a model call. +Status: local behavior tested on Linux x64 through WSL2. This is the whole-application package behind the desktop's export and restore. It is not a single-project subset, and it does not start Pi or dispatch a model call. Primary demo target: Linux x64. The writer uses same-filesystem rename, directory fsync, and `O_NOFOLLOW`. Those were not re-checked on Windows or macOS. @@ -79,7 +79,7 @@ A successful restore writes phase `complete` and then clears the journal. If tha `interruptAfter` of `prepared` or `first-replaced` is a test seam. Production callers omit it. -## Parent integration +## Application integration `PackagesConfig` is the whole dependency set: @@ -105,20 +105,18 @@ Suggested export and restore order: `hasReviewBearingRows(filename)` opens one short read-only connection and does not take the semaphore. Call it while the semaphore is already held, before the legacy database-file restore. If it returns true, refuse that database-only restore. This package module does not replace that path. -This module does not edit `application.ts`, the RPC contracts, or migrations. A second runtime is not created. +A second runtime is not created. ## Limitations - Schema drift in `review_execution_runs`, `review_execution_attempts`, or `review_execution_events` fails closed. The column list is the 0003 list, not a guessed superset. - There is no remote-owner column. Foreign means a different `owner_generation`, or no generation was supplied. -- `paused` remains resumable by the current execution service until the parent blocks that call. - Database-only packages are refused here even when they contain no review rows. - Orphan artifacts and journals are not deleted. - The retained pre-restore directories are not pruned. - Windows and macOS package behavior was not run. -- Full `bun run verify`, packaged E2E, and the desktop shell wiring are left to the parent. -## Council recovery corrections +## Recovery corrections A replacement journal now blocks public review mutations, note operations, and package/database maintenance until startup recovery completes. The barrier is diff --git a/docs/research/sam-integration.md b/docs/architecture/sam-integration.md similarity index 92% rename from docs/research/sam-integration.md rename to docs/architecture/sam-integration.md index 531486d2..1b98bb01 100644 --- a/docs/research/sam-integration.md +++ b/docs/architecture/sam-integration.md @@ -2,16 +2,16 @@ `backend/inference/segmentation.ts` provides `ReviewSegmentation.layer(config)`. Its public operations are `segment(fence, request)` and `list(projectId)`. Both -return Effect values with `ExecutionError`; the parent owns their ManagedRuntime +return Effect values with `ExecutionError`; the application owns their ManagedRuntime and cancellation. No background runtime, fallback provider, automatic retry, accepted fact, finding mutation, or new artifact directory is introduced. -## Parent integration +## Application integration The configuration is: - `dataDirectory`: the directory containing migrated `app.sqlite`. -- `queries`: the parent's shared `Semaphore.Semaphore` for SQL and recovery. +- `queries`: the application's shared `Semaphore.Semaphore` for SQL and recovery. - `owner`: the held `DataDirectoryOwnership.Service` with its generation. - `apiKey`: `Redacted.Redacted`, unwrapped only for Authorization. - `loadArtifact(projectId, artifactId): Effect`. @@ -39,7 +39,7 @@ allowed rendered `artifactId` and `prompt`, a discriminated union: Points are limited to 32, coordinates to 0–2048, and the official prompt builder checks actual image bounds and nonempty boxes. The supported subset deliberately omits the experiment's multi-object prompt. The model chooses a rendered source -and guidance; the parent combines the result with the same source image for +and guidance; the application combines the result with the same source image for subsequent model interpretation. SAM returns proposed instances and masks only. An empty completed result is a zero-match proposal, never a coverage guarantee or a deterministic no-issue finding. @@ -74,7 +74,7 @@ than encoding the immutable source again. An existing uncertain intent, including `sent=0`, never dispatches on retry or reopen. Failures after admission persistently mark the run uncertain and append an execution event, retaining a cancelled state if cancellation already won. -The parent must not convert an uncertain SAM error into ordinary model progress. +The application must not convert an uncertain SAM error into ordinary model progress. Malformed provider output retains the raw HTTP body and any successfully parsed response ID in the attempt. SQL connections are scoped to short transactions under `queries`; each reads back foreign key enforcement and FULL synchronous @@ -94,9 +94,9 @@ finishes; the service does not detach a callback or resend to recover it. The full review package already contains `app.sqlite`, including these private requests and raw responses. No additional export artifact type is needed. -### Required parent SQL changes +### SQL changes outside the adapter -The parent owns these changes outside this adapter's assigned files: +The application owns these changes: 1. `ExecutionStore.status`: add the following count to the existing unresolved execution-attempt count, for the same run: @@ -126,7 +126,7 @@ The parent owns these changes outside this adapter's assigned files: 3. Completion and driver error handling must preserve an uncertain run or an existing cancellation after a SAM failure, including a tool error the model would otherwise treat as recoverable. Shared `admitted_calls` already includes - SAM; do not add its count a second time. The parent also owns maintenance + SAM; do not add its count a second time. The application also owns maintenance admission while an operation or unresolved intent exists. ## Geometry and parser evidence @@ -161,5 +161,4 @@ malformed responses; paused/cancelled/old-owner receipt replay; uncertain intent reopen; correction/cancellation/revision/owner changes during inference; project, finding, source, journal and owner admission; shared and SAM-specific call limits; pre-send revalidation; concurrent delivery; and Effect abort during fetch headers -and body consumption. No paid request runs in these tests. The parent owns live -SAM validation, final integration, full verification, council, and publishing. +and body consumption. No paid request runs in these tests. diff --git a/docs/cloud-reviews.md b/docs/cloud-reviews.md index 1807f3fe..eeac982f 100644 --- a/docs/cloud-reviews.md +++ b/docs/cloud-reviews.md @@ -118,8 +118,7 @@ finding. Upload time is set by the reviewer's uplink, about 1.8 MiB/s on the measured development host, so Send no longer uploads files that Modal already holds. See -[the fast initial review research](research/fast-initial-review.md) for the -measurements. +[the fast initial review measurements](validation/fast-initial-review.md). Each workspace has its own Modal Volume (v2), named `-cache-`, where `` is a random 16-hex-digit identifier saved in `.cloud/cache.json`. @@ -333,17 +332,12 @@ The previous image is `moke-review-engine:v0.2.0`, desktop fails the identity probe against `latest`, before any transfer, so it must select that immutable tag in its private operator configuration. -Protocol 3 was deployed from `d7ba4f874` as `moke-review-engine:d7ba4f874`. -That immutable tag, `moke-review-engine:latest` and this checkout's private -operator selection resolve to `im-01M630XTWWQWB3FM0K4NY8NR0D`. -The published image passed strict identity, migration-hash and native PDF -readiness checks. The bounded protocol-3 live run verified ownership fencing, -backend close/reopen, RPC progress and automatic verified-package return. -Its single sent Meta request remains uncertain, with no successful response -receipt, counts or final answer. It was not retried. The strict live validator -exited nonzero; successful inference remains unverified. See -[the deployment record](research/protocol-3-deployment.md) for identities, -receipts, cleanup and the retained rollback image. +Protocol 3 was deployed from `d7ba4f874` as `moke-review-engine:d7ba4f874`, +image `im-01M630XTWWQWB3FM0K4NY8NR0D`. The image passed strict identity, +migration-hash and native PDF readiness checks. Its bounded live run verified +ownership fencing, backend close/reopen, RPC progress and automatic +verified-package return. Its single sent Meta request remained uncertain, with no +successful response receipt, and was not retried. `cloud-reviews.test.ts` exercises persisted ownership, duplicate commands, ambiguous dispatch, stop failures, wrong job identity, required checkpoints, @@ -377,7 +371,7 @@ proposed **2 / 3** counts, plus its final conversation response, in 34.1 seconds Recovery of that completed review through its stopped sandbox's exit snapshot also passed. A separate live probe with zero model calls verified stale candidate skipping, rejected sandbox cleanup, name-based claim recovery, and cold fallback. -See [the recorded cloud validation](research/cloud-validation.md). +See [the recorded cloud validation](validation/cloud-validation.md). ## Troubleshooting @@ -508,8 +502,6 @@ describes the entire workspace, although cached files are not uploaded again: a it could safely replace the local database. The selected project, model, scope, package bytes and live sheet/finding/coverage totals identify the active job. Terminal receipts stay in their originating conversation and can be dismissed. -See [Audit 3 cloud research](research/audit-3-cloud.md) for resource pricing, -capacity decisions and regression evidence. Cloud requests retain the selected scope through transfer and automatic return. Local steering and editing remain fenced while the cloud diff --git a/docs/core-implementation-plan.md b/docs/core-implementation-plan.md deleted file mode 100644 index b4042494..00000000 --- a/docs/core-implementation-plan.md +++ /dev/null @@ -1,238 +0,0 @@ -# Moke core implementation plan - -Build one complete drawing review with a supported issue, an engineer-corrected false alarm, a targeted second look, and a revision recheck. Bun and Effect provide the application infrastructure. Pi Durable supplies agent execution behind a narrow application service. Start locally; add the selected Modal handoff after the local workflow passes its recovery checks. - -This is an implementation plan, not a claim that the review engine exists. The working baseline is the Electron shell, compiled Bun backend, Effect RPC, SQLite recovery, and a separate UI study. Bun is pinned to 1.4.4 and Effect to 4.0.4. Source evidence and unresolved compatibility questions are recorded in [Core implementation research](research/core-implementation-sources.md). - -## Decisions and scope - -The current repository and [runtime decision](review-runtime.md) supersede the older planning packet's pnpm/Vite+ toolchain, main-process application services, and deferred Bun engine. The approved chat and findings study supersedes that packet's drawing-first window layout. Preserve project/conversation navigation, the findings popover, and the source pane. A review conversation is a view of real application work, rather than an unrestricted assistant. - -The initial case contains one engineering rule, multiple instances, one floor, about two related sheets, and one revised PDF. Select the rule and object class using actual drawing evidence. Panel clearance remains a candidate. Record applicability and a verified source before encoding a code threshold. If those cannot be established, choose a supported coordination check and label it accordingly. - -Required local outcomes: - -- Import permissioned PDFs and retain immutable originals, sheet coverage, and source hashes. -- Render an overview and a legible detail region directly from the original PDF. -- Let Muse request a specific missing observation; use SAM where the development experiment establishes a useful contribution. -- Present evidence, accepted inputs, rule applicability, and repeatable arithmetic for each result. -- Correct an accepted input, withdraw a false alarm when warranted, and leave unrelated check runs unchanged. -- Apply Rev B with confirmed sheet correspondence, mark affected results stale, and recheck them without losing history. -- Pause, cancel, reopen, and recover without duplicate domain mutations or automatic reissue of an uncertain paid call. - -Accounts, sync, arbitrary drawing registration, autonomous redesign, a general rule library, a second agent framework, and automatic updates are outside this core. Maintain existing platform CI; choose one primary demo platform before investing in PDF packaging. Windows x64 is the packet's proposed target, not a newly confirmed requirement. - -## Runtime ownership and primitives - -| Responsibility | Implementation choice | Boundary and reason | -| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Domain values and external parsing | Effect Schema, branded IDs, tagged states and precise tagged errors | Parse RPC, database rows, provider payloads, worker messages and imported manifests. JSON or model confidence alone never establishes truth. | -| Application operations | `Context.Service`, `Layer`, `Effect.fn` | Reuse the existing composition root; application services enforce project scope, fact versions, policy and effect order. Avoid services per table. | -| Domain persistence | `effect/sql` with `@effect/sql-sqlite-bun` | Retain scoped connections per operation and the existing recovery semaphore. Short transactions never enclose network or PDF work. | -| Agent execution | Pinned Pi Durable and Pi AI, wrapped by `ReviewExecution` | Pi owns turns, context, tool scheduling and its execution journal. Do not run a second Effect Workflow loop around the same turns. | -| HTTP calls | Effect `HttpClient` with `FetchHttpClient` over Bun's fetch, or Pi's provider adapter behind the same admission policy | Use Pi's existing Meta/Baseten adapters first. A controlled provider wrapper must intercept every paid dispatch, including compaction and retries. | -| Progress | Persisted application events plus Effect `Stream`/`PubSub`, delivered with Effect RPC | Persist authoritative progress; in-memory broadcasts only wake subscribers. Reconnect from a saved event cursor. | -| Work admission and concurrency | Durable rows, scoped fibers, Effect `Semaphore`, bounded `Queue` where needed | A queue is an in-process wake-up mechanism. SQL and Pi checkpoints determine recoverable work. Begin with one active review and one PDF job. | -| CPU processing | Effect worker/RPC support with `@effect/platform-bun` | Use Bun workers for the first PDF experiment; keep the same schemas if a subprocess is required for reliable termination. | -| Images and files | `Bun.Image`, `Bun.file`, `Bun.write`, `Bun.CryptoHasher("sha256")`, filesystem rename/sync | Bun 1.4.4 exposes the image API locally. Use it for bounded image encoding and provider resizing; PDF rasterization remains a separate engine. File publication requires more than buffered writes. | -| Clock, retries and lifecycle | Effect `Clock`/`DateTime`, `Schedule`, `Scope`, `acquireRelease`, one managed application runtime | Apply retries only to classified, safely repeatable operations. Dispose workers, streams, the Pi owner and database resources explicitly. | -| Desktop transport | Existing Electron MessagePort Effect RPC and authenticated Bun loopback RPC | Electron owns windows, dialogs, OS credentials and the trusted bridge. The renderer receives no provider credential, raw path or backend bearer token. | -| Builds and tests | Bun scripts, `bun:test`, existing real Electron Playwright tests | Preserve `svelte-check --tsgo`, Oxlint and its vendored rules. Imports come from installed packages, never `repos/`. | - -The matching subtree has `BunWorker`, `BunWorkerRunner`, `BunServices`, `FetchHttpClient` and worker RPC protocols. Add `@effect/platform-bun` at the matching version only when used. Its current child-process spawner re-exports the shared Node implementation; do not claim it uses `Bun.spawn`. If native Bun process control is needed, isolate `Bun.spawn` in the processing adapter under an Effect scope, with schema-checked messages, abort, exit handling and cleanup. - -## Application contracts and data - -Keep shared contracts under `src/lib/review/` using `#lib` imports. Put implementation under `backend/review/`, `backend/documents/` and `backend/inference/` as behavior warrants. These are cohesive modules, not a mandatory hierarchy. Extend `backend/application.ts`, the shared backend RPC group, and Electron's authorized proxy. The renderer sees only user operations; import/export paths and credential operations remain shell capabilities. - -In unit 1, port the approved project/conversation navigation, findings popover and source pane into `src/routes/+page.svelte` and reusable renderer components. Unit 2 connects the evidence/calculation inspector. The minimal production workbench replaces the running notes screen; rebuilding the study alone does not satisfy either gate. Persist user/reviewer messages and references to Pi transcript entries; derive progress rows from application events rather than storing a second copy of every agent message. - -### Persistent records - -| Record family | Required information and invariants | -| ------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Project and review conversation | Stable project/conversation IDs, selected review scope and associated run IDs. Conversation text is separate from evidence. | -| Document revision and sheet | Immutable PDF hash, original artifact, page index, sheet identity, revision association, page boxes, rotation, user unit and coverage status. Sheet labels are proposals until confirmed. | -| Artifact, region and observation | Content hash, media type, dimensions, source revision/page, render settings, full transform chain and producing operation. Observation stores provider/model/prompt versions and request/response identity. | -| Fact version and calibration | Proposed/accepted/rejected state, typed value or geometry, units, source observation, actor, rationale and superseded version. Calibration belongs to a view/region and its dimensional source. | -| Rule definition and check run | Versioned deterministic logic, required inputs, source/citation status, jurisdiction/edition/applicability when relevant, exact input fact versions, result and missing evidence. | -| Finding and dependency edges | Stable issue identity, historical check runs, evidence references and explicit edges from facts/calibrations to derived measurements/checks. Finding closure and user task completion remain separate. | -| Review run, work item and operation receipt | Engine/schema/prompt versions, input snapshot, durable request ID, stable tool operation ID, owner generation, state, committed result and event sequence. A unique operation ID returns its stored result on redelivery. | -| Provider attempt and budget reservation | Logical request key, unique attempt ID, route/model, input fingerprint, intent, provider response ID if known, status, observed usage and conservative reserved cost. Unknown cost remains unknown. | - -Use new numbered migrations after `0001_notes.sql`; applied migrations remain immutable. Add constraints for operation IDs, fact version uniqueness and project-qualified references. Initialize and verify `PRAGMA foreign_keys=ON` before transactions on every newly opened domain or journal connection that relies on those constraints. Apply and read back connection-local durability settings too; the migration connection's configuration does not configure later connections. Do not use a transcript as the canonical finding database or introduce an event-sourcing framework. Append-only facts and receipts plus current-result pointers and an event table are sufficient. - -### State and concurrency rules - -Keep separate state axes for evidence acceptance, check outcome, citation verification, reviewer closure and user completion. Check outcomes include `pending`, `running`, `needs_evidence`, `supported_issue`, `checked_no_issue`, `stale`, and `outside_scope`. Execution distinguishes queued/running, pausing/paused, waiting for reviewer, uncertain attempt, cancelling/cancelled, completed and failed. Define legal transitions and resume conditions explicitly in schemas and pure domain functions. - -Each mutation supplies a command ID, input fingerprint and expected fact/scope version. A transaction first looks up an existing command receipt: matching input returns the stored result even if the expected version is now old; reuse with different input is rejected. For a new command, verify the expected version, write new facts and dependency invalidations, store the command result, and append its durable event together. A late model result may be stored historically, but cannot replace a current result when its source revision, accepted-fact versions, rule version or owner generation has changed. - -Corrections can arrive while inference is in flight: commit the correction first, mark affected results stale, and reject stale promotion on completion. On a global source revision change, stale every result that depends on a changed or unconfirmed sheet. Preserve unchanged results only after correspondence and unchanged dependencies are established. An old PDF hash alone cannot stand in for Rev B evidence. - -## PDF and evidence pipeline - -Start with `pdfjs-dist` and its first-party Node rendering approach using `@napi-rs/canvas`. This is a candidate pending Bun and compiled packaging tests, not an established compatibility claim. Evaluate the actual native addon, PDF worker asset, CMaps, standard fonts and any WASM assets in the installed package. Pin exact versions and record licenses. Do not add Sharp or a second image library when Bun's image pipeline covers the operation. - -The unit 0 packaging probe inspects the selected published package's canvas dependency, optional platform binary and bundler resolution. Current PDF.js source already uses `@napi-rs/canvas`; test an override only if that exact package demonstrates a resolution failure. The compiled probe must locate the addon and all render assets without a developer `node_modules` directory. - -Import is a shell-selected capability: Electron chooses a file, Bun validates and stages it, hashes its bytes, copies the immutable original, inventories the pages and commits the manifest. Renderer and model arguments use project-qualified artifact IDs and bounded regions, never arbitrary filesystem paths or URLs. Reject inaccessible/encrypted/malformed/oversized PDFs as explicit states. Bound source bytes, page count, decoded pixels, crop area, render time and concurrent jobs in parsed configuration. - -Store canonical geometry in PDF page coordinates together with the page box, rotation and user unit. Record affine forward/inverse transforms through the PDF viewport, region origin, output pixels, provider resize and padding. Store actual submitted dimensions and the provider's grounding/mask coordinate convention. Do not assume all returned coordinates are normalized alike or normalize away rotation. Round only for display. - -For the current SAM protocol, preserve source-pixel boxes with inclusive `x2`/`y2`, the submitted image's displayed `w`/`h` after orientation, and the mask's own raster dimensions. Convert inclusive box edges deliberately when a renderer uses half-open rectangles. Pin the official `@meta-sam/parser` and supported mask encoding; use its parser/decoder rather than hand-written token parsing. Validate confidence-bearing output support for the selected parser version. Geometry tests include single-pixel boxes, edge pixels, orientation and the crop-to-PDF inverse transform. - -Render small overview images for navigation. Render detail directly from the original selected region, initially testing 300 DPI. Clip the render viewport to the requested region rather than allocating a full 24-by-36-inch page at that DPI. Use `Bun.Image` for deliberate final encoding/resizing with dimensions recorded. Avoid hidden Pi image-tool resizing: Moke supplies image blocks and owns their provenance. - -Publish artifacts to a temporary file on the destination filesystem, close/flush and sync as required, verify hash/dimensions, then rename into a content-addressed location before committing its database reference. Distinguish process-crash recovery from stronger power-loss claims. On startup, reconcile temporary files and committed manifests; keep unreferenced completed artifacts until safe collection. A missing artifact produces a recoverable error, not a check with empty evidence. - -The UI reads bounded artifact chunks or previews through an authorized application capability and creates renderer-local Blob URLs. Add a renderer-only PDF.js build for interactive page viewing if needed; keep canvas/native modules in Bun. The backend's current 64 KiB setting limits incoming HTTP request bodies, not responses. Keep import commands small and shell-mediated. Bound artifact response chunks separately for memory and cancellation, and test serialization on both RPC hops. Preserve loopback origin rejection and top-frame authorization. - -## Review loop and deterministic checks - -The agent receives selected scope, rule requirements, current proposed/accepted evidence and remaining budget. Register only these capabilities, with application-owned schemas: - -| Tool capability | Policy | -| ------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Inspect selected sheet/context | Returns bounded text, overview and coverage from the allowed project and revision. | -| Render source region | Requires an allowed sheet, valid coordinates and a reason tied to a missing fact. Uses a content-addressed cache and stable operation receipt. | -| Segment source region | Sends the exact image to SAM; decodes bounded masks/boxes into proposals with their transform and receipt. A recovered paid SAM attempt uses the provider ledger. | -| Propose observations | Stores typed candidates and evidence references. Cannot accept a fact or overwrite a reviewer correction. | -| Evaluate selected check | Runs deterministic code against current accepted input versions and applicability. Returns exact arithmetic or the missing prerequisites. | -| Ask reviewer / finish review | Persists a question or terminal summary and yields execution. Neither completes engineering evidence by itself. | - -Effect Schema is authoritative. Start descriptor generation with Effect 4's `Schema.toJsonSchemaDocument`, which emits draft 2020-12 for the schema's JSON codec, then adapt only the subset the pinned Pi/TypeBox validator supports. Align excess-property rejection on both sides. Probe branded IDs, tagged unions, bounds, `$ref`, `additionalProperties` and the JSON codec's encoded/decoded differences, including examples rejected on either side. Simplify tool payloads when equivalent conversion is unavailable; do not maintain two independent hand-written validation systems. Decode arguments again into domain types and validate project membership, source versions, region bounds and allowed operations before execution. Drawings and model text are untrusted data, not tool authority. Do not install Pi `CodingTools`, arbitrary shell, filesystem reader or unrestricted URL fetch capabilities. - -Declare `replay: "safe"` explicitly for render, segment, propose, evaluate and every other tool expected to recover by redelivery. That declaration is justified only by stable operation IDs and stored results. A replay before the domain write can perform the operation once; after the domain commit it returns the receipt. A segment replay with an uncertain paid attempt blocks for reconciliation instead of resending. Tools without a proven safe replay strategy remain explicitly unsafe and surface interruption. - -Proposed starting limits are two additional acquisition rounds per check, twelve total tool calls per run, one model request at a time and one PDF job at a time. Set `toolExecution: "sequential"` explicitly, matching the current Pi source, and verify the pinned package accepts it. Parse a mandatory elapsed-time ceiling, output-token ceiling and cost budget before admitting a run. Calibrate actual numeric time/token/cost defaults in the first real provider experiment. Repeated identical unproductive requests terminate with `needs_evidence`. All repair requests and SAM calls count toward the same ledger; compaction remains disabled initially. - -A model never supplies the final engineering arithmetic or acceptance decision. Missing calibration, conflicting dimensions, unverified applicability, unreadable evidence or unsupported geometry produces `needs_evidence` or `outside_scope`. A model-only observation remains proposed. The engineer explicitly accepts the operands used by the first demonstration's check. SAM masks help locate boundaries; they do not establish a physical dimension. - -## Pi adapter, recovery and paid-call policy - -### One execution owner - -Expose `ReviewExecution` operations to submit, pause, resume, cancel, inspect and observe a run. Its Layer acquires the Chord/Pi lifecycle, registers tools/models and disposes them under the application scope. The current backend uses a top-level `Effect.runPromise`; unit 4 introduces one `ManagedRuntime` for application services and callback entry, preserving stdin ownership, readiness and the bounded shutdown behavior. Imperative Pi callbacks enter that runtime. Propagate cancellation both ways and finish cleanup before closing storage. Do not create an unmanaged runtime for every tool call. - -Use a dedicated Pi journal per review session under the application data directory. Keep domain SQLite and Pi storage ownership separate. Before opening application databases or journals, the backend must acquire an exclusive process ownership resource for the canonical local data directory. Use a dedicated SQLite lock file outside exported/restored project state, with zero busy wait and a held `BEGIN IMMEDIATE` write transaction; a second process fails before execution, and process death releases its OS-managed lock. The lock connection is scoped to backend ownership and performs no application queries. Record this deliberate lifetime scope in `docs/data.md`; domain queries remain scoped per operation. Never unlink/restore the live lock file, and limit this mechanism to supported local filesystems. Test two independent compiled backends and crash/reacquire. Electron's single-instance lock and a process-local session registry are additional guards, not the enforcement mechanism. Cloud ownership is a separate protocol below. - -The first storage candidate is Pi's existing `openNodeSqliteStorage` adapter under Bun's `node:sqlite` compatibility API. Moke already uses that compatibility API for migration and backup. Test the existing adapter on compiled Bun before writing a new one. Its async transaction executor explicitly serializes operations and issues `BEGIN IMMEDIATE`/`COMMIT`/`ROLLBACK`; it does not pass an async callback to Bun's synchronous transaction helper. Pin exact Pi Durable, Pi AI and Chord versions and record their published exports. The journal connection is scoped to the active `ReviewExecution` session and closes before maintenance. Document this distinct harness resource lifetime in `docs/data.md`; Moke's domain data continues through scoped Effect SQL operations. - -Only if the existing adapter fails a recorded Bun conformance or packaging case should a custom facade be considered. Prefer Effect SQL when it can preserve transaction-bound context across the Promise bridge: scope standalone operations, route every callback query through its transaction executor, queue unrelated work, and reject expired handles. Never re-enter an unscoped client that waits on the transaction's own lease. Verify multi-statement `exec`, positional bindings, row/null conversion, rollback failures and close/drain behavior. Never pass an async callback to Bun's synchronous `Database.transaction`. Do not implement a second harness to evade a failed gate. - -Start from the existing adapter's documented WAL and `synchronous=NORMAL` guarantee and verify process-crash behavior. Do not claim power-loss preservation of its newest commits. `synchronous=FULL` is a separate configuration/custom-adapter experiment when needed, not a reason to replace the adapter first. Domain mutations and Pi receipts still span two stores: commit the domain result and operation receipt atomically first; a replay returns that receipt rather than repeating the mutation. A process crash after the domain commit and before Pi's receipt must be a dedicated test. - -### Attempt ledger - -Before every paid dispatch, reserve budget and commit an intent with a stable logical key derived from the run and durable generation/tool identity, including compaction purpose. Hash inputs for diagnostics; identical prompts in distinct legitimate turns are not the same operation. Persist a returned response ID and completed response receipt as soon as available. Promote output only after schema/source/version validation. - -Persisted intent with no final receipt becomes uncertain after an interrupted dispatch. A known retrievable response ID can be reconciled through a verified route. If the provider cannot retrieve it, present the uncertainty and permit an explicit recorded new attempt; do not promise exactly-once billing or treat abort as a refund. An HTTP timeout or local cancellation after submission is not proof that no billable work occurred. - -Pi can resume unfinished generation. Enforce admission in the registered model-provider dispatch seam and count sends at the underlying HTTP transport. Set Pi provider-internal retry counts explicitly to zero initially, including the Responses adapter's `retryProviderRequest` path, and disable harness-level automatic retries until an application decision has reconciled the attempt. A wrapper around a logical model call is insufficient if a lower layer can issue another HTTP request. - -Unit 0 must demonstrate a supported way to carry durable generation task identity and attempt purpose from the Pi task boundary to that seam. Pi's provider `sessionId` is conversation affinity, not a request ID. Do not substitute process-local counters or prompt hashes for durable identity. If the published API cannot supply this correlation, the Pi gate fails pending a concrete adapter change. Disable automatic compaction in the initial short-context core; reaching the context ceiling yields an explicit limit. Enabling compaction later requires the same durable identity and send-count tests for compaction requests. SAM calls use their stable tool operation IDs. Unresolved attempts must produce zero additional HTTP sends after timeout, close, crash and reopen. Hook failures may be skipped by Pi, so hooks alone cannot enforce authorization, budgets or the uncertain-call rule. - -RPC disconnect only detaches observation; cancelling a Pi wait is not a job cancellation. Pause durably closes an application admission gate, serialized with the start of every model/tool operation, then drains the operation already admitted to a committed boundary. Prove that no next operation crosses the gate. Reopen reads the paused state before starting Pi's scheduler. Cancel persists intent first, invokes explicit Pi conversation/task abort, aborts processing, and attempts provider cancellation where supported. A late receipt may update history and cost without making the cancelled run current. - -Desktop shutdown uses the existing three-second backend termination deadline. Stop admission and request Pi close to preserve pending work; do not interpret close as conversation abort or wait indefinitely for uncooperative code. If cleanup misses the deadline, the forced stop leaves unfinished provider dispatches uncertain and pending work recoverable. Unit 0 tests disconnect, wait cancellation, pause, abort, graceful close and forced kill separately. Reopen reconciles artifacts and attempts before enabling resume; schema/prompt/engine incompatibility blocks with an explicit recovery action. - -### Backup and restore - -Database-only backup no longer describes a complete review once originals, artifacts and Pi journals exist. Before enabling durable reviews, reject legacy database-only Restore for review-bearing data or any restore that omits its matching journals/checkpoint. Label database-only exports as partial. This guard lands with journal admission, rather than waiting for unit 5. Add a versioned review/project package that identifies all required artifacts and journals by hash. Preserve database-only recovery for data whose semantics remain self-contained. - -Maintenance first blocks new work, drains or pauses active runs, closes Pi owners, then acquires the existing database recovery exclusion. Back up each database consistently at that quiescent boundary and write the package manifest last. Never hold the domain SQL semaphore while waiting for an active tool that needs it. Restore validates all hashes, versions and database integrity in staging before replacement; retain a pre-restore package and use a recoverable replacement journal so partial multi-file replacement rolls back on restart. A failed restore preserves the usable existing project. - -Restored historical runs do not automatically resume their scheduler: an older package cannot prove that no newer external dispatch or cloud activation occurred. Open them as historical/non-executing state. Starting new work from restored facts requires an explicit new run identity and budget; an offloaded run additionally requires remote ownership reconciliation. Ordinary crash/reopen of the current live data follows the attempt-ledger policy above. - -Split the current `Application` wrapper when review operations are added: it presently holds one semaphore around each entire application operation. Retain that permit only around short domain SQL reads/writes and recovery. Rendering, network waits and Pi work run outside it. A separate maintenance admission gate stops new review work before draining; only afterward may maintenance acquire the SQL exclusion. Record this lock order beside the composition root and test correction/export/restore races through the public operations. - -## Provider integration and credentials - -The primary development and main-track demo route is Meta `muse-spark-1.3` through Pi's Meta Responses adapter. The second evaluated route is Baseten `meta-models/muse-spark-open-1026` through Chat Completions, using the same fixture and application tools. Probe both before unit 4. Promote Spark Open to the primary demo only if it passes those gates and performs useful review work; the open-weight bonus does not justify weakening the main workflow. If the Meta route is unavailable, Baseten is a fallback only after passing the same acceptance cases. Do not switch provider silently during a run. - -Use explicit model and endpoint configuration from [Model reference](models.md). Map Moke's `MODEL_API_KEY` to the adapter directly rather than relying on Pi's differently named environment variable. Select one documented context mode per route, test its tool-result and any supported reasoning continuity across reopen, and preserve required opaque protocol state without logging private reasoning. If the adapter changes image dimensions, record the actual transform or reject that path. - -Keep the exact organizer Baseten ID `meta-models/muse-spark-open-1026`. Public Baseten docs currently exclude Muse Spark Open from structured outputs and JSON mode. Use schema-validated tool proposals and bounded parse/repair behavior rather than assuming `response_format: json_schema` works. A malformed payload cannot become a successful empty review. Capability records are keyed by provider, endpoint, model and operation; a working text probe does not establish vision, SAM, cancellation or durable retrieval support. - -Before adopting either route, test image input, crop/tool round trip, malformed arguments, structured application results, stream interruption, usage, timeouts, response IDs and cancellation/retrieval where available. SAM is its own adapter and capability probe. Cap inline image bytes and account for base64 expansion. Meta background creation has a smaller documented body limit than ordinary media submission; choose background mode only if the actual adapter supports file references and durable IDs. A local loopback file URL is never a provider-accessible artifact. - -For packaged local use, Electron owns an OS-protected credential store and passes unwrapped keys only to the Bun backend over the private startup/control channel. Credentials are not renderer capabilities, command-line arguments, exports or logs. On platforms without secure credential storage, fail explicitly or use a user-selected session-only configuration. Development `.env` remains development-only. Add only this narrow credential path; account management is outside the core. - -## Modal handoff after local acceptance - -Modal remains the selected cloud destination. Preserve that decision while keeping its remote control plane out of the first local finding. Implement a real handoff only after the local correction, revision and recovery gates pass. - -The first handoff is deliberately one-way per execution: local checkpoint to one designated Sandbox, followed by remote completion and result import. Returning execution locally or replacing an uncertain live Sandbox requires confirmed termination and an explicit new ownership transfer. Network loss never restarts the local executor. - -Use a durable handoff state machine with a unique handoff ID, package hash and monotonically increasing owner generation. The authoritative record lives in the local domain ledger for this one-way handoff. Before activation, persist that local execution is relinquished, permanently stop its run, and bind activation to one recorded Sandbox ID and generation. Start the remote engine inert; it can accept an idempotent activation only for that exact handoff. If Sandbox creation acknowledgement is lost, reconcile by handoff identity before creating another. Never roll back ownership because an acknowledgement times out. - -This conservative protocol avoids time-based lease takeover and does not assume that copied owner fields, Modal Volumes or a dictionary provide compare-and-swap fencing. Test all activation/crash windows. Automatic remote replacement, multi-device ownership and cloud-to-local continuation require a separate authoritative conditional-write coordinator and remain outside the first handoff. - -The package includes the engine build identity, schema/prompt/tool versions, consistent domain checkpoint, closed Pi journal, outstanding attempt state and the originals/artifacts needed for future crops. Reject incompatible builds or uncertain paid attempts. Give a manifest-based preview of what leaves the device. Supply cloud keys separately through Modal Secrets, never the package. - -Run the same Linux Bun engine with a separate remote transport composition. Do not weaken the desktop's loopback-only endpoint to expose it remotely. The backend authenticates remote operations with scoped, expiring capabilities and keeps Modal/control credentials out of the renderer. Reviewer corrections route to the current owner; local evidence remains read-only until importing a version-checked result or a receipt of a remote correction. - -Use explicitly configured Sandbox lifetimes and persistent checkpoints on a Modal Volume or an equivalent selected durable store. Verify actual commit behavior at safe boundaries; an exit snapshot is best effort, not the only recovery copy. Persist remote checkpoints before reporting them durable. Reconnect reads a snapshot plus event cursor. Source/version/operation/generation validation makes result import idempotent. No Modal deployment or paid probe is part of writing this plan. - -## Implementation sequence and exit gates - -Use evidence-based gates rather than dates to admit implementation. The original packet's evaluation, freeze and submission targets remain planning context; re-estimate the remaining schedule after the feasibility results. Each unit is mergeable and demonstrates observable behavior through UI, RPC or a small CLI. - -### Unit 0 feasibility records - -Choose the primary demo OS in 0A before PDF packaging. Screen the highest risks in the first focused session, then record the measured work still needed. Passing all records is not promised within one session. Give each experiment a stop condition and a written result; do not turn a failed probe into an open-ended framework comparison. - -| Record | Work | Pass evidence | -| --------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 0A. Hero case and target | `docs/fixtures/hero-case.md`, permissioned fixture manifest, rule/source/applicability, Rev A/Rev B, primary OS, prompt provenance and budgets | One rule or coordination definition, multiple instances, known supported/corrected/revised outcomes, and recorded platform/scope choices. | -| 0B. Packaged source crop | Time-box PDF.js plus `@napi-rs/canvas`; update backend compile entrypoints and asset resolution, and `electron-builder.json` resources as needed | Installed-layout `app-backend` renders a direct region on the chosen OS without developer Bun, Node or `node_modules`. Include worker entrypoints, native binary, CMaps/fonts/WASM and license resources as required by the selected package. If it fails, test one licensed packaged renderer subprocess behind the same port first. | -| 0C. Pi storage, ownership and lifecycle | Try the existing Node SQLite adapter on Bun; run storage conformance and two-process/crash ownership checks; exercise pause/abort/close separately | Pinned Pi Durable/Pi AI/Chord versions, compiled reopen, exclusive backend ownership, rollback/close semantics, legacy-restore guard design and bounded shutdown. A custom facade requires a specific failure of the existing adapter. | -| 0D. Paid dispatch and schema boundary | Demonstrate durable generation identity at the provider/HTTP seam, zero internal retries, compaction disabled, outbound-send counting, safe tool receipts and schema-descriptor conversion | Timeout, close, crash and reopen send no new request for an uncertain attempt. Replayed domain tools write once. TypeBox descriptor and Effect JSON decoding agree for the supported input subset; pause admits no next operation. | -| 0E. Real provider route | Same small image/tool fixture against Meta Responses and the exact Baseten hackathon model; record route-specific context/retrieval/cancel limits | A real image, one validated crop-tool round, typed application observation and usage/receipt. Record the primary route and whether the second route passes or remains unavailable. SAM has a separate later usefulness gate. | - -All live experiments are explicit and budgeted. Unit 0 records capability and evidence; it does not implement a second application engine. - -### Local delivery and optional cloud - -| Unit | Concrete work | Exit evidence | -| ---------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1. Import and production workbench | Shared domain/RPC modules, next migration, document/artifact adapters, shell import, production navigation/findings/source pane | Original hash survives reopen; a real artifact larger than 64 KiB reaches the source pane through bounded chunks on both RPC hops; rotation/zoom overlays and a direct detail crop work. The running app replaces notes, rather than only rebuilding the concept. | -| 2. First supported result | Rule/fact/check modules, accepted-input persistence, evidence/calculation inspector | One supported issue and a checked/no-issue control from repeatable arithmetic. Missing calibration/applicability stays unresolved. A recorded provider observation may be used only with its actual receipt and a visible recorded status; fresh application calls require the attempt ledger first. | -| 3. Reviewer correction | Version-checked correction command, dependency invalidation, input editing and history; split the old all-operation semaphore | Withdraw the false alarm, retain history and the control's run ID/timestamp. Race correction against a controlled in-flight response without holding SQL resources over the wait. | -| 4A. Durable runtime integration | One application `ManagedRuntime`, Pi adapter, attempt ledger/admission, safe tool replay, persisted progress, pause/cancel, legacy-restore guard and packaged credential handoff | Prerequisites 0B-0D pass. A compiled run can pause/reopen, handle an uncertain attempt with zero extra sends and reject incomplete restore. Keys reach Bun through Electron's protected channel and never renderer state. | -| 4B. Muse-directed second look | Integrate the selected provider and crop tool, reviewer questions and supported-result promotion | Prerequisite 0E passes for the selected route. Muse identifies a missing fact and completes one genuine crop/tool round with visible receipts. Exhaustion/invalid input yields an explicit unresolved state. Add SAM only after its usefulness experiment and parser/coordinate tests pass. | -| 5. Revision and recovery package | Confirmed revision association, stale-result guards, full project-package export/restore and kill/restart fixtures | Rev B stales/rechecks only confirmed dependents; ambiguous matches stay unresolved; unrelated results keep identity. Recover all persistence boundaries and reject corrupt/incomplete packages before replacement. | -| 6. Evaluate and package | Bun evaluation runner, held-out manifests, evidence report, Electron/packaged tests, setup and provenance | Reproduce the four hero outcomes on a fresh packaged launch with real Meta activity. Export/reopen evidence and publish actual counts, latency/cost and known failures. Complete this before spending remaining time on optional cloud. | -| 7. Optional cloud slice | Versioned package, one-way ownership state, inert/active Sandbox bootstrap, remote transport and idempotent import | Begin only after units 0-6 pass. Disconnect/reconnect to the same review; lost create/activation/import receipts never yield a second executor. Cut the action from the demo if this gate fails. | - -Units 1-3 may proceed with deterministic fixtures while an integration gate is unresolved. Unit 4 requires the named passes; it does not inherit permission to bypass a failed experiment. Choose bounded artifact chunks as the first transport; add a streaming endpoint only if measured need justifies it. Land the credential path before claiming a packaged real-provider review. - -If Pi fails provider interception, Bun packaging or storage conformance, save the exact failure and estimate the native Effect AI/Workflow fallback before changing architecture. Keep coordinate contracts stable if switching the rendering adapter. Cut extra sheets, generic settings and visual extras before evidence inspection, correction or the real Muse round. Retain SAM when the crop-plus-SAM comparison supports its contribution. Spark Open uses the same fixture/tools; a bonus does not justify a second product. Modal, another rule and automatic registration cannot delay evaluation or the three-minute demonstration. - -## Verification and evaluation - -Use behavioral tests through service/RPC boundaries, temporary real SQLite files and the real compiled Bun process. Inject controllable provider/processing services through Layers. Do not mock SDK internals or add tests that mirror implementation details. Network-free fixtures drive normal CI; live probes are explicit and budgeted. - -Required acceptance cases: - -1. Coordinate round trips cover rotation, nonzero crop origin, page boxes/user units, different aspect ratios, resize/padding and both Muse/SAM conventions. Target at most one source-render pixel of geometric round-trip error; this measures software transforms, not model localization accuracy. -2. Invalid project/artifact references, out-of-bounds requests, malformed provider results and unexpected unit values are rejected. Truncated/missing evidence never becomes `checked_no_issue`. -3. A correction invalidates only transitive dependents. Concurrent stale output cannot become current; Rev B must not reuse unconfirmed Rev A geometry or measurements. -4. For tools explicitly declared `replay: "safe"`, crash after intent but before domain mutation and again after domain commit but before the Pi receipt. Recovery performs the mutation once or returns its stored result; no second fact/finding is created and no uncertain SAM call is resent. -5. Kill before dispatch, after dispatch without receipt, after a provider ID, during artifact publication, after fact commit, during Pi commit, during pause/cancel, and during package replacement. Reopen presents the correct recoverable or uncertain state. -6. Exercise Pi storage conformance, multi-statement execution/bindings, rollback and rollback failure, queued concurrent calls, expired transaction handles, read-only behavior where used, close/restore races, and journal incompatibility. Verify foreign-key rejection on a fresh connection after reopen and exclusive ownership across two compiled backend processes. Test the compiled executable, not only `bun run`. -7. Fill budgets, fail workers and disconnect subscribers. The event cursor recovers committed progress; memory use stays bounded; closing Electron terminates local work and leaves a recoverable checkpoint. -8. Keep existing origin/auth/top-frame tests and add the new import/artifact/credential boundaries. Cross-project access, arbitrary paths/URLs and secrets in renderer contracts must fail. -9. Export and restore a full review with originals, artifacts, fact history and Pi journal. Reject a corrupted or incompatible package before replacing current data. -10. Conditional on implementing unit 7: cloud tests cover duplicate handoff commands, lost create/activation/import receipts, local restart after relinquishment, stale generations, remote timeout and disconnect. If those fail, the cloud action is unavailable with a specific reason. This case does not block acceptance of the local core when unit 7 is deferred. - -Target 12-20 held-out instances across packets, separate from development/demo cases. Include real defects, valid cases, ambiguity, missing calibration and revisions. Compare overview/text, direct detail crops, and the same crops plus SAM using the same reasoning route, rule and declared budget. Report actual counts and denominators, unresolved checks, false alarms, missed in-scope defects, unsupported measurements, manual corrections, cold/warm latency, per-phase usage and known/unknown cost. A smaller completed sample is reported as such. - -Run `bun run verify` before handing back implementation changes, plus `bun run package:desktop` and `bun run test:packaged` when native PDF assets, worker compilation or packaged credentials change. Preserve existing cross-platform CI. Use the installed Svelte skills and autofixer when implementing components. Run the selected real-provider acceptance path against the packaged backend before making demo claims. - -## Completion criteria - -The local core is ready when a fresh packaged launch imports the fixture, completes an evidence-backed check with real Meta activity, survives reopening, supports the false-alarm correction and revision recheck, and exports a reproducible evidence package. All current findings have explicit source/input versions and all unresolved work is visible. Recorded UI actions and tool receipts match the submitted build. - -Remaining user-facing choices are the permissioned drawing case, verified rule and applicability, primary demo platform, and actual probe budgets. They block their respective implementation gates, not writing or reviewing this plan. Cloud credentials and deployment approval become relevant only when implementing the optional handoff against an external account. diff --git a/docs/data.md b/docs/data.md index a4426ae3..14c571a5 100644 --- a/docs/data.md +++ b/docs/data.md @@ -163,8 +163,8 @@ persistent historical engine marker and cannot resume their copied Pi journals; no model or SAM work starts on reopen. This applies to packet reviews and selected-count runs. Uncertain model and SAM attempts and their accounting are retained. -See [package recovery](research/review-packages.md) and -[execution safety](research/pi-runtime.md) for exact contracts and limitations. +See [package recovery](architecture/review-packages.md) and +[execution safety](architecture/pi-runtime.md) for exact contracts and limitations. ## Offline review reports diff --git a/docs/deployment.md b/docs/deployment.md index 0895fcf8..80710a91 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -28,9 +28,7 @@ bun run package:desktop ``` Increment package.json's version, commit the release, then push its matching v tag -when ready to publish. The generator itself never creates release tags. -The initial homepage assumes `sc-frederick/`; update package.json to -the actual project URL before release. Linux desktopName and application identity +when ready to publish. Linux desktopName and application identity are generated from the project name. ## Bun backend and build tools @@ -72,10 +70,8 @@ app. Backend code does not run in Electron's embedded Node runtime. ## Update delivery -Choose the updater and download access for each application. The user often hosts -release files on R2 and presents notices, downloads, and installation inside the -app. Evaluate aecPermitMonitor's implementation before reusing it. The baseline -does not hardcode a bucket, feed, public-download policy, or universal installer. +Choose the updater and download access for each application. The baseline does +not hardcode a bucket, feed, public-download policy, or universal installer. An updater must select the matching OS/architecture artifact, verify it before execution, report progress and failure, preserve data, and distinguish download, @@ -95,9 +91,7 @@ secrets and avoid logging user data or credentials. Hosted monitoring is optiona Store persistent credentials through the OS credential facility; never bundle provider API tokens or store them in renderer localStorage. -Cloud features are optional. Add Cloudflare + Alchemy with development, staging, -and production separation when required. Authentication, sync ownership, offline -behavior, and conflict policy are chosen for the application. +Cloud review is optional; see [cloud reviews](cloud-reviews.md). macOS artifacts receive an ad-hoc signature on the runner, which requires no Apple developer account. They have no Developer ID signature or notarization, diff --git a/design/DESIGN.md b/docs/design.md similarity index 84% rename from design/DESIGN.md rename to docs/design.md index 7ca933d2..3b007eac 100644 --- a/design/DESIGN.md +++ b/docs/design.md @@ -1,12 +1,9 @@ # Workbench design -MOKE uses [Forge](https://design.uiverse.io), an industrial dark design system -from Uiverse Design. `design/selection.json` pins it from -`sc-frederick/uiverse-templates`. `project-kit design forge .` imports the -licensed pack, with its DESIGN.md, CSS, HTML, screenshots and license intact, -into the ignored `design/reference/forge/` directory. The pack's license allows -adapted application use. It forbids republishing or repackaging the pack, so the -reference source stays out of the repository and application distributions. +MOKE adapts [Forge](https://design.uiverse.io), an industrial dark design system +from Uiverse Design. Forge's license allows adapted application use and forbids +republishing or repackaging the pack, so the pack itself stays out of the +repository and application distributions. Only this adaptation ships. ## What the app takes from Forge @@ -36,7 +33,7 @@ layout. Components use the tokens directly; there is no second UI system. `tests/smoke/icons.test.ts` fails if any path drifts from `@tabler/icons`. Disclosure rows and select menus draw the Tabler chevron in CSS instead of the platform triangle. Sheet rotation uses the paired `rotate-2` and - `rotate-clockwise-2` icons at 20px. See `TABLER-LICENSE`. + `rotate-clockwise-2` icons at 20px. See the [Tabler license](../src/lib/review/TABLER-LICENSE). - **Tooltips.** Explanations belong in tooltips, not subtext. `tip()` in `src/lib/review/tooltip.ts` marks an element and `ReviewTooltips` shows one Slate panel with a hairline, 8px radius and the floating shadow, on hover and diff --git a/docs/development.md b/docs/development.md index bea5624a..e760ca19 100644 --- a/docs/development.md +++ b/docs/development.md @@ -72,7 +72,7 @@ images. The packaged command requires exactly one application under `release/`, so remove obsolete build output before repackaging. Live checks are opt-in and may spend API credits. See [inference setup](inference.md) -and the [recorded validation](research/core-validation.md). A passing offline +and the [recorded validation](validation/core-validation.md). A passing offline suite establishes application behavior with controlled responses, not current provider availability or model accuracy. @@ -96,15 +96,13 @@ The captures include the [workbench](screenshots/workbench.png), [correction history](screenshots/correction.png), and [revision history](screenshots/revision.png). -Inspect the generated PNGs before committing them. Keep historical live-validation -screenshots under `docs/research/screenshots/` with the original evidence notes. +Inspect the generated PNGs before committing them. ## Design and experiments -The workbench uses the Forge design system described in -[the design notes](../design/DESIGN.md). Run `project-kit design forge .` to -restore its licensed reference under the ignored `design/reference/` directory. -Reference source stays outside application distributions. +The workbench adapts the Forge design system, described in +[the design notes](design.md). The licensed Forge pack is not in this repository +and is not needed to build the app. SAM and adaptive-review commands are separate research tools. Their recipes and captured results live under `experiments/`; generated runs and private PDFs are diff --git a/docs/adaptive-meta-review.md b/docs/experiments/adaptive-meta-review.md similarity index 97% rename from docs/adaptive-meta-review.md rename to docs/experiments/adaptive-meta-review.md index 575dd2c8..78ef9106 100644 --- a/docs/adaptive-meta-review.md +++ b/docs/experiments/adaptive-meta-review.md @@ -49,7 +49,7 @@ metadata and is excluded from model-visible tool results. The work-area annotation write completed at 22:51:25 UTC, before the run started; an evaluator read at 22:51:45 also precedes the blind response at 22:51:54. Formatting at 22:52:13 replaced the JSON inode, so its later filesystem birth -time is not its first-write time. The parent execution records and a controlled +time is not its first-write time. The execution records and a controlled formatter reproduction are retained in `.local/sam/iteration-4/workarea-parent-chronology.json` and `formatter-inode-evidence.json`. Annotation timing is explicit in the fixture; the evaluator labels absent timing as unknown rather than inferring it. @@ -77,8 +77,7 @@ the evaluator labels absent timing as unknown rather than inferring it. The hosted contract uses ordinary per-object box/point guidance. This does not implement SAM's research image-exemplar mode that discovers every similar object. See Meta's [tool calling documentation](https://dev.meta.ai/docs/tool-calling) -and [Responses protocol](https://dev.meta.ai/docs/protocols/responses). These were -checked through Executor Exa; Firecrawl could not scrape the developer site. +and [Responses protocol](https://dev.meta.ai/docs/protocols/responses). Only `tool_choice: "auto"` is supported by the checked contract. The adapter uses `store: false` with `include: ["reasoning.encrypted_content"]`, replaying encrypted reasoning, prior calls/results and images on subsequent turns. A @@ -128,7 +127,7 @@ The earlier frozen-center evaluation against the hash-linked extras / zero misses; exam two hits / nine extras / two misses; fresh lighting 10 hits / zero extras / zero misses. Procedure annotations are a post-run visual audit; exam and lighting expectations were recorded before their SAM runs. These -are implementation-agent annotations independently audited by Sol, not a human +are annotations by an AI agent, checked by a second model, not a human benchmark or an estimate of performance on unseen drawings. The initial adaptive attempt stopped after its blind observer used all 7,000 diff --git a/docs/plumbing-segmentation.md b/docs/experiments/plumbing-segmentation.md similarity index 92% rename from docs/plumbing-segmentation.md rename to docs/experiments/plumbing-segmentation.md index 1fedf60d..f58253c7 100644 --- a/docs/plumbing-segmentation.md +++ b/docs/experiments/plumbing-segmentation.md @@ -1,6 +1,6 @@ # SAM for Moke plumbing coordination -This study applies Moke's [plumbing review instructions](https://github.com/sc-frederick/skills/blob/8a0e2648738018631eaa6d24ee0b6ebaa239afb1/moke-review/references/disciplines/plumbing.md): reconcile fixture marks and quantities, room uses, and existing/new scope across drawings. +This study applies Moke's [plumbing review instructions](../../backend/review/prompts/assets/disciplines/plumbing.md): reconcile fixture marks and quantities, room uses, and existing/new scope across drawings. ## Result @@ -86,7 +86,7 @@ has been completed, so review-speed improvements remain unmeasured. ## Correcting evidence Four additional live tests are preserved in -[plumbing-refinement-cases.json](../experiments/sam/plumbing-refinement-cases.json): +[plumbing-refinement-cases.json](../../experiments/sam/plumbing-refinement-cases.json): - A fresh positive toilet click and negative wall click recovered the private restroom fixture after the earlier Muse pixel box selected a wall. @@ -114,8 +114,8 @@ or accuracy savings. Check held-out rooms, overlapping crop coverage, missed Muse candidates, visible marks, and new/removed scope. Keep schedule quantities out of grounding prompts. -Moke source: `sc-frederick/skills` revision -`8a0e2648738018631eaa6d24ee0b6ebaa239afb1`, master instructions, shared method, +Moke source: the [review instructions](../../backend/review/prompts/README.md) at revision +`8a0e2648738018631eaa6d24ee0b6ebaa239afb1`: master instructions, shared method, and plumbing discipline. API grounding follows Meta's [normalized coordinate guidance](https://dev.meta.ai/docs/image-understanding#perception-grounding) and [SAM location prompts](https://dev.meta.ai/docs/sam/segmenting). @@ -123,9 +123,9 @@ Local records, raw responses, and focused evidence are in `.local/sam/iteration-2/`. This iteration used six Muse calls and 97 SAM calls. No billing total is established. -## Council-directed experiments +## Follow-up experiments -Sol recommended testing imperfect geometry. Four fixtures each received a box +The first follow-up tests imperfect geometry. Four fixtures each received a box expanded 20% around its center and one shifted horizontally 20% of its width. All eight masks retained fixture association, but two private-restroom shifts also absorbed substantial wall geometry. This misses the proposed pilot bar of @@ -148,6 +148,6 @@ Points alone clipped the patient and staff toilets and fell just outside the 15-pixel tolerance on the private toilet. Counts may still be usable with those partial masks; solid equipment footprints need a stricter review. -The single pipe-exclusion probe is in `plumbing-pipe-case.json`. Every council +The single pipe-exclusion probe is in `plumbing-pipe-case.json`. Every follow-up experiment keeps original colored inputs, object IDs, raw responses, and failed outcomes. The recommended two-person review-time comparison has not been run. diff --git a/docs/sam-drawing-expansion.md b/docs/experiments/sam-drawing-expansion.md similarity index 95% rename from docs/sam-drawing-expansion.md rename to docs/experiments/sam-drawing-expansion.md index d5ea87a3..0c54760f 100644 --- a/docs/sam-drawing-expansion.md +++ b/docs/experiments/sam-drawing-expansion.md @@ -4,8 +4,8 @@ Can the current SAM 3.1 API help Moke count electrical devices, inspect other drawing objects, and review revisions? This continues the -[plumbing study](plumbing-segmentation.md). Research used Executor with Exa -search and Firecrawl scraping on 2026-10-27. Drawing experiments use the original +[plumbing study](plumbing-segmentation.md). Research dates from +2026-10-27. Drawing experiments use the original anonymous local PDFs, rendered directly with Poppler. No application integration or complete permit review is claimed. @@ -17,9 +17,9 @@ The fixture pack has these relevant sheets: | C | E-2, page 7 | E-3, page 8 | M-2 page 5; P-1/P-2 pages 1/2 | | B | No electrical sheets in supplied pack | None | B-2 page 2, sparse residential architecture | -Moke's [electrical method](https://github.com/sc-frederick/skills/blob/8a0e2648738018631eaa6d24ee0b6ebaa239afb1/moke-review/references/disciplines/electrical.md) +Moke's [electrical method](../../backend/review/prompts/assets/disciplines/electrical.md) calls for fixture marks, circuits, controls, room use and schedule coordination. -Its [mechanical method](https://github.com/sc-frederick/skills/blob/8a0e2648738018631eaa6d24ee0b6ebaa239afb1/moke-review/references/disciplines/mechanical.md) +Its [mechanical method](../../backend/review/prompts/assets/disciplines/mechanical.md) calls for terminal airflow, equipment marks and system tracing. An object count is useful evidence for these tasks, while classification, scope and coverage remain separate checks. @@ -57,7 +57,7 @@ geometric candidates a general electrical classifier: procedure false positives were a sink drain and the `@` in `USB@42`; exam false positives included furniture and other devices. -The local visual audit includes an independent Sol pass. These are development +The local visual audit includes an independent second-model pass. These are development experiments on a few drawings, not measured performance on a representative benchmark or a human timing study. Exam, pre-op lighting and resolution-control expectations were generated before their SAM calls. Procedure-room centers were @@ -135,7 +135,7 @@ better geometry and source-specific checks before becoming measurements. ## Reproduce and inspect -Recipes are under [experiments/sam/symbols](../experiments/sam/symbols). +Recipes are under [experiments/sam/symbols](../../experiments/sam/symbols). Captured Muse proposals in `candidates.json` are denormalized crop-pixel boxes, converted from Muse's original 0–1000 coordinates. Original normalized integers remain in `.local/sam/iteration-3/grounding--response.json` and its parsed @@ -162,7 +162,7 @@ recipes contain supplied locations, so their output cannot establish discovery recall. Never overwrite earlier runs to hide failures. Raw requests, image hashes, responses, result summaries, overlay SVGs and reports -are retained in `.local/sam/iteration-3/`. Exploratory Muse scripts and Exa/Firecrawl +are retained in `.local/sam/iteration-3/`. Exploratory Muse scripts and source records are there too. The study does not establish billing totals or production accuracy. diff --git a/docs/sam-expanded-sets.md b/docs/experiments/sam-expanded-sets.md similarity index 91% rename from docs/sam-expanded-sets.md rename to docs/experiments/sam-expanded-sets.md index f351bf97..57380387 100644 --- a/docs/sam-expanded-sets.md +++ b/docs/experiments/sam-expanded-sets.md @@ -11,8 +11,8 @@ fire alarm, sprinklers and related coordination inputs. Two source-only agent audits saved electrical/alarm and structural observations at 23:15:00 and 23:15:27 UTC on 2026-10-27, before those crops' first inference. -The parent separately recorded civil observations before its probes. These are -implementation-agent observations, not a human benchmark. Annotation points, +Civil observations were recorded separately before the civil probes. These are +AI agent observations, not a human benchmark. Annotation points, quantities and timing are retained in `experiments/sam/expanded/expectations.json`. Inference used original source images; it did not read that evaluation file. @@ -101,10 +101,10 @@ calls. Historical scores for the 15 branches remain in `captured-locations.json` and `expected-metrics.json`; the test suite no longer reruns the scoring helper. Source review methods: Moke's -[electrical](https://github.com/sc-frederick/skills/blob/8a0e2648738018631eaa6d24ee0b6ebaa239afb1/moke-review/references/disciplines/electrical.md), -[structural](https://github.com/sc-frederick/skills/blob/8a0e2648738018631eaa6d24ee0b6ebaa239afb1/moke-review/references/disciplines/structural.md), -[civil](https://github.com/sc-frederick/skills/blob/8a0e2648738018631eaa6d24ee0b6ebaa239afb1/moke-review/references/disciplines/civil.md), -[fire alarm](https://github.com/sc-frederick/skills/blob/8a0e2648738018631eaa6d24ee0b6ebaa239afb1/moke-review/references/disciplines/fire_alarm.md) -and [sprinkler](https://github.com/sc-frederick/skills/blob/8a0e2648738018631eaa6d24ee0b6ebaa239afb1/moke-review/references/disciplines/fire_sprinkler.md) +[electrical](../../backend/review/prompts/assets/disciplines/electrical.md), +[structural](../../backend/review/prompts/assets/disciplines/structural.md), +[civil](../../backend/review/prompts/assets/disciplines/civil.md), +[fire alarm](../../backend/review/prompts/assets/disciplines/fire_alarm.md) +and [sprinkler](../../backend/review/prompts/assets/disciplines/fire_sprinkler.md) instructions. This study tests selected supporting workflows; it is not a full permit review or a claim of complete sheet coverage. diff --git a/docs/segmentation-test.md b/docs/experiments/segmentation-test.md similarity index 99% rename from docs/segmentation-test.md rename to docs/experiments/segmentation-test.md index 65ef5a22..831dcacc 100644 --- a/docs/segmentation-test.md +++ b/docs/experiments/segmentation-test.md @@ -54,7 +54,7 @@ at full size to inspect edges. ## Cases -The editable [case list](../experiments/sam/cases.json) contains: +The editable [case list](../../experiments/sam/cases.json) contains: | Case | Question | | ------------------------- | ---------------------------------------------------------------- | diff --git a/docs/firm-memory-plan.md b/docs/firm-memory-plan.md deleted file mode 100644 index 15d23b8f..00000000 --- a/docs/firm-memory-plan.md +++ /dev/null @@ -1,79 +0,0 @@ -# Firm memory: what MOKE remembers now and what waits - -Decided 2026-10-30. MOKE should learn each engineering firm's way of working: -its details, calculation methods, sheet standards and review habits. One example -is suggesting a detail the firm used on a similar project. A full firm memory -system was researched and designed, and then deferred. MOKE ships three smaller -pieces that improve the current findings and revisions workflow and the -requirements pass, and that collect the data a later memory system would need. - -Sources: the MOKE code at `8e94b578a`, the T3 thread "How Gradon Builds Its Firm -Brain" (2026-10-17), [agent memory research](research/memory-sota-notes.md), -[AEC firm knowledge research](research/aec-firm-knowledge-notes.md) and the -Audit 3 records. - -## What MOKE remembers now - -| Piece | What the reviewer or firm does | What later reviews get | -| ------------------ | ------------------------------------------------------------------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Finding decisions | Marks a finding valid, false alarm, not applicable or duplicate, with a reason | Follow-ups in the conversation receive the decision and reason, and are told not to raise a dismissed issue again without new evidence. The report shows each decision. | -| Firm checklist | Writes the firm's own checks in Review settings; each save is a revision | New reviews apply the checklist as firm practice. A deviation is a Recommendation unless a verified governing source makes it a code issue. | -| Jurisdiction leads | Nothing; the reviewing model records the project's jurisdiction as before | Sources, citations, recorded code editions and agency comments from earlier reviews in the same state or county/city, labeled as leads | - -All three keep MOKE's rule that past work can suggest a check but only the -current documents and governing sources establish a finding. Decisions and the -checklist are written by people. Jurisdiction leads come from earlier reviews -and never become the new review's sources, citations or prior comments; the -model must read a source again before citing it. See -[review runtime](review-runtime.md#what-later-reviews-learn-from-earlier-ones) -and [data lifecycle](data.md) for how each is stored and used. - -## Why the full system waits - -- **The base review is still settling.** In Audit 3, single-conversation - full-set reviews recorded a fraction of their required checks: Brewery 7 of - 1,276, Laundry 16 of 572, Porch 29 of 396 and Civil 31 of 176 - ([Audit 3 validation](research/audit-3-validation.md)). Parallel sheet review - has since recorded 1,276 of 1,276 Brewery checks on this computer and 1,246 - in the cloud ([parallel sheet review](research/parallel-sheet-review.md), - [fast initial review validation](research/fast-initial-review-validation.md)), - but those runs cover one drawing set. Reading accuracy is still a limit: the - recheck found reviews calling complete tables "not located" or "clipped" - because the model's own crop cut them off - ([Audit 3 recheck](research/audit-3-recheck.md), M3-18). Agent memory studies - found that agents copy similar stored records, so stored mistakes spread. -- **There was nothing to learn from.** Reviewers could only tick Completed. - Learning firm practice also needs a firm's past projects, comment letters and - standard details; MOKE has sanitized sample sets. -- **The expected gain is modest and unmeasured for drawings.** Learning from - experience added +10.6% (AppWorld) and +8.6% (finance) in the ACE study, on - tasks with an automatic success signal. Retrieval of similar AEC details has - no public benchmark. -- **Shipped products mostly use checklists.** UpCodes, Groundbook, Structured AI - and others apply uploaded firm checklists. Claims that a product learns from - reviewer corrections (SWAPP, Pirros, Pelles) had no independent test. -- **The cost is large.** A second database, a local embedding worker that is - unproven inside the compiled Bun backend, model calls to index detail sheets, - and playbook and approval screens. - -## Deferred design - -The deferred design had four kinds of memory: a firm detail library, review -history, an approved firm playbook, and project profiles. It retrieved -precedents with SQLite FTS5 plus a patch-level visual retriever such as -ColModernVBERT, proposed playbook edits from reviewer decisions for a person to -approve, and froze a memory snapshot at review start. The research notes above -keep the evidence, and the design is in this document's history in the pull -request that replaced it. - -Revisit it when all of these hold: - -1. Full-set reviews finish their required checks across several drawing sets. -2. Saved finding decisions show repeated false alarms or firm-specific issues - across projects. -3. A firm supplies past projects with agency comment letters and its standard - details, so retrieval can be measured. - -Start then with a retrieval test on that firm's data: FTS5 alone against FTS5 -with a visual retriever, run inside the compiled backend on Windows, macOS and -Linux. diff --git a/docs/hackathon-bonuses.md b/docs/hackathon-bonuses.md deleted file mode 100644 index 7c912e98..00000000 --- a/docs/hackathon-bonuses.md +++ /dev/null @@ -1,59 +0,0 @@ -# Hackathon bonus priorities - -Checked October 27, 2026 against the [participant guide](https://cerebralvalley.ai/e/meta-ai-developer-hackathon/details) in the authenticated browser and the [Official Rules](https://cerebralvalley.ai/e/meta-ai-developer-hackathon/waiver). - -Prioritize **Best Agentic Use-Case** and preserve a genuine **Meta open-weight review mode**. Reuse the finished demo for the viral and community opportunities. Do not add unrelated creative features to pursue another category. The main-track first-place entry remains the priority: impact 30%, Meta tooling 25%, execution 20%, and demo 25%. - -The guide lists five bonuses totaling $50,000. The Official Rules reserve discretion over their categories, allocations and awards. The listed amounts are opportunities, not expected revenue; the reviewed material does not establish that multiple bonus wins can be combined. - -## Opportunity assessment - -| Bonus in the current guide | Amount | Fit for Moke | Decision and evidence needed | -| ------------------------------------- | ------- | ------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Best Agentic Use-Case | $5,000 | Strongest product fit | Target directly. Show Muse deciding what evidence is missing, requesting a crop/SAM operation, interpreting the result and completing a useful check. Preserve actual tool receipts and the changed outcome. | -| Best Build on Meta Open Weight Models | $10,000 | Strong if the model performs useful review work | Retain an evaluated Muse Spark Open mode. The guide names Spark Open or Glimmer, served on own hardware or partner inference, and calls for verification of eligible model use. | -| Most Viral Project | $20,000 | Distribution opportunity with little additional product scope | Prepare a concise public demo post for X after the working submission is stable. Submit its post link through the hackathon platform. The guide counts likes, reposts and comments through November 8 EOD; it does not specify the timezone for EOD. | -| Community Vote | $10,000 | Distribution and usability opportunity | Make the public gallery entry, video, sample and setup easy to understand. The guide limits voting to one per verified account and lists November 16 for the winner announcement. | -| Best Creative Build | $5,000 | Weak fit with the current engineering scope | Do not pursue as a separate workstream. It rewards an original creative project using Muse Image or Muse Wonder. Using Wonder for reasoning alone does not establish that Moke meets the creative-project criterion. | - -These categories and judging descriptions come from the guide's bonus-prize section. The guide says open-weight and creative projects are flagged during judging; it gives an explicit post-link action for the viral award. Check the final submission fields and organizer announcements for any additional bonus-selection action. Do not assume that an SDK or infrastructure provider automatically enters a category. - -## Best Agentic Use-Case - -The bonus evaluates autonomy, effectiveness in completing a meaningful task and practical usefulness. Moke's proposed inspection loop provides a direct demonstration: - -1. Muse finds that a candidate finding lacks a required fact. -2. Muse chooses a specific source region or evidence operation. -3. Moke validates the request and renders a crop directly from the original PDF. -4. SAM supplies candidate geometry where useful; the tool preserves source coordinates and evidence provenance. -5. The accepted facts drive a deterministic check. Moke produces a supported finding or a specific unresolved condition. -6. A reviewer correction or new revision changes the affected outcome while unrelated results remain intact. - -The Meta model must make a meaningful inspection decision. A fixed sequence that invokes SAM and asks Muse to summarize the output would give weaker evidence for the award. Pi Durable supplies execution infrastructure; the new contribution is Moke's review behavior and evidence system. Record the actual sequence, prompt/model versions, timings and results for the demo and evaluation. - -## Best Build on Meta Open Weight Models - -The current configured partner route is Baseten's `meta-models/muse-spark-open-1026`. Keep that organizer-specific identifier explicit. Modal Endpoints or own-hardware inference are alternatives, but migrating the inference provider is not required to run the review engine in a Modal Sandbox. [Model setup](models.md), [inference setup](inference.md). - -An eligible model should complete an end-to-end review case, making real inspection/tool decisions and producing a source-linked result. Keep provider/model identities, request and usage records, prompt versions, fixture hashes and the corresponding evidence artifacts. These records support the infrastructure partner's model-use verification described in the guide. - -Compare Spark Open with the hosted Muse route on the same representative cases, using correct findings, unsupported claims, elapsed time and actual cost. Retain Spark Open as a supported mode only if it completes useful work reliably enough to demonstrate. A token greeting, an unused model toggle or a nominal call is not a credible open-weight build. SAM alone does not meet the named model requirement, and Pi/Modal usage does not replace it. - -Do not weaken the main-track demo to chase this bonus. If the open-weight path cannot meet the evidence gate in time, document the limitation and present the strongest working Meta-powered review. Preserve the optional adapter boundary so this decision does not require a second product architecture. - -## Launch preparation - -Prepare one clear engineering story and reuse its evidence for the three-minute submission video, gallery entry and a shorter social clip. Lead with the drawing problem, show the actual finding, correction and revision recheck, and identify the Meta model's inspection decisions. Include the cloud handoff only when it is working and useful to the story. - -For the viral category, plan an X post and submission of its URL through the portal. For community voting, provide clear setup instructions and a sample with appropriate sharing rights. Follow the platform's participation and voting rules. This documentation task does not publish social posts or submit an entry. - -## Decisions carried into implementation - -- Pi Durable owns the agent lifecycle; Effect owns application services, contracts and engineering state. -- Reviews execute locally by default. An explicit Continue in cloud action transfers a checkpoint to Modal Sandboxes with one active execution owner. -- Hosted inference and execution location remain independent choices. A local review may call Meta or Baseten; a Modal review may use those same endpoints. -- Best Agentic Use-Case is the primary bonus target. A genuine evaluated Spark Open review mode preserves consideration for the open-weight award. -- Viral and community work reuse the main demo. Creative-build work stays outside the planned scope. -- Keep Meta models central, original work identifiable, the repository public, and dependency/asset rights documented. - -The [runtime decision](review-runtime.md) defines the local/cloud boundaries and validation gates. The [SDK comparison](research/agent-harness-comparison.md) records the alternatives and independent critique. The selected architecture and these source links are also recorded in `.project-kit/manifest.json`. diff --git a/docs/inference.md b/docs/inference.md index 4e35a4fe..24a58da4 100644 --- a/docs/inference.md +++ b/docs/inference.md @@ -10,19 +10,18 @@ OPENCODE_API_KEY= Bun loads the root `.env` when `bun run dev` starts. Restart after changing it. Configured providers are available together in the Model menu, except that a Go -key makes Go the only chat choice; see [OpenCode Go](#opencode-go). Keep each +key limits chat to Go and any saved custom endpoint; see [OpenCode Go](#opencode-go). Keep each key with its own provider and endpoint; keys never enter renderer code or logs. The repository ignores `.env` and `.env.*` except the blank `.env.example`. ## Models and routes -| Model | Provider | Canonical endpoint | Credential | -| ------------------------------------------------ | ----------- | ------------------------------------- | ------------------ | -| `muse-spark-1.3` | Meta | `https://api.meta.ai/v1` | `MODEL_API_KEY` | -| `muse-wonder-1.0` | Meta | `https://api.meta.ai/v1` | `MODEL_API_KEY` | -| `meta-models/muse-spark-open-1026` | Baseten | `https://inference.baseten.co/v1` | `BASETEN_API_KEY` | -| `muse-spark-1.3-contributor` | OpenCode Go | `https://opencode.ai/inference/go/v1` | `OPENCODE_API_KEY` | -| `mimo-v2.6-flash`, `glm-5.3-flash`, `gpt-6-luna` | OpenCode Go | `https://opencode.ai/inference/go/v1` | `OPENCODE_API_KEY` | +| Model | Provider | Canonical endpoint | Credential | +| ---------------------------------- | ----------- | ------------------------------------- | ------------------ | +| `muse-spark-1.3` | Meta | `https://api.meta.ai/v1` | `MODEL_API_KEY` | +| `muse-wonder-1.0` | Meta | `https://api.meta.ai/v1` | `MODEL_API_KEY` | +| `meta-models/muse-spark-open-1026` | Baseten | `https://inference.baseten.co/v1` | `BASETEN_API_KEY` | +| `muse-spark-1.3-contributor` | OpenCode Go | `https://opencode.ai/inference/go/v1` | `OPENCODE_API_KEY` | ## OpenCode Go @@ -33,19 +32,19 @@ Meta's Muse Spark 1.3 otherwise. Go is a subscription with monthly dollar allowa **Muse Spark Contributor trains on your drawings.** Go sells it at a discount in exchange for using prompts and completions to train Meta's models, and it is not zero-retention. An owner must turn on **Allow paid endpoints that train on request data** in the workspace's Settings > Privacy, or Go -refuses it. The other three Go models here do not train: MiMo-V2.6-Flash and GLM-5.3-Flash keep -nothing, and GPT-6 Luna keeps abuse logs for 30 days. They have not been tried on drawings. +refuses it. - **Traffic.** Go is designed for coding agents and monitors traffic for abuse. Parallel drawing reviews are not typical coding traffic. Moke identifies itself as `moke/`, sends a session - ID per conversation and runs 3 sheet reviewers at a time. See [OpenCode Go](#opencode-go). + ID per conversation and runs 3 sheet reviewers at a time. See [OpenCode Go](#opencode-go-1). - **Images.** Go has no Files API, so images travel inline. Reviews keep the eight-image window Baseten uses, so a reviewer inspects a batch before requesting the next. - **The only chat route once connected.** With a Go key present, the model menu and the Initial Review - button offer only Go models, so a review cannot spend Meta or Baseten credits by accident. + button offer only Muse Spark 1.3 Contributor and any saved custom endpoint, so a review cannot + spend Meta or Baseten credits by accident. - **What stays on Meta.** SAM runs only on Meta, so object counting needs `MODEL_API_KEY`. Selected count checks run on Meta or Baseten, because their table admits only those two; a check started while - a Go model is selected uses that default route. + the Go model is selected uses that default route. - **Search.** There is no built-in web search; reviewers use `search_sources` and `read_source`. Keep the `-1026` suffix for the supplied Baseten hackathon key. Public examples @@ -57,7 +56,7 @@ key is configured, packet reviews use it with either chat provider; see `MOKE_PROVIDER` chooses the preferred startup provider, with `meta` as the default. Each review saves the model selected in the workbench. Changing the menu does not change an existing run. When Go is connected, a follow-up whose earlier -model the menu no longer offers moves to the first Go model. The +model the menu no longer offers moves to the first model it does. The other endpoint and model variables in `.env.example` support experiment scripts; they do not override the desktop's canonical routes. @@ -184,7 +183,7 @@ least 90% of prompt tokens served from cache (`cacheRead / (input + cacheRead)`) one `x-session-affinity` value was never reused by requests with a different value, so each extra key would pay the full prefix again. Twelve parallel 4-turn conversations on one key had 91.7% of prompt tokens cached, against 88.7% with no - header. See [Baseten rate limits and caching](research/baseten-rate-limits.md). + header. See [Baseten rate limits and caching](validation/baseten-rate-limits.md). - **Shared fields first.** A sheet reviewer's opening message lists the request, scope, inventory, project configuration and sources before its own sheet, so peers reuse the same prefix. @@ -213,17 +212,13 @@ share, and **Copy review diagnostics** includes it for every finished review. Review settings → OpenCode Go connects an [OpenCode Go](https://opencode.ai/docs/go/) subscription, so reviews need not spend Meta or Baseten credits. Paste a key from -the Console harnesses page (`oc_sk_...`) and pick the model to test it with. In -development, `OPENCODE_API_KEY` in `.env` does the same; Pi and OpenCode read the -same variable. The model menu then offers these Go models, which read images and -call tools: - -| Review model | OpenCode Go wire format | -| ---------------------------- | ----------------------- | -| `glm-5.3-flash` | Chat Completions | -| `muse-spark-1.3-contributor` | Responses | -| `mimo-v2.6-flash` | Chat Completions | -| `gpt-6-luna` | Responses | +the Console harnesses page (`oc_sk_...`). In development, `OPENCODE_API_KEY` in +`.env` does the same; Pi and OpenCode read the same variable. The model menu then +offers `muse-spark-1.3-contributor`, which reads images and calls tools over Responses. + +Moke offers only Meta's model through Go. Earlier reviews on MiMo-V2.6-Flash, +GLM-5.3-Flash or GPT-6 Luna stay readable, and a saved connection that names one of +them still opens, but no menu offers them for new reviews. Requests go through Pi's built-in `opencode-go` provider, which OpenCode lists as a supported harness. Its catalog picks each model's wire format, reasoning field @@ -266,10 +261,10 @@ least typical part of this traffic. Model images are at most 2048 pixels on thei long edge; OpenCode's catalog suggests 2000, so a model that receives them resized could place an image box up to about 2% off. -Some models need Console settings first. Muse Spark 1.3 Contributor needs "Allow paid endpoints that train on request data" under Settings › Privacy. Meta may then train on those reviews' drawings and -text. **Test and save** sends one tool-calling Chat Completions request (OpenCode +text. Without that setting Go refuses the model; turn it on, or remove the Go +connection to return to Meta's and Baseten's models. **Test and save** sends one tool-calling Chat Completions request (OpenCode translates the format for every model) and reports a refused key, a refused model or a usage limit. The key is stored and removed like a custom endpoint's, in `opencode-go-credential.json`. Count checks stay on Meta and Baseten, and SAM @@ -279,7 +274,7 @@ in `opencode-go-credential.json`. Count checks stay on Meta and Baseten, and SAM Review settings → Model connections chooses the model the Initial Review button and new conversations use. Until it is set, a connected OpenCode Go key selects -Muse Spark 1.3 Contributor, then a connected custom endpoint, then Muse Spark 1.3. +Muse Spark 1.3 Contributor, then Muse Spark 1.3, then a connected custom endpoint. The choice is stored in the window's local storage and holds no credential. A review records the model it started with. Initial Review runs on this computer whenever its model's key diff --git a/docs/models.md b/docs/models.md index e3ca4ff7..1bc221ea 100644 --- a/docs/models.md +++ b/docs/models.md @@ -87,7 +87,7 @@ it is not served by Meta Model API. See | Input | Text, image, video, audio | | Output | Text, with reasoning support | -The organizer's supplied `image.png` is the source for the `-1026` model ID and +Baseten's hackathon onboarding is the source for the `-1026` model ID and the restriction that the hackathon key works only with this model. Public Baseten examples use `meta-models/muse-spark-open`. Replace that public ID with `meta-models/muse-spark-open-1026` in every hackathon request. diff --git a/docs/research/aec-firm-knowledge-notes.md b/docs/research/aec-firm-knowledge-notes.md deleted file mode 100644 index 68b12e84..00000000 --- a/docs/research/aec-firm-knowledge-notes.md +++ /dev/null @@ -1,239 +0,0 @@ -# AEC firm knowledge: domain notes for a firm-memory feature - -Research date: October 30, 2026. Status: domain evidence only. This note does not propose a MOKE design. - -Question: how do architecture, engineering and construction (AEC) firms record and reuse their own ways of working (typical details, calculation templates, sheet and title block standards, QA/QC checklists, recurring review comments), and what do AI products already do with firm-specific knowledge? - -Conventions. Each claim carries a source link and a date. "Accessed 2026-10-30" marks an undated page. **Shipped** means the vendor's own release notes, help documentation or general-availability announcement describes the feature as available. **Preview** means the vendor labels it beta, preview or labs. **Announced** means a dated plan with no availability evidence found. **Marketing claim** means a capability stated on a product page that I could not check against documentation or an independent test. Vendor blogs that argue a product category (InspectMind, Helonic, Structured AI, Nomic, SWAPP) are marked as vendor sources even when the point seems reasonable. - -## Summary of decision-relevant findings - -1. Drawings already carry structured, standard metadata a firm memory could key on: UDS sheet identifiers (discipline, sheet type, sequence), detail indicators that reference a detail number and a sheet, drawing block titles and scales, sheet keynotes and reference keynotes tied to MasterFormat sections, and revision clouds with deltas. Firms adapt these conventions, and the NCS leaves sequence numbering and some sheet types user-defined (Section 1.1). -2. Firms that keep detail libraries separate a vetted "typical" set from historical project details. They retire superseded details with a reason instead of deleting them, and revisit them on code cycles and after RFIs or field problems (Pirros help; Caltrans Revised Standard Plans; Section 1.3). The most common way reuse goes wrong is the copied detail or note that no longer fits: a different seismic design category, a wrong jurisdiction, a superseded code edition, delegated-design language left in by mistake (STRUCTURE magazine 2021–2022; Section 2.2). -3. Under professional-practice rules, reuse does not transfer responsibility. The NCEES Model Rules require a licensee who adapts a standard design to do the code research and design file again. The licensee who seals takes "full professional responsibility" (Section 2.3). Standard details normally remain the firm's property and can be reused across clients under AIA B101 and EJCDC terms. Client confidentiality limits what may flow between projects, and the AIA's 2024 Code of Ethics and its AI toolkit treat client drawings as confidential data (Sections 2.4–2.5). -4. Among the products checked, only Pirros ships retrieval of a firm's past details by the condition they solve, with firm-approved and reference-only status. Its Quality Management System checks whether a detail in the set matches the firm-approved version. Pirros works in Revit, not on PDF sets. Nomic generates a reusable "Drawing Review Standards" document from past markups and RFI logs. Most other reviewers, including UpCodes, Groundbook, InspectMind, CodeComply, Structured AI and Bluebeam Max, add firm knowledge as uploaded checklists or standards documents rather than learning it from history (Section 3). -5. Correction memory, meaning learning from reviewer dismissals and fixes, appears only in marketing: SWAPP "workflow memory", Pirros "learns the firm's tolerance", Pelles "each fix ... teaches it". I found no independent evaluation of any of these claims (Section 3, Section 6). -6. Bluebeam Max went generally available on May 19, 2026, but Smart Review is still labeled preview. It runs a fixed set of checks, is tuned for US vertical commercial floor plans and supports only imperial units. Procore's agent packages have been generally available since July 23, 2026. The Procore feature that teaches agents company standards, Skills, was announced to roll out from August 2026 (Section 3). -7. Research and benchmarks show models read drawing text reliably but read symbols and follow cross-sheet references poorly. AECV-Bench reports text questions at up to 0.95 accuracy and symbol counting at roughly 0.40–0.55. On LED-BENCH, the best model averages 52.65%. Detail-to-detail similarity retrieval for AEC typical details has no public benchmark that I found (Section 4). -8. Practitioners and surveys stress the same requirements. A finding should name its sheet and location and cite its basis (code section, spec paragraph, the sheets that disagree). It should carry a severity, and the reviewer's accept or reject should be recorded with a reason. False positives cost more than their count suggests because triage time grows faster than their number. AIA (2025) and Autodesk (2025) surveys report near-universal accuracy concern and falling trust in AI (Section 5). - -## 1. How firm standards are encoded - -### 1.1 US National CAD Standard (NCS) and Uniform Drawing System (UDS) - -- **NCS version.** NCS V7 "released in 2025" adds a floor-numbering format, survey and mapping discipline designators and revised notation text ([NCS V7 what's new](https://www.nationalcadstandard.org/ncs7/new.php), accessed 2026-10-30). The ordering page lists the V7 online document under NIBS product code 92014-7 with a 2024 ISBN date ([NCS V7 ordering](https://www.nationalcadstandard.org/ncs7/ordering.php), accessed 2026-10-30). The UDS is CSI's module within the NCS ([CSI NCS page](https://www.csiresources.org/standards/ncs), accessed 2026-10-30). -- **UDS modules.** The UDS modules cover drawing set organization (set order, sheet identification, file naming), sheet organization (drawing area, title block, coordinate grid), schedules, drafting conventions (including cross-referencing), terms and abbreviations, symbols (indexed by MasterFormat 2004 number plus a three-digit extension) and notations ([NCS V7 content](https://www.nationalcadstandard.org/ncs7/content.php), accessed 2026-10-30). -- **Sheet identifier format.** A sheet ID is a discipline designator (one letter plus a hyphen, or two letters), a one-digit sheet type and a two-digit sequence number. Sheet types: 0 general, 1 plans, 2 elevations, 3 sections, 4 large-scale views, 5 details, 6 schedules and diagrams, 7 and 8 user-defined (type 7 explicitly "including typical detail sheets"), 9 3D. Sequence numbers start at 01, need not be consecutive, and inserted sheets may take a suffix of up to three characters ([NCS V5 UDS Module 1](https://www.nationalcadstandard.org/ncs5/pdfs/ncs5_uds1.pdf); [NCS V6 UDS Module 1](https://www.nationalcadstandard.org/ncs6/pdfs/ncs6_uds1.pdf), accessed 2026-10-30). The NCS FAQ notes that sequence numbers are user-defined. Matching floor numbers to sheet numbers (A-102 as the second floor) is an option, not a rule ([NCS V7 FAQ](https://www.nationalcadstandard.org/ncs7/faqs.php), accessed 2026-10-30). -- **Layer names.** Layer names follow discipline, then a mandatory four-character major group, up to two optional minor groups and an optional status field, for example `A-WALL-FULL-TEXT`. Annotation goes under `ANNO` (`DIMS`, `TEXT`, `TTLB` for title block, `REVS` for revisions) ([NCS V5 layer format](https://www.nationalcadstandard.org/ncs5/pdfs/ncs5_clg_lnf.pdf); [Spangler, NIBS 2017](https://www.nationalcadstandard.org/ncs7/pdfs/spangler_02_09_2017.pdf)). Layers exist only in CAD files. A PDF packet usually preserves them as optional content groups or loses them. That is my inference, not a sourced fact. -- **Symbols and notes.** The symbols module includes detail indicators (dashed circle, dashed rectangle, and a 45° arrow for small conditions), the drawing block title and the door mark ([NCS V6 UDS Module 6](https://www.nationalcadstandard.org/ncs6/pdfs/ncs6_uds6.pdf), accessed 2026-10-30). Sheet keynotes must use a hexagon. A reference keynote is "the root of the specification section number and a user-defined suffix". A revision is shown by "a cloud and a delta symbol" ([NCS V7 FAQ](https://www.nationalcadstandard.org/ncs7/faqs.php)). The notations module says notes "have important legal consequences" and should avoid vague references such as "SEE SPECIFICATIONS" ([NCS V6 UDS Module 7](https://www.nationalcadstandard.org/ncs6/pdfs/ncs6_uds7.pdf)). -- **Agency adaptations.** Agencies amend the NCS. The VA Drawing Deliverable Requirements limit scales to the NCS common-scales list, require a graphic scale for each scaled view and make keynoting optional ([VA DDR](https://www.cfm.va.gov/til/bim/DwgDelivRqmts.pdf), accessed 2026-10-30). The Los Alamos CAD Standards Manual numbers details and letters sections independently on each sheet, reading from upper left to lower right. References carry the sheet number, and deleted numbers may be left blank. Standard details carry their own `ST-` numbers assigned by the Engineering Standards Manager ([LANL CSM 200 R5](https://engstandards.lanl.gov/CSM/pdfs/CSM200-R5.pdf), accessed 2026-10-30). This is the convention behind a reference like "5/S-501": detail 5 on sheet S-501. - -### 1.2 Firm CAD and BIM standards files and markup libraries - -- **AutoCAD standards checks.** AutoCAD audits drawings against a `.dws` standards file covering layers, dimension styles, linetypes and text styles. CHECKSTANDARDS checks one drawing and the Batch Standards Checker writes an HTML report. Users can fix or ignore each violation ([AutoCAD 2026 help](https://help.autodesk.com/cloudhelp/2026/ENU/AutoCAD-Core/files/GUID-F8B47FBB-1276-4D74-BA89-0440CB4E4866.htm); [Autodesk blog, 2026-01-06](https://www.autodesk.com/blogs/autocad/cad-standards-tuesday-tips-with-frank/)). -- **Autodesk Assistant standards check.** Autodesk Assistant in AutoCAD 2027 adds "ability to check standards of the current drawing against a standards file" with the caveat "Mistakes are possible" (**shipped**; [AutoCAD 2027 what's new](https://help.autodesk.com/cloudhelp/2027/ENU/AutoCAD-WhatsNew/files/GUID-99E2D8F7-D4D9-4B3A-9E57-7B73D6BB12AD.htm), accessed 2026-10-30). In Civil 3D 2027 the same check runs as a technology preview. It compares the drawing to a company `.dwt` and produces a report, and the user must make the changes manually ([IMAGINiT, 2026-08-05](https://resources.imaginit.com/civil-solutions-blog/autodesk-assistant-standards-checker)). -- **Bluebeam Tool Chest.** Bluebeam Revu stores standard markups in Tool Chest tool sets (`.btx`). A shared tool set on a network drive is locked by default, checked out for edits and cached for offline use. Bluebeam describes sharing "for setting standard markups for different projects, job functions, clients, and more" ([Bluebeam Manage tool sets](https://support.bluebeam.com/user-manual/menus/window/manage-tool-sets.html); [Share tool sets](https://support.bluebeam.com/revu/how-to/share-tool-sets.html), accessed 2026-10-30). Firms use these libraries to standardize review comments. Markup lists can be exported, and Smart Review exports map to the Markups List columns (Section 3). -- **Revit tagging templates.** BIMLOGIQ Smart Annotation saves firm tagging setups as exportable templates and can generate one from an existing view ([BIMLOGIQ](https://bimlogiq.com/product/smart-annotation.html), accessed 2026-10-30). SWAPP describes typical firm guideline documents. They cover tag placement, door-mark numbering derived from room numbers, completeness of tagging, dimension-string rules and sheet and view conventions, and SWAPP says they usually "sit in a PDF nobody opens" ([SWAPP, 2026-10-16](https://swapp.ai/enforce-revit-drawing-standards-firm-guidelines/), vendor source). - -### 1.3 Standard and typical detail libraries - -- **Organization.** Detail libraries are commonly organized by CSI MasterFormat. Land F/X ships MasterFormat-based division lists that update every 30 days while keeping office edits separate ([Land F/X docs](https://www.landfx.com/docs/details/organize/5962-standard-library-division-list.html), accessed 2026-10-30). CSI's MasterFormat guide allows lower-level numbers for "multiple versions of the same specification section for different clients or for different types of projects" in a firm library, presented on projects under the standard number ([Using MasterFormat](https://theconstructionstandard.com/guides/masterformat-using.pdf), accessed 2026-10-30). -- **Vetted versus historical details.** Pirros help pages describe a vetted "Typical Details" set separate from "All Details" drawn from uploaded projects. A "Reference Only" status lets staff see past conditions "without the risk of reusing Details that haven't been reviewed". Search covers view names, sheet names, titles on sheets and annotation text. Project filters cover location, market sector, client and building material ([Pirros: search for details](https://help.pirros.com/en/articles/14503491-how-to-search-for-details), accessed 2026-10-30). -- **Vetting and maintenance.** Pirros recommends updating a typical when "a code cycle changes (e.g., new IBC edition)", when "an RFI or field issue reveals a detailing error", when the firm adopts a new material or method, or when a team member flags a problem. Retired details stay in the library as historical references with a note such as "Superseded by updated version per 2024 IBC". Suggested cadence: review flags monthly, audit quarterly, and review against current codes annually. Pirros notifies anyone who downloaded the old version ([Pirros: managing your library](https://help.pirros.com/en/articles/14502009-managing-your-detail-library-over-time), accessed 2026-10-30). Its bootstrap guide treats a detail that appears five or more times across a firm's 10–25 best projects as a strong typical candidate, "effectively been reviewed eight times". It routes a redline review to the person who handles construction administration, because "they know which detail draws an RFI every time" ([Pirros, 2026-06-27](https://help.pirros.com/en/articles/15676361-build-a-typical-detail-library-from-scratch-stash-vet-and-refine-with-mira)). -- **Public-agency analog.** Caltrans publishes Standard Plans by edition plus "Revised Standard Plans" (RSP), each with a plan number and date. Canceled plans are listed explicitly ("Canceled Plan P8, 4/20/2026"), and each RSP sheet states that it "supplements the Standard Plans book dated 2024" ([Caltrans Revised 2025 Standard Plans](https://dot.ca.gov/programs/design/xxxx-ccs-standard-plans-and-standard-specifications/revised-2025-standard-plans-toc), last updated 2026-04-20; [2024 RSP PDF](https://dot.ca.gov/-/media/dot-media/programs/design/documents/entire-2024-revised-standard-plans-5-sheets-locked-a11y.pdf)). - -### 1.4 Master specifications - -- **MasterSpec and office masters.** MasterSpec is owned by the AIA and published exclusively by Deltek. Specpoint lists more than 900 specifications ([Deltek help](https://help.deltek.com/Products/Specpoint/AE/MasterSpec.html), accessed 2026-10-30). Deltek defines an office master as a custom section that "enables a firm to preserve its knowledge". Firms may keep several office-master versions of one section "to cater to projects of different project groups, owners, and submission and regulatory requirements" ([Deltek Office Masters](https://help.deltek.com/products/Specpoint/AE/OfficeMasters.html), accessed 2026-10-30). An AIA and Deltek article warns that Word-based office masters lead teams to work "from the latest authored section from the last project rather than standardized vetted information" ([AIA, 2022-09-13](https://www.aia.org/resource-center/proven-specification-strategies-better-built-projects)). -- **UFGS and SpecsIntact.** The Unified Facilities Guide Specifications (USACE, NAVFAC, AFCEC) are published for SpecsIntact and updated quarterly. The 2026 releases were February 19, May 28, August 27 and November 19 ([WBDG UFGS Masters](https://www.wbdg.org/dod/specsintact/masters), accessed 2026-10-30). "Archived" sections are "prohibited for use in new projects" ([WBDG UFGS library](https://www.wbdg.org/dod/ufgs), updated 2026-08-25). NASA withdrew from UFGS on September 30, 2023 ([WBDG NASA](https://www.wbdg.org/nasa/master-specifications), posted 2023-11-30). Agency masters thus carry explicit effective dates and active or archived status, which firm masters often lack. - -### 1.5 Calculation templates and design guides - -- **IStructE spreadsheet guidance.** The Institution of Structural Engineers says "a well-checked and validated spreadsheet, reused across multiple projects, can enhance structural safety. However, if a spreadsheet contains errors, reusing it can significantly increase risk." It asks for cited clause references, separated inputs, calculations and outputs, edge-case tests, and a recorded creator, last modifier and last checker ([IStructE](https://www.istructe.org/resources/technology-digital/calculation-models-spreadsheets/), accessed 2026-10-30). -- **CalcTree.** CalcTree separates per-project design inputs, company templates (versioned, "change rarely") and project calculations that are reviewed and signed. It argues for engineer-controlled rather than automatic propagation of input changes ([CalcTree, 2026-06-21](https://www.calctree.com/blog/project-level-calculation-management); [Templates 2.0, 2026-10-08](https://www.calctree.com/blog/templates2-0), vendor source). - -### 1.6 QA/QC checklists and review records - -- **CASE 962-D.** The Council of American Structural Engineers (CASE) publishes _A Guideline Addressing Coordination and Completeness of Structural Construction Documents_ (CASE 962-D, 2020 edition) plus drawing review checklists ([ACEC CASE publications](https://www.acec.org/member-center/get-involved/coalitions/case/resources/case-publications/), accessed 2026-10-30). The earlier edition's appendix checklist of more than 350 items was meant "as a starting point in establishing your own in-house checklist", used "as a 'to do' list rather than a 'did I complete' list" ([STRUCTURE, February 2004](https://www.concreteproductivity.org/portals/19/Images/D-CASEbusinesspractices1.pdf)). -- **Engineers and Geoscientists BC.** The BC regulator requires documented checks whose record states who checked, when, what was checked, the issues found and the corrections made ([EGBC documented checks guide](https://www.egbc.ca/getmedia/5dea60e7-a232-4905-a064-b0468006e953/EGBC-QMG-Documented-Checks.pdf), accessed 2026-10-30). Firm quality practice there includes retaining project documentation for at least 10 years ([ACEC-BC, 2022-11-02](https://acec-bc.ca/wp-content/uploads/2022/11/2022-11-02_Budget-Guidelines-for-Eng-Services-Doc-2_Buildings-1.pdf)). -- **Owner checklists.** Owner checklists ask whether previous submission comments were addressed. The Pennsylvania Turnpike discipline quality verification checklist includes "Addresses previous submission comments" and "Independent Peer review comments reviewed and addressed" ([PA Turnpike](https://files.paturnpike.com/production/docs/default-source/resources/policies-standards/engineering-standards/dom-resources/discipline-quality-verification-checklist.pdf), accessed 2026-10-30). -- **Stutzman and Medcalf (2018).** In a 2018 NIBS BEST5 paper, two specification consultants wrote that firm standards "all too often, relate primarily to formatting rather than content". They found that "there is not a plan for periodic reviews of standard details", so details no longer meet current requirements. They recommend Pareto-style checklists informed by QC reviewer and construction administration feedback, early checking of initial details "to avoid the 'ripple' effect of copying mistakes", and libraries of "Lessons Learned" technical bulletins. They also "advocate destruction of documents with review comments and checklists to avoid unnecessary 'second-guessing'" while keeping a permanent record that the review happened ([Stutzman and Medcalf, NIBS BEST5, 2018-04-16](https://www.conspectusinc.com/blog/2018/04/construction-document-quality-crisis); [PDF](https://www.brikbase.org/sites/default/files/MO1A.1Stutzman.pdf)). This recommendation runs directly against retaining a comment history. - -### 1.7 Metadata that exists on a typical PDF packet - -Sources: Sections 1.1–1.6 plus AEC-Bench task definitions ([Nomic AEC-Bench](https://huggingface.co/datasets/nomic-ai/aec-bench), accessed 2026-10-30). - -| Item | Where it appears | Structure | Reliability notes | -| ----------------------------- | ------------------------------------------------------ | --------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------ | -| Sheet number and title | Title block; sheet index on cover or G-series sheet | UDS format, often firm-varied | Index and title blocks can disagree (Bluebeam Smart Review and AEC-Bench both test this) | -| Project, issue and revision | Title block; revision schedule | Free text plus dates and deltas | Revision clouds may not cover every change. Trunk Tools reports 10–20% of meaningful changes left unclouded (vendor data, Section 3) | -| Seal and responsible licensee | Title block or seal area | Name, license, discipline note | NCEES requires a note scoping each seal when several licensees sign (Section 2.3) | -| Detail or drawing block | Title mark under each block | Number, title, scale | Titles may not match what is drawn (AEC-Bench "Detail Title Accuracy") | -| Cross-references | Detail, section and elevation indicators | Detail number / sheet number | Broken references are common copy-forward symptoms (AEC-Bench "Cross-Ref Resolution", 51 instances) | -| Keynotes | Hexagon sheet keynotes with legend; reference keynotes | Reference keynotes are a MasterFormat section plus suffix | Links drawings to the specifications | -| General notes | G-series and discipline notes sheets | Free text | Carry code editions and design criteria (often stale, Section 2.2) | -| Legends and schedules | Symbol legends, door, fixture and equipment schedules | Tables | Legends support symbol matching (Section 4) | -| Layers | CAD only | NCS fields | Usually absent or unreliable in PDFs (inference) | -| Project context | Code summary sheet, title block address | Free text | Jurisdiction, code edition, occupancy and seismic design category are needed to judge whether a past detail applies | - -## 2. Firm knowledge management in AEC - -### 2.1 Lessons-learned research - -- **Caldas et al. (2009), 70 organizations.** Lessons-learned programs were multiplying, but "the potential for savings and improvement has not been fully met". The study identified seven characteristics of an effective program ([J. Constr. Eng. Manage. 135(6), 2009](https://ascelibrary.com/doi/10.1061/%28ASCE%29CO.1943-7862.0000011)). -- **Graham and Thomas (2007), Irish contractor case study.** Reported challenges include "failure to deliver lessons when and where they are needed" and "lack of proper classification system". The firm needed notifications of new lessons and training on its lessons database ([WIT repository, 2007](http://repository.wit.ie/526/1/LESSONS_LEARNED_PRACTICES-_CASE_STUDY_OF_A_LEADING_IRISH_CONSTRUCTION_ORGANIZATION.pdf)). -- **Paranagamage et al. (2012), UK contractors.** There is "a disjoint in the effort spent obtaining lessons learned and their dissemination and use". Respondents named time pressure, reluctance to share problems, and exercises "too generic to be of value" ([Built Environment Project and Asset Management, 2012](https://doi.org/10.1080/21573727.2012.681643)). -- **Wiewiora and Murphy (2013).** Lessons are often "documented as simple, line-item statements devoid of context". The authors report poor visibility and "overloaded LL spreadsheets" ([Knowledge Management Research & Practice, 2013](https://doi.org/10.1057/kmrp.2013.26)). -- **Australian contractor survey (2013).** The top barriers were employee time, resources and clear guidelines. Only a minority of firms retained lessons-learned documentation for each project ([Construction Innovation 14(1)](https://www.ingentaconnect.com/content/mcb/ci/2013/00000014/00000001/art00007)). -- **Debs and Hubbard (2023), US construction companies.** Post-project evaluations are frequent, but the information "is not effectively reutilized". Their review cites Carrillo, Ruikar and Fuller (2013), in which lessons-learned documents ranked sixth of seven sharing tools, and lists legal issues and lack of trust as barriers ([AJCEB 23(1/2), 2023-07-12](https://doi.org/10.5130/ajceb.v23i1/2.8390)). - -### 2.2 Why reuse fails - -- **Practitioner account (STRUCTURE, December 2021 and January 2022).** A delegated-connection engineer lists these failure modes ([Part 1](https://www.structuremag.org/article/the-hidden-cost-of-copy-and-paste-2/); [Part 2, 2022-01-01](https://www.structuremag.org/article/the-hidden-cost-of-copy-and-paste/)): - - connection details copied from a Seismic Design Category E building to an SDC A project - - copied details that reference delegated design when none is delegated - - a steel stair specification on an Arkansas project requiring a California-licensed engineer - - general notes citing the 1979 welding code, the 1997 building code and the 7th and 8th edition AISC manuals - - an EOR who demanded joist calculations to the 1989 9th edition while the state had adopted AISC 360-10 -- **Spec and checklist consultants (2018).** Standard details go unreviewed between projects, and copy errors "ripple" through a set ([Stutzman and Medcalf, 2018](https://www.conspectusinc.com/blog/2018/04/construction-document-quality-crisis)). -- **Propagation mechanism (vendor article).** A pasted detail "transfers the assumption of review without performing any review". Its callouts may point to sheets that no longer exist and its spec sections may be missing from the project manual. Revisions to the source detail do not reach the copies ([InspectMind, 2026-08-26](https://www.inspectmind.ai/resources/articles/how-copy-paste-detail-errors-spread-across-a-drawing-set)). -- **Agency practice.** Agency libraries mark archived sections as prohibited for new projects and attach dates to every revision (Sections 1.3 and 1.4). Firm libraries often lack the equivalent fields: code edition, jurisdiction, date and status. That conclusion is my synthesis, not a single source. - -### 2.3 Engineer of record and responsible charge - -- **Sealing.** NCEES Model Rules §240.20 (August 2024) require the seal "only when [work] was under the licensee's responsible charge". Responsible charge means authority to change, reject or approve the work, personal awareness of the project's scope and limitations, the ability to answer questions about engineering decisions, and full responsibility. Revisions to sealed documents must be "described and dated" ([NCEES Model Rules, August 2024](https://ncees.org/wp-content/uploads/2024/09/Model-Rules_August-2024_web.pdf)). -- **Standard designs.** When "the work is a site adaptation of a standard design plan", a successor licensee "may take responsible charge by performing all professional services to include developing a complete design file with work or design criteria, calculations, code research, and any necessary and appropriate changes" (same source, §240.20.G). -- **State example.** Texas lets engineers seal "standards or general guideline specifications that they have reviewed and selected. Upon sealing, engineers take full professional responsibility for that work" ([22 Tex. Admin. Code §137.33](https://www.law.cornell.edu/regulations/texas/22-Tex-Admin-Code-SS-137-33), accessed 2026-10-30). -- **Ethics opinion.** The NSPE Board of Ethical Review found it unethical to seal manufacturer drawings not prepared under the engineer's direction and control ([NSPE BER Case 19-11, approved 2020-01-30](https://www.nspe.org/sites/default/files/BER19-11-APPROVED.pdf)). - -### 2.4 Ownership and liability of reused documents - -- **AIA B101-2017.** The architect and consultants retain copyright in instruments of service. The owner receives "a nonexclusive license to use ... solely and exclusively for purposes of constructing, using, maintaining, altering and adding to the Project", and unauthorized use is "at the Owner's sole risk" (§7.2–7.4; [B101-2017 sample](https://bidcondocs.delaware.gov/DTCC/DTCC_18001Asblghvac_sample_1.pdf); [AIA Contract Docs help](https://help.aiacontracts.com/hc/en-us/articles/1500009310881-the-rights-of-an-architect-s-instruments-of-service), accessed 2026-10-30). -- **Keeping standard details.** The AIA advises excluding "all drawings, designs, details, specifications, and all other materials you prepared prior to commencing services" from any copyright transfer, so the firm can keep reusing them ([AIA, 2023-12-01](https://www.aia.org/resource-center/understanding-copyright-protection-architects)). A model clause preserves the consultant's "practice detail library" for reuse "for other clients" ([ConstructionRisk contract guide](https://www.constructionrisk.com/wp-content/uploads/2014/11/Contracts-for-Design-Professionals-by-C-Risk-Module-6.pdf), accessed 2026-10-30). EJCDC language keeps "the right to reuse at the discretion of the Engineer" ([IRMI, 2002](https://www.irmi.com/articles/expert-commentary/copyright-infringement-of-design-documents)). -- **Insurer guidance.** The insurer Victor O. Schinnerer recommends that "the instruments of service shall not be reused ... for any new project without an evaluation of the documents in relation to applicable codes and standards" ([Schinnerer management advisory, via Professional Underwriters, 2024](https://www.profunderwriters.com/wp-content/uploads/2024/04/Electronic-Transfer-Issues.pdf)). - -### 2.5 Client confidentiality - -- **AIA Code of Ethics (2024).** Rule 3.401 bars knowingly disclosing information "that would adversely affect their client or that they have been asked to maintain in confidence" ([AIA Code of Ethics, April 2024](https://www.aia.org/sites/default/files/2024-04/aia-code-of-ethics-april-2024.pdf)). -- **AIA AI Firm Toolkit.** The toolkit classifies "Firm standards, internal templates" as Internal data, limited to approved enterprise AI tools. "Project drawings not yet public" are Confidential, allowed only in tools under a verified data-handling agreement. Security-critical drawings (data centers, courthouses, embassies, utilities) go into no AI tool "without explicit legal and client sign-off". Restricted uses include "relying on AI-generated code references ... without independent verification" and "using AI in ways prohibited by an active client contract" ([AIA AI Firm Toolkit](https://aifirmtoolkit.aia.org/), accessed 2026-10-30). -- **ACEC guidelines (July 2024).** ACEC recommends human QA/QC by the person in responsible charge, and keeping AI outputs from growing "so large or complex that employees cannot understand and review them thoroughly" ([ACEC AI guidelines](https://www.acec.org/wp-content/uploads/2024/07/Artificial-Intelligence-Guidelines-for-ACEC-AI-Risk-Subcommittee-Technology-Committee-July-2024.pdf)). A 2024 SGH and Donovan Hatem memo advises avoiding inputs "that could potentially enable an outsider to identify the specific project" ([ACEC-MA, 2024-04-16](https://www.acecma.org/wp-content/uploads/SGH-And-Donovan-Hatem-Guidelines-For-AI-Use.pdf)). -- **Contract terms.** Contracts can impose project-specific confidentiality. Document Crunch's contract playbook asks "Are there confidentiality requirements on this project?" ([Document Crunch, 2025-01-29](https://www.documentcrunch.com/blog/project-playbooks)). -- **Gap.** I found no source that addresses whether showing one client's project detail to a team working for another client is permitted. See Section 6. - -### 2.6 Vendor data terms - -- **Autodesk.** Autodesk's December 2025 Acceptable Use Policy update says customers "may train on your models and data", provided they do not copy Autodesk products ([Autodesk, 2025-12-08](https://adsknews.autodesk.com/en/news/autodesk-terms-changes/); [AEC Magazine analysis, 2026-03-03](https://aecmag.com/features/rewriting-the-rules/)). Autodesk says some features train on "elements of customer content in aggregated and de-identified form" and that it does not share customer data for third-party model training ([Autodesk, 2026-08-17](https://adsknews.autodesk.com/en/stories/how-autodesk-builds-trusted-ai/)). -- **Specialist vendors.** These say they do not train on customer data: Pirros ("Zero AI training on your data"), SWAPP ("not shared across firms"), UpCodes, Structured AI and Pelles (product pages cited in Section 3). - -## 3. AI products and firm-specific knowledge - -Status uses the definitions at the top. "Firm knowledge" lists how a product takes in firm-specific practice. - -| Product | Drawing or document capability | Firm knowledge mechanism | Status and evidence | -| ----------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Bluebeam Max** (Nemetschek) | Smart Review runs a fixed list of checks: sheet blank, missing from set or index, duplicate sheet number, referenced sheet missing, gridline consistency, door tags against the schedule, plumbing tags in plan. Also Smart Overlay, and Claude over MCP | No custom or firm checks documented. Firm consistency comes from Tool Chest markups (Section 1.2) | Max **shipped** 2026-05-19 ([press](https://press.bluebeam.com/2026/05/bluebeam-max-launches-globally-bringing-ai-powered-productivity-to-aec-teams-everywhere/)). Smart Review and Overlay remain **preview**, "optimized for US-based vertical commercial construction" ([Max page](https://www.bluebeam.com/bluebeam-max/)). Imperial units only, works best on floor plans ([Smart Review FAQ](https://support.bluebeam.com/revu/resources/smart-overlay-review-faq.html), accessed 2026-10-30). Built on Firmus AI, acquired 2025-09-04 ([press](https://press.bluebeam.com/2025/09/bluebeam-acquires-firmus-ai-to-accelerate-intelligence-for-drawing-reviews-and-financial-risk-mitigation/)) | -| **Autodesk** Assistant, Forma (formerly ACC), AutoCAD | Assistant searches specs, RFIs, issues and submittals. The Sheets tool extracts sheet numbers and titles from PDFs. AutoSpecs builds submittal logs | Saved prompt library. AutoCAD standards-file check (Section 1.2). AutoSpecs "Suggested Submittals" compares a spec against "historical project data", trained "on thousands of projects" | Assistant for construction out of beta ([Autodesk, 2026-03-25, updated 2026-06-03](https://www.autodesk.com/blogs/construction/meet-autodesk-assistant-ai-native-intelligence-in-forma/)). AutoSpecs **shipped** ([AutoSpecs](https://construction.autodesk.com/tools/autospecs-construction-submittal-log/), accessed 2026-10-30). Assistant in Revit, AutoCAD and Civil 3D **announced** at AU 2025 ([Autodesk, 2025-09-16](https://www.autodesk.com/blogs/construction/au-2025-top-autodesk-construction-announcements/)), later shipped in AutoCAD 2027 | -| **Procore AI** | 20 agents. Drawing Analysis (coordination gaps, missing details). Drawings & Specs Specialist. RFI Triggered Agent checks for duplicate RFIs and answers already in the documents | Contract Review agent checks contracts "against your internal standards". **Lessons Learned Agent** mines RFIs, change events, meetings and daily logs into "reusable lessons learned with preventative actions". Plans list "agentic memory". **Skills** let customers "teach Procore AI their own processes, standards, and best practices" | Digital Coworker packages **shipped** 2026-07-23 ([press](https://www.procore.com/press/procore-introduces-digital-coworker-packages-expands-ai-agent-library-and-previews-skills-to-help-construction-teams-put-ai-to-work)). Agent list from [Procore support](https://v2.support.procore.com/faq-which-agents-are-included-in-my-procore-ai-plan) and [plans](https://www.procore.com/ai/plans), accessed 2026-10-30. Skills **announced** for an August 2026 rollout; current status not verified | -| **Trimble** | ProjectSight AI Title Block Extraction and Auto-Submittals | Agent Studio (customer-built agents) | Title block extraction **shipped** in North America; Agent Studio **pilot** ([Trimble, 2025-11-10](https://investor.trimble.com/news/news-details/2025/Trimble-Highlights-AI-Strategy-and-Innovation-at-Dimensions-User-Conference/default.aspx)) | -| **UpCodes** Plan Review and Copilot | Discipline reviews against 11 million adopted code sections in 6,000+ jurisdictions. Issues are linked to the page and the code section | "Checklist-based review using their own QA/QC templates". Copilot "remembers your jurisdiction, code year, and project details". Custom issues can cite code sections | Plan Review **shipped** 2026-06-03 ([PR Newswire](https://www.prnewswire.com/news-releases/upcodes-adds-ai-native-plan-review-to-its-aec-qaqc-platform-302789640.html)). Executive summary and discipline filters by sheet prefix ([release roundup, 2026-10-01](https://cms-v2.up.codes/a/release-roundup-october-1-2026)). "93% weighted accuracy" on 43 questions is a **marketing claim** ([Copilot page](https://cms-v2.up.codes/features/ai)) | -| **CodeComply** | Code checks with cited findings, VersionVue revision comparison | "Custom Checks" for local amendments and client standards. PlanChat builds a knowledge base from uploaded PDFs | Product pages ([CodeComply, 2026-01-28](https://codecomply.ai/product/)). Also resold by CivicPlus ([2026-03-16](https://www.civicplus.com/community-development/codecomply/)) | -| **Archistar AI PreCheck** (formerly eCheck) | Municipal zoning and code prechecks with pass/fail per rule | "Custom rule configurations" per city | **Shipped** to 30+ governments ([Archistar](https://www.archistar.ai/aiprecheck/government/), accessed 2026-10-30). Burnaby checks "over 50 zoning rules" and "checks objective zoning rules, not ones that need judgment" ([City of Burnaby](https://www.burnaby.ca/our-city/projects/ai-precheck-digital-building-permit-review-tool)) | -| **Pirros** | Detail and family search in Revit using computer vision over geometry and annotations. The Quality Management System checks a set against codes, specs and firm standards | Firm-approved versus reference-only details. Library Builder "recommends new standards based on project history, reuse, and detail quality". QMS flags "when a detail doesn't match the version your firm approved". "Reviewers dismiss with a reason, and the system learns the firm's tolerance" | Detail management and vetting **shipped** (help center, Section 1.3). QMS and learned tolerance are **marketing claims** ([Pirros QMS](https://www.pirros.com/quality-management-system), accessed 2026-10-30). Revit-native, so details come from models, not PDFs | -| **Nomic** | A drawing-review agent dispatches sub-agents by issue class across the set. Each finding records drawing citation, standard citation, severity, discipline and status (New, Accepted, Exported, Addressed, Ignored) | **Drawing Standards Creation** "turns markups, RFIs, and existing standards into a structured Drawing Review Standards artifact". A closing quality pass marks unsupported findings Ignored | Documented workflows ([Nomic docs](https://docs.nomic.ai/docs/capabilities/drawing-review/details); [quality pass](https://docs.nomic.ai/help/reviews/review-accept-reject-export-drawing-issues), accessed 2026-10-30) | -| **SWAPP** (Frank) | Generates and audits Revit and ArchiCAD documentation | A two-week onboarding "encodes your guidelines". "Every correction your team makes becomes workflow memory". The guideline audit returns pass or fail per rule and cites the rule | Audit and onboarding described in vendor articles ([SWAPP, 2026-10-16](https://swapp.ai/enforce-revit-drawing-standards-firm-guidelines/)). Learning from corrections is a **marketing claim** ([FAQ](https://swapp.ai/faq/)) | -| **InspectMind** | Multidiscipline PDF plan check with snippet and code evidence | Upload a QA manual. Examples include "Common mistakes: Issues the firm has caught on past projects" | Product pages ([custom standards](https://www.inspectmind.ai/education/custom-qa-standards-checking), accessed 2026-10-30). Its FAQ says findings "do not certify compliance" ([home](https://www.inspectmind.ai/)) | -| **Groundbook, Structured AI, ArchiBoost** | PDF and/or BIM QA/QC with source-linked findings | Firm checklists are applied to every project. Structured lists checks such as "Find sheets with an outdated title block" and a "Universal QAQC Standard" | Product pages, accessed 2026-10-30 ([Groundbook](https://groundbook.ai/), [Structured](https://getstructured.ai/), [ArchiBoost](https://archiboost.ai/qaqc)). Structured's "9 in 10 findings ... confirmed" is a **marketing claim** | -| **Part3** | Construction administration: submittal review against specs with cited findings | Lists "prior-comment retrieval" and resubmittal comparison as AI uses. The reviewer owns the response | Submittal Assistant **shipped** per product pages ([Part3](https://www.part3.io/submittal-assistant); [knowledge center](https://www.part3.io/knowledge-center/ai-submittal-review-for-architects), accessed 2026-10-30) | -| **Trunk Tools** | A per-project knowledge graph (Cortex). TrunkReview finds clouded and unclouded revision changes | TrunkRFI checks prior RFIs. "Roughly 15 percent of RFIs ... are preventable" (platform data) | Vendor claims ([Trunk Tools, 2026-09-08](https://trunktools.com/blog/ai-construction-documents/)) | -| **Provision, Document Crunch** (contractor-side) | Contract, spec and scope review with citations | Provision compares clauses with the customer's "default positions" and preferred templates. Document Crunch offers configurable checklists and playbooks to "codify your experts' standards" | Product and help pages ([Provision](https://provision.com/risk-review); [Document Crunch help](https://help.documentcrunch.com/articles/9706789692-single-document-playbooks), accessed 2026-10-30) | -| **Pelles** (trade contractors) | Q&A, revision compare, reports in the customer's own templates | "Each fix your team approves ... teaches it how you build", with "no models trained on your data" | **Marketing claim** ([Pelles FAQ](https://www.pelles.ai/faq), accessed 2026-10-30) | -| **Helonic** (formerly Articulate, renamed 2026-03-11) | PDF review across 10 issue categories with draft RFIs | Requires agreement between several models before surfacing a high-severity finding | Vendor blog ([rebrand](https://helonic.com/blog/articulate-rebrand-helonic); [false positives, 2026-08-07](https://helonic.com/blog/ai-drawing-review-false-positives)) | -| **ArchiLabs, BIMLOGIQ** | Revit automation (sheets, tags, dimensions) | Saved and shared "Recipes" or commands, and tagging templates | Product pages ([ArchiLabs](https://archilabs.ai/), [BIMLOGIQ Copilot, v1.25.13 2025-10-28](https://apps.autodesk.com/RVT/en/Detail/Index?appLang=en&id=1840006799718333764&os=Win64)) | -| **Graphisoft / Nemetschek AI Assistant** | Archicad chatbot covering product knowledge, BIM model queries and visualization | None found | Preview announced 2025-01-15. "Public release with our October product updates" was planned ([Graphisoft newsroom](https://newsroom.graphisoft.com/nemetschek-group-unveils-ai-assistant-at-bau-2025); [community, 2025-01-16](https://community.graphisoft.com/t5/Graphisoft-Insights/Nemetschek-AI-Assistant-announced/ba-p/646256)). Release not verified | -| **Augmenta, Hypar, Togal, Kreo** | Electrical raceway routing; space-planning suggestions; AI takeoff. Kreo maps drawing zones (legend, tables, title block, notes) | Project specifications and routing zones (Augmenta). No firm-standard learning found | [Augmenta, 2025-09-11](https://www.globenewswire.com/news-release/2025/09/11/3148579/0/en/augmenta-unveils-comprehensive-agentic-design-environment-making-ai-powered-design-a-reality-for-electrical-contractors.html); [Hypar, 2025-07-01](https://docs.hypar.io/what's-new-or-product-updates/july-1st-2025); [Kreo](https://www.kreo.net/); [Togal](https://www.togal.ai/) | - -Patterns across products (my synthesis of the table): - -- Firm knowledge enters these products in four ways: - 1. uploaded standards documents or checklists (most reviewers) - 2. curated libraries with approval status (Pirros, Deltek office masters) - 3. standards synthesized from past markups and RFIs (Nomic; Procore's Lessons Learned Agent) - 4. correction memory (SWAPP, Pirros, Pelles; marketing only) -- I found no shipped PDF reviewer that suggests "your firm used detail X on a similar project" during review. Pirros comes closest, but it works inside Revit. -- Nearly every product cites a sheet location and keeps a human accept-or-reject step. Several say they do not train on customer data. - -## 4. Research on drawing and detail retrieval - -**Title block and sheet parsing** - -- **Lombardi et al. (2025, Buro Happold and Birmingham City University).** A lightweight CNN plus GPT-4o detects title blocks and extracts metadata from CAD and hand-drawn drawings. A search interface built on the results was deployed on real projects, and the work includes an expert-annotated dataset ([arXiv 2504.08645, 2025-04-11](https://arxiv.org/abs/2504.08645); [EC3 2025](https://doi.org/10.35490/ec3.2025.344)). -- **Peng et al. (EC3 2026).** Qwen2.5-VL-3B fine-tuned with LoRA locates title blocks by grounded prompting, then extracts scale, drawing name and route from historical bridge drawings. Dense machine-printed title blocks are harder than handwritten ones ([EC3 2026 paper](https://ec-3.org/wp-content/uploads/2026/08/EC32026_252.pdf)). -- **Earlier work.** Kashevnik et al. reached about 74% accuracy on title block detection and metadata for good-quality scans ([IEEE Access, 2023-02-14](https://doi.org/10.1109/access.2023.3244603)). Sajadfar et al. applied Faster R-CNN to title blocks in construction documents ([SSRN, 2024](https://doi.org/10.2139/ssrn.4845684)). - -**Drawing similarity and retrieval** - -- **Dai et al. (WWW 2019).** In engineering diagram search, a query part matches "only a small region". Local matching beats global matching, and a gating network suppresses "visual metadata such as callouts" ([ACM, 2019-05-13](https://doi.org/10.1145/3308558.3313500)). -- **DeepPatent (WACV 2022).** The dataset has 350,000+ design-patent drawings. The authors found commercial image retrieval performs poorly and "retrieval of technical drawings remains an open challenge" ([CVF](https://openaccess.thecvf.com/content/WACV2022/papers/Kucer_DeepPatent_Large_Scale_Patent_Drawing_Recognition_and_Retrieval_WACV_2022_paper.pdf)). A sketch-based transfer-learning approach for diagram retrieval appears in [Bhattarai et al., CVPRW 2020](https://www.openaccess.thecvf.com/content_CVPRW_2020/papers/w8/Bhattarai_Diagram_Image_Retrieval_Using_Sketch-Based_Deep_Learning_and_Transfer_Learning_CVPRW_2020_paper.pdf). -- **Manufacturing precedents.** Several systems retrieve past drawings with partial-design queries or multimodal inputs: [Van Daele et al., AAAI 2021](https://ojs.aaai.org/index.php/AAAI/article/view/17783) (deployed tool, query by part of a drawing, table or text); [CoLIBRi](https://dbs.uni-leipzig.de/research/publications/colibri-supporting-quotation-through-multi-modal-retrieval-and-conversational) (OCR, segmentation, visual embeddings and a VLM for title blocks; code and weights released; accessed 2026-10-30); a CLIP vector search ([IET, 2026-06-01](https://doi.org/10.1049/icp.2026.1931)); and [Baba et al., 2005](https://doi.org/10.1109/ism.2005.105). -- **BIM detail reuse.** An association-rule recommender built from more than 30,000 BIM views of a case-study firm reported 80% efficacy and 15% workflow time saved ([2021 case study](https://www.academia.edu/65122864/Artificial_intelligence_for_BIM_content_management_and_delivery_Case_study_of_association_rule_mining_for_construction_detailing)). BIM search with geometric, semantic and topological embeddings is described in [Huang et al., Buildings, 2025-03-18](https://doi.org/10.3390/buildings15060951). A case-based reasoning system over BCF coordination issues reached over 60% top-1 retrieval of similar cases and 70% acceptance of the resolutions it proposed ([TUM thesis, 2025](https://mediatum.ub.tum.de/1788972)). - -**Symbols and legends** - -- **Few-shot detection.** Few-shot symbol detection works with fewer than ten training examples per class ([Jamieson et al., 2024-09-29](https://doi.org/10.1080/08839514.2024.2406712)). -- **Legend-informed recognition.** Contrastive learning on legend entries reaches over 80% accuracy on building-services diagrams and rejects symbols that are not in the legend ([Hain et al., 2025-10-03](https://doi.org/10.1007/978-3-662-72243-5_23)). Legend-table extraction followed by symbol counting is described in [Sarkar, arXiv 2204.13277](https://ar5iv.labs.arxiv.org/html/2204.13277). -- **BlueprintSymVL.** This benchmark gives the model a one-shot visual exemplar because "standardized symbology ... rarely hold[s]" in practice ([Chalmers, accessed 2026-10-30](https://research.chalmers.se/publication/549362/file/549362_Fulltext.pdf)). - -**Datasets and benchmarks** - -- **FloorPlanCAD.** 15,663 SVG floor plans (November 2021 update). The project shut down in 2022 ([site](https://floorplancad.github.io/)). -- **ArchCAD-400K (NeurIPS 2025).** 5,538 drawings, 413,062 chunks and 27 categories including notation, with annotations derived from CAD layer and block structure. A 40K-sample subset was released first ([NeurIPS paper](https://proceedings.neurips.cc/paper_files/paper/2025/file/b96ce7d38339874a8704e8895f743284-Paper-Conference.pdf); [GitHub](https://github.com/ArchiAI-LAB/ArchCAD)). -- **AEC-Bench (Nomic, 2026-03-31).** 196 tasks on public project PDFs across three scopes: within a sheet, across a drawing set, and across project documents. Tasks include detail title accuracy, note callout accuracy, cross-reference resolution and tracing, sheet index consistency and spec-drawing sync, with defects injected by domain experts. General coding agents "fail on visual grounding, exhaustive traversal, and cross-document coordination". Released under Apache 2 ([arXiv 2603.29199](http://arxiv.org/abs/2603.29199); [dataset](https://huggingface.co/datasets/nomic-ai/aec-bench)). -- **AECV-Bench (2026-01-08).** OCR and text questions reach up to 0.95 accuracy. Counting doors and windows "remains unsolved (often 0.40–0.55)" ([arXiv 2601.04819](https://arxiv.org/abs/2601.04819)). -- **DrawingVQA (CVPR 2026 Findings).** 33 issued-for-construction structural drawings and 92 questions, with "a substantial gap between model and expert performance" on expert reasoning ([CVF](https://openaccess.thecvf.com/content/CVPR2026F/papers/Jung_DrawingVQA_A_Real-World_Benchmark_for_Multi-Depth_Visual-Textual_Reasoning_on_Construction_CVPRF_2026_paper.pdf)). -- **LED-BENCH (2026-10-17).** 107 large-format sheets. The best model averages 52.65%, and all models do worse when evidence is spread across a sheet ([arXiv 2610.19571](https://arxiv.org/html/2610.19571)). -- **ArchPlanVQA.** General VLMs reach 33–38% on architectural floor-plan CAD questions ([J. Comput. Civ. Eng., 2026-06-02](https://doi.org/10.1061/jccee5.cpeng-7571)). - -**Mining review comments and RFIs** - -- **Panahi et al. (2024).** NLP, ChatGPT and computer vision identify prior-project RFIs likely to recur on a project under design review, tested on 19 healthcare projects ([ASCE i3CE, 2024-01-25](https://doi.org/10.1061/9780784485224.020)). -- **Fokwa Soh et al. (2020).** Across 26 steel projects, the recurring RFI theme was missing or poorly presented connection information ([OTMCJ, 2020](https://doi.org/10.2478/otmcj-2020-0005)). -- **Elshaboury et al. (EC3 2025).** LLM extraction from design inspection reports structures each finding as checked document, referenced document, defect and building object. The authors note such reports are usually "inaccessible due to confidentiality constraints" ([EC3 2025](https://doi.org/10.35490/ec3.2025.337)). -- **Afzal et al. (2023).** LDA topic modeling of RFIs ([IEEE CASE, 2023-08-26](https://doi.org/10.1109/case56687.2023.10260517)). - -## 5. What reviewers want from AI findings and suggestions - -**Surveys** - -- **AIA (2025-03-11).** Only 6% of architects use AI regularly. Ninety percent are concerned about inaccuracy, unintended consequences, security, authenticity and transparency ([AIA press release](https://www.aia.org/about-aia/press/new-research-explores-perceptions-and-opportunities-artificial-intelligence)). One respondent wrote that AI output "can look good but be very wrong" ([AIA Architect, 2025-03-03](https://www.aia.org/aia-architect/article/architects-are-excited-about-potential-ai-concerns-abound)). -- **Autodesk State of Design & Make 2025.** Trust in AI among AECO leaders fell 14 points year over year ([Autodesk report PDF, April 2025](https://adsknews.autodesk.com/app/uploads/2025/04/2025-SDM-Report-%E2%80%93-AECO-Final.pdf)). -- **Architects' Journal (2025-10-03).** Inaccuracy was the top barrier, cited by 55%, and "many saying they had to thoroughly check any materials produced" ([AJ](https://www.architectsjournal.co.uk/news/ai-in-practice-chatgpt-use-far-outstrips-midjourney-aj-survey-shows)). -- **RIBA AI Report 2025.** Half of respondents disagree that AI has improved specification accuracy. The report warns it is "unlikely to be a legitimate defence to claim the genAI tool is at fault" ([RIBA](https://www.riba.org/media/uqmlay5m/riba-ai-report-2025-final.pdf)). -- **BST Global with ACEC.** 84% of AEC executives are concerned about accuracy, including "incomplete or outdated data" ([BST report](https://bstglobal.com/resources/download/AI_and_Data_Insights_Report/20260226/BST_Global-AI_and_Data_Insights_Report_2025.pdf), survey Oct–Dec 2024). -- **RICS (2025-09-12).** 45% of 2,200+ construction professionals report no AI implementation ([RICS](https://www.rics.org/news-insights/artificial-intelligence-in-construction-report)). - -**Evaluation guidance and practitioner writing** - -- **redline.best checklist.** A useful finding names a sheet and location, cites its basis, ranks severity, can be accepted or rejected with a reason, and records "which person accepted or rejected each finding". Evaluators should time triage, run the same set twice to measure variability, and score each discipline separately ([redline.best, 2026-09-25](https://redline.best/guides/evaluate-ai-drawing-review-tools)). -- **False positives.** Their cost "is nonlinear": past a point, sorting noise "eats up more time than the tool saved", and a finding that is traceable can be dismissed in seconds ([Structured AI](https://getstructured.ai/education/false-positives-in-ai-drawing-review/), vendor source, accessed 2026-10-30). RedlineBench scores a false alarm at −1 against +1 for a correct find, on a project with 55 known issues ([RedlineBench, 2026-03-16](https://redlinebench.benfeicht.com/)). -- **Industrial P&ID deployment.** Dzhusupova et al. treat false positives as tolerable because engineers review every output, and false negatives as missed design errors ([J. Softw. Evol. Process, 2023-02-15](https://onlinelibrary.wiley.com/doi/10.1002/smr.2543)). -- **Accountability.** A framework separates reading, comprehension, cross-sheet understanding and application. It argues that a judgment at the application level needs an attributable sign-off before it drives action ([KP Reddy, 2026-07-16](https://kpreddy.substack.com/p/a-framework-for-evaluating-ai-in)). -- **What QC reviewers value.** Identifying "what is not shown, but is required" is often the most valuable thing a reviewer provides. Checklists should be short and fed by QC and construction administration feedback ([Stutzman and Medcalf, 2018](https://www.conspectusinc.com/blog/2018/04/construction-document-quality-crisis)). -- **Professional duty.** The AIA restricts relying on AI code references without verification, and ACEC warns against output too large to review (Section 2.5). - -Requirements these sources imply for suggestions drawn from prior work (my inference from the evidence above, not a design): - -- Each suggestion should identify its source: project, sheet, detail, revision, approval status, code edition and jurisdiction. -- A suggestion should say why it applies here. -- Nothing should reach a reviewer that the reviewer cannot check quickly. -- The person who accepts or rejects a suggestion should be recorded. - -## 6. Open uncertainties - -1. **Learning from corrections is unverified.** SWAPP, Pirros (learned tolerance) and Pelles describe it, and I found no independent test, documentation of the mechanism, or failure data. Nomic's standards-from-markups workflow is documented, but I found no measure of its quality. -2. **Cross-client reuse inside AI tools.** The AIA code, the AIA toolkit and ACEC guidance address disclosure to AI vendors. None addresses showing client A's past project detail to a team working for client B, or what "similar project" retrieval does under per-project confidentiality clauses. B101 lets firms reuse their own standard details, but project-specific content is licensed to the owner for that project only. -3. **Retaining review comments.** Stutzman and Medcalf (2018) advocate destroying commented review sets once the review is recorded. EGBC-style rules require documented checks and roughly 10-year retention. I found no legal source on whether an AI-searchable history of internal review comments increases discovery exposure. -4. **NCS adoption and version.** I found no data on what share of US firms follow the NCS and UDS sheet format faithfully. The V7 "released in 2025" statement and the 2024 ISBN date conflict slightly. Firm deviations such as three-digit sequences (LANL) and floor-number alignment are documented anecdotally only. -5. **Vendor status may have moved.** Bluebeam Smart Review is still labeled preview, and its accuracy outside US commercial floor plans is unknown. I did not verify whether Procore Skills reached customers after August 2026, or whether Graphisoft's AI Assistant was released in October 2025. -6. **No benchmark for detail similarity in AEC.** No public dataset labels typical details by the condition they solve (for example "1-hr metal stud partition head at deck") across firms. Pirros's condition search is proprietary. Existing benchmarks measure reading and cross-reference tasks, not whether a retrieved past detail applies to a new project. -7. **No evidence on what applicability metadata firms actually record.** Code edition, seismic design category, jurisdiction and fire rating are what a reviewer needs to judge whether a past detail still applies. No source showed how often firm libraries record these fields; Pirros's project filters (location, market sector, client, material) are the only documented example. -8. **Most 2026 product evidence is vendor-authored.** Accuracy figures from Structured AI, UpCodes, Provision, Helonic and Trunk Tools are self-reported. Independent comparisons such as RedlineBench and AEC-Bench are small, with 196 or fewer items. diff --git a/docs/research/agent-harness-comparison.md b/docs/research/agent-harness-comparison.md deleted file mode 100644 index aca24396..00000000 --- a/docs/research/agent-harness-comparison.md +++ /dev/null @@ -1,125 +0,0 @@ -# Agent SDK options for Moke - -Research date: October 27, 2026. Status: comparative recommendation; no runtime or dependency change. - -Decision update: the selected direction is Pi Durable with local execution by default and explicit handoff to Modal Sandboxes. See [Review runtime and cloud handoff](../review-runtime.md) for the architecture, verified hackathon requirements and scoring priorities. Cloudflare PiHarness remains a researched alternative. - -Moke needs a local agent runtime for lengthy drawing reviews: inspect selected PDF crops, request additional evidence through SAM, run deterministic checks, accept reviewer corrections, and recover after interruption. Electron owns the desktop interface and trusted bridge; the compiled Bun backend owns application services and SQLite. The project currently pins Effect 4.0.4 and Bun 1.4.4. - -For the hackathon, **Pi Durable behind an Effect application service is the preferred candidate**. It already supplies the agent lifecycle and its recovery policy. **Effect AI plus Effect Workflow is the strongest native alternative**, but assembling the conversation loop, durable work admission, recovery policy, context management and progress projection is more work than adding provider Layers. The choice remains conditional on a small recovery and packaging experiment with Moke's actual provider routes. - -## What the SDK must own - -Three responsibilities need separate evaluation: - -1. Model access: images, tool schemas, streaming, structured output, reasoning continuity, and provider-specific request options. -2. Agent execution: conversation history, repeated model/tool turns, limits, cancellation, context management, and visible progress. -3. Durable execution: persisted work identity, completed results, recovery of unfinished tasks, human waits, and replay policy. - -A persisted transcript alone does not make an interrupted tool safe to execute again. A durable workflow cannot determine whether a paid provider call succeeded if the process died before recording its response ID. The harness also cannot replace Moke's source hashes, coordinate transforms, accepted-fact versions, dependency invalidation, or deterministic engineering checks. - -## Comparison - -The fit judgments below are architectural assessments, not measured performance results. - -| Option | Existing model and agent features | Persistent execution | Integration cost for Moke | Assessment | -| --------------------------------------- | -------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------- | -| Effect AI and Workflow | `LanguageModel`, `Chat`, `Tool`, `Toolkit`, Effect Schema and provider Layers | `Workflow`, `Activity`, `DurableDeferred`; SQL-backed `ClusterWorkflowEngine` with `SingleRunner` | Application loop plus durable admission, recovery policy, context limits and activity design | Strongest native alternative; substantial harness assembly remains | -| Pi Durable | Complete agent turns, tool tasks, queued inputs, retries, compaction, streams, usage and extensibility | Committed transcripts and task state; SQLite, JSONL and custom storage | Chord/Effect lifecycle bridge, TypeBox/Schema boundary, Bun storage facade, domain-state reconciliation | Preferred hackathon candidate; experimental API is the main concern | -| LangGraph JS | Explicit graph execution, conditional routing, interrupts and state inspection; model access is separate | Checkpointers, including SQLite; `sync` durability stores checkpoints before the next step | Introduces graph state and a second runtime model; native storage packaging and actual provider routes need validation | Strong candidate when branching and human intervention dominate | -| Mastra durable agents | Agents, tools, harness features, memory, observability and workflows | Native durable-agent recovery is opt-in; libSQL can use local files | A broader framework and storage system; embedded startup/recovery integration remains application work | Credible broader alternative, especially if its harness features save substantial UI work | -| AI SDK `ToolLoopAgent` | Provider abstraction, tool loop, stop conditions and structured output | In memory without another execution layer | Straightforward inference integration; durable execution still needs a separate choice | Useful model/loop library, insufficient by itself for the intended recovery contract | -| AI SDK `WorkflowAgent` and Workflow SDK | Durable agent loop and persistent approvals | Workflow steps and a configured World; local and community Turso/libSQL Worlds exist | Build transforms, generated queue/webhook handlers, World configuration and compiled-backend packaging | Viable with Bun, but adds build/runtime integration to this starter | -| Cloudflare Agents with PiHarness | Pi Durable harness hosted in a Durable Object | Durable Object SQLite and alarm-driven wake-up | Requires a hosted execution component and a protocol for accessing local drawing evidence | Attractive for later remote execution or reviews that must continue after desktop exit | -| Restate with an agent SDK | SDK integrations plus journaled model and tool steps | Restate Server coordinates durable service handlers | Adds a separate server/service lifecycle | Strong server-side durability option; less suitable for the current desktop-only architecture | - -Sources: [Effect AI](https://effect.website/docs/v4/api/effect/ai/LanguageModel), [Effect SingleRunner](https://effect.website/docs/v4/api/effect/cluster/SingleRunner), [Pi Durable](https://github.com/earendil-works/pi/blob/main/packages/durable/README.md), [LangGraph checkpointers](https://docs.langchain.com/oss/javascript/langgraph/checkpointers), [Mastra durable agents](https://mastra.ai/docs/harness/durable-agents), [AI SDK agents](https://ai-sdk.dev/docs/agents/building-agents), [WorkflowAgent](https://ai-sdk.dev/docs/agents/workflow-agent), [Cloudflare Pi](https://developers.cloudflare.com/agents/harnesses/pi/), [Restate durable agents](https://docs.restate.dev/ai/patterns/durable-agents). - -## Effect already provides substantial infrastructure - -The vendored reference at Effect revision `e17ecdbe902aa7108f775c8ab9c0ddf9f38801ab`, recorded in `.project-kit/manifest.json`, contains the following APIs in Effect 4.0.4: - -- `effect/ai`: `LanguageModel`, `Chat`, `Tool` and `Toolkit`, with Schema-based tool parameters and results. -- `@effect/ai-openai-compat`: a dedicated Chat Completions and embeddings integration with configurable `apiUrl`. This is the relevant initial adapter for OpenAI-compatible chat routes. -- `@effect/ai-openai`: the OpenAI-specific provider, including Responses-oriented functionality. Compatibility with Meta's particular Responses implementation must be tested rather than inferred from a matching endpoint name. -- `effect/workflow`: `Workflow`, `Activity`, `DurableDeferred`, `DurableClock` and `WorkflowEngine`. -- `effect/cluster`: `ClusterWorkflowEngine` and `SingleRunner`, allowing SQL-backed workflows in one embedded process. A distributed deployment is not required. - -Effect's own chat example implements repeated model/tool turns with a short explicit loop. Tool execution and chat history are provided; application policy decides when to stop. `Chat` persistence saves conversation history, while `Workflow` and activities supply the execution journal. A memory-only engine does not satisfy crash recovery. - -The short loop is not the total integration cost. Moke would also need durable input admission, retries and uncertain-attempt policy, interrupted-tool behavior, context limits, and a recoverable progress view. `SingleRunner` brings SQL-backed cluster mailboxes and a long-lived runtime. The current application opens scoped SQL connections per operation and serializes backup/restore through one semaphore. Integrating a persistent workflow runner requires an explicit ownership design; it is not a drop-in addition to the current database Layer. - -For Moke, each model request and each evidence operation should have its own durable identity. Keep model generation separate from tool execution when those need distinct recovery boundaries. `LanguageModel` exposes `disableToolCallResolution` for this separation. Wrapping an entire multi-call review in one activity would replay too much unfinished work. - -Activity results are memoized after completion. An unfinished or suspended activity can run again, so external calls and domain writes still need reconciliation or idempotency. AI and workflow modules are marked unstable: pin versions and use the matching reference source. This is a native solution, not a promise of a turnkey durable agent product. - -Sources: [Effect Chat](https://effect.website/docs/v4/api/effect/ai/Chat), [Toolkit](https://effect.website/docs/v4/api/effect/ai/Toolkit), [OpenAI-compatible provider](https://effect.website/docs/v4/api/ai-openai-compat), [ClusterWorkflowEngine](https://effect.website/docs/v4/api/effect/cluster/ClusterWorkflowEngine). Matching local references: `repos/effect/ai-docs/src/71_ai/30_chat.ts`, `packages/effect/src/workflow/Activity.ts`, `packages/effect/src/cluster/SingleRunner.ts`, and `packages/ai/openai-compat/src/OpenAiClient.ts` under `repos/effect`. - -## Pi Durable is a serious alternative - -Pi Durable is separate from the Pi coding-agent SDK. It is intended for general agentic applications and already owns the conversation/tool lifecycle that Moke would otherwise assemble. Its README explicitly marks the API experimental. The fetched main-branch package manifest reports version 1.4.0; that does not establish the version currently published to the package registry. - -Its durable task model is particularly relevant: input admission, model generation and tool execution are persisted as distinct work. A repeated input `requestId` returns the same submission. On restart, completed committed results remain available; interrupted model requests can be retried. An interrupted tool reruns only when declared `replay: "safe"`; otherwise the model receives an interruption result. A safe declaration still requires the tool to implement safe behavior. - -The portable SQLite core can run on Bun with an asynchronous `SqliteDatabase` facade. The documented ready-made local adapter is for Node; a Bun integration must implement and verify the facade or use another supported storage path. Storage requires a single owning process. The Node SQLite adapter uses WAL with `synchronous = NORMAL`, so process-crash recovery and power-loss durability are different guarantees. - -Pi AI already includes separate Meta and Baseten providers. The Meta factory uses Responses and reads `META_API_KEY`; Moke currently uses `MODEL_API_KEY`, so the integration must map credentials explicitly. Baseten uses Chat Completions and `BASETEN_API_KEY`. The organizer-specific `meta-models/muse-spark-open-1026` model must remain explicit even if a catalog uses a different default. - -Pi uses Chord Context for cancellation/telemetry and TypeBox for tool schemas. Effect would still own Moke's service lifecycle, domain operations, errors and validation. One adapter must translate between the two systems. Do not run an Effect durable agent loop and a Pi durable loop over the same work: choose one execution owner. - -Pi storage also changes resource ownership. Prefer a dedicated harness database initially, with one owner, rather than bypassing Moke's existing recovery serialization on `app.sqlite`. Domain updates and harness receipts then span two stores: every mutating tool needs a stable operation ID and idempotent result lookup so a crash between those writes cannot duplicate an accepted fact or finding. Project export/restore must include a consistent harness snapshot and stop or quiesce the owner before replacement. Separating databases simplifies connection ownership; it does not provide cross-database atomicity. - -Keep PDF rendering and image coordinates in Moke's application pipeline. Pi's optional coding-tool image reader has its own resizing rules and is not the intended entry point for engineering crops. Pi's hooks can block a tool, but the checked Cloudflare integration documents no built-in approval/permission workflow; persistent reviewer decisions remain explicit application state. - -Sources: [Pi Durable README](https://github.com/earendil-works/pi/blob/main/packages/durable/README.md), [author's introduction](https://earendil.com/posts/pi-durable/), [Pi AI](https://github.com/earendil-works/pi/blob/main/packages/ai/README.md), [Meta factory](https://github.com/earendil-works/pi/blob/main/packages/ai/src/providers/meta.ts), [Baseten factory](https://github.com/earendil-works/pi/blob/main/packages/ai/src/providers/baseten.ts), [package manifest](https://github.com/earendil-works/pi/blob/main/packages/durable/package.json). - -## Important qualifications for other options - -LangGraph's persistent checkpointer and its durability mode are separate settings. For preserving each completed boundary before the next step, use `sync`, not `exit`. SQLite is documented for local workflows; Moke's compiled Bun distribution still needs an actual driver and packaging test. Graph checkpoints do not automatically version accepted drawing facts or invalidate downstream engineering results. - -Mastra now documents a dedicated durable-agent facility, beyond ordinary workflow suspend/resume. Default `recovery.durableAgents: 'off'` skips normal per-step running checkpoints. Automatic recovery requires `recovery.durableAgents: 'auto'` and the corresponding startup path. Its docs explicitly warn that recovery can reissue paid model calls and tools, and that a delegated subagent can replay as a whole. Local libSQL is supported, but an embedded host must own recovery and shutdown. - -The current AI SDK durable class is `WorkflowAgent` from `@ai-sdk/workflow`, replacing `DurableAgent`. It requires Workflow 5. There is a documented Bun integration with generated handlers and an SWC transform, and the Worlds catalog lists an embedded Turso/libSQL backend. It is therefore inaccurate to describe it as Vercel-only or incompatible with Bun. Its integration footprint is simply larger than adding an ordinary model SDK to the current compiled backend. - -The current first-party Local World documentation also describes file-backed workflow state and startup re-enqueue of active runs, enabled by default through `WORKFLOW_LOCAL_RECOVER_ACTIVE_RUNS`. Interrupted inline steps are released and retried according to their retry limits. The host assumes one active owner for the data directory. This corrects the critic's claim that Local World's queue cannot recover after restart. The Local World is positioned for development; these recovery features alone do not establish suitability for a packaged production desktop app. [Local World configuration and recovery](https://workflow-sdk.dev/worlds/local) - -Cloudflare's `PiHarness` is a separate deployment option for Pi Durable. Its documented limits include no built-in tool approval step, snapshot-based event reconnect, and a 15-minute alarm invocation limit that can interrupt a single very long streaming model call. The desktop application would need to serve selected local evidence to hosted work. This option becomes more useful if review execution must continue while the desktop is closed. - -Sources: [LangGraph persistence](https://docs.langchain.com/oss/javascript/langgraph/persistence), [Mastra durable recovery](https://mastra.ai/docs/harness/durable-agents#crash-recovery), [Mastra local libSQL](https://mastra.ai/integrations/databases/libsql), [WorkflowAgent migration](https://ai-sdk.dev/docs/agents/workflow-agent), [Bun integration](https://workflow-sdk.dev/docs/how-it-works/framework-integrations), [Worlds catalog](https://workflow-sdk.dev/worlds), [Cloudflare Pi limits](https://developers.cloudflare.com/agents/harnesses/pi/). - -## Recommended decision and proof - -Use **Pi Durable as the first candidate to validate for the hackathon**. Keep Effect as the application runtime and expose the harness through an application-owned service. This buys an existing recovery-aware agent lifecycle while retaining the selected SQL, RPC, Schema and domain services. Effect AI plus Workflow remains the alternative if Pi's Bun facade, provider behavior or lifecycle bridge proves more costly than the native assembly. Both have unstable APIs; architectural consistency alone is not evidence of lower delivery effort. - -Pi owns model turns, conversation context, tool scheduling, task retries and its journal. Effect owns project authorization, PDF rendering, SAM operations, accepted facts, rule calculations and dependency invalidation. The harness sees narrow tools such as render region, segment region and run check. A SAM call remains a dedicated provider operation inside a tool; neither a generic Chat Completions adapter nor Pi's Muse provider should be assumed to implement SAM's response format. The UI consumes Moke's typed progress and result contracts. - -For a longer-lived product, keep that boundary while the review behavior becomes clear. Reconsider a native Effect harness if concrete maintenance experience shows the Chord, TypeBox and experimental API costs exceed the native implementation cost. The subsequent runtime decision selects Modal Sandboxes for explicit cloud handoff; Cloudflare PiHarness remains an alternative, not a second cloud runtime. - -The deciding experiment should use the same narrow operation on each serious candidate: ask Muse to inspect an overview, request a crop, call a fake deterministic evidence tool, then produce a typed result. First use a controllable local provider for crash injection; then verify a small real image-and-tool round against Meta and Baseten with explicit model IDs. Provider compatibility must include tool results, structured output, reasoning continuity, cancellation and response-ID handling, not just a successful text greeting. - -Kill and reopen at these boundaries: before model submission, after submission before receipt persistence, after a committed model tool call, during a tool, and after its domain write before the harness receipt. A reviewer correction must create a new fact version, stop stale output from becoming current, and recheck only dependent work. Verify the compiled Bun executable on the target desktop platform. - -Pass criteria: completed paid work is not deliberately repeated; uncertain paid attempts are visible and reconciled before retry; committed findings are unique; interrupted operations obey their replay policy; cancellation reaches owned work; original evidence transforms survive; reopening persisted state requires no hosted orchestration service. Subsequent model calls still require provider access. Local execution resumes when the backend is running again. Continuing while the app is closed requires a separately selected daemon or hosted runtime. - -## Independent critique - -Grok 4.6 independently reviewed the comparison and the matching Effect source. The findings and their disposition follow. - -Accepted findings: - -- The draft understated the work remaining after composing Effect AI and Workflow. Existing components do not by themselves provide Pi's complete durable agent lifecycle. This changed the hackathon recommendation to Pi first. -- SQL ownership is an integration concern for both candidates. A persistent harness must cooperate with the current scoped queries, backup and restore behavior. A separate journal still needs reconciliation with domain writes. -- Provider choice must be per operation. Muse reasoning, Baseten's hackathon route and SAM segmentation must each be verified; a working text greeting does not establish compatibility. -- A framework's durable features need their actual startup and persistence configuration. Mastra's recovery default and Workflow SDK's required transforms materially affect integration. - -Rejected or narrowed findings: - -- Grok said Local World cannot recover an in-memory queue. The current first-party page explicitly documents startup recovery of active runs and abandoned inline steps. The report retains WorkflowAgent as viable and judges its build/runtime cost separately. -- Lack of a turnkey Effect agent does not imply lack of durable resume primitives. Effect Workflow does provide resume, activities and durable waits; the missing piece is an assembled agent lifecycle. -- Pi's Node SQLite settings do not establish the power-loss behavior of a future custom Bun facade. The facade's settings and transaction behavior must be specified and tested. - -The critic proposed a day per candidate. For the hackathon, begin with a bounded Pi integration experiment and use native Effect as a fallback candidate, rather than spending two full days benchmarking frameworks before producing review evidence. This sequencing is a delivery judgment, not a measured estimate. - -## Research scope - -Discovery used Exa and page retrieval used Firecrawl through Executor. Substantive claims rely on first-party documentation and the project's matching Effect source. No inference SDK was installed, no paid model probe was run, and no performance ranking was measured. Remaining uncertainty concerns provider conformance, cross-platform compiled Bun packaging, crash behavior in Moke's actual integration, and the adoption cost of each runtime. diff --git a/docs/research/audit-2-validation.md b/docs/research/audit-2-validation.md deleted file mode 100644 index f84873c3..00000000 --- a/docs/research/audit-2-validation.md +++ /dev/null @@ -1,383 +0,0 @@ -# MOKE Audit 2 implementation and validation - -## Cloud integration (2026-10-29) - -Approved audit commit `9aa92c1f041a7c589d011f378451a330f35bed24` was -rebased onto PR10's `c359e522f06f1a73888057233b4854193e2da9b2` as -`2596c9ecd06b3525c444949054bdb45b6cfee45b`. The three conflicts combine -Modal setup with trusted PDF drop imports, cloud ownership controls with the -audit composer/canvas, and cloud-return assertions with saved-answer, -source-generation and count-selection regressions. Auto-merged RPC, fixtures, -package scripts and subscription paths were inspected. - -Local steering now enters the application's existing cloud ownership fence. -The centered import dialog, scope/reviewer controls, count edits and mirrored -finding completion also honor cloud ownership. Existing backend reconciliation, -automatic package return and immutable history remain in use; remote steering -is unavailable. Explicit scope travels through the shared cloud request and -worker JSON contracts. Offline round trips cover legacy requests and whole-set, -selected-sheet and discipline scopes, excluding later revisions as requested -and retaining sheet discipline metadata. - -Cloud protocol 3 rejects PR10 worker images before model dispatch because those -images would silently discard the new scope fields. A matching worker image -must be built and published separately before cloud use. No image build, cloud -operation, credential change, dependency upgrade or migration was performed. -`bun install --frozen-lockfile` installed upstream's pinned Modal dependency. -The hardened isolation runner and packaged-executable support are unchanged -from the approved audit commit. The isolated launch adapter also strips ambient -Modal configuration so it cannot reintroduce credentials removed by each test. - -Integration evidence is recorded in `.local/audit-2/integration-verify.log`, -`.local/audit-2/integration-svelte-autofixer.log`, -`.local/audit-2/integration-desktop/manifest.json` and the ignored review ledger. -Final `bun run verify` passes 316 smoke tests across 33 files (1,988 assertions), -all builds and checks, and all seven Electron tests. Both GUI runs passed the -hardened preflight. All five Svelte components/routes have zero autofixer issues; -the existing dialog-ref and Map/Set suggestions remain advisory. The 24 existing -class-instance-spread lint warnings remain. Fresh cloud-owner, mirrored-finding -and missing-source screenshots were inspected from the final run. No live -provider/cloud validation ran. Windows/macOS hosted CI and packaged execution -remain external validation; no inference-quality claim is added. - -The focused comparison is -`git range-diff 0fe032e3e..9aa92c1f0 c359e522f..HEAD`, preserved in -`.local/audit-2/integration-range-diff.txt`. Parent round 3 reviews this -integration; publication remains with the parent. The sections below record -the pre-integration audit approval and its historical evidence. - -## Approved audit baseline - -Work resumed on `fix/moke-audit-2` from base -`0fe032e3e1b9aac35967545065a8c7677e089e44`. The existing implementation diff -was retained and completed. No migration, dependency upgrade, deployment, PR, -push, merge, or paid provider probe is part of this change. - -## Isolation and reproducibility - -All GUI evidence for this handoff comes from `scripts/isolated-desktop.ts`. -Earlier screenshots and GUI logs under `.local/audit-2` are **not isolation -proof** and are excluded from the acceptance evidence below. - -The parent-hardened runner uses private user, mount, network, IPC and PID -namespaces, kills children when its parent exits, mounts a private X11 socket -directory, hides `/mnt/wslg`, and brings up only private loopback networking. -It starts Xvfb explicitly on `:99`, verifies 1600×1000 with `xdpyinfo`, and -checks that desktop `:0` is unreachable before Playwright can launch Electron. -Desktop buses, portals, audio, notifications and inherited display variables are -disabled. Electron tests use temporary data and profile directories and one -worker. No user desktop, live MOKE profile, clipboard, input, browser, or provider -endpoint is used. - -Run the required verification with Bun 1.4.4 and the prepared local Xvfb: - -```sh -MOKE_XVFB_BINARY="$PWD/.local/audit-2/xvfb/usr/bin/Xvfb-private" \ -LD_LIBRARY_PATH="$PWD/.local/audit-2/xvfb/usr/lib/x86_64-linux-gnu" \ -nice -n 10 bun run verify -``` - -The ordinary E2E and screenshot package scripts now route through this runner. -`test-packaged.ts` delegates to `test:e2e`; it was inspected but packaging was -not required or run. Tests and builds ran sequentially at reduced CPU priority. - -## Acceptance evidence - -A01–A07 and A09–A13 are implemented and verified by the fixtures below. A08 has -verified lifecycle recovery; the original live-profile transport trigger remains -unconfirmed, as described under limits. - -| Item | Implementation | Concrete proof | -| ---- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| A01 | Settings labels and the optional call-budget input use a full-width grid with border-box sizing. Text inputs use a single focus underline; other controls retain keyboard-visible focus. | `audit-2.spec.ts` measures the input width and absence of an outline; `polish.spec.ts` exercises keyboard selectors. `audit-settings.png` was inspected. | -| A02 | Centered import dialog supports a native chooser and real file drops. Preload resolves the actual OS file path with Electron `webUtils`; main validates the trusted top-level frame, absolute PDF path and regular file, then issues a short-lived opaque token. Only Electron/backend see the path. The successful import directory is remembered in a private preference file for both entry paths. Invalid or superseded drops cannot retain a previously selected file. | Real native selection and an OS-backed dropped file from a different directory pass through the isolated preload/main/backend. The next chooser defaults to the dropped file's directory. `audit-import.png` and `audit-revision-import.png` were inspected. | -| A03 | Later imports request `Rev 1`, `Rev 2`, etc. The Revision tab contains an actionable Import revision button. The selected revision changes after import; old sources, findings and explicit correspondence remain intact. | E2E checks both default labels and the selected imported revision. Backend regression imports a second PDF, rejects mixed-revision selection and verifies that import creates no correspondence. Existing revision/reconciliation suites cover immutable history and explicit mapping. | -| A04 | Compact sheet rows show optional discipline assignment and selection. Bulk assignment saves selected sheets in one command. Count verification lives behind a disclosure and remains separate from ordinary review. | A 52-sheet Electron fixture bulk-assigns Fire alarm, verifies compact row height, changes selected-sheet scope to 51, and reopens with assignments retained. SQLite regression confirms assignments do not mark coverage or confirmation. `audit-52-sheets.png`. | -| A05 | Explicit Whole set, Selected sheets and Selected discipline scopes carry a revision and sheet selection through shared Schema/RPC contracts. Backend freezes only eligible sheets. All-disciplines modes show that fact in the composer; a discipline scope requires a named reviewer and assigned sheets. Explicit scope wins over conflicting prose. Distinct fire responsibilities are preserved. | SQLite/Pi regressions check selected-sheet inventory, electrical-only context, contradictory whole-set wording under explicit discipline, explicit whole-set inventory after another revision, mixed-revision rejection, empty discipline rejection and persistence. E2E checks 51/52/0 scope counts for selected sheets/Fire alarm/Fire sprinkler. | -| A06 | Source views occupy the main work surface. Zoom/pan and Back to conversation remain available. Compact floating findings open readable central details, source images and PDF-coordinate bounding boxes; focused locations can request a high-DPI crop. Obsolete source requests cannot reopen a closed pane. | Electron asserts a canvas wider than 1100px and taller than 550px, uses zoom and keyboard pan, and opens a persisted finding with its bounding-box overlay. `audit-large-drawing.png` and `audit-finding-canvas.png` were inspected. | -| A07 | Conversation and composer expand to 1120/1180px. Review summaries have a short default presentation with full notes in a disclosure. Saved response text is not duplicated next to the same review summary. Compact finding actions open details centrally. HTML export is removed from the main review UI; saved report data and backend export remain compatible. | Packet E2E opens a persisted summary and findings, checks absence of the HTML-report action, changes conversations and reopens. `packet-review-workbench.png` and `audit-conversation.png` were inspected. | -| A08 | A terminated subscription previously set a permanent disconnect flag, retained a dead controller, swallowed its error and never recovered. The owned subscriber now reloads durable records and resubscribes with bounded abortable backoff. A warning appears after repeated failures and clears on resumed events. Refresh does not block stream acknowledgement. No inference work is restarted by reconnecting. | A real SQLite/Pi service test interrupts the stream three times, records completion during a gap, verifies saved-state backfill and a fourth live subscription, clears the warning and aborts cleanly. Electron logs contain no renderer errors or progress warnings in ordinary flows. See the limitation below about the original transport trigger. | -| A09 | Enter submits; Shift+Enter adds a newline. IME/composition and key-code 229 are respected. Pointer dropdown opening avoids forced focus while keyboard navigation remains visible. New count check is a secondary header action. | Electron exercises Shift+Enter, Enter and a composing Enter that preserves the draft. The polish test exercises keyboard dropdown navigation. Settings and conversation screenshots were inspected. | -| A10 | Visible workbench, page title, settings, summaries, native export labels, saved packet reports and auxiliary generated reports use MOKE. Vendored instruction sources and historical records retain their provenance. | Electron asserts the brand text; screenshots show MOKE. Existing report suites exercise generated output. | -| A11 | Render requests fit source regions into the existing 16-million-pixel budget. Actual fractional DPI and forward/inverse source transforms travel with the artifact. Worker-side limits, original metadata validation, source-region checks and requested-DPI bounds remain enforced. Detail crops retain 300 DPI when they fit. | Real PDF regressions render a 36×48-inch coordinate page with UserUnit 2 at rotations 0 and 90. The former fixed 150-DPI request exceeds the ceiling; fitted output succeeds under it, a focused 300-DPI crop succeeds, inverse coordinates round-trip and actual PNG dimensions match. A large source is also rendered through Electron. | -| A12 | `SteerPacketReview` saves the reviewer message and pending delivery atomically, then submits to the active conversation's installed Pi Durable 1.4.0 `root.submit({type:"input",requestId,whenBusy:"steer"})`. Retried command IDs reuse Pi admission. Delivery states are persisted and visible. Settlement closes steering admission and waits for admitted steering before finalizing. Drafts survive rejected delivery. Explicit resume carries unresolved instructions into its durable continuation and updates their delivery status after the resumed answer. Existing ownership, budget and provider-attempt controls apply. | Real SQLite/Pi tests queue steering during both a tool round and a final answer, retry the same command concurrently, reject another conversation, inspect the next provider fixture request for the instruction, assert exactly two model attempts and one message, then reopen with no dispatch. An additional budget-exhaustion test verifies that explicit resume answers the queued instruction with exactly two paid-attempt fixtures in total and corrects its delivery state. Installed README Busy Conversations and Request IDs sections were read. No live inference claim is made. | -| A13 | Adjacent fixes include bulk-assignment event lifetime, central finding overlays, source-request cancellation, invalid-drop state, explicit-scope precedence, saved project-version consistency, accurate prior-scope labels, accessible finding buttons, and separating active steering from next-review settings. | Full verification covers import, review/pause/resume/cancel/uncertainty, event delivery, persistence, evidence transforms, corrections, revision isolation and complete-package recovery. The final suite and evidence paths are recorded below. | - -## Verification results - -Final `bun run verify` passed with Bun 1.4.4 at reduced CPU priority: -282 smoke tests across 29 files, 1,727 assertions, all seven isolated Electron -tests, type checking, concept checks, lint, formatting, frontend build, Electron -build and compiled Bun backend/PDF worker builds. The complete result is -`.local/audit-2/sol-g1-verify.log`. All GUI launches in this run used the -hardened wrapper and passed its private-display and unreachable-desktop probes. -There are 24 preexisting class-instance-spread lint warnings and no new warning -category or lint error. - -Focused evidence includes real SQLite/Pi finish-boundary tests for successful, -budget-limited, cancelled and interrupted steering admission; concurrent token -claims and backend receipt replay; real Electron lost-response/retry and delayed -source delivery in both orderings; and accepted-count refreshes on a 36×48 sheet. -The targeted large-count run passed in `.local/audit-2/repair-count-desktop.log`. -The Svelte autofixer reported no issues in the five changed components/routes. -Its existing focus/dialog refs and local Map/Set suggestions remain advisory. -No dependency or migration was added, no model provider was contacted, and no -commit, push, PR, reviewer, publication, or deployment was started. - -Earlier verification logs and screenshots are historical context, not the GUI -isolation evidence for this handoff. Impeccable was not repeated for these -behavioral fixes; its earlier static report is not new acceptance evidence. - -## Screenshot and diff inventory - -All listed screenshots were regenerated in the passing G1 follow-up full run -through the hardened runner. The saved-results and missing-source captures were -inspected with `view_image`. Supporting captures remain available for parent -review. Files are preserved under this absolute directory: - -`/home/scfred/Work/moke-meta-hackathon/.local/audit-2/sol-g1-desktop/` - -- `audit-large-count-refresh.png`: refreshed 36×48 source at fitted fractional DPI. -- `audit-large-count-saved.png`: persisted corrected count and source provenance. -- `audit-source-race.png`: second source image and distinct box after stale delivery. -- `audit-import-retry.png`: lost-success response with the retained request/label. -- `audit-settings.png`: legible call-budget label, field and single underline. -- `audit-import.png`: centered drop/picker modal. -- `audit-revision-import.png`: real dropped PDF with the Rev 1 label. -- `audit-52-sheets.png`: compact rows, selected sheets and bulk assignment. -- `audit-large-drawing.png`: large source canvas with zoom and keyboard focus. -- `audit-finding-canvas.png`: readable finding details, source image and bounds. -- `packet-review-workbench.png`: expanded conversation and compact saved results. -- `audit-missing-source-finding.png`: missing-source finding clears prior source/count. -- `audit-conversation.png`: saved Enter/Shift+Enter message after revision import. -- `polished-workbench.png`: narrower desktop navigation/interaction regression. -- `review-workbench.png`: count correction and revision-history workflow. - -`manifest.json` records the full-run log associated with these files. The -uncommitted tracked and untracked file inventory and exact insertion/deletion -counts are in `.local/audit-2/sol-g1-diff-stat.json`. Both the existing task edits -and the parent's hardened isolation wrapper are included; no unrelated changes -were discarded. The base/HEAD is unchanged from the SHA at the top of this note. - -## Limits and decisions for independent review - -- The original user's transport-level disconnect trigger was not reproduced - against their live app or profile. The permanent failed-subscription lifecycle - was reproduced with deliberate termination through the real service seam and - fixed. A continuing transport defect remains visible through diagnostics and - the reconnect warning; it is not treated as success or used to restart work. -- Model/SAM inference uses deterministic external transport fixtures. No real - provider request ran inside the isolated network, and no new live inference - quality or provider behavior is claimed. Electron proves the actual desktop - result/import/evidence surfaces; service tests prove durable inference and - steering mechanics. -- Whole-set and selected-sheet scopes inspect all applicable disciplines for - their selected revision. Selected discipline filters by persisted assignment. - Blank assignments are deliberately excluded from a discipline scope and remain - included in whole-set/selected-sheet review. Scope does not authorize automatic - revision correspondence or count verification. -- Large full-page renders trade DPI for bounded memory. A focused crop retains - detail resolution. The hard resource limit remains 16 million output pixels. -- Local GUI proof requires private Xvfb and namespace support and fails closed. - Disposable GitHub-hosted runners have a separate path requiring both - `GITHUB_ACTIONS=true` and `RUNNER_ENVIRONMENT=github-hosted`. Generic CI and - self-hosted runners cannot enter it. Windows/macOS use their hosted native - desktop; hosted Linux uses the workflow's Xvfb. Every path uses private - profiles, one worker, muted audio and disabled notifications. The hosted - platforms were not executed locally. -- The parent owns independent Grok review, Sol-loop, PR work and any publishing. - This handoff is an uncommitted implementation diff for that review. - -## Accepted review repairs (G1–G4, P1) - -- **G1:** Installed Pi 1.4 admits input asynchronously on its session line. An - idle `whenBusy: "steer"` creates a new run; checking a host flag immediately - before `submit` cannot make it a busy-only operation. The existing pending - lease now explicitly owns that continuation through settlement, including - errors after Pi admission. Completion includes every instruction's outcome - and converts undelivered durable intents to `unanswered` instead of claiming - completion from the first run alone. Concurrent identical command admissions - share one result; changed input conflicts. Existing provider admission still - enforces the persisted call budget, cancellation and ownership. Once the - original submission settles, new drafts are rejected clearly; already staged - instructions drain within the owned lifecycle. Failed Pi runs withdraw queued - instructions as unanswered before the host awaits their admission leases. - The deterministic regression - pauses the real host runtime after SQL intent, waits for the original Pi - submission to be durably `done`, then releases admission. Success, budget, - cancellation and admission interruption preserve one message and bounded - provider calls. Existing explicit-resume and reopen regressions still apply. -- **G2:** A dropped PDF capability claims the complete import command before - asynchronous backend work. Other command IDs or changed versions/labels - conflict; identical retries retain the path and successful receipt for the - desktop session. Claimed entries are not evicted; a bounded session capacity - refuses further drops instead of forgetting claims (native selection remains - available). The real backend's receipt handles a response lost before shell - caching. A cached success needs no source reread. The import UI retains the - command after failure, locks its label and chooser, refreshes project version, - and requires an explicit new drop/cancel to start a new request. The desktop - transport regression loses one successful response and retries the exact - payload, proving one added revision. Same-version duplicate imports already - had backend optimistic concurrency; that broader reproduction was declined. -- **G3:** Intended overlays travel into `loadSheetSource` and commit with the - accepted image/artifact generation. Packet fallback reuses its existing - request generation; focus requests no longer write overlays after a stale - loader returns. Real Electron RPC responses are held while another finding - on another sheet renders, then released in both focus/fallback orderings. - The selected image and second box remain together. Returning to conversation - while a render is held cannot reopen its source pane. Project/conversation - switches and component disposal retain their request invalidation. -- **G4:** The accepted source request remembers integer requested DPI separately - from the fitted artifact's fractional actual DPI. Count acceptance still - saves first, then refreshes at the intended 96 or 300 DPI. A 36×48 two-page - source proves both refreshes, visible evidence overlays, fractional fitted - DPI, the 16-million-pixel bound, and PDF/pixel transform round trips. G3 also - exercises 150-DPI fallback and 300-DPI focus. The former issue broke the - refresh after saving; it did not lose the accepted count. -- **P1:** A narrow disposable-hosted-runner path preserves the existing - Ubuntu/Windows/macOS workflow and packaged test entrypoint. All tests honor - `PROJECT_KIT_EXECUTABLE`, including audit tests. Non-GUI guard regressions - reject generic CI, self-hosted and incomplete/spoofed hosted flags, missing - profiles, inherited local displays and Wayland. No desktop safety control - was weakened for local runs. Windows/macOS/package execution is left to - external CI; no package or hosted session was launched here. - -Repair logs use `.local/audit-2/repair-*.log`. The first repair verification -passed all 281 smoke tests (1,710 assertions) and builds, then exposed two new -GUI fixture issues: reload before initial navigation and a finding locator in -the hidden conversation. The targeted rerun passed source races and lost-response -recovery, and exposed an assertion made before count-refresh RPC dispatch. -Those test synchronizations were corrected without changing desktop isolation. -A later assertion incorrectly assumed all 36×48 renders must fit below 96 DPI; -96 DPI already fits, while a 300-DPI full-page request fits to approximately -96.1769 DPI. The corrected regression distinguishes requested/actual DPI and -passes. Final verification passed all 281 smoke tests, 1,710 assertions and -seven Electron tests. Its log and screenshot manifest are listed above. - -## Sol round 1 G1 follow-up - -The remaining G1 failure was reproduced through the real packet service and Pi -1.4 journal. The fixture holds the provider after dispatch, queues a stable -instruction, then throws. Before the repair, the review became uncertain but -the instruction stayed queued. The bounded regression failed in -`.local/audit-2/sol-g1-red.log`. - -The production change selects Pi's existing `afterFailedRun: "withdraw"` -policy. Installed `README.md` Failures and Recovery and Busy Conversations, -`dist/harness/types.d.ts`, `generation.js` and `inbox.js` were inspected. -Pi withdraws the queued inputs in the failed-run settlement transaction, before -the host waits for steering admission leases. No new queue or retry mechanism -was added. The original submission is `unanswered` with `model_error`; steering -is `unanswered` with `run_failed`. The saved instruction and reviewer message -remain, and the host review retains `uncertain` with one sent provider attempt. - -The new non-GUI regression also proves bounded cleanup, reopen, same-command -start and steering replay, rejected resume and exactly one provider fixture -call. All 17 focused service tests passed with 226 assertions in -`.local/audit-2/sol-g1-focused.log`, including successful tool/final boundaries, -normal failed admission, call budgets, cancellation and explicit resume. - -The parent's S1-S3 behavior and test changes were preserved. Initial full -verification found three missing blank lines in those changes, plus five in the -new regression. Only the three required blank lines were added to -`ReviewWorkbench.svelte` and `tests/e2e/packet.spec.ts`. That failed lint run is -preserved in `.local/audit-2/sol-g1-verify-lint.log`. The Svelte autofixer still -reports zero issues in `.local/audit-2/sol-g1-svelte-autofixer.log`. - -Final `.local/audit-2/sol-g1-verify.log` passes all checks and builds, 282 smoke -tests with 1,727 assertions, and all seven Electron tests. Its preflight confirms -private Xvfb `:99` at 1600x1000 and desktop `:0` unreachable. The parent's S1-S3 -persisted-result regression also passes: count comparison clears packet -selection, earlier saved answers remain visible, and a finding without source -evidence clears prior source/count state. New screenshots are recorded above. -The only remaining lint warnings are the same 24 class-instance-spread warnings. -Sol round 2 approved the behavior diff with no open findings. Its independent -non-GUI checks passed 11 tests with 113 assertions and the Svelte analyzer -reported zero issues. G1 and S1-S3 are resolved; G2-G4 and P1 remained resolved. -The two Sol rounds used GPT-6.1 Sol on xhigh with priority service. Grok 4.6 on -xhigh supplied the initial adversarial findings. Neither reviewer exposed usage. - -## Sol integration round 3 S4 recovery repair - -S4 reproduced a recovery regression after the protocol-3 scope handshake: an -already-dispatched protocol-2 worker's progress failed decoding before the -result directory could be snapshotted. Reopening or retrying could not return -the paid work. Publishing a new image cannot change that existing worker. - -New ownership receipts now record `workerProtocol: 3`. The actual Modal progress -decoder permits protocol 2 only for unmarked saved `dispatching`/`running` -receipts, preserving protocol-3 recovery for receipts saved before the marker. -Engine, count prompt, packet prompt, instruction revision, job ID, phases and -progress payloads remain validated. Requests, warm admission and newly admitted -jobs remain strict at protocol 3. No worker is dispatched by recovery, no model -call is replayed, no SQL migration changes, and no existing receipt is rewritten -on load. Persisted incompatible sync pauses still require explicit **Retry sync**. - -The offline regression uses the decoder called by Modal and the real cloud -services, synchronization and application package restore. It reopens old -dispatching and running receipts, decodes protocol-2 ready progress, creates a -separate local result snapshot, downloads it once, confirms stop, verifies and -restores the actual package, and proves local writes return. A paused old receipt -recovers after Retry sync. A corrupt old result fails hash validation and retains -the fence. A packet-service/Pi fixture also returns its saved answer across reopen -with exactly one model-transport fixture call and one dispatch. Reopening again -and replaying the original command do not download, dispatch or infer again. -Negative cases cover the wrong job, unrelated protocols, each identity field, -invalid phase/execution/message, protocol-2 warm admission and new-job progress. -Existing strict protocol-3 request and scope tests remain. - -Before repair, `.local/audit-2/s4-red.log` recorded two passing current-protocol -package cases and three failing legacy cases with the actual unreadable-progress -error. Focused validation is in `.local/audit-2/s4-focused.log`; full verification -is recorded in `.local/audit-2/s4-verify.log`. - -Final focused validation passes 34 tests with 398 assertions. Full verification -passes all type/design checks, formatting and builds, 321 smoke tests with 2,152 -assertions, and all seven Electron tests (54.1 seconds). The same 24 baseline -class-instance-spread lint warnings remain. Desktop preflight again confirms -private Xvfb `:99` at 1600x1000 and desktop `:0` unreachable. Fresh screenshots -and their hashes are retained in `.local/audit-2/s4-desktop/manifest.json`. -The first lint-only failures are retained as `s4-verify-lint.log` and -`s4-verify-receipt-spacing.log`; the earlier complete passing run before tightening -the persisted dispatching-receipt assertion is `s4-verify-before-receipt-assertion.log`. - -All checks use Bun 1.4.4 at `nice -n 10`, sequentially. Desktop proof runs only -when full verification invokes the unchanged hardened `scripts/isolated-desktop.ts` -with the prepared private Xvfb. No inherited/user desktop, Windows/Helium browser, -development app, live provider/Modal request, image build/publish, external write, -credential change, push, PR or merge is part of this repair. Protocol-3 image -deployment for new reviews and live/Windows/macOS/package validation remain -separate work. - -Sol round 4 approved the recovery delta with S4 resolved and no new scoped -findings. Its independent non-GUI validation passed 29 tests with 386 assertions. -The recovery review concluded after four rounds with GPT-6.1 Sol on xhigh and priority -service; usage was not exposed. All accepted Grok, Sol and parent findings are -resolved. The final behavior diff against the integrated cloud base spans 56 -files with 4,565 insertions and 740 deletions before this approval note. Growth -from the initial 43-file audit includes regression evidence, desktop isolation, -cloud scope integration and recovery of existing cloud work. - -## Packaged CI launcher repair - -PR #11's first Linux, macOS and Windows jobs passed checks, builds and all development -desktop tests, then failed the standalone reconciliation report in the packaged -suite. The isolated launch adapter replaced that report's custom Electron -entrypoint with the packaged MOKE executable. The report assertions therefore -looked for worksheet controls in the main application. - -The adapter now substitutes the packaged executable only for the repository's -`.` entrypoint. Existing explicit executable selections remain intact, while -standalone report scripts use Electron. Isolation checks, private profiles and -desktop-service restrictions are unchanged. - -The failure reproduced with a local Linux package under private Xvfb in -`.local/audit-2/ci-11-packaged-red.log`; the same report test passed after repair -in `ci-11-packaged-focused.log`. Full `bun run verify` then passed 321 smoke -tests, 2,152 assertions and seven isolated Electron tests in `ci-11-verify.log`. -All seven packaged tests also passed in `ci-11-packaged.log`. Both complete -desktop runs confirmed private `:99` and user desktop `:0` unreachable. - -Sol round 5 approved the launcher repair, checked every caller, and confirmed -the isolation controls were unchanged. C1 is resolved with no new findings. -This final review was static only; usage was not exposed. diff --git a/docs/research/audit-3-cloud-preflight.json b/docs/research/audit-3-cloud-preflight.json deleted file mode 100644 index 123a94b9..00000000 --- a/docs/research/audit-3-cloud-preflight.json +++ /dev/null @@ -1,78 +0,0 @@ -{ - "checkedAt": "2026-10-29T15:31:58.500Z", - "imageName": "moke-review-engine:audit3-fixes-20261029", - "imageId": "im-01M644B1KBNWT5KTS160NMF596", - "identity": { - "protocol": 3, - "engine": "review-pi-1.4.0-v1", - "prompt": "moke-selected-coordination-v2", - "instructions": "8a0e2648738018631eaa6d24ee0b6ebaa239afb1", - "packetPrompt": "moke-packet-review-v2" - }, - "resources": { - "cpu": 4, - "cpuLimit": 4, - "memoryMiB": 8192, - "memoryLimitMiB": 8192 - }, - "providerChecks": [ - { - "provider": "meta", - "requiredKey": "MOKE_API_KEY", - "available": true - }, - { - "provider": "baseten", - "requiredKey": "MOKE_API_KEY", - "available": true - } - ], - "preflightElapsedMs": 2098, - "cachedPreflightElapsedMs": 396, - "activePreflightSandboxes": [], - "migrations": [ - { - "name": "0001_notes.sql", - "sha256": "8a93c50f8a57fd9225a5877e795a72ee152e9e01001291d9073dede385f8a635" - }, - { - "name": "0002_reviews.sql", - "sha256": "7fb660362355eb918d6a8c3ebb9c60486fd9e474d65895c8bbd28d39c31a3cee" - }, - { - "name": "0003_review_execution.sql", - "sha256": "2b7da3d906bcc74145b497d88c033c0ef3ac362fa67175b31965988e95ddb3de" - }, - { - "name": "0004_sam_observations.sql", - "sha256": "b978415b7695ba5344bbd91f4b626b24b4d5cdcbbd2378d859023909defa3dd6" - }, - { - "name": "0005_execution_commands.sql", - "sha256": "8c784304701a1b4af4e556e085b7e2067beeae24a79e84a5cd96f21bf145fb31" - }, - { - "name": "0006_immutable_sheet_correspondence.sql", - "sha256": "040b9b44d7cfb10d9075bbfdf7accabf7eee9f74c7c265a1d1f5bdca555a13af" - }, - { - "name": "0007_review_prompt_context.sql", - "sha256": "77ef4649b91c3507280a83e000f4833f1ff887c440880062fc9540d2f572f0e8" - }, - { - "name": "0008_execution_timing.sql", - "sha256": "b3df500c8f4ca9d8ade9b44b716f206ee8eab6e562c67ec8478f8fc094d7215b" - }, - { - "name": "0009_packet_reviews.sql", - "sha256": "ac8f46ff7b9f6d87bd973901fb56ccf990a2173966df71e9cac676ae7db135c1" - }, - { - "name": "0010_revision_dispositions.sql", - "sha256": "d23b5362d62fbcfb894b64a2212797a008ef482e8891476232294692e98962a3" - } - ], - "productionConfigurationUntouched": true, - "bundleSha256": "0803c2ed24883143e6f8b32b1c470718f91114de35357512dd90b045362e3b0a", - "warmPool": null -} diff --git a/docs/research/audit-3-cloud-worker-proof.json b/docs/research/audit-3-cloud-worker-proof.json deleted file mode 100644 index 774cd599..00000000 --- a/docs/research/audit-3-cloud-worker-proof.json +++ /dev/null @@ -1,20 +0,0 @@ -{ - "checkedAt": "2026-10-29T15:32:12.259Z", - "imageName": "moke-review-engine:audit3-fixes-20261029", - "imageId": "im-01M644B1KBNWT5KTS160NMF596", - "sandboxId": "sb-01M644MD5PDKQ7TSJEBX87PXGD", - "hashes": [ - { - "sha256": "0803c2ed24883143e6f8b32b1c470718f91114de35357512dd90b045362e3b0a", - "path": "/opt/moke/worker.mjs" - }, - { - "sha256": "147424f9592bc029d7a2b9d111d9de40f5084d607b886f7bb4361737dc61f765", - "path": "/opt/moke/pdf-worker.mjs" - } - ], - "workerMatchesCurrentSource": true, - "pdfWorkerMatchesCurrentSource": true, - "networkBlocked": true, - "terminated": true -} diff --git a/docs/research/audit-3-cloud.md b/docs/research/audit-3-cloud.md deleted file mode 100644 index 38a464ce..00000000 --- a/docs/research/audit-3-cloud.md +++ /dev/null @@ -1,111 +0,0 @@ -# Audit 3: cloud readiness, capacity, and progress - -Date: 2026-10-29. Question: how should MOKE address M3-03, M3-04, M3-11, -M3-12, M3-20, and M3-24 while allowing roughly 3–5 times the previous sandbox -compute cost? - -## Evidence and decisions - -The Windows audit observed a missing named Baseten secret, an acknowledged job -that survived reopen while the workbench displayed another project, repeated -maintenance errors during normal refresh, and a 271,986,285-byte workspace that -exceeded the former 256 MiB transfer cap. The preserved evidence is in -`.local/audit-3/report/findings.json`; the original audit changed no app code. - -Modal's [Secret API](https://modal.com/docs/sdk/js/latest/Secret) exposes -`fromName` with `requiredKeys`. The installed `modal@0.11.1` source confirms -that it issues `secretGetOrCreate` with the requested key names. Its -[Image API](https://modal.com/docs/sdk/js/latest/Image) resolves published -images by name to immutable image IDs. MOKE now checks every configured outbound -provider secret and image before package preparation and ownership recording. -A credential-free, network-blocked, one-minute probe runs the strict worker -identity command. Successful identity validation is cached by immutable image ID -for the runtime, while name and secret lookups repeat before every start. The -probe is always released; its hard lifetime also bounds an ambiguous allocation. -Cloud setup reports the safe actionable failure instead of saying Connected. -No provider key is injected into the probe and no drawings are uploaded to it. - -Cold and optional warm review workers now request and cap **4 physical CPU cores -and 8 GiB RAM**, retaining no GPU and no automatic warm pool. Old warm offers -without the new resource profile are rejected and stopped, so they cannot -silently preserve the old one-core configuration. - -The complete-package cap is now **1 GiB**, preserving the 5,000-file and strict -path/hash checks. This admits the actual 259.4 MiB audit workspace without -discarding projects or histories. The package size appears during transfer; -limit failures describe the measured manifest size and safe reduction path. -Whole-workspace transfer and atomic replacement remain explicit: transferring -only a selected project requires a separately designed merge protocol. Omitting -unrelated rows from the existing replacement package would lose local projects -on return. Project-scoped cloud sync is therefore unresolved future work, -rather than an unsafe shortcut in this patch. - -Read-only workbench snapshots share SQL replacement exclusion and check pending restoration before opening SQLite. They do not wait behind network maintenance or emit expected-maintenance command errors. Local mutations remain -fenced during cloud ownership. The root workbench restores the active job's -project and conversation on reopen. Public cloud status now retains the requested -conversation, model and scope before worker acknowledgement, plus transfer bytes. -Worker progress publishes opened and total sheets, findings, recorded passes and -last review activity. The card shows elapsed and remaining time with an explicit -Ends by deadline. Terminal receipts appear only in their originating project and -conversation and can be dismissed. Disconnected observers do not show a reconnect -warning for a terminal receipt. - -## Cost calculation - -Research used **Exa through Executor**, fetching the official pricing page and -resource/API guides. [Modal pricing](https://modal.com/pricing) currently lists -Sandbox CPU at **$0.00003942 per physical core-second** and memory at -**$0.00000667 per GiB-second**. These are the Sandbox rates, including the -non-preemptible multiplier; using the lower Function list rates would understate -the cost. [Sandbox resources](https://modal.com/docs/guide/sandbox-resources) -and [resource configuration](https://modal.com/docs/guide/resources) say billing -uses the greater of requested and actual CPU/memory. Hard caps make the following -estimates bounded for the configured resources; transfer, storage, regional -multipliers and provider inference are separate. - -| Review worker | Per hour | 60-minute review + 2-minute cleanup | 6:48 audit runtime | -| ---------------------- | -------: | ----------------------------------: | -----------------: | -| Previous 1 CPU / 2 GiB | $0.18994 | $0.19627 | $0.02153 | -| Selected 4 CPU / 8 GiB | $0.75974 | $0.78507 | $0.08610 | - -The selected resource cost is exactly **4 times** the earlier worker at equal -runtime, within the requested 3–5 times allowance and comfortably below $25 for -one hour. A full one-minute 1 CPU / 2 GiB readiness probe costs at most about -$0.00317 in these CPU/memory rates; normal identity validation should finish much -sooner. Setup validates an image once per runtime, not once per status update. - -More CPU and memory should help local PDF rasterization, image processing, -package transfer overhead, and memory headroom. It does **not** make the remote -Muse model reason faster or establish that provider timeouts/accuracy are solved. -The exact performance improvement remains an inference pending live measurement. -No GPU or additional always-on service was added. - -## Validation and limits - -`bun test tests/smoke/cloud-reviews.test.ts tests/smoke/cloud-sync.test.ts -tests/smoke/cloud-workspace.test.ts` covers preflight failure before ownership or -compute, the actual accumulated-workspace size, the raised bound, responsive reads -during held uploads/downloads, real complete-package return, write fencing, -reopen recovery, and legacy in-flight observation. Recovery accepts the prior known packet-prompt -identity only for already-dispatched jobs, preserving earlier paid work without -relaxing new worker admission. Existing strict image identity, -path limits, snapshot requirements and uncertain-request protections remain. -The Svelte autofixer reported no issues for `CloudReviewControls.svelte`. - -The unique audit-only image `moke-review-engine:audit3-fixes-20261029` was -last rebuilt as `im-01M644B1KBNWT5KTS160NMF596`, retaining the existing production -tag and private operator configuration. The real preflight checked both provider -secrets and the new worker identity in 2098 ms. Its cached repeat took 396 ms, -and an independent SDK listing showed zero active preflight sandboxes afterward. -The image was rebuilt after the Baseten eight-image request-window fix and Sol's -plural-absence guard correction, then after the native cloud-return, admission-publication and SDK output-policy fixes. Fresh current-source bundles match the uploaded -files exactly. A separate network-blocked sandbox read the remote worker bytes: -[both remote hashes match](audit-3-cloud-worker-proof.json), and that proof sandbox -was confirmed terminated. [The safe preflight receipt](audit-3-cloud-preflight.json) records the -image, prompt identities, resource bounds and migration hashes. This preflight -issued no model requests and does not prove successful inference. The parent -validation report records the observed live Windows/cloud outcomes separately. - -## Reopened native acceptance - -The pre-send composer now shows a read-only estimate of whole-workspace MiB, file count, project count and limits. Final package preparation still enforces exact bounds. Upload/download counters advance per completed file, including verified reused downloads, rather than showing a static package size. Host-owned tool activity identifies active/finished tool, controlled stage and the drawing when applicable. Terminal cards collapse their details. Native transfer proof and remaining limits are recorded in [the corrected acceptance report](audit-3-validation.md). diff --git a/docs/research/audit-3-provider-timeout.md b/docs/research/audit-3-provider-timeout.md deleted file mode 100644 index 15babc99..00000000 --- a/docs/research/audit-3-provider-timeout.md +++ /dev/null @@ -1,184 +0,0 @@ -# Reopened provider timeout assessment - -Checked 2026-10-29. Read-only source and evidence investigation using Exa through -Executor for primary documentation. No external inference request, replay, code -change or deployment was performed. One local SDK serialization capture used an -injected transport returning a fabricated HTTP response; it sent no network call. - -## Finding - -**M3-01 cannot be declared resolved by the available evidence.** The image-count -HTTP 400 cause was fixed and demonstrated separately. The reopened three-sheet -cloud review still stopped without a complete third provider receipt. There is -one concrete, minimal adapter policy correction worth making, but no evidence -that it alone will fix the observed timeout: the application does not actually -leave the Baseten output parameter at its API default. - -The five successful live sets returning findings also do not establish complete -coverage of the larger sets. The later bounded admission demonstration proves -three admitted requests, including a sent third attempt; it does not prove three -completed model responses or completed review coverage. - -## Directly observed local evidence - -- `reopened-live-worker-read.jsonl`: review - `03bcd2b5-7f6b-4b64-822d-2ffa2670578f` has two completed model receipts and a - third `sent=1`, `status=uncertain`, no response receipt. Two of three sheets were - rendered; the last rendering activity finished. This rules out an active PDF - render at the recorded checkpoint. The parent separately observed the native - 300-second timeout and safe no-replay return; this intermediate checkpoint - alone does not establish what the provider did internally. -- `reopened-live-admission-worker-read.jsonl`: review - `11ebfa0f-31ad-44ee-90c7-a54edadcf9dc` similarly has two completed receipts and - a sent third attempt without a complete receipt. The parent explicitly stopped - that bounded admission test. Its third request is not evidence of another - independently observed provider timeout. -- `backend/inference/provider.ts:41`: capabilities for the exact required - `meta-models/muse-spark-open-1026` are `contextWindow=1048576`, - `maxOutputTokens=null`. The fallback SDK model at line 266 nevertheless sets - `maxTokens=1048576` because the SDK requires a numeric field. - -## Concrete SDK output-policy mismatch - -Current call path: - -1. Pi Durable `dist/harness/generation.js:353` calls `runtime.models.streamSimple`. -2. Our `provider.ts:519` wrapper forwards to `base.streamSimple`. -3. Installed pi-ai `dist/api/openai-completions.js:520` calls - `buildBaseOptions` before its ordinary `stream` implementation. -4. `dist/api/simple-options.js:23` substitutes `model.maxTokens` when - `options.maxTokens` is absent, then subtracts estimated context plus a - 4096-token reserve. -5. `openai-completions.js:592` serializes that value. The fallback Baseten model - has no explicit `compat.maxTokensField`; automatic compatibility detection - chooses **`max_completion_tokens`**, not `max_tokens`. - -The existing application intentionally omits `result.maxTokens` for packet -reviews on Baseten (`provider.ts:494`). This does **not** survive `streamSimple`. -A local capture through the actual wrapper and SDK, with only the synthetic text -`Capture request policy.`, produced: - -```json -{ - "model": "meta-models/muse-spark-open-1026", - "max_tokens": null, - "max_completion_tokens": 1044473, - "reasoning_effort": "high", - "stream": true -} -``` - -The `null` above records an absent JSON property; the request did not send literal -null. The exact context-fit number changes with the transcript. This also means -`docs/inference.md` currently overstates the API-default behavior. - -[Baseten Chat Completions](https://docs.baseten.co/reference/inference-api/chat-completions) -documents `max_tokens` default 4096 and range 1–262144; larger requested output -budgets can slightly reduce scheduling priority. The checked reference does not -establish that the dated hackathon variant enforces that same range for -`max_completion_tokens`. Its first two successful receipts used the same adapter -policy. Therefore it would be incorrect to infer either that the alias was -rejected or that its value caused the third-request timeout. - -## Small correction alternatives - -| Alternative | Effect and limitation | -| -------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Preferred: use the existing `base.stream` path for Baseten packet work in the `streamSimple` callback** | Bypasses SDK injection of a context-sized output ceiling. Keep the existing explicit `reasoningEffort: high`, signal, timeout, telemetry, no-retry transport and image window. Our ordinary `stream` callback already demonstrates this route. Both output-limit keys should be absent on the captured wire. This restores the documented application policy; it is not a demonstrated timeout cure. | -| Strip both output fields with the SDK `onPayload` hook | Also restores API defaults, but adds a payload mutation hook and must remove both `max_tokens` and `max_completion_tokens` after SDK serialization. More machinery than using the existing ordinary stream path. | -| Explicit 4096-token budget | Uses the public documented default as a deliberate application limit. This changes policy and includes reasoning tokens, so it can truncate a high-reasoning tool call or answer. It is not required to repair the accidental SDK default injection. | - -Proof without external inference: use `ProviderConfiguration.transport` with a -fixture key and `route: recorded`; capture only the decoded outbound JSON model, -reasoning, streaming and output-limit fields, then return a local SSE fixture or -fabricated HTTP rejection. Exercise both `models.complete` and -`models.completeSimple` under the real admission telemetry. Assert both limit -keys are absent for the API-default Baseten policy, while Meta's selected native -output ceiling remains present. This proves request shape only. A future live -acceptance test must be a new, explicitly bounded request, never a replay of the -uncertain third attempt. - -## Other assessed explanations - -**Image count and size:** the current adapter retains at most eight image parts, -and `render_sheet` caps each fresh batch at eight. The supplied timeout checkpoint -has only two rendered pages. Model images are resized to at most 2048 pixels per -side and at most 1 MiB (`backend/review/operations.ts:291`). The SDK sends base64 -`image_url` objects without an explicit detail value. Baseten documents eight -images, a 100 MiB base64 request limit, and optional `auto` detail. No evidence -supports another image-count violation or mandatory resolution/detail fix for -this checkpoint. Reducing resolution could impair dense drawing interpretation. -[Baseten vision](https://docs.baseten.co/inference/model-apis/vision) - -**Reasoning:** current requests select `high`, a documented Muse Spark Open value. -Higher effort may increase latency; the model always reasons. That supports a -latency hypothesis, not a finding that `high` is invalid. A reasoning-only token -cap is documented as enforced only by a different model family, so adding one -here is unsupported. [Baseten reasoning](https://docs.baseten.co/inference/model-apis/reasoning) - -**Streaming timeout:** the adapter requests SSE with retries disabled. The -installed OpenAI SDK's `client.js:991` deadline wraps fetch and is cleared when -fetch returns headers; it is not an overall stream-completion watchdog. A -`timeout` diagnostic in the adapter's `dispatch` phase with no HTTP status would -identify a wait before headers. A post-header incomplete stream is a separate -failure category. The intermediate worker reads do not expose these wire timing -boundaries or prove provider generation was idle. The SDK buffers the normalized -response until its durable receipt commits, so the absence of a completed -receipt is not proof that no partial SSE chunks were received. - -Baseten advises sizing timeouts for actual latency and checking both stream -completion and stop reason. Its dedicated deployment timeout examples do not -establish a guaranteed response time for this shared hackathon Model API. Neither -another timeout increase nor automatic replay is justified by this read-only -assessment. [HTTP clients](https://docs.baseten.co/inference/http-client-configuration), -[streaming](https://docs.baseten.co/inference/streaming), -[inference errors](https://docs.baseten.co/inference/errors) - -**Compute:** no active local rendering tool appears in the recorded stall. More -Modal CPU or memory would not establish that the separately hosted Baseten model -responds within the desktop deadline. This is an inference from the observed -process boundary, not a provider uptime claim. - -## Implemented follow-up - -The preferred narrow stream-path correction is now implemented. Both real SDK -entrypoint captures omit both output-limit fields, retain high reasoning and keep -Meta's native output policy. The old simple-path assertion failed on1044474; -the new assertions pass. Sol independently approved the change after34tests and -375assertions. This is request-policy proof only. A new bounded native Fire alarm -review is recorded separately in the acceptance report. - -## Remaining acceptance - -Correct the output-policy mismatch if retaining the API-default intent; preserve -all unresolved-attempt records and the no-replay boundary. Keep M3-01 open until a -new real native/cloud review demonstrates the required completion and coverage. -Safe request shape, returning partial records, and three admitted calls establish -specific improvements, not complete provider reliability. - -## Recheck: what the 300-second limit measures - -Checked 2026-10-29 during the native recheck. Source reading plus one small streamed -request; no review attempt was replayed. - -The packet deadline is passed to the OpenAI SDK as `timeout`. In the installed SDK, -`fetchWithTimeout` (`node_modules/openai/client.js`) arms the timer around `fetch` and -clears it in `finally` once the response headers arrive. `parseResponseWithTimeout` -skips its body timer for streamed requests. Pi Durable adds no per-call deadline. The -300-second limit therefore bounds the wait for the first response headers, not the -length of a streamed answer. - -A synthetic streamed Muse Spark Open request (`reasoning_effort: high`, 3,000-token -cap, no images) returned headers in 0.71 s. Reasoning deltas started at the same -moment, content at 5.7 s, and the stream finished at 6.2 s, with no gap over 142 ms -across 380 chunks (`.local/audit-3-fixes/opus-followup/stream-timing.json`). - -So the earlier 300-second stop was a request that produced no response headers for -five minutes, and slow answers already finish once streaming starts. A 259-second -call during the recheck finished normally. Raising the limit would only delay the -failure report for a stalled request, so it stays at 300 seconds. Nothing here shows -why Baseten held that request; it may have queued under load or stalled. - -In the recheck's fresh workspace, five local Muse Spark Open and Muse Spark 1.3 reviews -ran concurrently for 17 to 46 minutes each, and none hit a provider timeout, HTTP 400 or -other provider failure. diff --git a/docs/research/audit-3-recheck.md b/docs/research/audit-3-recheck.md deleted file mode 100644 index 6edfed39..00000000 --- a/docs/research/audit-3-recheck.md +++ /dev/null @@ -1,221 +0,0 @@ -# Audit 3 recheck: native walkthrough after the fixes - -Checked 2026-10-29. This replaces the acceptance claims in -[audit-3-validation.md](audit-3-validation.md), which leaned on earlier screenshots -and test results. - -## Question - -Do the Audit 3 fixes hold up when the packaged Windows app is used the way Audits 1 -and 2 were done: a full walkthrough of the real UI, with real drawing sets and real -Muse Spark Open reviews, rather than a spot-check of each finding? - -## Method - -- The app was built from branch `fix/audit-3` plus this pass's fixes and installed as a - separate Windows copy. It ran on a **fresh, empty workspace**; the earlier audit - workspaces already contained confirmed sheets and saved mappings, which hid - first-run problems. -- Cua Driver 0.38 drove the window in the foreground. Every action saved a screenshot, - and each observation was journaled with its frame and a box on the screenshot. -- The five Audit 3 sets were imported through the native file dialog. Reviews ran on - Muse Spark Open (Baseten), plus one fire-alarm review on Muse Spark 1.3 using the - default model a new user gets: brewery 29 sheets, laundry 13, porch 9, civil 5 - (Civil discipline scope), and fire alarm 3. Four ran concurrently. -- A civil revision was built that keeps four sheets, removes C-4 and adds one page, so - removed and added sheets are both exercised. -- One cloud review (fire alarm, Muse Spark Open) ran on the audit worker image, with an - app restart while the job was running. It uploaded 311.7 MiB (191 files), ran 36 calls, - recorded 132/132 passes with 11 findings in 13:04, and synced back 340.5 MiB. Modal - reported no running sandboxes afterwards. -- Fixes found in this pass were made, rebuilt into the same install, and re-checked in - the native app. Those frames are marked AFTER FIX on the board. - -The deliverable is an Excalidraw board in the format of `MokeAudit1/2.excalidraw`: -`.local/audit-3-recheck/MokeAudit3Recheck.excalidraw` (copied to the Windows Desktop). -It has screenshots in walkthrough order, red boxes and arrows to red notes for -problems, green for what works, and blue for fixes re-checked after a rebuild. - -## Results by original finding - -| ID | Recheck result | -| --------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| M3-01 full-set reviews stop early | None of the five fresh reviews hit a provider failure, timeout or HTTP 400. All returned findings (17–46 min). Large sets still stopped early: brewery 25/29 sheets and 4/1,276 passes while summarizing "completed". **Fixed in this pass:** unbudgeted reviews that call `finish_review` with unopened sheets or unrecorded passes are sent back (see below). The brewery re-run opened 29/29 sheets and recorded 220 passes. | -| M3-02 error offers only Cancel | Not re-exercised; no provider failure occurred in this pass. | -| M3-03 Connected before preflight | Settings reads Connected after the image and secret preflight. A missing secret was not re-tested. | -| M3-04 reopen lands on the wrong project | Holds. A restart during the cloud job reopened on the job's conversation, model and location with edits locked. The card now names the job's project from the local list instead of `Project `. | -| M3-05 scope leaks across projects | Scope stays per conversation. **Fixed:** after a restart the model fell back to the default, and one brewery re-run went to Muse Spark 1.3 by mistake. A conversation now restores the model of its last review. Rechecked after a restart. | -| M3-06 invisible count form | Focus and scroll hold. **Was broken on a fresh project:** the sheet lists only offered "confirmed" sheets, so they were empty. **Fixed:** all sheets are listed and unverified ones are marked. The form asks the user to confirm the chosen sheets, and changing a sheet clears that confirmation (Sol S2). | -| M3-07 generic page names | Holds for the five audit PDFs, which carry embedded page labels. **Open:** without page labels (a merged revision) the printed-text fallback named 1 of 5 pages. Rows are still tall (three per screen) and labels repeat the page number. | -| M3-08 revision mapping | Removed, saved and paired-thumbnail mapping hold. **Fixed:** explicit "Added in this revision" state; Escape in the comparison no longer closes the panel; pre-apply copy. **Open:** no automatic matches for real (non-identical) revisions; the "larger" comparison is an upscaled thumbnail. | -| M3-09 sheet navigation | Previous/next/jump hold. **Fixed:** zoom leaked into unrelated sheets and crops; the jump popover opened off the window edge. | -| M3-10 evidence layout | Holds. **Fixed:** a saved crop is now labeled "partial view of this page". | -| M3-11 cloud progress | Holds: tool, drawing, file counters and stage. **Fixed:** missing spaces in the job line, raw IDs, repeated status lines, a finished upload bar left on screen, and no status between Send and the first receipt. **Open:** the local review card shows only counts. | -| M3-12 stale receipts | Scoped per conversation. **Fixed:** Dismiss is remembered across restarts. | -| M3-13 cancel rewrites duration | Holds: pause freezes the clock, and cancel keeps the active time and saved work. | -| M3-14 notices leak | Holds across projects. **Fixed:** "Review status refreshed." stayed stacked above a changed state. | -| M3-15 coverage wall | Discipline table and filters hold. **Fixed:** the "coverage gaps" headline sat above "132 of 132 recorded" with no explanation; raw `camelCase` configuration keys and Title Case values. | -| M3-16 compact window | Sheets and More stay reachable. **Fixed:** threads opened at the top after a resize or project switch. They now stay at the latest message unless the reader scrolls up, even during rapid updates (Sol R1). At 900×900 the active cloud card pushed Send below the window; its details now scroll within a height that fits the window. Both rechecked at 900×900. | -| M3-17 Windows copy | Ctrl K and plurals hold. **Partly fixed:** search covers only the current conversation and the project's count checks; the field now says so. | -| M3-18 false absence claims | Both reviews read the battery and NAC calculations correctly. **New P1, seen twice:** the local Muse Spark 1.3 review said the NAC#2 table continuation was "not located". The cloud Muse Spark Open review said the matrix input column was "truncated/clipped in supplied PDF" and made up row labels. Both tables are complete on FA-1 (checked against the PDF at 150 DPI), and each time the model's own crop had cut them off. **Partly fixed:** "not located/not found/unable to locate" now trigger the absence gate, and round 2 adds crop-edge guidance to `render_sheet`. Model reading accuracy remains a limit. | -| M3-19 report export | Holds (native Save, 12.5 MB HTML). **Fixed:** the export had the same unexplained gap headline, raw keys and ISO timestamps. | -| M3-20 maintenance error | Not seen during upload or while browsing other projects. | -| M3-21 Pause | Pause, Refresh and Resume work. **Partly fixed:** Resume right after Pause failed with internal "draining" wording; the message is now plain, but the card still cannot show that a request is in flight. | -| M3-22 invalid package | Holds for the original empty-folder case ("The package manifest is missing"). | -| M3-23 governing sources | Holds: URL, retrieval time, Open in browser, saved text. | -| M3-24 cloud capacity | The estimate and limits hold. **Fixed (P1):** Send was enabled while other projects had local reviews running. The request was saved as an orphan, and the error named nothing. Send is now disabled, and the panel names the blocking projects. Returning to a conversation set to cloud re-measures the workspace (Sol S1). **Open:** a fresh five-project workspace reached 252–363 MiB within an afternoon, and every cloud job still transfers all of it. | - -## Review and verification - -Five Sol review rounds (gpt-6.1-sol, xhigh) covered this pass. The user accepted -the final correction using parent validation, without another review. Round 1 found -two problems. The pre-send blocker list could go stale after switching -conversations (S1). A ticked source confirmation survived changing the sheet -(S2). Round 2 found that the thread's own automatic scroll could cancel -follow mode when updates arrived quickly (R1). All three were fixed with -regression tests. The S2 and R1 tests failed before their fixes. - -Round 3 confirmed R1 but found T1: manually returning to the same bottom -position did not restore following. Commit `185c6f315` clears the old automatic -scroll marker after reader input. Its new regression failed twice without the -fix and passed three times with it. Round 4 confirmed T1, then found U1: the -numeric sentinel `-1` is within the two-pixel tolerance of scroll position zero. -If a resize makes the thread fit, that scroll event can be ignored, leaving -following disabled when the window shrinks again. U1 is fixed with an absent marker and a resize regression. The new case -failed before the fix, 5,915 pixels behind the latest message, then passed in -three consecutive desktop runs. Round 4 is not approval of the current -change. The loop extends to five rounds for these evidenced UI state and event -ordering findings; no additional round is authorized beyond that budget. - -Usage was not exposed. The private Xvfb runner is available to the parent at -`/tmp/moke-invisible-display/runtime/usr/bin/Xvfb`; the round-4 reviewer reported -it unavailable in its own environment, so its reproduction used headless Chromium. - -## Coverage gate - -`finish_review` previously accepted a finish at any time. The brewery review stopped -after opening 25 of 29 sheets and recording 4 of 1,276 required passes, and summarized -the work as complete. - -When the user has set no call budget, the host now returns an early finish while sheets -are unopened or required passes are unrecorded. The message lists the unopened sheets -and tells the reviewer to open them and record passes with `record_coverage_batch`, -marking disciplines that do not apply as not applicable with a reason. Budgeted reviews -are never sent back, because every retry spends a call against the user's limit. - -With the first version, which returned a finish up to twice, the brewery re-run opened -29/29 sheets and recorded 220 passes in 33:34. After the second return, the model ended -without calling `finish_review` again, so the run showed "Review stopped". The gate now -returns a finish once and says the next finish will be accepted with gaps recorded. - -The audit cloud worker image predates the gate; cloud reviews get it with the next image -build. - -## Provider timeouts - -See [audit-3-provider-timeout.md](audit-3-provider-timeout.md#recheck-what-the-300-second-limit-measures). -The 300-second limit applies only until the response headers arrive. Baseten returned -headers in 0.7 s, and the stream showed no gap over 142 ms. A 259-second call -therefore completes normally, and the earlier stop was a request that sent no headers -for five minutes. The limit is unchanged. - -## Open items - -- Disciplines are not proposed from sheet prefixes (C-, S-, M-, E-, P-), so discipline - reviews on large sets need manual assignment. -- The sidebar does not show which projects have reviews running. -- There is no workspace-wide search and no project-only cloud transfer. -- Revisions get no automatic matches unless the PDFs are byte-identical, and the - comparison view is low-resolution. -- The local progress card does not show the current tool or drawing. -- Compact windows leave little room for the thread. -- Polish: light native menu bar in dark mode; the settings dot is a status indicator - shown without a label; the revision label defaults to "Rev 1" on a second import; the - Save dialog suggests "MOKE-review" as the filename. - -## Closing the desktop during CI - -The final full run reproduced the old macOS failure on Linux: -`audit-2.spec.ts:133` completed its UI assertions, saved its final screenshot -and empty console diagnostics, then timed out inside `app.close()`. The worker -also timed out during teardown. Twelve other desktop tests passed. This was -not evidence of a failing import or count interaction. - -The desktop had prevented `before-quit` while disposing the RPC runtime, keeping -renderer windows and their streaming connections alive during shutdown. Cleanup -now runs at `will-quit`, after the windows close and their ports disconnect. The -same runtime disposal and backend cleanup remain awaited before process exit; -no process-kill workaround or longer test timeout was added. A private compiled -probe of that event change passed all three Audit 2 desktop flows twice, including -the previously hanging flow in 4.9 and 5.7 seconds. - -[Electron's app event documentation](https://www.electronjs.org/docs/latest/api/app), -checked through Exa via Executor on 2026-10-29, confirms this event ordering and -that `will-quit` supports `preventDefault`. A fresh complete verification and -macOS CI remain required; the Linux probe alone does not establish macOS success. - -Final local verification for U1 and the shutdown change passed on Bun 1.4.4: -`bun run verify` completed all 359 smoke tests (39 files), 13 desktop tests, -typecheck, lint, formatting and builds. The formerly hanging flow finished in -4.7 seconds. The Svelte autofixer reported no issues. Final Sol approval and -fresh macOS CI are still pending. - -The latest Civil cloud follow-up returned and imported successfully: 22 calls, -5/5 sheets opened and 14/220 passes recorded. The native receipt says Results -synced. That is successful transfer, with incomplete review coverage. The earlier -separate Opus delegate failed with `Provider turn failed`; it is not counted as an -approval. The subsequent fresh-workspace walkthrough was performed by Opus 5.5 -in this parent thread, then reviewed and corrected through the Sol rounds above. - -The packaged Windows resize check used two saved audit notes, without starting -inference. After scrolling up in a 900×700 window, enlarging to 1633×1600 made -the entire thread fit (frame 1606); shrinking back to 900×700 kept the end of -note B visible (frame 1610). An earlier 1132-pixel-high attempt did not establish -that the thread fitted, so it is not counted as acceptance. File → Exit closed -the rebuilt app, its PID disappeared from the running-app list, and reopening -retained both notes (frame 1593). The audit copy is idle; the user workspace is -unchanged. - -## Fifth review: exact-top resize case - -Round 5 returned CHANGES, not approval. It accepted C1 (the shutdown order) and -found V1: U1 remained open when the reader was already at scroll position zero. -Making the whole thread fit then need not produce a scroll event, so the earlier -nullable marker fix could not restore following on its own. - -The observer now follows when following is already enabled **or the whole thread -fits in the viewport**. The nullable automatic-scroll marker stays in place. -The regression keeps the prior cases and adds an explicit scroll to zero before -the fit/shrink sequence. It verifies full fit before shrinking; the meaningful -pre-fix run failed at 6,315 pixels behind after shrinking, then three post-fix -runs passed. An initial probe that did not reach full fit is excluded. - -Five rounds exhaust this recheck's budget. The current correction has parent -validation but has not been approved by a further Sol review. Nothing is pushed -or merged on the basis of the exhausted budget. Usage was not exposed. - -The exact-top case was then repeated in the packaged Windows app, using the -same two saved notes. The previous build remained on note A after the -fit/shrink sequence (frame 1647). In the corrected build, Ctrl+Home established -the top (frame 1677); both full notes fitted in the enlarged window (frame 1681); -after shrinking, the end of note B stayed visible (frame 1685). No inference was -sent for these checks. This is direct native evidence, independent of the test. - -The final V1 implementation passed `bun run verify` on Bun 1.4.4: all 359 smoke -tests (39 files), 13 desktop tests, typecheck, lint, formatting and builds. The -Svelte autofixer reported no issues. The scoped regression also passed three -consecutive runs. The latest native board contains 71 selected screenshots. -Sol round 5 accepted shutdown fix C1; its remaining resize finding is corrected -and parent-verified, with any further independent recheck awaiting the user's -review-budget decision. Fresh macOS CI is still pending. - -## Delivery decision - -The user chose **Finish using completed validation** after round 5. The final -exact-top correction in `7461428ec` is accepted on its parent verification: -native before/after frames, a regression that fails without the fix, three -consecutive scoped passes and the complete 359-smoke/13-desktop verification. -No sixth review is requested, and no new Sol approval is claimed. C1 was accepted -by Sol; its remaining resize concern is corrected with the recommended observer -condition. The PR can be updated and watched for fresh CI. Merge still requires -all checks and the remaining merge gate to pass. diff --git a/docs/research/audit-3-review-reliability.md b/docs/research/audit-3-review-reliability.md deleted file mode 100644 index cada5e0a..00000000 --- a/docs/research/audit-3-review-reliability.md +++ /dev/null @@ -1,107 +0,0 @@ -# Audit 3 review reliability and source evidence - -Question: what explains repeated unresolved Muse Spark Open reviews, and what -changes improve reliability without replaying uncertain paid requests or treating -model coverage as proof of understanding? - -Research used Exa search and page retrieval through Executor on 2026-10-29. -Primary sources were retrieved in this session, rather than inferred from search -snippets. - -## Evidence and decisions - -- The audited adapter applied a 60 second request timeout to both selected-count - and full-packet reviews. Audit records include failures whose final requests - ran for approximately that duration. This is consistent with a client timeout, - but does not establish that every observed failure shared that cause. -- [Baseten HTTP client configuration](https://docs.baseten.co/inference/http-client-configuration) - says to include expected inference, capacity wait and a buffer in the timeout. - Streaming needs enough time before headers and between chunks. The full-packet - adapter now allows five minutes per request, while explicit test configuration - still takes precedence. The small selected-count adapter retains its minute. -- [Baseten reasoning](https://docs.baseten.co/inference/model-apis/reasoning) - documents that Muse Spark Open always reasons and that deeper reasoning can - increase latency. Packet reviews already request high reasoning. Increasing - Modal CPU cannot accelerate the separately hosted inference model; it can help - PDF rendering and local work only. No unverified claim of an inference speedup - is made. -- [Baseten inference errors](https://docs.baseten.co/inference/errors) distinguishes - HTTP errors from incomplete streams. A new request after an interrupted stream - is a retry, not resumption. The application's existing no-automatic-replay - policy remains in force. Safe receipts now retain category, phase, elapsed - duration and HTTP status, never the raw provider body, prompts or credential. -- The actual Pause operation compared its displayed version with a version - incremented by ordinary progress. Pause/cancel now tolerate intervening progress - only while their current state is compatible. Resume remains strict and cannot - replay an unresolved provider attempt. -- Active execution time now accumulates across running intervals. Pauses, - uncertainty and cancellation retain that duration. Cancelling preserves the - failure explanation and safe diagnostics instead of replacing their meaning. - -## Source accuracy - -The live audit proved supplied page 2 contained calculations and an input/output -matrix that one model run called absent. A better prompt alone cannot establish -that this is fixed. Packet prompt v2 requires a literal page-content inventory, -legible table crops, and an explicit search for contradictory content before an -absence claim. The record tool requires fresh inspection of every frozen page, -a targeted detail at the claimed location, and a structured search and -permit-stage basis. It rejects absence claims whose own search reports -contradicting evidence, including Needs input recommendations. A plain final -model response without the structured finish record is retained as a partial -summary and the review stops as failed; it cannot silently become a completed -source review after a rejected finding. - -These host checks enforce evidence collection, not semantic truth. The live -acceptance rerun must compare returned claims with visible battery, NAC voltage -and matrix content. No additional reviewer count is invented: this harness still -uses sequential discipline passes. An independent model verifier would be a -separate orchestration feature and is not represented as present. - -## Regression proof - -`tests/smoke/audit-3-runtime.test.ts` uses the actual packet service, durable -model adapter and SQLite state. It proves safe HTTP diagnostics and no replay, -version-tolerant Pause, waiting-time exclusion on cancellation, and rejection of -a missing-calculation finding after only a whole-page glance without its source -search and detail inspection. It covers missing, not shown, omitted and does not -contain language without exempting recommendations, and diagnoses a real SDK -deadline as timeout rather than an explicit caller cancellation. The existing packet and selected-execution suites -also exercise the unchanged replay fences. - -## Remaining uncertainty - -The prior audit did not retain provider failure categories, so it cannot identify -all original external failure causes. A five-minute timeout and better diagnostics -must be evaluated against the same live drawing sets. Source-search records are -model-authored; satisfying the collection gate does not prove that a visible -table was correctly read or that any engineering finding is adequate. - -## Native retest isolated the HTTP 400 root cause - -The follow-up Windows retest retained safe status diagnostics and showed HTTP -400 after roughly 9–10 full pages, and after repeated crops on the three-page -set. This is a distinct, now reproduced cause rather than evidence for the -initial timeout hypothesis. - -[Baseten's vision limits](https://docs.baseten.co/inference/model-apis/vision) -cap Muse Spark Open at **eight images per request**, counting historical image -parts in the conversation. The old packet adapter retained every image. A fresh -synthetic diagnostic with nine tiny tiles returned HTTP 400 with the explicit -reason `Too many images: 9 (max 8)`; a separate eight-tile request returned HTTP 200. Neither diagnostic replayed an unresolved review. - -The corrected Baseten adapter retains the newest eight images in each wire -request. All source metadata remains in context and all original pixels remain -in the durable journal. Omitted pixels have an explicit notice requesting a fresh -render before a new claim. The rendering tool rejects a ninth image in one tool -batch before generating or recording that evidence, so newly requested batches -cannot silently lose their first images. Other model routes retain their own -existing image policy. - -A live diagnostic through the actual SDK and corrected admission adapter started -with twelve source images and sent eight, returned HTTP 200 on one request, and -left the original transcript unchanged. Regression tests prove the same wire -bound, preservation of the newest image bytes and all metadata, the eight-render -batch limit, and safe image/context rejection classification. A bounded provider -error parser stores only an allowlisted reason and numeric request shape; raw -error text and echoed inputs are discarded. diff --git a/docs/research/audit-3-sol-loop.md b/docs/research/audit-3-sol-loop.md deleted file mode 100644 index 94fe50c6..00000000 --- a/docs/research/audit-3-sol-loop.md +++ /dev/null @@ -1,192 +0,0 @@ -# Audit 3 Sol review ledger - -Base: `4f17b8b5492c5293996ae19fe5e6fe3c8609faf8`. -Review command for every round: -`git diff 4f17b8b5492c5293996ae19fe5e6fe3c8609faf8 -- backend desktop src scripts migrations tests`. - -Reviewer: Codex provider, `gpt-6.1-sol`, verified `xhigh` reasoning. -Default budget: three rounds. The native return audit exposed a state/concurrency issue; the skill permits extending to five rounds for this evidenced should finding. Usage was not exposed. - -| Round | ID | Severity | Outcome | -| ----- | --- | -------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | S1 | should | Fixed plural absence nouns, including calculations, tables, schedules and matrices. Real packet-tool regressions reject these claims without the required source search. | -| 1 | S2 | should | Fixed historical display replacing the latest execution owner. The latest panel and controls stay present; history uses a separate read-only panel. Desktop regression preserves paused-run controls and blocks a new request while reading history. | - -Round 1 verdict: CHANGES. No declined findings. Before fixes: 52 behavior/test -files, 2,817 insertions and 347 deletions. Later rounds review changed code and -these findings only. No push or PR during the loop. - -Round 2 verdict: APPROVE. S1 and S2 resolved; no new in-scope issues. The reviewer -observed two failures in an earlier findings-menu step before the new paused-history -assertions. Investigation found a delayed source refresh closing a newly reopened -menu. The redundant asynchronous close was removed (the initial selection already -closes it). A real MessagePort regression now holds and releases that snapshot, -checks that the reopened menu remains visible, and reaches the paused-history -assertions. The focused desktop flow passes in 21 seconds. This behavior change -is submitted for round 3; round 2 alone does not approve that final change. - -Round 3 verdict: APPROVE. No new S3 finding; S1/S2 remain resolved. The reviewer -independently passed the focused desktop flow in 22.4 seconds, including the -delayed snapshot/menu and paused-history assertions. Svelte analysis found no -issues. Final behavior/test diff: 53 files, 2,926 insertions and 365 deletions. -All three rounds used gpt-6.1-sol on verified xhigh; usage was not exposed. -Nothing was declined. The branch was not pushed during review. - -Round 4 scope: repeat Windows acceptance found snapshot reads waiting behind the -entire cloud download (eventually a generic load error), and interrupted SDK -copies restarting hundreds of completed downloads and leaving an unlisted -scratch file. These are state/concurrency and workspace-return failures. - -- S3 (should, own audit): snapshot reads now use the existing SQL exclusion - shared with package replacement, rather than waiting for network maintenance. - The pending-restore guard runs inside that SQL exclusion before opening SQLite. - Held upload/download regressions require reads before transfer release while - writes remain fenced. -- S4 (should, own audit): retries reuse only size/SHA-256 verified files from the - immutable result manifest. After copy completes, cleanup removes only Modal - 0.11's exact scratch-file pattern from this job's download directory. Other - unexpected files remain subject to full package validation. - -Native proof: the same 482.8 MiB input / 503.8 MiB result recovered without -re-running inference, reached imported with seven findings and 12/12 passes, -released local controls, and left zero running Modal sandboxes. The focused -cloud tests pass 33 tests / 429 assertions. Full verification passes 343 smoke -tests and nine desktop tests; final typecheck/lint/format also pass. Round 4 -reviews only these changes and confirms the prior resolved findings stay fixed. - -Round 4 verdict: APPROVE. S1/S2 remain resolved; S3 and S4 resolved. No new -in-scope concurrency or recovery issues. The reviewer independently passed -33 focused cloud tests and 429 assertions. Final behavior/test diff: 53 files, -3,087 insertions and 393 deletions (round 1 began at 52 files, 2,817 insertions -and 347 deletions). All four rounds used gpt-6.1-sol on verified xhigh. Usage -was not exposed; no findings were declined. Push is authorized after this -approval by the user's full-send request. - -Round 5 scope: the first PR Windows run failed a five-second source-selection -help assertion before asynchronous rendering had finished. Only the polish -desktop test changes: wait for the prior image before starting selection, hold -and release its real RenderSource response, assert rendering and readiness, -then retain the pointer-drag/PDF-coordinate/coverage checks. Readiness uses a -15-second bound. Application behavior remains exactly as approved in round 4. -This final focused recheck covers asynchronous test-state ordering. - -Round 5 verdict: CHANGES. S5 (should) found that the prior image could still be -visible while scheduled acceptance was pending. A 500 ms delayed acceptance -response caused the render hold to capture acceptance's refresh instead of -selection. The smallest suggested fix is implemented: wait for the Select area -button to be enabled before checking the prior image and arming the hold. -A strict private probe fails the old ordering on the misidentified response and -passes the fix, with all three held responses belonging to Select area. -Application behavior is unchanged from round 4. The five-round budget is -exhausted; final recheck or a user-authorized waiver remains pending. Usage -was not exposed; nothing was declined and this follow-up has not been pushed. - -Post-S5 proof: full `bun run verify` passes 343 smoke tests and nine desktop -tests, with polish in 9.9 seconds. The strict delayed-acceptance probe fails -without the enabled-control wait and passes with it. The application diff is -unchanged. This test follow-up remains local pending the user's review-budget -choice; the PR still points to the round-4 approved application commit. - -The user explicitly authorized one additional focused Sol recheck after the -five-round budget was exhausted. Round 6 checks only the S5 enabled-control -ordering fix and confirms that prior application approvals remain applicable. -The test follow-up stays unpushed until this recheck returns. - -Round 6 verdict: APPROVE. S5 resolved: the side-specific Select area button -must be enabled before the prior image check and render hold; the same locator -is clicked afterward. The reviewer confirmed the strict red/green probe -evidence and preservation of readiness, coordinates and coverage assertions. -No new blocker or should finding. No tests were rerun by this reviewer. - -Final result: S1–S5 resolved, no declined findings. Six rounds used -gpt-6.1-sol on verified xhigh; usage was not exposed. Final behavior/test diff -is 54 files, 3,106 insertions and 398 deletions, compared with 52 files, -2,817 insertions and 347 deletions entering round 1. The user authorized the -sixth round. The CI follow-up is approved for push after this agreement. - -## Reopened native acceptance: new review scope - -The user reopened acceptance because the report paired original problems with -passing checks without demonstrating the visible fixes. The resulting behavior -changes are reviewed separately from the six approved rounds above. - -Base: `694109c58728dbb3be4b1c012e6d8d5fd9d630c3`. -Same command each reopened round: -`git diff 694109c58728dbb3be4b1c012e6d8d5fd9d630c3 -- backend desktop src scripts migrations tests`. -Default budget: three rounds. Reviewer: one `gpt-6.1-sol` on verified `xhigh`. - -The configured T3 MCP connection returned `thread_credential_required`: it is -signed in outside a thread. Its supported ACP fallback exited 2 because -`T3_ACP_MCP_ENDPOINT` and `T3_ACP_MCP_AUTHORIZATION` are absent. The skill's -outside-T3 native Sol route was used with the same verified model. No substitute -model, extra reviewer, or new top-level thread was used. Usage was not exposed. - -Round 1: CHANGES. Initial behavior/test diff34 files,3520 insertions,497 deletions, -including the two newly staged tests. - -- **R1 (should):** model and SAM admission saved the incremented call count - without publishing it. The live worker SQLite receipt had three admitted calls - while published progress and the native UI still showed two during the third - pending provider request. Both admission paths now publish after their - transaction commits and before dispatch. Completed receipt replay skips the - new-admission publication path. -- A real observer regression holds the third provider response. It fails before - the fix because the third admission never reaches the observer, and passes - afterward while that response is still held. Fourteen focused tests and 120 - assertions passed; full verification passed 350 smoke tests and 10 desktop flows. -- No nits were declined and no other blocker/should was reported. A suspected - overlapping-render issue was disproved: the worker was waiting on its third - provider request, with no active tool. - -Round 2 checks only R1 and the changed code. Submitted behavior/test diff34 files, -3529 insertions,497 deletions. Native final admission-counter acceptance remains -separate from source review and is still being recorded. No push during the loop. - -Round 2 verdict: APPROVE. R1 resolved, no new should finding. Sol independently -passed all 14 focused tests/120 assertions, including count 3 observed while the -third response remains held. Admission publication is after commit and before -dispatch; model/SAM replay guards remain intact. Native acceptance remains with -the parent and is not counted as complete by this review. Usage was not exposed. - -Native R1 acceptance completed: the packaged Windows card and published worker -receipt both displayed three admitted calls while the third sent request had no -completed response (frame1637). The bounded probe was explicitly stopped; it was -not replayed and is not completion evidence. - -Round3: APPROVE. The later provider investigation found an independent SDK -policy mismatch: `streamSimple` injected a context-sized output limit despite -the intended API-default policy. The narrow Baseten packet branch now uses the -existing ordinary stream implementation, with explicit high reasoning and all -admission, image, telemetry, cancellation and no-replay guards retained. The -real SDK serialization regression failed before this correction and passed after -it for both entrypoints. No R2 finding and no nits were reported. - -The T3 connection became available before round3. It ran through `delegate_task` -with verified Codex/gpt-6.1-sol/xhigh. Its brief used the full PR base -`4f17b8b5492c5293996ae19fe5e6fe3c8609faf8` for surrounding context, while limiting -review to the new SDK policy and resolved R1; the reopened change remains measured -against694109c. Sol independently passed34tests and375assertions across five -files. Usage was not exposed. This approves the current behavior diff, not live -provider completion. The parent owns the fresh native3-sheet/12-pass acceptance. - -Reopened behavior/test diff:36files,3573insertions,537deletions, versus34files, -3520insertions,497deletions entering reopened round1. Nothing was declined; -R1 is resolved. No push occurred during the review loop. The complete current -verification passed351smoke tests and10desktop tests with Bun1.4.4. - -## Final review of the parallel-review commits - -Scope: commits after `0602085` that no earlier round reviewed (parallel sheet review, Meta -console settings, 429/529 resends, the on-disk workspace estimate and CI timeouts): -`git diff 0602085 -- backend desktop src scripts migrations tests`. Reviewer: Codex -`gpt-6.1-sol` on verified `xhigh` through T3 `delegate_task`. Usage was not exposed. - -| Round | ID | Severity | Outcome | -| ----- | --- | -------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | P1 | should | Fixed. A sheet's SAM call was refused while a peer sheet's model request was on the wire, and an in-flight SAM call had no owner, so peers' model requests failed admission. SAM now uses the same conversation-scoped check and records its owning conversation. | -| 1 | P2 | should | Declined as a nit. Compaction attempts have no owner, but a sheet conversation compacts only past about 917k of 1,048,576 tokens, and an unowned failure stops the whole review as uncertain without a resend. Follow-up. | -| 1 | P3 | should | Fixed. A sheet that settled without `finish_sheet` was marked completed and handed the coordinator no notes. It is now failed, and resume retries it. | - -Two new smoke tests fail on the original service and pass with the fixes; the SAM test also -fails with only the guard fixed and no owner recorded. Round 2 verdict: APPROVE, P1 and P3 -resolved, no new findings. `bun run check`, 365 smoke tests and `bun run build` pass with Bun 1.4.4. diff --git a/docs/research/audit-3-validation.md b/docs/research/audit-3-validation.md deleted file mode 100644 index 846a35ff..00000000 --- a/docs/research/audit-3-validation.md +++ /dev/null @@ -1,314 +0,0 @@ -# Audit 3: implementation and native acceptance - -> The fresh-workspace recheck in [audit-3-recheck.md](audit-3-recheck.md) supersedes the acceptance -> statements below. It found several of them incomplete on a new install. - -The earlier report copied the original problems and proposed fixes, then followed them with passing-test statements. That was insufficient evidence that each visible problem had been fixed. The reopened Windows audit found missing behaviors and fixed those directly. This report distinguishes implementation, observed native behavior and remaining limits. - -The original 24 findings remain unchanged in the private source audit. The corrected per-finding report is `.local/audit-3-fixes/report/index.html`; its JSON separates `originalProblem`, `originalSuggestedFix`, `actualChange`, `nativeObservation`, before/after evidence and `remaining`. It has no blanket all-fixed status. - -## What the reopened audit changed - -Existing imports can now request and apply sheet-name proposals without reimporting or changing confirmation/count coverage. Revision mapping now pairs old/new thumbnails and opens a larger comparison. Long finding text has its own scrolling column beside a fitted source canvas. Coverage has a discipline summary, exception filters, sheet search and bounded detail pages. - -Cloud status now carries file-completion byte/file counters and host-owned active/finished tool activity. The composer estimates the entire workspace before Send. Submission has its own state so read-only navigation can continue through upload, while edits remain fenced. Terminal cards collapse transfer details. - -The native diagnostic-copy action failed through the renderer clipboard. Electron now loads and copies only the saved safe receipt through the trusted RPC boundary. Both short and long review notes use the escaped formatter. Cancelled historical errors hide the obsolete cancel instruction in the panel, and runtime cleanup preserves an existing known failure instead of replacing it with generic interruption copy. - -## Per-finding acceptance - -### M3-01: Full-set local reviews stop before usable results - -**Status:** Provider rejection resolved; coverage remains incomplete. - -**Changed:** Muse Spark Open packet requests retain at most eight recent images, replacing older request pixels with an omission notice while retaining durable source metadata and artifacts. Packet requests use a five-minute deadline. Safe failure receipts classify timeout, abort, HTTP, network and response failures and retain request image/byte counts and HTTP details without raw provider bodies. Uncertain paid attempts remain non-replayable. - -**Observed:** Five supplied drawing sets returned findings after the eight-image request limit was enforced. This does not establish full-set completion: Brewery opened 27/29 sheets and recorded 7/1276 passes; Laundry 16/572; Porch 29/396; Civil 31/176. The fire-alarm discipline run recorded 12/12. The reopened native Brewery panel explicitly says Review finished with coverage gaps. - -**Native frames:** 0886.png. - -**Limits:** Transport admission and usable findings do not establish full-set coverage. A completed run can still have unopened sheets or unreviewed passes; the UI and report now name those coverage gaps rather than forcing additional inference. The rolling image window requires the reviewer to render an omitted source again before relying on its pixels; durable retention does not mean all pages are simultaneously visible to the model. - -### M3-02: The error offers Cancel while telling the user to resume - -**Status:** Native copy recheck pending. - -**Changed:** Stopped packet reviews display the known provider failure, state that saved work is retained and the request may have been charged, and tell the user to cancel before starting a separate review. Cancel preserves the failure explanation and diagnostics. Copy review diagnostics exposes the safe saved receipt, and the composer uses the same cancel-to-unlock guidance. The UI does not automatically resend an uncertain request. The Copy action now asks Electron to load and copy the saved safe receipt, rather than depending on the unavailable renderer clipboard. Cancelled historical failures no longer display an obsolete cancel-to-unlock directive in the panel; the original saved diagnostic is retained. - -**Observed:** The saved HTTP400 failure and 1:29 duration survived cancellation. Reopening the native Copy action exposed a renderer clipboard failure; it has been replaced with a trusted Electron action. The final packaged-app clipboard recheck is pending. - -**Native frames:** 0195.png, 1145.png. - -### M3-03: Cloud says Connected before checking the required provider secret - -**Status:** Observed readiness behavior; inference availability remains separate. - -**Changed:** Cloud setup reports Connected only after resolving the selected image, checking configured provider secrets for MOKE_API_KEY and verifying the worker identity in a bounded preflight sandbox. Start repeats preflight before preparing or uploading the workspace. Missing image/secret access returns a configuration-specific message instead of waiting for an unacknowledged worker. - -**Observed:** Native settings showed Connected after the real image/worker/secret preflight. The audit-only image and both provider-secret names were checked before transfer. This does not prove the credential will succeed: a separate bounded Meta probe returned HTTP404 before any tools ran. - -**Native frames:** 0997.png. - -**Limits:** Preflight verifies worker compatibility and secret availability, not successful inference with the provider credential or future provider availability. - -### M3-04: Reopening shows the wrong project and blocks returning to the live job - -**Status:** Reopen observed; upload navigation recheck pending. - -**Changed:** At startup an active cloud receipt selects its project and conversation, restores the packet model when available and sets cloud execution location. The receipt identifies project, model and scope. Snapshot reads use SQL replacement exclusion without waiting behind network transfer, allowing read-only navigation while local mutations remain fenced by cloud ownership. The send/upload request has its own submission flag so the sidebar can navigate during that request while mutation controls stay disabled; a late start response does not move the user back to the sending conversation. - -**Observed:** The prior live cloud job reopened into its own fire-alarm conversation/model and allowed navigation during return. The reopened audit found the send request still disabled navigation during upload; that UI state is fixed in source and its new native upload check is pending. - -**Native frames:** 0515.png, 1132.png. - -### M3-05: Review scope leaks across projects - -**Status:** Observed fixed in native Windows app. - -**Changed:** Conversation switches save and restore scope mode, revision, selected sheets, discipline, model, execution location, call limit, draft and scroll position for that conversation. A fresh conversation defaults to whole-set scope, no selected sheets and automatic discipline instead of inheriting another project's selection. The composer separately names the scoped PDF and sheet count. - -**Observed:** Switched from Fire alarm / Selected discipline / Push to cloud to Civil / Whole set / On this computer, then back. Fire alarm and cloud choices were restored only in their own saved conversation; Civil retained Whole set. - -**Native frames:** 1180.png, 1188.png. - -**Limits:** These conversation view preferences are kept in renderer memory for the session; a general durable preference store across application restarts was not added. Active cloud startup restoration is a separate path. - -### M3-06: New count check creates an invisible form below the sheet list - -**Status:** Observed fixed in native Windows app. - -**Changed:** Opening New count check creates the source-selection form and attaches focus behavior to Check name that scrolls the field into the working viewport and focuses it. The form contains the drawing and schedule selectors rather than requiring another inventory visit to choose them. - -**Observed:** Clicked New count check in the long sheet inventory. The Check name form was brought into the visible work area rather than left below the inventory. - -**Native frames:** 0230.png. - -### M3-07: Generic page names make real drawing sets hard to navigate - -**Status:** Observed fixed in native Windows app. - -**Changed:** Fresh PDF inspection proposes advisory names from embedded labels or an unambiguous printed identity while preserving original page references. Inventory rows have lazy thumbnails, text search and collapsed count-verification controls. Existing generic labels now have Read sheet names, a current/proposed-name preview and an explicit Apply suggested names action. Applying those proposals preserves custom labels, confirmation and count-check coverage. - -**Observed:** On the existing three-sheet fire-alarm import, Read sheet names proposed G-1 and FA-1 and preserved the custom FA-2 label. Apply changed the two generic labels without confirming them; closing and reopening the panel retained them. The before/proposal/after frames show this on an existing import, not only a new fixture. - -**Native frames:** 0824.png, 0828.png, 0832.png, 0838.png. - -**Limits:** Name extraction uses PDF labels and text heuristics, not OCR or a general title-block recognition model. Ambiguous/scanned pages can remain Page N and require manual naming. Existing labels are not silently rewritten; the user must request and apply proposals. Non-generic custom labels are intentionally excluded from replacement. - -### M3-08: Revision mapping blocks a removed sheet and obscures saved matches - -**Status:** Removed-sheet flow and previews observed; Added state remains limited. - -**Changed:** Revision correspondence supports changed, unchanged, removed and unresolved prior sheets, permits partial decisions and keeps saved associations visible. Byte-identical PDFs propose page-index matches. Mapping rows now pair previous/new page previews with inline decision controls, keep checkbox text beside its checkbox and open a larger paired comparison dialog. Unmatched new pages remain visible as requiring fresh review. - -**Observed:** The native Civil mapping shows compact paired old/new thumbnails and a larger comparison dialog. Saved retained-four/removed-one decisions remained after reopening. A separate persisted Added disposition is still unavailable; unmatched new pages are derived from correspondence. - -**Native frames:** 0858.png, 0855.png, 0861.png. - -**Limits:** Automatic matching is limited to byte-identical PDFs; retained pages inside a changed PDF are not independently matched by page content. New pages are derived and displayed as new or unmatched; an explicit persisted Added disposition is not implemented. - -### M3-09: Reviewing successive sheets requires returning to the inventory - -**Status:** Observed fixed in native Windows app. - -**Changed:** The source viewer has previous/next sheet controls, current position and a searchable Jump to sheet picker. The workbench carries the zoom factor into the next source render, and Reset view/Fit sheet restores the fitted view. - -**Observed:** Moved between original pages using Previous/Next in the source viewer, with the page position and Fit control visible. The searchable sheet picker is available alongside them. - -**Native frames:** 0210.png. - -**Limits:** Loading a new image resets pan and rotation; only zoom factor is preserved across source loads. - -### M3-10: Long finding text pushes the source below the working viewport - -**Status:** Observed fixed in native Windows app. - -**Changed:** Packet-finding evidence now uses a bounded layout with an independently scrolling issue column and a separate fitted source canvas. Observation/correction and evidence notes are collapsed initially. At narrow widths the issue area is bounded above the canvas rather than allowing long text to push the image below the working area. Source navigation remains with the viewer. - -**Observed:** Reopened the same Porch C1 finding. Before the new layout its source began near the bottom of the viewport and was cut off. Afterward the issue stays in a separately scrolling left column and the fitted source crop, toolbar and provenance remain visible on the right. - -**Native frames:** 0791.png, 0810.png. - -### M3-11: Cloud progress shows calls but not what the reviewer is doing - -**Status:** Transfer counters observed; native tool-activity check pending. - -**Changed:** Cloud receipts now carry uploaded/downloaded completed bytes and file counts with a progress bar, plus named preparation/transfer/review/result stages. Host-owned tool activity records identify the active or last finished tool and its drawing when applicable; the card separately labels the last recorded drawing opened. It also shows opened-sheet, finding and pass totals, recent activity age, elapsed time, remaining deadline time and Ends by. The configured review duration remains visible in settings. - -**Observed:** The real 506.1MiB upload advanced from 50.1MiB/21 files to 148.0MiB/105 files, 256.4MiB/244 files and 297/297 files. The 506.3MiB return showed advancing download counters. The Meta request failed before tools, so it does not verify the new active tool/drawing surface; the explicit Baseten check is pending. - -**Native frames:** 1122.png, 1125.png, 1127.png, 1145.png. - -**Limits:** Transfer counters advance after each complete file, or after a downloaded file is verified and reused. They do not show partial bytes within a large file, transfer speed or a transfer-specific ETA. Tool/drawing activity describes host tool work; it does not expose what the model is reasoning about between tool calls. Preflight and local package preparation finish before the first persisted cloud job receipt is created. The card's elapsed clock and transfer counters do not cover that earlier work. - -### M3-12: Old cloud receipts persist across every project - -**Status:** Observed fixed in native Windows app. - -**Changed:** Synced or cancelled cloud receipts display only in their owning project/conversation and expose Dismiss. A failed job that still owns the workspace remains globally visible until stopped, alongside active ownership, with project/model/scope context. Disconnect/reconnect copy is suppressed when the receipt no longer locks the workspace. - -**Observed:** Dismissed the terminal receipt and moved to Civil. The unrelated cloud card stayed absent. Active ownership stayed visible while browsing a different conversation during the live download. - -**Native frames:** 1167.png. - -**Limits:** Dismissal is held in component memory rather than persisted; reopening the application can show the owning conversation's terminal receipt again. - -### M3-13: Cancelling rewrites the displayed failure duration - -**Status:** Failure duration observed; final failure-message recheck pending. - -**Changed:** Packet receipts accumulate active running intervals and stop adding elapsed work while paused, failed or cancelled. Cancellation retains the existing error/provider-failure details. Previous reviews now open their saved result/failure panel, with a Back to latest review action, instead of presenting only a timestamp and state. Export cleanup and reopen now preserve an existing known failure when the reconciled state is unchanged, rather than replacing an HTTP failure with generic interruption wording. - -**Observed:** The native cancelled HTTP400 review retained its 1:29 active duration. A new cloud probe revealed that export cleanup replaced its specific HTTP404 explanation with generic interruption text; the host reconciliation rule now preserves the known failure. The saved providerFailure receipt retained the HTTP404 details. - -**Native frames:** 0195.png, 1132.png. - -**Limits:** The UI does not present waiting time or a separate cancellation duration; its duration is active work. Older receipts without interval fields use their available legacy timestamps. - -### M3-14: A notification survives into an unrelated empty project - -**Status:** Observed context clearing in native Windows app. - -**Changed:** Project/conversation transitions and source closure clear transient notices and selected source/finding context. A queued-follow-up or selected-region notice therefore does not continue as the status of an unrelated conversation. - -**Observed:** After a status-refresh notice and a stopped review, opening another conversation/project removed the old transient notice rather than displaying it as that context’s current status. - -**Native frames:** 1082.png, 1093.png, 1167.png. - -### M3-15: Expanded coverage becomes a wall of repeated status rows - -**Status:** Observed fixed in native Windows app. - -**Changed:** The coverage panel explains the requested sheet × discipline × pass denominator and shows aggregate status counts by discipline. Detailed records default to Needs attention, with status/discipline filters and sheet search before opening a sheet. Only one sheet's records open at a time and detail pages contain at most 20 rows. Earlier-scope records are opt-in, requested totals do not change with filtering, and Not applicable stays separate from checked work. Completed reviews with unopened sheets or unsettled coverage explicitly say Review finished with coverage gaps; exports give actual opened/recorded/unopened/unreviewed counts before the narrative summary. - -**Observed:** Opened the real Brewery result: the denominator remains 1276, the discipline filter reduced matching details to 116 Civil records, sheet search narrowed the list, and Next records moved through a sheet in pages of at most 20. Unreviewed work stays explicit and Not applicable has a separate column. The headline now names the coverage gap. - -**Native frames:** 0886.png, 0918.png, 0921.png. - -**Limits:** This change makes existing coverage navigable; it does not perform the unreviewed passes or validate the engineering correctness of self-recorded statuses. The standalone export retains its full static coverage table and follow-up list; the workbench's interactive filters and pagination do not apply to that export. - -### M3-16: The direct Sheets entry disappears in a smaller desktop window - -**Status:** Core navigation observed; compact receipt recheck pending. - -**Changed:** Compact layouts retain a direct Sheets control and an active-project label. A More menu keeps New count check and Import PDF available when secondary top-bar controls are hidden. Synced/cancelled cloud receipts can be dismissed and are scoped to their owning conversation. Terminal cloud cards now show a compact status/context summary; transfer and review details require explicit expansion. - -**Observed:** At a native900×900 window, the active project and Sheets remained visible. More exposed Import PDF and New count check. Terminal receipt details are now collapsed in source; its packaged compact view recheck is pending. - -**Native frames:** 0969.png, 0972.png. - -### M3-17: Windows shortcuts and singular counts use the wrong copy - -**Status:** Native shortcut observed; short-note recheck pending. - -**Changed:** Windows uses Ctrl K and Mac uses the command symbol. Counts use the appropriate singular/plural form. Both short summaries and full notes now render supported headings, bullets, bold and inline code through escaped Svelte text. - -**Observed:** The Windows sidebar shows Ctrl K. Native single-call receipts show 1 review call. Short-note formatting was still missing in the earlier implementation; it now uses the escaped formatter and is awaiting the final native package check. - -**Native frames:** 1145.png. - -**Limits:** ReviewText supports a small safe formatting subset rather than all Markdown. - -### M3-18: The completed cloud review calls supplied calculations and matrix missing - -**Status:** Observed corrected fire-alarm result; semantic limits remain. - -**Changed:** Review instructions require a content inventory, printed sheet identity, targeted detail inspection and a distinction between present-but-inadequate and absent calculations/matrices. Recognized absence claims must include absenceCheck data; the host rejects them unless every frozen source page was inspected in the current run, a cited location received a detail inspection and no contradicting evidence is declared. Recorded coverage is presented as an audit trail rather than a correctness certificate. - -**Observed:** The repeated local and completed Baseten cloud fire-alarm reviews recognized the supplied battery/NAC tables and matrix rather than calling them absent. The saved cloud output reads the 8.90 Ah requirement and distinguishes incomplete/existing-system verification from absence. This is evidence for that supplied case, not a general guarantee of engineering correctness. - -**Native frames:** 0552.png. - -**Limits:** The gate checks host inspection records and the model's declared source-search/contradiction evidence. It does not independently determine whether a table is adequate or whether a location's descriptive label matches its pixels. Absence-claim detection uses a phrase pattern; this is not a general semantic verifier for every paraphrase of a missing-content claim. - -### M3-19: There is no visible action to export the completed review report - -**Status:** Observed fixed in native Windows app. - -**Changed:** Saved packet-review panels expose Export review report beside the review actions. The trusted desktop chooser saves a standalone HTML report using the existing application export path, including scope/provenance, source and citation status, actual coverage, structured findings and embedded available evidence. Complete-workspace backup remains a separate action. - -**Observed:** Used Export review report and the native Save dialog to save a9,771,955-byte standalone HTML file with embedded fire-alarm evidence. It is linked below and remains distinct from complete-workspace backup. - -**Native frames:** 0552.png. - -### M3-20: Normal cloud preparation surfaces an internal maintenance error - -**Status:** Download navigation observed; upload recheck pending. - -**Changed:** Read-only snapshots no longer use write admission or wait behind the network maintenance semaphore. They share the SQL exclusion needed for replacement and pending-recovery checks instead. Cloud preparation/sync can continue while snapshot refresh and project navigation read local data, so ordinary observer reads do not surface the write-maintenance error. - -**Observed:** During the new live download, opened another saved conversation and inspected its failure while transfer counters continued advancing. The internal maintenance error did not appear. The upload sidebar issue was separately found and is awaiting its new native check. - -**Native frames:** 1132.png, 1145.png. - -### M3-21: Pause repeatedly fails with a stale-progress error and no Refresh action - -**Status:** Observed fixed in native Windows app. - -**Changed:** The backend accepts compatible idempotent pause/cancel controls across older progress versions. On a remaining control conflict the workbench fetches the current receipt and retries pause/cancel once only if that action is still compatible. A visible Refresh review action fetches the latest receipt directly. Resume and uncertain paid inference remain outside this control-command retry path. - -**Observed:** Started a live local review and clicked Pause while it was running. The panel changed to Review paused; Refresh retained that state. Cancel then changed it to Review cancelled without resending inference. - -**Native frames:** 1074.png, 1082.png. - -### M3-22: An invalid package folder asks for recovery instead of explaining validation - -**Status:** Observed fixed in native Windows app. - -**Changed:** The restore chooser validates the selected package before asking to replace local projects. Missing/invalid manifests return a specific invalid-package explanation, say that the workspace has not changed and direct the user to choose another folder. Recovery wording is reserved for a restore operation that reports interruption after replacement work begins. - -**Observed:** Selected the exported-report folder through the native restore chooser. The app said the folder is not a valid review package, identified its unexpected directory, directed another selection and said the workspace had not changed. No replacement confirmation appeared. - -**Native frames:** 0966.png, 0969.png. - -### M3-23: Governing sources hide the URL and have no open action - -**Status:** Observed fixed in native Windows app. - -**Changed:** Expanded governing-source records show the exact URL and retrieval time, an Open source in browser action for permitted HTTPS URLs and a Saved retrieved text disclosure. The trusted shell validates the external URL and opens it through the desktop. Source availability remains separate from each citation's substantive verification status. - -**Observed:** Expanded the saved source text and inspected the exact URL and retrieval time. Clicked Open source in browser; the native browser loaded the matching Florida Building Code resource URL. Only the opened tab was then closed. Browser chrome is excluded from this report. - -**Native frames:** 0948.png. - -### M3-24: Over-limit Send to cloud silently saves the request and leaves an old synced receipt - -**Status:** Capacity/feedback observed; project-only transfer unavailable. - -**Changed:** Cloud packages have a bounded 1 GiB/5,000-file allowance and measured-size errors. Selecting cloud refreshes a read-only workspace estimate; a pre-send panel shows estimated MiB, files, project count and limits with Recheck size. Send is disabled while the estimate is missing or over limit. Final preparation still enforces exact package bounds. Start failures remain visible after snapshot refresh, while unrelated old terminal receipts are hidden by project/conversation scoping. - -**Observed:** Before Send, the native composer displayed the whole-workspace scope, estimated504.5MiB,295files,7projects and limits. A later506.1MiB package actually uploaded and its506.3MiB return imported. Whole-workspace transfer remains mandatory; project-only transfer and an in-app reduction tool are not implemented. - -**Native frames:** 1093.png, 1127.png, 1151.png. - -**Limits:** Transfer still includes every project and its history. Project-only transfer, an in-app package reduction tool and a workspace-selection/creation flow were not added; over-limit guidance tells the user to export a backup and use a smaller workspace. The read-only estimate is not the final serialized package size and can become stale as the workspace changes. Exact preparation can still reject a package that exceeds the bound. - -## Review completeness and provider evidence - -The nine-image Baseten request reproduced HTTP400; eight images succeeded. The adapter retains the latest eight request images while durable metadata and artifacts retain all source history, and uses a five-minute request deadline. Uncertain paid requests are not replayed automatically. This resolves that reproduced provider rejection; it does not establish complete engineering coverage. - -| Live local set | Opened / supplied | Recorded passes | Findings | Calls | -| --------------------- | ----------------: | --------------: | -------: | ----: | -| Brewery | 27 / 29 | 7 / 1,276 | 11 | 39 | -| Laundry | 13 / 13 | 16 / 572 | 11 | 34 | -| Porch | 9 / 9 | 29 / 396 | 10 | 19 | -| Civil latest revision | 4 / 4 | 31 / 176 | 11 | 37 | -| Fire alarm discipline | 3 / 3 | 12 / 12 | 8 | 25 | - -The completed Baseten fire-alarm cloud run returned seven findings and 12/12 passes in 30 calls. It recognized the supplied 8.90 Ah battery requirement, NAC tables and matrix. The native report export saved 9,771,955 bytes with embedded evidence. Larger results still have substantial coverage gaps; the workbench and exported headline now state the actual gap before the narrative. - -A separate bounded reopened-cloud probe used the new conversation’s default Meta model and returned HTTP404 after 171 ms, before any tool execution. Its 506.1 MiB upload and 506.3 MiB / 298-file return demonstrate real transfer counters and result recovery, not a completed review or active-tool acceptance. The returned safe provider-failure receipt retained the HTTP status. That probe exposed the cleanup-message overwrite fixed above. - -## Research, cost and proof limits - -Research used Exa through Executor and primary sources. See [provider evidence](audit-3-review-reliability.md) and [Modal configuration](audit-3-cloud.md). The isolated audit image uses 4 CPU / 8 GiB hard caps, approximately $0.76/hour at the researched Sandbox rates, four times the former worker. Provider inference is separate. No production image tag or operator configuration was changed. - -[Electron’s clipboard documentation](https://www.electronjs.org/docs/latest/api/clipboard), retrieved through Exa/Executor during the reopened audit, documents the asynchronous desktop clipboard methods and deprecated direct renderer access. The implementation awaits the trusted main-process write; it exposes no arbitrary clipboard reader or provider configuration to the renderer. - -Workspace transfer remains atomic whole-workspace replacement. Project-only transfer and an in-app reduction tool are not implemented; the native composer now shows that scope, size and limits before Send. Transferring a project subset through the existing whole-workspace replacement protocol would risk dropping unrelated data. This limitation is explicit rather than counted as an implemented scope feature. - -Transfer counters advance after each completed file or verified reused download. They are not within-file byte streaming, transfer throughput or a transfer-specific ETA. Host activity is the actual tool/stage/drawing record, not a claim about hidden model reasoning. - -Automated verification and Sol review supplement the recorded native behaviors; they do not close a visual finding by themselves. The earlier six-round approval covers the previously submitted change. The reopened behavior diff requires a separate focused review before shipping. Final verification/review status is recorded at closeout. - -All Windows observations used the isolated packaged audit app with Cua Driver 0.38.0, fresh accessibility snapshots and native file dialogs. The original workspace and original recordings remain preserved. Private raw evidence is under `.local/audit-3-fixes` and Windows Downloads `Moke-audit-3-fixes-2026-10-29`. diff --git a/docs/research/audit-4-harness.md b/docs/research/audit-4-harness.md deleted file mode 100644 index c4ffaf98..00000000 --- a/docs/research/audit-4-harness.md +++ /dev/null @@ -1,392 +0,0 @@ -# Audit 4 — inference and review harness - -Audited commit: `68deb49100638edc41f8da925ac11d763b7f9627` on `fix/audit-4`, -2026-10-31. Read-only audit; this ledger is the only file written. No provider, -Modal, search, or other live service requests were made. No commits. - -## Evidence and limits - -Read the project instructions, Effect reference, coding-standards skill, runtime, -inference and model documentation, parallel-review and initial-review research, -Baseten measurements, and prior audit records. Traced packet tool registration, -stage orchestration, request admission, rendering, source retrieval, SAM, -selected-count execution, segmentation experiments, and cloud worker review/export. - -`CONFIRMED` means the mechanism is established by source or a local reproduction; -it does not claim a new measured production incidence or performance improvement. -All 12 entries below are CONFIRMED. None relies on a speculative provider limit. - -Local verification used Bun **1.4.4**, matching `.bun-version`: - -- Inline `bun run -` probes imported the actual workspace provider and pacer with - recorded transports and fake credentials. Dependencies resolved from Bun's - existing package cache. No test/source files were created. Captured results: - - Nine Baseten images, one historical plus eight new: only indices **4–8** - reached the recorded transport; indices **1–3** were new crops. - - Forty recorded HTTP 429 refusals: one admitted request ultimately ended - with **`uncertain: true`**, HTTP 429, despite no accepted generation. - - `pacer.refused("overloaded", 0, 120000)` returned **60000**. - - Aborting a Meta generation while its recorded Files upload was held left - the upload's signal **unaborted**. Once released, generation returned an error - without dispatching a Responses request. -- Attempted `bun test` on `packet-provider`, `request-pacing`, `packet-parallel`, - `packet-sam-focus`, `meta-review`, `adaptive-review`, and `packet-sources` smoke - files. All seven failed to load with **Cannot find package 'effect'** because - the checkout has no `node_modules`. No assertions ran. Did not install or change - dependencies. The known WSL clock flakes were not reached. -- Dead-export candidates were checked with whole-repository `rg -l -w`, excluding - `repos/`, `bun.lock`, and this audit. The apparent one-file exports (`RateLimits`, - `reportedRateLimits`, `JurisdictionPlace`, `SourceAddress`, `PdfSourceText`, - `SourceSearchResult`) have internal uses. Experiment modules have script/test - consumers. No substantial dead-code deletion is claimed. -- Excluded prior fixed timeout/output-policy, affinity, parallel SAM-admission, - source-verification and missing-notes findings. The prior unowned-compaction - observation was already recorded in Audit 3 and is not re-reported. -- Excluded a suspected Meta 50-image ceiling: newer `docs/inference.md:57` - records acceptance of 300 small tool-result images. Also excluded a suspected - omitted-system-prompt pacing bug: a recorded 90,000-character system prompt - reserved 22,501 tokens through the real SDK normalization path. - -Counts: **12 findings; high 4, medium 8, low 0**. Categories: **inference 3, -perf 3, correctness 5, standards 1, dead-code 0, slop 0**. H-12 requires a product -decision. The other entries restore existing boundaries or remove redundant work. - -## Packet service and prompt owner - -Keep H-01–H-04 and H-11 with one implementer because they edit -`backend/packet-review/service.ts`. H-12 shares that file but is decision-gated. - -### H-01 — Setup exposes the whole review workflow and has no enforced completion boundary - -- **Category:** inference. **Severity:** high. **Verdict:** CONFIRMED. **Size:** M. -- **References:** `backend/packet-review/service.ts:778`, `:1251`, `:1278`, - `:1490`, `:1653`, `:1926`, `:2078`, `:2178`; - `backend/review/prompts/review-prompt.ts:103` (coordinator guidance); - `docs/research/baseten-rate-limits.md:65` (recorded Brewery run). -- **Evidence:** One registry installs every tool for every conversation. Setup - receives `render_sheet` for any frozen sheet, coverage/finding tools, SAM and - other review mutations. `outsideSheet` constrains only assigned children. - `finish_review` rejects setup calls, but neither findings nor coverage do. - The host starts children only after the coordinator voluntarily yields. The - full review method and discipline instructions precede the short setup rule. - Thus an otherwise valid tool sequence can inspect the whole packet during - setup. The recorded run's 63 requests / approximately 25 minutes / 24 rendered - sheets / 10 findings follows a path explicitly permitted by this surface. -- **Concrete fix:** Make setup an explicit tool capability set: inventory, - project configuration, source research and a typed `finish_setup`. Exclude - coverage, findings, presentation summaries, count segmentation and review - completion. Enforce stage permissions at execution too, including replay and - steering. Scope setup rendering to a host-recorded selection of setup sheets; - provide an explicit bounded selection step when labels are ambiguous. Persist - setup completion and transition immediately to sheet work, allowing unknown - configuration fields to remain Needs input. Use a small setup-specific prompt - and a deterministic setup turn/render ceiling as a fallback; reaching that - ceiling must preserve unknowns, never fabricate configuration or completion. - Preserve distinct stable tool lists per role/stage instead of mutating a tool - list on every turn. Stronger wording alone does not address this mechanism. -- **Proof test:** Extend `tests/smoke/packet-parallel.test.ts` with an adversarial - setup script requesting coverage, a finding, SAM and a non-setup sheet. Assert - these are absent from its wire tools and rejected if emitted anyway; no such - mutations/paid SAM dispatch occur. Assert a saved setup result or bounded - setup exit starts children and survives restart. Existing scripted setup - cooperates; it does not test refusal of setup-stage review work. - -### H-02 — Sheet assignments do not fence finding writes or segmentation - -- **Category:** correctness. **Severity:** high. **Verdict:** CONFIRMED. **Size:** M. -- **References:** `backend/packet-review/service.ts:784`, `:1494`, `:1543`, - `:1660`, `:1686`, `:1723`, `:1804`, `:2028`. -- **Evidence:** Rendering and coverage call `outsideSheet`; `record_finding` - does not. Its location validation checks membership in the entire packet, - and evidence validation checks the shared packet ledger. A child can create - a finding on another sheet or supply another child's finding ID and overwrite - it, including changing the summary's owner to itself. A nonempty evidence note - permits the write without an artifact. `record_finding_summary` checks owner - only after that ownership has already been replaceable. `segment_objects` - similarly accepts any shared evidence artifact, without checking assignment. - This defeats the documented one-sheet boundary under ordinary model mistakes. -- **Concrete fix:** For children, require every finding location and referenced - evidence artifact to belong to the assigned sheet. Before replacing a finding, - verify its existing ownership as well as the new locations; reject attempts - to adopt a peer's record. Apply the same assignment guard before segmentation. - Leave multi-sheet creation/updates to the coordinator. Centralize this policy - beside the existing render/coverage guard so new tools cannot omit it. -- **Proof test:** In `packet-parallel.test.ts`, have child A submit a finding - located on B, update B's existing finding by ID, and segment B's artifact. - Assert all fail before mutation or SAM dispatch; coordinator updates still - work. Existing assignment tests attempt an unauthorized render, not these - writes. No existing test found covering these paths. - -### H-03 — One sheet's failed SAM request stops every sheet and the coordinator - -- **Category:** correctness. **Severity:** high. **Verdict:** CONFIRMED. **Size:** M. -- **References:** `backend/packet-review/service.ts:1017`, `:1028`, `:1091`, - `:1104`, `:1206`, `:1244`, `:2644`; `docs/review-runtime.md:51`. -- **Evidence:** The model adapter marks only the owning sheet failed for an - uncertain child request. SAM's catch instead sets the whole packet state to - `uncertain`, regardless of `conversationId`. It is also used by automatic - finding zooms. `focusFinding` catches the rejection and reports a skipped zoom, - but the global state has already changed; peers then fail `assertActive` and - coordination cannot proceed. This is separate from Audit 3's fixed - conversation-scoped _admission_ conflict: dispatch isolation works, failure - isolation does not. Even a definite SAM HTTP refusal takes this path. -- **Concrete fix:** Resolve the owning sheet from `assignments` in the SAM failure - path and mark that child failed, retaining its unresolved receipt and blocking - only its further work. Allow peers and coordination to complete, then derive - the overall uncertain outcome as for model failures. Preserve global handling - for coordinator SAM and never resend an uncertain SAM attempt. Classify known - HTTP refusals separately from ambiguous post-dispatch failures. -- **Proof test:** Extend the parallel SAM test to return an incomplete/failed SAM - response for A while B's model is held. B must finish, coordination must run, - and A's attempt must not be resent. Also cover host-triggered focus SAM. - Existing `packet-parallel.test.ts:494` covers successful concurrent SAM/model - admission, not failed SAM isolation. - -### H-04 — Repeated source retrieval duplicates network work and the persisted source ledger - -- **Category:** perf. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **References:** `backend/packet-review/service.ts:1412`, `:1424`, `:1257`, - `:2263`; `backend/packet-review/sources.ts:476`, `:484`, `:492`. -- **Evidence:** Every `read_source` invocation fetches again and assigns a new - UUID, even for the identical URL in the same review. `upsert` keys by UUID, so - repeated calls append complete duplicate source text. Children initially get - only source IDs/URLs/titles and have no tool to read the already-retrieved body - by ID; retrieving a known source again is a natural way to inspect it. Parallel - children can all perform the same fetch. The full coordinator inventory later - contains every copy, and every packet save serializes the enlarged ledger. -- **Concrete fix:** Provide review-scoped access to a saved source by ID and - deduplicate same-review retrieval by canonical URL including the PDF page - selector. Share in-flight retrievals, return one stable source receipt, and - give an explicit refresh operation if needed. Do not silently reuse another - review's historical source as current verified evidence. Keep failed reads - retryable rather than permanently caching unavailability. -- **Proof test:** Through packet tools with a recorded source transport, make - two children read the same URL concurrently and subsequently read it by ID. - Assert one fetch and one saved source with identical IDs/content; separate - `#page=` selectors must stay distinct. Existing source tests exercise retrieval - and parsing, not shared packet-level deduplication. Performance magnitude was - not benchmarked in this audit. - -### H-11 — The supposedly shared child prefix is read from mutable global review state - -- **Category:** inference. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **References:** `backend/packet-review/service.ts:1453`, `:1473`, `:2210`, - `:2252`, `:2262`, `:2325`, `:2437`; - `tests/smoke/packet-parallel.test.ts:312`. -- **Evidence:** Each queued child reloads the current review and serializes its - current `projectConfiguration` and `sources` before its assignment. Earlier - children can call `record_project` and `read_source`, changing both between - child launches. Children can therefore review against inconsistent project - assumptions and get different bytes in the advertised shared prefix, despite - using the same affinity key. The append-only test compares a conversation - with its own previous request; it does not compare openings across children - after a peer changes shared state. The stable system/tool prefix still hits; - this finding does not claim loss of the entire approximately 20k-token prefix. -- **Concrete fix:** At the setup-to-sheets transition persist a canonical setup - snapshot and serialize every child opening from it in stable order. Children - should report proposed project-fact corrections in notes for coordinator - reconciliation, rather than overwrite global configuration. Newly discovered - sources remain available by ID without changing later children's opening - prefix. Persist the snapshot for restart/resume; do not merely cache it in RAM. -- **Proof test:** Hold a queued child until an earlier child adds a source and - attempts to alter occupancy. Compare serialized opening prefixes before the - assigned-sheet field, and verify every child uses the same setup version. - Repeat across restart. Existing cache tests do not mutate setup between child - openings. Actual cache-rate improvement remains unmeasured. - -## Review operations and store owner - -### H-05 — Every model crop loads the entire workbench snapshot - -- **Category:** perf. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **References:** `backend/review/operations.ts:54`, `:56`, `:65`; - `backend/review/store.ts:229`, `:261`, `:268`, `:281`; - `backend/packet-review/service.ts:1323`, `:2036`. -- **Evidence:** `render` needs one sheet, its revision and an operation receipt, - but uses `snapshot(projectId, 0)`. That loads all project artifacts, findings, - accepted facts, checks, conversations, messages, correspondence and up to - 2,000 historical events, as well as the project list. Every crop and automatic - focus render repeats this under the shared SQL exclusion. `segment_objects` - also loads a full snapshot to find one artifact. Data volume and parsing cost - grow with project history; the work is repeated across parallel children. -- **Concrete fix:** Add narrow scoped store reads for render inputs and - operation receipts, and use the existing artifact lookup for segmentation. - Preserve project ownership, source version and render-input checks. Keep the - workbench snapshot for actual workbench callers. This removes serialized SQL - and Schema work without weakening scoped resource ownership. -- **Proof test:** Populate unrelated messages/artifacts and instrument the SQL - adapter for a render and segment lookup. Assert only targeted records are read - while the same output and source rejection behavior remain. Existing rendering - tests verify output and fencing, not this query shape. No timing claim made. - -### H-06 — Identical crops are rasterized again under every new tool task - -- **Category:** perf. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **References:** `backend/packet-review/service.ts:1326`, `:1329`, `:1167`; - `backend/review/operations.ts:65`, `:82`; - `backend/documents/documents.ts:159`, `:161`, `:171` (called boundary only). -- **Evidence:** Render reuse matches only `operationId`. Packet tools derive that - ID from the Pi task, so re-opening a previously rendered page/detail in a later - task misses reuse. The documents adapter always calls the PDF worker before - storing content-addressed bytes. On-disk deduplication therefore saves storage - but not rasterization, worker-slot time or base64 conversion. Re-opening is - explicitly required after image eviction and occurs between setup/children/ - coordination as well. -- **Concrete fix:** Reuse immutable render results by source hash, page identity, - rotation/user unit, exact region, effective DPI and renderer version. Preserve - a distinct operation receipt/association so replay and evidence provenance - remain correct. Share identical in-flight renders. Implement the lookup in - review operations/store, coordinating with the documents auditor if a renderer - version is needed; no broad documents rewrite is required. -- **Proof test:** Request the same crop with two distinct operation IDs through - review operations and assert one worker render and valid evidence for both - tasks. Changes to source, rotation, region or effective DPI must render again. - Existing idempotency tests cover operation identity, not equivalent crops - across different tasks. Savings were not benchmarked. - -## Provider and pacing owner - -### H-07 — Stepped image eviction discards fresh images before their first model inspection - -- **Category:** correctness. **Severity:** high. **Verdict:** CONFIRMED. **Size:** M. -- **References:** `backend/inference/provider.ts:292`, `:299`, `:599`; - `backend/packet-review/service.ts:1304`, `:1336`, `:1358`; - `tests/smoke/packet-provider.test.ts:312`. -- **Evidence:** With nine total images, `ceil((9 - 8) / 4) * 4` drops four. - A locally captured real-SDK Baseten request with one old image plus eight new - images contained only indices 4–8: three of the eight fresh images were - replaced with the omission notice. The render tool explicitly allows eight - per generation and marks all rendered sheets inspected before provider - serialization. Consequently the host's inspection ledger can get ahead of - images actually made available for inspection. The existing test uses twelve - images, where the arithmetic happens to preserve the last eight. -- **Concrete fix:** Track the latest unconsumed image batch and protect it from - eviction. Drop historical images in steps only while all fresh images fit; - reduce the step at that boundary. If a fresh batch itself exceeds byte/count - bounds, split or reject it before recording delivery. Preserve the useful - stepped cache behavior for previously seen images. Do not equate rendering - alone with successful model delivery when coverage depends on delivery. -- **Proof test:** Add real-SDK recorded requests for total counts 9, 10, 11, 13 - and 17 with varying old/new splits. Every fresh image must appear on its first - request, no request exceeds eight images, and the durable transcript remains - unchanged. Add one packet-tool test that verifies the delivered crop set before - coverage can be recorded. Existing 12-image test misses the failure. - -### H-08 — Exhausted definite refusals are persisted as ambiguous paid attempts - -- **Category:** correctness. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **References:** `backend/inference/provider.ts:760`, `:763`, `:793`, `:806`, - `:813`; `backend/inference/response.ts:52`; - `backend/packet-review/service.ts:2474`, `:2490`, `:2678`. -- **Evidence:** After 40 explicit 429/529 refusals or the refusal time budget, - the loop hands an error response to `parseResponseReceipt`. Its successful - stop-reason schema rejects it, and the catch calls `end(true, ...)`. A recorded - all-429 probe reproduced this exactly. SQL retains an uncertain attempt and - future admission blocks it; the user-facing path says it may have been charged. - The adapter itself has already established these responses generated nothing. - This is the terminal-state bug after the new retry budget, not the prior - fixed six-retry pacing policy. -- **Concrete fix:** Persist a distinct definitive refusal outcome with safe - status/detail before ending the run. Stop or pause on prolonged provider - unavailability, and allow an explicit later resume without treating that - attempt as an unresolved charge. Continue to block genuine network/stream - ambiguity. Do not implement this by marking an error as a successful receipt - or by automatically replaying a previously uncertain dispatch. -- **Proof test:** Use recorded refusals and a manual clock to exhaust both limits. - Assert a definitive non-success receipt/disposition, no success usage, accurate - messaging and safe explicit resume. Existing provider tests recover after one - or nine refusals; they do not exhaust the new policy. - -### H-09 — Provider Retry-After is truncated and HTTP-date values are ignored - -- **Category:** correctness. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** S. -- **References:** `backend/inference/pacing.ts:192`, `:197`; - `backend/inference/provider.ts:120`, `:171`. -- **Evidence:** Both shared refusal cooldown and routing-refusal delay apply - `min(60000, requested)`. A local pacer call requesting 120,000 ms returned - 60,000 ms. `retryAfterMs` only accepts a positive numeric string, so an - HTTP-date is treated as absent. This conflicts with the documented promise - to honor provider waits and can send every waiting sheet into a known refusal. -- **Concrete fix:** Cap only locally generated exponential backoff. Honor a - validated provider delay in full; if it exceeds the allowed refusal budget, - terminate as a definitive refusal without an early resend. Parse both - delta-seconds and HTTP-date, converting once into a monotonic deadline with - injected wall/monotonic time for tests. -- **Proof test:** Extend `request-pacing.test.ts` and recorded provider tests with - `Retry-After: 120`, an HTTP-date, zero, and malformed headers. No dispatch - before the valid requested deadline; cancellation still removes waiters. - Existing tests cover short numeric waits only. - -### H-10 — Files uploads are detached from review cancellation and lifetime - -- **Category:** standards. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **References:** `backend/inference/provider.ts:458`, `:471`, `:485`, `:489`, - `:519`, `:880`, `:901`. -- **Evidence:** `onPayload` awaits image uploads, but those use only a new - 60-second timeout. The review abort signal is forwarded to the model SDK, not - the upload. The recorded held-upload probe showed the upload still active after - cancelling generation; generation could finish only after the probe released - it. Recursive `Promise.all` can start many such uploads. This violates explicit - runtime cleanup: cancelled work can continue transferring source images and - hold shutdown/drain for the upload timeout. -- **Concrete fix:** Give the review's image-upload pool an explicit lifetime and - abort it on review cancel/close. Because peers share the same image promise, - do not tie the shared upload solely to one child's cancellation; use pool - ownership or waiter accounting. Make each cancelled waiter stop awaiting the - upload promptly, and bound upload concurrency. Retain byte-identity reuse and - avoid turning cancellation into a five-minute cached upload failure for peers. -- **Proof test:** Hold a recorded Files upload, cancel its only review, and assert - its signal aborts and drain completes without release. With two live child - waiters, cancelling one must not cancel the other's upload. Existing file-ID - tests cover reuse and fallback, not cancellation/lifetime. - -## Refactoring guidance without a separate style finding - -The service's length alone is not a finding. While fixing the issues above, -three useful boundaries reduce actual duplicated policy: - -- A stage/role capability module shared by wire tool selection and execution - authorization (H-01/H-02/H-11). -- A packet SAM dispatcher owning intent, receipt, failure disposition and - conversation isolation, called by both explicit segmentation and finding - focus (H-03/H-12). Keep geometry preparation/decoding in the existing `sam.ts`. -- A coverage reducer shared by single and batch tools: the two handlers at - `service.ts:1490` and `:1537` duplicate inventory, inspection, reason and upsert - rules. Extract only those rules when touching them; preserve atomic batch - rejection and durable tool receipts. - -Keep SQL mutation/replay ownership in one service/store boundary. Splitting every -tool into a tiny wrapper would spread dependencies without simplifying policy. -No standalone slop or dead-code findings are counted for these suggestions. - -## Needs decision - -### H-12 — Every qualifying finding synchronously buys a SAM presentation zoom - -- **Category:** inference. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **References:** `backend/packet-review/service.ts:1125`, `:1135`, `:1162`, - `:1206`, `:1775`, `:987`; `docs/inference.md:107`. -- **Evidence:** After saving a finding, `record_finding` always awaits - `focusFinding`. With a Meta key and a location under half a sheet, this renders - another crop and makes a paid SAM call even for a prose note, code citation or - table discrepancy. No object-identification need is required. Calls consume - the shared review budget and block that conversation's next tool/model work. - Existing focus tests explicitly expect this behavior. The finding is already - saved and the model-facing result includes only an artifact ID and segment - count; this work principally serves presentation, not additional reasoning. - H-03 separately covers its current global-failure consequence. -- **Decision:** Keep automatic SAM zooms for every qualifying finding, or make - them selective/on demand? This changes visible evidence presentation and the - established SAM product behavior, so it is not an automatic audit fix. -- **Concrete fix if approved:** Retain the mandatory model-requested SAM path - for device counts. For other findings, persist a deterministic PDF highlight - immediately, and request SAM only for an explicit object-location need or - reviewer action. If retaining automatic enrichment, move it to an owned, - bounded post-save queue with visible status, explicit budget accounting and - cancellation; package export must drain or record pending enrichment. -- **Proof test:** Update `packet-sam-focus.test.ts`: a textual finding should - require no presentation SAM call under the approved policy, while a device - count still requires its evidence SAM. Verify deterministic highlights, optional - enrichment failure, budget handling and cloud export. Existing tests cover - today's unconditional focus generation; they must change only after approval. - No claim is made about accuracy or measured cost savings from removing it. diff --git a/docs/research/audit-4-platform.md b/docs/research/audit-4-platform.md deleted file mode 100644 index b66ab2f7..00000000 --- a/docs/research/audit-4-platform.md +++ /dev/null @@ -1,391 +0,0 @@ -# Audit 4 — platform ledger - -Audited 2026-10-31 at `68deb49100638edc41f8da925ac11d763b7f9627`, branch -`fix/audit-4`. This is a read-only platform audit; this ledger is the only -repository write. No provider requests, deployment, publication, or commits. - -Read `AGENTS.md`, the installed coding-standards skill, `repos/effect/LLMS.md`, -`docs/data.md`, `docs/deployment.md`, `docs/development.md`, -`docs/cloud-reviews.md`, and the relevant Audit 3/product-audit records. -Historical provider/product gaps are not repeated below. Fixes must not edit -applied migrations. SQL index changes require a new migration. - -Evidence uses source inspection, whole-repository ripgrep excluding `repos/`, -build configuration and installed package inspection, and disposable local Bun -probes. Probes used Bun 1.4.4 and packages already available in Bun's cache. -Temporary probe data was removed. No source/test/config files were added. - -The attempted existing-suite command was: - -```sh -bun test ./tests/smoke/backend.test.ts ./tests/smoke/database-files.test.ts ./tests/smoke/documents.test.ts ./tests/smoke/credential-files.test.ts ./tests/smoke/cloud-reviews.test.ts ./tests/smoke/cloud-cache.test.ts ./tests/smoke/cloud-transfer-progress.test.ts -``` - -It stopped during module loading: this checkout has no installed `node_modules`, -and all seven files reported `Cannot find package 'effect'`. Result: 0 pass, -7 load failures; no test assertions ran. Dependencies were not installed during -this audit. This is a validation limitation, not a source regression. No full -verify, packaged run, or macOS timing measurement is claimed. Existing coverage -mentioned below was inspected, not represented as passing here. - -## Backend documents and artifacts - -### P-01 — Render accepts changed PDF bytes under the original source identity - -- Category: correctness. Severity: high. Verdict: CONFIRMED. Size: M. -- Files: `backend/documents/documents.ts:127`, - `backend/documents/pdf-worker-main.ts:65`, `backend/review/operations.ts:82`. -- Evidence: `Documents.render` validates the hash's syntax and reads the file's - size, but never verifies its contents against `originalSha256`. The worker - verifies geometry and rotation, not content identity. The caller records the - resulting image against the original revision. Offline reproduction used - `createHeroPdf("A")`, imported it, rendered its first page, replaced its stored - hash-named PDF with `createHeroPdf("B")`, then rendered again with the original - sheet and hash. Output was - `{"tamperedRenderSucceeded":true,"renderPixelsChanged":true,"inspectRejected":true}`. - Thus the separate `inspectStored` integrity check rejects exactly the file - that rendering accepts. No concurrent mutation or malicious PDF was needed. -- Fix: include the expected source hash in the private worker render contract - and verify the exact buffer supplied to PDF.js before rendering. Return a - typed integrity failure and do not publish/store a render on mismatch. A - parent-only precheck would still leave a check/read race. -- Proof test: add the A-to-B replacement reproduction to `documents.test.ts` - and exercise `RenderSource` to show no artifact is committed. Existing - `audit-3-documents.test.ts:30` covers corruption for label reinspection only; - geometry/malformed-input tests do not cover a different valid PDF. - -### P-02 — Every artifact chunk rereads and hashes the entire file synchronously - -- Category: perf. Severity: medium. Verdict: CONFIRMED. Size: M. -- Files: `backend/documents/artifact-store.ts:119`, - `backend/review/operations.ts:305`, `src/lib/review/client.ts:396`. -- Evidence: `readArtifactChunk` performs `readFileSync(path)` and SHA-256 over - the full buffer before slicing the requested range. Both the model-image - reader and renderer loop in 256 KiB chunks. A permitted 16 MiB model image - therefore causes 64 full reads/hashes: 1 GiB of synchronous processing for - one image, before image conversion. The endpoint is bounded in response size, - but its work and allocation per response are not bounded by the chunk size. - This runs in the Bun API process and blocks other callbacks during each read. -- Fix: give the backend image reader a verified whole-file operation; provide - renderer chunks from a bounded verified read session/cache with explicit - invalidation on replacement and file-identity changes. Preserve corruption - detection; do not simply stop checking hashes or trust the filename. -- Proof test: read a multi-chunk artifact and demonstrate linear total file - processing, while altered/truncated files still fail. Keep bounded response - and cancellation checks. `documents.test.ts:282` checks a single 128-byte - response, not repeated full-file work. - -## Backend composition and HTTP transport - -### P-03 — Application-layer and background logs bypass the file logger - -- Category: correctness. Severity: medium. Verdict: CONFIRMED. Size: S. -- Files: `backend/main.ts:60`, `backend/cloud/service.ts:310`, - `backend/cloud/sync.ts:84`, `desktop/backend.ts:47`. -- Evidence: `Logger.layer([logger])` is a sibling in `Layer.mergeAll`, not a - provider for `Application.layer`. A local Effect 4 probe using this exact - composition shape printed `acquisition-probe` through the default stdout - logger; the supplied logger captured only a subsequent runtime call. - Cloud seeding/reconciliation fibers are forked during layer construction and - inherit that acquisition context. Their warnings can therefore miss - `userData/logs`. Stdout is also the readiness channel: Electron treats its - first line as `BackendReady` and closes the reader afterward. A startup - warning can interfere with readiness; the complete startup race was not - separately reproduced, so that consequence remains PLAUSIBLE. -- Fix: provide the logger to application-layer construction, including scoped - fibers, and expose it to runtime operations. Keep stdout exclusively for the - readiness envelope. The cloud worker already demonstrates provider-style - logger wiring in `backend/cloud/worker-main.ts:96`. -- Proof test: use an offline application/cache seam that logs on acquisition - and in a background fiber; assert both enter the log sink and stdout contains - only valid readiness. `backend.test.ts` does not exercise startup/background - logging. - -### P-04 — Valid shared-schema requests exceed the backend HTTP body limit - -- Category: correctness. Severity: medium. Verdict: CONFIRMED. Size: S. -- Files: `backend/main.ts:104`, `src/lib/review/contracts.ts:39`, - `src/lib/review/contracts.ts:313`, `src/lib/review/packet.ts:458`. -- Evidence: the server caps requests at 65,536 bytes, while messages and steering - allow 32,000 characters. A real `BackendApi` probe created a project and - conversation, decoded a valid `PostMessage` containing `"界".repeat(24000)`, - then sent it through RPC. Its command JSON was 72,210 bytes and the typed call - failed with `RpcClientDefect: Received empty HTTP response from RPC server`. - JSON escaping can also expand otherwise valid text; this is not limited to - that language. Shared validation succeeds before the transport rejects it. -- Fix: reconcile request-byte limits with every public payload's encoded - maximum, including framing, Unicode and escaping. Bound currently unbounded - array inputs where appropriate; surface an application input error instead - of a transport defect. Preserve authentication and the origin rejection. -- Proof test: send maximum supported ASCII, Unicode and escape-heavy messages - through the actual backend HTTP RPC, then reject an explicitly over-limit - request without losing subsequent API availability. Existing backend tests - cover an empty invalid name, not large valid payloads. - -## Cloud persistence and application queries - -### P-05 — The cloud writer can persist state that prevents the next app launch - -- Category: correctness. Severity: high. Verdict: CONFIRMED. Size: M. -- Files: `backend/cloud/files.ts:83`, `backend/cloud/files.ts:104`, - `backend/cloud/service.ts:318`, `backend/cloud/service.ts:546`. -- Evidence: reads refuse state above 8 MiB; writes have no corresponding byte - check. Admission permits up to 1,000 receipts. Each receipt stores both the - complete request and its JSON fingerprint, including selected sheet IDs. - An offline probe schema-validated 500 terminal receipts with 200 selected - sheets each, wrote them through `writeCloudState`, then read that same file. - The successful write produced 8,650,412 bytes; read failed with - `Cloud ownership records exceed the supported limit.` Application layer - construction awaits this read, so the whole backend fails to start, even - though all these jobs are terminal. Removing preview data does not prevent it. -- Fix: enforce a write/read-compatible storage budget before replacing the - last valid state and reserve room for active-job progress. Separate/archive - terminal receipt bodies or version a compact fingerprint representation while - retaining durable command deduplication. Provide safe recovery for existing - oversized files; never discard active ownership or replay old command IDs. -- Proof test: add this 500-receipt round trip and an active-job progress update - near the byte ceiling. A refusal must leave the previous state readable and - must happen before external dispatch. Existing cloud tests cover corrupted - ownership and previews, not aggregate valid-history growth. - -### P-06 — Cloud setup scans every attempt per historical count run - -- Category: perf. Severity: medium. Verdict: CONFIRMED. Size: S. -- Files: `backend/application.ts:275`, `backend/application.ts:281`; - add a NEW migration after `0011_firm_checklist.sql`. -- Evidence: `localWorkBlockingCloud` uses a correlated lookup on - `review_execution_attempts(run_id,status)`, but the only key there begins with - `journal_id`. It also reads JSON state from every packet payload. An in-memory - SQLite probe applied all existing migration SQL unchanged and ran - `EXPLAIN QUERY PLAN` on the two exact application queries. Results included - `CORRELATED SCALAR SUBQUERY 1` / `SCAN a` and `SCAN packet_reviews`. - Adding temporary in-memory indexes on `(run_id,status)` and - `json_extract(payload,'$.state')` changed those to indexed searches. The count - query otherwise has a history-sized nested scan; packet payloads can be large. -- Fix: add both indexes in a new immutable migration. Keep query behavior and - uncertainty/acknowledgement conditions unchanged. Rebuild the cloud image - through the separately authorized deployment workflow when shipping it. -- Proof test: migration upgrade preserves rows/history; query results stay the - same for active, terminal and acknowledged/uncertain cases. Record plans and - an offline large-history comparison. `cloud-local-work.test.ts` covers fencing - semantics, not the query plan. Do not modify migration 0003 or 0009. - -## Credentials and Modal admission - -### P-07 — Accepted provider URLs are rejected only after cloud upload - -- Category: correctness. Severity: medium. Verdict: CONFIRMED. Size: S. -- Files: `desktop/credential-files.ts:20`, `desktop/credential-files.ts:28`, - `backend/cloud/modal.ts:615`, `backend/cloud/service.ts:588`. -- Evidence: credential validation strips trailing slashes for comparison but - returns the original URL. Modal dispatch compares that original string to - the slash-free endpoint. A local parse probe accepted - `https://api.meta.ai/v1/`, retained the slash, and failed the exact cloud - canonical comparison. Preflight checks image/secrets, not this URL; `start` - allocates, uploads and saves the dispatching fence before calling dispatch. - A valid imported credential can thus leave a paid sandbox and fenced workspace - for a configuration inconsistency knowable before any cloud action. -- Fix: normalize the accepted URL in the credential boundary and enforce the - same canonical rule before allocation/upload for non-shell callers. Keep - endpoint restrictions; do not broaden allowed hosts. -- Proof test: accepted slash variants normalize identically for Meta/Baseten; - an invalid route fails an offline cloud start before allocate/upload. Existing - `credential-files.test.ts` lacks this cross-boundary case. - -## Packaging and development startup - -### P-08 — Entire PDF.js legacy distribution ships for one worker module - -- Category: perf. Severity: medium. Verdict: CONFIRMED. Size: S. -- Files: `scripts/prepare-pdf-assets.ts:66`, `electron-builder.json:22`, - `backend/documents/pdf-worker-main.ts:31`. -- Evidence: preparation recursively copies `legacy/`; builder includes all - staged assets. The installed pinned distribution contains 16,493,936 bytes - under `legacy`, of which 9,540,392 are source maps. The runtime's explicit - external legacy asset is `build/pdf.worker.mjs` (2,418,244 bytes). The main - PDF.js module is already bundled into the compiled worker. Whole-repository - ripgrep and build config inspection found no application loader for the - legacy web viewer, its images/CSS, declarations, minified duplicates, webpack - wrapper or source maps. These increase staging, archive/compression work and - installed size on all three CI platforms, including macOS. No elapsed-time - saving or compressed-size saving is claimed. -- Fix: stage an explicit runtime asset allowlist: retain fonts, CMaps, WASM, - the worker module and licenses; exclude unused viewer/development variants. - Verify any image-decoder dependency before removing it. At least the measured - 9.54 MB of source maps can be excluded without a runtime feature change. -- Proof test: packaged PDF imports/renders with fonts, rotated pages and image - codecs succeed without source `node_modules`; inspect archive contents/size. - Existing packaged flows exercise rendering but do not assert an asset budget. - -### P-09 — Modal and its transitive packages are also shipped inside Electron - -- Category: perf. Severity: low. Verdict: CONFIRMED. Size: S. -- Files: `package.json:39`, `package.json:47`, `electron-builder.json:8`. -- Evidence: Modal is a production dependency. The only SDK import sites found - by whole-repository ripgrep are Bun backend/tooling/tests; Electron imports - only shared schemas for Modal configuration. `build:backend:main` compiles - dependencies into the backend executable without externalizing Modal. - Electron Builder 26.15.3's installed `platformPackager.js` separately adds - production node-module file sets, even with the positive application `files` - allowlist. Its runtime package copy is unnecessary for the compiled backend. - Modal itself depends on cbor-x, long, nice-grpc, protobufjs, smol-toml and uuid. - This is a redundant packaged copy, not an unused application SDK. -- Fix: move Modal to build/dev dependencies, preserving the pin/lock and all - backend functionality. Keep Effect/electron-effect-rpc production packages - because the desktop bundle externalizes them. -- Proof test: inspect a built asar for the absent Modal dependency tree and - run packaged startup plus offline cloud tests. Existing packaged tests do not - check dependency contents; a package build was not run in this audit. - -### P-10 — Development startup compiles a backend it never launches - -- Category: tooling. Severity: low. Verdict: CONFIRMED. Size: S. -- Files: `scripts/desktop-dev.mjs:12`, `desktop/main.ts:99`, `package.json:33`, - `package.json:39`. -- Evidence: every `bun run dev` runs the complete `build:backend`, including - compilation of `dist/backend/app-backend`. The unpackaged launch path then - starts `bun --no-orphans backend/main.ts`, not that executable. It does use - the compiled PDF worker and staged assets. Ripgrep/build-config checks - confirmed the app-backend executable is needed for packaged/compiled proof, - not this development path. -- Fix: have the development preparation build the PDF worker/assets and desktop - entrypoints only; retain full backend compilation in production build/verify. -- Proof test: a clean-output development launch reaches readiness and imports - a PDF without `dist/backend/app-backend`; normal packaged build still includes - and launches it. No measured startup-time reduction is claimed. - -## Agent tooling and documentation - -### P-11 — Local lint-rule regression entrypoint still invokes pnpm - -- Category: tooling. Severity: low. Verdict: CONFIRMED. Size: S. -- Files: `tools/oxlint/anti-slop/rules/require-readable-spacing-cli.test.ts:15`, - `package.json:25`, `package.json:28`, `bunfig.toml:9`. -- Evidence: the CLI regression calls `spawnSync("pnpm", ["exec", "oxlint", ...])`. - There is no pnpm installation/declaration in either CI workflow. Application - tests are deliberately rooted at `tests/smoke`, lint ignores `tools/oxlint`, - and whole-repository script/workflow search found no explicit rule-test - command. An agent changing a local rule has no documented Bun-only way to - run this CLI regression; normal verify does not execute it. This does not - imply that every vendored test belongs in the ordinary application suite. -- Fix: change the local regression's executable to the pinned Bun/Oxlint route; - add a focused opt-in rule-test command and document it for local customization - work. Preserve vendored licenses/provenance and avoid unrelated rule changes. -- Proof test: run the CLI regression with no pnpm on PATH; bad input is rejected, - `--fix` produces the expected files, and a second pass is stable. Existing - assertions already cover those behaviors but the launcher is stale. - -### P-12 — Current cloud compatibility instructions contradict the implementation - -- Category: docs. Severity: medium. Verdict: CONFIRMED. Size: S. -- Files: `docs/data.md:175`, `docs/cloud-reviews.md:395`; - references: `backend/cloud/contracts.ts:21`, `backend/cloud/progress.ts:22`, - `backend/cloud/service.ts:559`. -- Evidence: data.md calls protocol 4 / packet prompt v4 current; the cloud guide - correctly introduces protocol 5, then says new ownership receipts save - `workerProtocol: 3`. Code admits protocol 5, stamps that protocol into new - receipts, and has explicit separate recovery schemas for protocols 2–4. - These conflicting current instructions can lead an implementer to publish - the wrong worker or incorrectly narrow recovery compatibility. -- Fix: separate historical deployment evidence from current admission/recovery - rules; document new protocol-5 receipts and the actual older-worker matrix in - one current section. Retain historical evidence with its date/version. -- Proof test: compare documentation to the literal identity and all progress - schema branches; keep `cloud-reviews.test.ts` identity/recovery regressions. - Existing historical reports do not verify current documentation. - -### P-13 — Desktop test instructions describe the removed native-session route - -- Category: docs. Severity: low. Verdict: CONFIRMED. Size: S. -- Files: `docs/development.md:12`, `docs/development.md:22`, - `scripts/isolated-desktop.ts:8`, `scripts/isolated-desktop.ts:41`, - `scripts/desktop-test-environment.ts:6`. -- Evidence: development.md recommends `DISPLAY=:0` and says Windows/macOS use - native desktop sessions. The actual local runner rejects non-Linux hosts and - requires private Linux user/mount/network/PID/IPC namespaces, Xvfb, xdpyinfo - and other command-line tools; native sessions are allowed only on disposable - GitHub-hosted runners. Inherited displays are explicitly rejected/hidden. - The command table also says `check` includes a design study, but package.json - only runs typecheck, lint and format checking. These are active setup - instructions, not the clearly dated research records. -- Fix: document actual local requirements, CI-only native sessions and current - command composition. Keep the isolation protections; do not make the code - accept the obsolete instructions. -- Proof test: walk through the documented command on a prepared Linux host; - inspect the intentional early failure on a local macOS/Windows host. Existing - environment guards test enforcement, not prose accuracy. - -## Needs decision - -These entries require a product/retention choice. They are separated from fixes -that restore the existing advertised behavior. Do not silently change their -policies while implementing the entries above. - -### P-14 — The advertised one-minute cloud configuration cannot dispatch - -- Category: correctness. Severity: medium. Verdict: CONFIRMED. Size: S. -- Files: `src/lib/backend-rpc.ts:52`, `backend/cloud/service.ts:567`, - `backend/cloud/modal.ts:591`, `backend/cloud/contracts.ts:48`, - `docs/cloud-reviews.md:252`. -- Evidence: configuration accepts 1–60 minutes. The saved deadline starts before - allocation/upload, while dispatch floors the remaining time to seconds and - requires at least 60,000 ms. With a one-minute setting, even 1 ms of setup - yields 59,000 ms and refusal. The worker schema also requires a minimum - 60,000 ms. This follows directly from the three admission boundaries; no - live sandbox was started to demonstrate an arithmetic invariant. -- Decision/fix: choose whether one minute means a full minute of review after - setup (requiring coordinated deadlines/warm admission), or raise/reject the - unsupported configuration before allocation. Do not silently grant additional - paid time. Until that decision, fail this impossible setting early with a - specific explanation instead of uploading and leaving a dispatching fence. -- Proof test: offline one-minute start with nonzero allocation/upload time; - assert the chosen policy, no late surprise, and no unauthorized extra runtime. - Existing deadline-precision tests cover fractional seconds, not this setting. - -### P-15 — Terminal cloud packages accumulate without local retention controls - -- Category: perf. Severity: medium. Verdict: CONFIRMED. Size: M. -- Files: `backend/cloud/service.ts:553`, `backend/cloud/service.ts:641`, - `backend/cloud/service.ts:700`, `backend/cloud/service.ts:716`. -- Evidence: each new job gets full `input/` and later `output/` package - directories under `.cloud/`. Successful import and cancel - save a terminal receipt but do not delete either directory. Whole-repository - searches for these staging paths and removal/pruning found cache-volume and - interrupted-download cleanup, but no terminal local-package retention path. - At the supported near-1-GiB workspace size, one completed job can retain about - 2 GiB of transfer packages, apart from the intentional pre-restore backups. - The 1,000-receipt limit is not a storage budget. This is source-confirmed - accumulation; no user workspace was measured or cleaned. -- Decision/fix: define how long terminal transfer packages remain recoverable - and whether cleanup is automatic or explicit. Retain small replay/ownership - receipts; never remove active, paused, failed-but-owned or interrupted-restore - inputs/outputs. Treat pre-restore backup retention as a separate decision. -- Proof test: under the selected policy, terminal staging shrinks while command - replay remains non-dispatching and active/paused recovery remains intact - across reopen. Existing cloud-cache pruning tests concern remote cache files, - not these local directories. - -## Audit conclusions and split notes - -15 findings, all CONFIRMED for the stated defect or cost. P-03 explicitly marks -its unexercised startup-race consequence PLAUSIBLE. Categories: correctness 6, -perf 5, tooling 2, docs 2; dead-code/security/standards/slop 0. Severities: -high 2, medium 9, low 4. - -Suggested independent implementation ownership: documents P-01/P-02; backend -bootstrap P-03/P-04; cloud persistence/query P-05/P-06; credential normalization -P-07; packaging/dev/tooling P-08–P-11; docs P-12/P-13. P-14 and P-15 await policy -decisions and overlap the cloud implementer's files. - -No confirmed trusted-frame/authentication bypass was found. Loopback binding, -private startup credentials, browser-origin rejection, narrow HTTPS shell URLs, -schema boundaries and migration-hash checks are present. This is not a claim -that every security property was exhaustively proved. Existing migrations must -remain byte-for-byte unchanged. - -Dependency/export/experiment searches did not establish a worthwhile whole -feature removal. Some exports are only used within their defining module, but -removing `export` alone is trivia. Manually invoked research scripts and recorded -experiment recipes are not declared dead merely because the production app -does not import them. `@opentelemetry/api`, despite no direct app import, is an -optional SvelteKit peer; no removal is recommended without separate build proof. diff --git a/docs/research/audit-4-renderer-tests.md b/docs/research/audit-4-renderer-tests.md deleted file mode 100644 index c3fd1aa2..00000000 --- a/docs/research/audit-4-renderer-tests.md +++ /dev/null @@ -1,155 +0,0 @@ -# Audit 4: renderer and tests - -Read-only audit of `fix/audit-4`, HEAD `68deb4910`, on 2026-10-31. Source, tests and configuration were not changed. This ledger is the only written file. - -## Verification and limits - -- Read `AGENTS.md`, `PRODUCT.md`, `design/DESIGN.md`, `repos/effect/LLMS.md`, the installed coding and Svelte skills, and the prior Audit 3 and hackathon audit findings. The ignored Forge reference pack is not present in this checkout; `design/DESIGN.md` is the checked-in adaptation specification. -- Bun matches `.bun-version`: **1.4.4**. Ran `bun test ./tests/smoke` once. It ended in **119 ms: 1 pass, 49 fail, 48 errors**, overwhelmingly import failures because `node_modules` is absent. `forge-tokens.test.ts` passed in 13.90 ms; `icons.test.ts` failed in 2.32 ms because its Tabler dependency was absent. These are environment results, not application regressions or useful suite timings. -- Ran `bun run build:ui`: failed immediately with `Script not found "vite"`. **No bundle-size measurements are available.** Did not install packages, call live providers, run cloud scripts, or run Electron e2e. No claims below rely on the known WSL clock-jump failures. -- The Svelte MCP autofixer checked the complete `ReviewWorkbench`, `PacketReviewPanel`, `SourceViewer`, `FindingDecision`, `SheetThumbnail` and `ReviewSelect` components against 5.57.2. All returned zero compiler issues. Workbench and picker returned generic suggestions; these are not treated as findings. An initial truncated workbench submission produced a parse error; resubmitting the complete file removed it. -- Executed two dependency-free harnesses with Bun's TypeScript transpiler, extracting the actual workbench function bodies. They reproduced R-01 and R-02. These establish handler behavior, not a complete Electron reproduction. -- `CONFIRMED` means the cited defect or test limitation is established by source tracing or those harnesses. It does not mean the full test suite passed. Runtime severity and performance measurements still need verification after dependencies are available. - -**13 findings:** correctness 4, perf 2, dead-code 1, slop 2, useless-test 1, test-gap 1, slow-test 2. Severity: high 3, medium 9, low 1. All 13 are CONFIRMED, with the measurement limits above. - -## Workbench implementation, behavior-preserving work - -### R-04: Every packet update launches another complete project snapshot - -- **Category:** perf. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **Files:** `src/lib/review/components/ReviewWorkbench.svelte:469`, `:481`, `:791`, `:809`; `backend/packet-review/service.ts:3921`; `backend/review/store.ts:229`, `:257`. -- **Evidence:** Every `onReview` calls `void refresh(projectId)`, including each existing review replayed when a subscription starts. There is no coalescing or in-flight cancellation. `refreshRequest` discards superseded responses only after their RPC work completes. `afterSequence` limits events, but the snapshot still reads and decodes every revision, sheet, artifact, finding, fact, check, conversation and message. Applying it replaces all these renderer collections and rebuilds the accumulated event array. Starting a subscription with N saved reviews therefore schedules N full snapshot calls. A progress burst also schedules one call per update. This is verified work amplification; no frame-time or latency improvement is claimed without a runnable build. -- **Concrete fix:** Coalesce snapshot invalidation per selected-project generation. Allow one request in flight and one pending refresh. Packet-only progress should update the packet state without fetching unrelated records; refresh messages and evidence on their relevant transitions. Keep R-01's project-generation protection in the same implementation. -- **Proof test:** Replay many saved reviews and emit a controlled progress burst while holding the first snapshot response. Assert bounded snapshot requests, eventual newest messages/evidence, and no application of the previous project's data. Measure snapshot traffic and interaction latency on a large saved project. - -### R-07: Source-view ownership is duplicated inside the 4,406-line workbench - -- **Category:** slop. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **Files:** `src/lib/review/components/ReviewWorkbench.svelte:493`, `:717`, `:1244`, `:1383`, `:1481`, `:1863`, `:2720`. -- **Evidence:** `openPacketEvidence` and `loadSheetSource` separately implement loading state, blob ownership, request-generation checks, stale-response disposal, error presentation, zoom initialization and overlay setup. Finding selection, import completion, close and component teardown each repeat pieces of invalidation and disposal. The component contains 1,870 lines of script plus source-panel markup and the rest of the application. The issue is repeated ownership logic, not an assumption that Svelte rerenders the entire component on every change. Current blob cleanup is present and is not reported as a leak. -- **Concrete fix:** Extract a source-view controller with `openArtifact`, `renderRegion`, `close` and `dispose`, sharing a single generation and owned URL. Extract the source-panel markup into a component taking its selected finding/sheet and controller actions. Keep snapshot/subscription orchestration together for R-01/R-04. Do not split by arbitrary line counts or introduce a general store framework. -- **Proof test:** Retain existing source-selection e2e tests and add delayed artifact/render completion after close, finding switch and project switch. Verify no stale image appears and each created URL is released once. This refactor must preserve selection geometry, zoom and evidence identity. - -## Packet coverage panel - -### R-05: Coverage grouping repeatedly scans the full ledger for every sheet - -- **Category:** perf. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **Files:** `src/lib/review/components/PacketReviewPanel.svelte:159`, `:170`, `:188`, `:197`, `:204`, `:212`, `:446`, `:566`; `src/lib/review/packet-coverage.ts:110`. -- **Evidence:** `coverageSheets` filters `matchingCoverage` for each sheet and filters `detailCoverage` again for its counts. Whole-set rows scale with sheets × disciplines × four passes, so these scans grow quadratically with sheet count. The search query also invalidates this grouping. The outer native `
` does not conditionally mount its contents, so its closed state does not defer these calculations; `matchingCount` and the sheet list still consume them. Pagination limits records within an expanded sheet, not grouping work. This is an algorithmic finding, not a measured frame-rate claim. -- **Concrete fix:** Build a per-sheet coverage index and counts in one pass when coverage changes. Apply discipline/status/search filters to that indexed representation. If deferring closed details, preserve filter state and accessible summaries. A focused `PacketCoveragePanel.svelte` is a useful split because it owns these filters, pagination and grouping independently of run controls. -- **Proof test:** Compare filtered counts and rows with existing coverage expectations across scopes and earlier coverage. Use a large synthetic packet to verify one grouping pass per coverage change, and benchmark typing in sheet search with the panel open and closed. - -## Retired conversation submission code - -### R-06: The old count-review send module has no production callers - -- **Category:** dead-code. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** S. -- **Files:** `src/lib/review/conversation.ts:27`, `:61`; `tests/smoke/review-conversation.test.ts:33`, `:107`, `:485`, `:562`; `src/lib/review/components/ReviewWorkbench.svelte:959`, `:1041`. -- **Evidence:** Repository-wide `rg` for `sendReviewMessage`, `sendCloudReviewMessage` and `conversation.ts`, excluding the reference repository, found the definitions and only smoke-test consumers of `sendReviewMessage`. `sendCloudReviewMessage` has no caller at all. The real composer uses `postMessage`/`startDrawingReview` and packet APIs. Consequently the smoke test named “a real conversation send routes Moke disciplines…” exercises the retired helper, not the current composer. Its backend persistence/routing assertions still have value; the whole test is not useless. -- **Concrete fix:** Delete the unused production module. Preserve applicable legacy execution tests by calling the supported application operations directly or moving genuinely necessary fixture composition into tests. Add current packet-composer coverage rather than retaining a production API solely for tests. -- **Proof test:** `rg` shows no remaining imports of the deleted module; typecheck and retained legacy execution tests pass. Exercise current local/cloud packet submission, message identity and start-failure behavior through the actual composer. - -## Test organization - -### T-01: Audit-round filenames mix unrelated behaviors and obscure ownership - -- **Category:** slop. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **Files:** `tests/smoke/audit-2.test.ts:20`, `:65`, `:125`, `:415`, `:572`, `:662`; `tests/smoke/audit-3-runtime.test.ts:21`, `:121`, `:229`, `:343`, `:413`, `:478`; `tests/smoke/audit-3-documents.test.ts:30`, `:83`, `:356`; `tests/smoke/audit-3-workbench.test.ts:9`, `:23`; `tests/e2e/audit-2.spec.ts:13`, `:133`, `:311`; `tests/e2e/audit-3-workbench.spec.ts:7`; `tests/e2e/audit-3-sheets.spec.ts:12`, `:196`, `:246`, `:308`, `:429`. -- **Evidence:** These are active, often substantial regression tests. `audit-2.test.ts` alone combines display guards, drop claims, packet lifecycle, scope assignments, subscription recovery and steering. `audit-3-runtime.test.ts` combines provider diagnostics, runtime controls, absence checks, image budgets, coverage gates and crop metadata. Their filenames provide no behavioral ownership and encourage adding unrelated regressions to the same file. Do not delete these suites as “useless.” -- **Concrete fix:** Move smoke cases into `desktop-test-environment.test.ts`, `review-drop.test.ts`, `packet-lifecycle.test.ts`, `packet-steering.test.ts`, `packet-routing.test.ts`, `packet-subscription.test.ts`, `packet-provider.test.ts`, `packet-evidence-policy.test.ts`, `packet-coverage.test.ts`, `documents.test.ts`, `review-packages.test.ts` and `source-url.test.ts` as appropriate. Give e2e cases behavior owners such as `source-viewer.spec.ts`, `sheet-setup.spec.ts`, `revision-mapping.spec.ts`, `conversation-navigation.spec.ts` and `conversation-scroll.spec.ts`; merge overlapping setup with existing `polish.spec.ts`/`packet.spec.ts` only where it reduces duplication. Coordinate these moves with T-04/T-05 before editing shared files. -- **Proof test:** Preserve the collected test cases and assertions through the move; run all smoke tests and affected Electron flows. No audit-round test filenames remain. - -## PDF cancellation test - -### T-02: “Cancels in-flight PDF work” never proves worker work was cancelled - -- **Category:** useless-test. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **Files:** `tests/smoke/documents.test.ts:303`; `backend/documents/pdf-worker-main.ts:261`. -- **Evidence:** The test sets a 500 ms worker delay, forks import, sleeps 50 ms, interrupts the fiber and asserts only `Exit.isFailure(exit)`. It does not establish that the worker entered the job, inspect the interruption cause, observe process termination or check for late output. An immediate import failure satisfies the assertion. Removing underlying worker cancellation while keeping the parent fiber interruptible also satisfies it. Thus it cannot catch the worker-cancellation regression claimed by its name; the asserted parent-fiber failure has little independent value here. -- **Concrete fix:** Rewrite in `documents.test.ts`. Await an observable worker/job-start handshake, interrupt the import, verify an interruption exit and verify the owned worker terminates or releases the job without late artifacts. Assert a subsequent document operation succeeds so cancellation cannot silently wedge capacity. Replace the arbitrary 50 ms sleep with the handshake; TestClock alone cannot prove a native child started. -- **Proof test:** Deliberately disable worker abort/cleanup and confirm the rewritten test fails even though the parent fiber still reports interruption. Also make import fail before dispatch and confirm it cannot pass as a successful cancellation test. - -## Initial Review coverage - -### T-03: No test actually clicks Initial Review or proves its fixed submission contract - -- **Category:** test-gap. **Severity:** high. **Verdict:** CONFIRMED. **Size:** M. -- **Files:** `tests/e2e/initial-review.spec.ts:8`, `:71`; `tests/smoke/initial-review.test.ts:8`, `:16`, `:27`; `src/lib/review/components/ReviewWorkbench.svelte:1114`. -- **Evidence:** The e2e case checks visibility, enabled state, accessible description, split imports and removal after correspondence, but never calls `initial.click()`. The smoke cases validate static text, schema acceptance and phase calculation. `rg` across all tests found no other preset/Initial Review execution coverage. The button could have a no-op handler, use the wrong model, omit a first-set PDF, copy the optional composer budget or lose preset identity without these tests failing. -- **Concrete fix:** Extend the behavior-owned initial-review suite with an offline local provider/transport fixture and click the actual button. Verify one saved preset message and one admitted packet review containing every sheet from two initial PDFs, Muse Spark 1.3, null call budget and `initial-review-v1`. Separately exercise cloud selection and local fallback at an injected transport boundary, including duplicate-click protection and admission failure. Keep existing visibility tests; they are useful for a different contract. -- **Proof test:** Each of the wrong-model, omitted-PDF, inherited-budget and no-op-handler mutations must fail a relevant assertion. Reopen and verify persisted preset/message/review identity. No paid provider is needed. - -## Subscription recovery timing - -### T-04: The reconnect regression necessarily waits seven real seconds - -- **Category:** slow-test. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **Files:** `tests/smoke/audit-2.test.ts:572`, `:601`, `:638`, `:660`; `src/lib/review/packet-subscription.ts:5`, `:55`. -- **Evidence:** The test deliberately ends the first three subscriptions and waits for the fourth. Production `followPacketReviews` uses native `setTimeout` backoffs of 1, 2 and 4 seconds before that fourth connection. Therefore the test has a seven-second timer floor, plus fixture/SQL/provider work, and allows 80 seconds. This is derived from the exercised control flow, **not a measured timing from this checkout**; the dependency failure prevented execution. The regression itself is valuable. -- **Concrete fix:** Move the case to `packet-subscription.test.ts` under T-01. Make reconnect delay/clock an owned dependency, preferably Effect sleep/clock under the existing runtime, and drive it with TestClock. A small injected clock is also valid if the helper remains Promise-based. Keep one real persistence integration case without paying every backoff interval. -- **Proof test:** Advance 999 ms, 1 ms, 2 s and 4 s explicitly; assert no early reconnect, each expected reconnect, durable missed-state replay, warning recovery and no reconnect after abort. The test should finish without seven seconds of wall-clock waiting. - -## Provider and active-duration timing - -### T-05: Fixed sleeps substitute for readiness and elapsed-time control - -- **Category:** slow-test. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** M. -- **Files:** `tests/smoke/packet-provider.test.ts:459`, `:467`, `:494`, `:499`; `tests/smoke/audit-3-runtime.test.ts:44`, `:100`. -- **Evidence:** Provider cases use an injected pacer clock but still sleep 10/20 ms before asserting admission counts or advancing that clock. If the asynchronous SDK work has not reached acquisition/refusal by then, the assertion or subsequent timeline is wrong. The active-duration cases sleep 25 ms to create paused/cancelled time despite the application using Effect DateTime. These sleeps are confirmed; slowest-test rankings are unavailable. This finding does not classify the explicitly excluded review-conversation/cloud-activity clock failures as regressions. -- **Concrete fix:** Expose a test seam or Deferred signaling acquisition, refusal and next admission in the provider fixtures, then advance the pacer clock only after those signals. Drive active-duration tests through Effect TestClock, using a deliberately large paused interval to prove exclusion. Coordinate the latter moves into `packet-lifecycle.test.ts` under T-01. -- **Proof test:** Add artificial scheduling delay before acquisition/refusal and verify tests remain deterministic. Advance minutes of paused time instantly and verify active duration is unchanged. No fixed real sleep should be necessary for either assertion. - -## Checked areas without separate findings - -- No renderer imports of backend services, Bun APIs or `repos/` were found. `src/lib/backend-rpc.ts` contains private startup schemas, but a schema declaration under `src` is not itself a credential leak; no renderer entrypoint import of it was found. Renderer RPC uses shared Effect Schema contracts. Relative sibling imports exist, but import spelling alone is omitted as a style nit. -- No unused Svelte component was confirmed. No `nightdesk` or `--nd-` references remain in `src`. Six Forge declarations have no consumers: `--color-surface-sunken`, `--color-focus`, `--color-ember-wash`, `--leading-tight`, `--space-xl`, `--space-2xl` at `src/lib/forge.css:17`, `:24`, `:30`, `:47`, `:61`, `:62`. They are optional token cleanup, not a separate impact finding or evidence that Nightdesk remains. -- Source/thumbnail URLs, IntersectionObserver, ResizeObserver and component timers have cleanup paths. Important collection loops use keys. Index keys on immutable text fragments/overlays alone do not establish a defect. Packet panel timers continue in terminal views, but are a minor issue compared with R-04/R-05. -- Poppler-dependent tests in `adaptive-review.test.ts` and `segmentation.test.ts` use explicit `skipIf(Bun.which("pdftoppm") === null)` guards. They exercise external CLI rendering, so the guards have a real reason. CI does not explicitly install Poppler; ensure a designated lane runs them if that CLI remains supported. This checkout's import failures prevented a reliable skip count, and hosted runner availability was not checked. -- The stale-promotion half of `review-execution.test.ts:466` uses a fixture-owned fence implementation, but `review-fences.test.ts:24` separately exercises real `createReviewOperations`/`ReviewStore` promotion. The former still verifies runtime handling of a rejecting port; it is not labeled useless or counted as a missing production fence test. -- `playwright.config.ts` uses one worker; desktop script selection keeps the screenshots project separate. No unsupported claim of duplicate CI execution is made. Fixtures generally use real SQLite/PDF/service operations with injected external transports, which is a useful test boundary. - -## Needs decision - -These findings change visible behavior when fixed. Recommended outcomes are given explicitly. R-01 and R-03 correct violations of existing project/source ownership; they do not require a new product feature. R-02 needs a draft-preservation policy. Group these with the workbench work above when assigning implementers. - -### Workbench selection, composer and preview behavior - -### R-01: Previous-project progress can replace the newly selected project's snapshot - -- **Category:** correctness. **Severity:** high. **Verdict:** CONFIRMED. **Size:** M. -- **Files:** `src/lib/review/components/ReviewWorkbench.svelte:450`, `:469`, `:481`, `:791`, `:825`, `:918`, `:934`. -- **Evidence:** `selectProject(B)` changes `selectedProjectId` and starts `refresh(B)` without aborting A's subscription. That subscription is replaced only after a snapshot is applied. A subsequent A callback calls `receivePacketReview`, which correctly rejects A, **then unconditionally calls `refresh(A)`**. Its newer request number supersedes B's fetch. `refresh` checks only the request number and never verifies the requested/result project against the selected project. It can apply A's snapshot, choose A's conversation, and discard B's response. The dependency-free harness extracted the actual `refresh` body and resolved this ordering; output was `{"selectedProjectId":"B","snapshotProject":"A","selectedConversationId":"conversation-A","appliedB":false}`. The composer and sheet actions take `project` from the snapshot, so this is an ownership problem, not just a breadcrumb glitch. -- **Concrete fix:** Abort/invalidate A's subscription at selection time; ignore callbacks unless their subscription/project generation is still selected; reject snapshot application for a stale selection generation even if it is the newest RPC request. Handle initial null-project loading explicitly. Fence `selectProject`'s post-await conversation selection as well, so rapid B→C navigation cannot apply B's requested conversation to C. -- **Proof test:** Hold B's snapshot at the real RPC boundary, deliver A progress, then release responses in both orders. Assert project, conversation, sheets, composer target and subscription all remain B. Repeat B→C with explicit conversation choices and delayed snapshots. Existing packet source/menu delay tests do not cover this ordering. -- **Recommended visible result:** Navigation stays on the project the user most recently selected. - -### R-02: Saving a message erases text typed while its acknowledgement is pending - -- **Category:** correctness. **Severity:** medium. **Verdict:** CONFIRMED. **Size:** S. -- **Files:** `src/lib/review/components/ReviewWorkbench.svelte:959`, `:990`, `:1028`, `:1032`, `:1060`, `:2532`, `:2554`. -- **Evidence:** The composer textarea remains enabled while `busy` is true; its disabled condition checks project, conversation and cloud state only. Successful note save, steering delivery and the review `onSaved` callback clear `draft` unconditionally after awaiting RPC. A user can send text, type their next text while the save is pending, and lose the new draft when the first response arrives. Extracting the actual `postMessage` body and holding `runCommand` reproduced this for note mode: changing draft to `Second note still being written` before resolving the save returned `{"draft":"","notice":"Note saved."}`. Navigation being disabled while busy does not prevent editing the textarea. -- **Concrete fix:** Clear the submitted draft synchronously and restore it on failure only if no newer draft exists, or record a draft revision and clear only when it still matches the submitted revision. Preserve all text added after submission. Apply the policy consistently to note, initial packet send and steering paths. -- **Proof test:** Hold each send acknowledgement, type another draft, release success and failure, and verify the new text survives while the first message is saved exactly once. Include edits that restore the same string so a revision-based solution is tested correctly. -- **Decision:** Prefer allowing continued composition and preserving the new draft. Disabling the textarea during save is simpler but changes the current interaction and needs explicit product agreement. - -### R-03: Reopening previews creates durable artifacts until the project hits its record ceiling - -- **Category:** correctness. **Severity:** high. **Verdict:** CONFIRMED. **Size:** M. -- **Files:** `src/lib/review/components/ReviewWorkbench.svelte:1265`, `:1271`, `:1383`, `:1411`; `src/lib/review/components/SheetThumbnail.svelte:14`, `:21`; `backend/review/operations.ts:54`, `:65`, `:99`, `:107`; `backend/review/store.ts:146`, `:150`, `:261`, `:1164`, `:1197`. -- **Evidence:** Every thumbnail/source load generates a fresh random `operationId`. `Review.render` reuses an artifact only by that operation ID; otherwise it renders again and creates a fresh artifact ID. `saveRender` persists that new record. Sheet thumbnails reload when the Sheets or Revision pane remounts, so revisiting the same images grows durable artifact records even when their PNG hash is unchanged. `records` rejects more than 2,000 records per project/kind, and every normal snapshot reads artifacts through it. At 2,001 artifacts, subsequent snapshots fail with “This project exceeds the workbench record limit…”. Ordinary viewing can therefore make the project unreadable in the workbench. Content-addressed PNG storage does not deduplicate these metadata records. This consequence is verified by the source path; a 2,001-render runtime stress test was not run. -- **Concrete fix:** Reuse immutable preview identity keyed by project, revision/source hash, page, region, rotation and effective DPI, rather than issuing a new durable evidence operation per view. Reuse existing matching artifacts across pane remounts and app restarts. Keep separately identified model evidence/history intact. Coordinate any backend preview-cache change with the platform auditor; a renderer-only in-memory cache would not protect across restarts. -- **Proof test:** Repeatedly open/close sheets and revision comparisons and navigate the same source at the same DPI; assert stable artifact-record count and no extra render after cache warmup. Reopen the app and repeat. Change source revision/region/DPI and verify correct fresh pixels. Seed near the limit and demonstrate ordinary repeated viewing does not cross it. -- **Recommended visible result:** Browsing the same source remains repeatable and does not consume the project's durable record allowance. Do not silently delete existing evidence to solve this. - -### Shared select control - -### R-08: An already-open select menu ignores later changes to disabled - -- **Category:** correctness. **Severity:** low. **Verdict:** CONFIRMED. **Size:** S. -- **Files:** `src/lib/review/components/ReviewSelect.svelte:43`, `:84`, `:124`, `:130`; `src/lib/review/components/ReviewWorkbench.svelte:2575`; `src/lib/review/components/FindingDecision.svelte:40`. -- **Evidence:** `disabled` is applied to the trigger only. The open popover persists when that prop changes; each option uses only `option.disabled`, and its click handler invokes `onChange` without checking the component's disabled state. Asynchronous review/lock updates can therefore disable the displayed selector while its already-open menu still accepts a choice. This is a UI consistency issue, not a demonstrated backend authorization bypass. -- **Concrete fix:** Close the popover when the control becomes disabled, disable its options with `disabled || option.disabled`, and guard activation. Avoid stealing focus during an asynchronous disable. -- **Proof test:** Open the menu, flip disabled through a real component parent update, and attempt mouse/keyboard activation. Assert no change event and no active menu, then re-enable and verify normal selection. -- **Recommended visible result:** Disabled selectors stop accepting choices immediately, including through an already-open menu. diff --git a/docs/research/audit-4.md b/docs/research/audit-4.md deleted file mode 100644 index 0ed6fcce..00000000 --- a/docs/research/audit-4.md +++ /dev/null @@ -1,37 +0,0 @@ -# Audit 4 - -Read-only audit of main at `68deb4910` on 2026-10-31 by three gpt-6-astra -auditors on xhigh. Each wrote one ledger: - -- [Inference and review harness](audit-4-harness.md): H-01 to H-12. -- [Platform, desktop and scripts](audit-4-platform.md): P-01 to P-15. -- [Renderer and tests](audit-4-renderer-tests.md): R-01 to R-08, T-01 to T-05. - -## Decisions - -- **H-12, automatic SAM zoom on every finding:** kept. PR #21 made per-finding - zooms the intended behavior. H-03 still isolates a failed zoom to its sheet. -- **R-01, R-03, R-08:** fixed as the ledger recommends. They restore existing - ownership rules and change no intended behavior. -- **R-02, draft text:** the composer stays editable, and text typed while a send - is pending is kept. -- **P-14, one-minute cloud timeout:** the impossible setting now fails before - any allocation or upload, with a specific message. The setting's range is - unchanged until the user decides between a longer minimum and a full minute - of review after setup. -- **P-15, terminal cloud package retention:** not changed. It needs a retention - policy from the user. - -## Work split - -Fixes were implemented by Claude Opus 5.5 in separate worktrees, grouped so no -two implementers edited the same source file: - -| Owner | Findings | -| ------------------------------------------ | ---------------------------------------------- | -| Packet service | H-01, H-02, H-03, H-04, H-11 | -| Provider and pacing | H-07, H-08, H-09, H-10, T-05 (provider tests) | -| Review operations, store and documents | H-05, H-06, R-03, P-01, P-02, T-02 | -| Renderer | R-01, R-02, R-04, R-05, R-06, R-07, R-08, T-03 | -| Platform | P-03 to P-14 | -| Test organization, after the others merged | T-01, T-04, T-05 (active-duration tests) | diff --git a/docs/research/beta-implementation.md b/docs/research/beta-implementation.md deleted file mode 100644 index 966be978..00000000 --- a/docs/research/beta-implementation.md +++ /dev/null @@ -1,172 +0,0 @@ -# Moke beta implementation - -The beta extends the selected workbench design with persisted projects, -conversations, PDF sources, and bounded drawing inspections. Chat remains the -main work surface. The product scope is one selected drawn-to-schedule count -check at a time. A discipline choice supplies relevant instructions within that -boundary; it does not review the rest of the discipline or permit packet. - -This account follows the [product context](../../PRODUCT.md), -[implementation](../../src/lib/review/components/ReviewWorkbench.svelte), and -[project manifest](../../.project-kit/manifest.json). It records implemented -behavior. The [validation record](beta-validation.md) contains acceptance proof -and remaining limits. - -## Work surface and review flow - -Projects group their conversations in the left navigation. The breadcrumb names -the active project and conversation. Selecting a conversation restores its draft -and scroll position during the session. The top-right Findings popover holds the -current project's findings. Selecting one opens evidence on the right and keeps -the conversation visible; closing it returns that space to chat. Search filters -the project's findings. Ctrl/Cmd K opens search, and Ctrl/Cmd N creates a -conversation in the selected project. - -1. Create a project and import a drawing PDF through the desktop file dialog. - Import retains the original and opens the Sheets panel. It does not establish - that a sheet has been reviewed. -2. View each source, confirm its sheet label, and choose Unreviewed, Partial, or - Reviewer complete coverage. [Sheet setup](../../src/lib/review/components/SheetSetup.svelte) - exposes those choices explicitly. -3. Create a coordination check with a title, one object class and bounded room - or area, a confirmed drawn sheet, and a confirmed scheduled sheet. Confirm - that both sources describe the same object type and review boundary. -4. Choose the check and discipline in the composer. Review with Moke saves the - reviewer message before requesting an inspection. Save note records a message - without starting model work. Failed provider admission reports that the - message was saved and does not resend it automatically. -5. Inspect the proposed observations in Findings. Open a proposal's source - region or request a 300 DPI detail. Use proposal as draft copies the count, - statement, rationale, and region into editable fields and leaves coverage - unconfirmed. The reviewer accepts each count and confirms scope coverage. -6. Evaluate coordination compares the accepted drawn and scheduled counts. - Missing accepted evidence or coverage remains Needs evidence. Agreement or - disagreement describes the selected count comparison. Reviewer closed and - Completed task remain separate, explicit reviewer controls. - -[Conversation admission](../../src/lib/review/components/ReviewWorkbench.svelte), the -[finding inspector](../../src/lib/review/components/FindingInspector.svelte), and -the [deterministic check](../../src/lib/review/checks.ts) enforce these separate -steps. Model messages and completed inspections do not accept facts, confirm -coverage, close findings, or complete tasks. - -## Routed Moke instructions - -The embedded instructions come from `sc-frederick/skills/moke-review` revision -`8a0e2648738018631eaa6d24ee0b6ebaa239afb1`. The -[provenance manifest](../../backend/review/prompts/provenance.json) records the -unmodified source files and hashes. Prompt version -`moke-selected-coordination-v2` combines the master instructions, shared method, -General, and the selected discipline. The shared method's Report and release -section is excluded from model instructions. - -The [router](../../backend/review/prompts/review-prompt.ts) supports eleven -choices: General, Civil, Building design, Life safety, Structural, Mechanical, -Electrical, Plumbing, Fire alarm, Fire sprinkler, and Fire protection. The four -fire responsibilities remain separate routes. Each run freezes the reviewer -request, conversation, discipline, prompt version, and instruction revision. -Responses return to that conversation, and proposals retain provider, model, -response, and instruction provenance. See the -[prompt contract](../../backend/review/prompts/README.md). - -## Execution and beta boundary - -Review settings expose the configured provider and model without credentials. -The desktop imports credential configuration through its File menu. The local -Bun engine owns APIs, SQLite, and Pi Durable journals; Electron owns the trusted -bridge and file dialogs. Local execution can still send selected drawing -evidence to hosted Meta or Baseten inference. - -The UI offers limits of 4, 8, or 12 calls, with 12 selected initially. Each model -request reserves up to 64,000 text-input tokens and 4,096 output tokens. SAM, -when configured for the route, has a limit of two requests and proposes object -locations. Image-token usage and billing are unknown. Pause, resume, cancel, -state, and admitted calls are visible in the -[execution controls](../../src/lib/review/components/ExecutionControls.svelte). - -The model can inspect the selected PDF sheets, request original-PDF crops, -propose counts, ask for missing input, and evaluate accepted counts. It has no -browser or verified governing-code library. Printed drawing citations do not -verify an adopted edition or a regulatory claim. Relevant missing jurisdiction, -permit stage, code text, calculations, or prior comments remain Needs input. -This beta does not verify governing code, complete a permit or discipline -review, or generate a complete permit-review report. The offline fixture -worksheet is a separate report path. Modal execution remains deferred and -optional in the [runtime decision](../review-runtime.md). - -## Evidence, revisions, and recovery - -Evidence stays attached to original source sheets and PDF regions. The panel -shows the revision, page, render DPI, and source hash alongside the observation. -Accepted facts and evaluations retain versioned history. Corrections invalidate -dependent current results rather than erasing earlier evidence. - -Importing another PDF revision retains earlier originals. The Revision panel -maps old sheets to new sheets and asks the reviewer to confirm correspondence -and whether content changed. Confirmed correspondence is one-to-one and -immutable. Reusing baseline facts requires confirmed unchanged correspondence; -changed or unresolved inputs need fresh evidence. The -[data lifecycle](../data.md) defines the source and fact version checks. - -Reopen does not start model work automatically. Safe interrupted runs stay -paused until explicit resume. Unknown provider or SAM outcomes retain uncertain -evidence and are never resent automatically. An acknowledgement of a possible -prior charge permits a separate new run; it does not clear the earlier evidence. -Runs from incompatible engine or instruction versions cannot resume under the -new version. See [execution safety](pi-runtime.md). - -Export complete package includes the whole local application database, source -artifacts, review history, and closed execution journals. It excludes -credentials, logs, environment files, and the ownership lock. Restore validates -the package before replacement and retains a pre-restore snapshot. Startup -recovers interrupted replacement before opening databases. Restored runs remain -historical and cannot resume copied journals. A database-only export is not a -complete review backup. See [data recovery](../data.md) and -[package format](review-packages.md). - -## Existing design and responsive behavior - -The workbench follows the Forge design system recorded in -[the design notes](../../design/DESIGN.md) and pinned by the -[selection record](../../design/selection.json). Panels, wells and hairlines use -Forge's Graphite, Slate and Filament tokens over an Ink canvas, with Ember for -actions, focus and the current selection. Conversation text uses 14px Inter with -a 1.6 line height and a reading width of 68ch. Controls and supporting text -generally use 13px. Keyboard focus shows an Ember ring, and the running -indicator respects reduced motion. - -At widths from 701px through 1180px, opening evidence hides the sidebar to retain -chat width; the navigation toggle can reveal it as an overlay. At 700px and -below, navigation and evidence use dismissible overlays. Escape closes search, -navigation, or the source pane as appropriate; closing the source returns focus -to Findings. Narrow headers hide secondary context instead of shrinking text. -These are implemented behaviors, with live responsive acceptance still pending. - -The licensed Forge pack stays in the ignored `design/reference/forge/` directory. -Its license permits adapted application use and prohibits republishing or -repackaging the design-system pack, so the reference source stays outside public -distributions. - -## Validation - -Use Bun `1.4.4` from [.bun-version](../../.bun-version). Completed automated checks, -bounded live acceptance, responsive review, and actual limitations are recorded -in [beta validation](beta-validation.md). Council results belong to the pull request. -Relevant commands from [package.json](../../package.json) are: - -```sh -bun test tests/smoke/review-conversation.test.ts tests/smoke/review-execution.test.ts tests/smoke/review-fences.test.ts tests/smoke/review-packages.test.ts -bun run test:e2e tests/e2e/review.spec.ts -bun run verify -``` - -With the intended provider configured, the desktop review test also exposes an -explicit live path: - -```sh -MOKE_E2E_LIVE=1 bun run test:e2e tests/e2e/review.spec.ts -``` - -Record its actual provider receipts, accepted evidence, remaining uncertainty, -and council disposition separately. Synthetic transport tests and standalone -design-study builds do not establish live model acceptance. diff --git a/docs/research/beta-validation.md b/docs/research/beta-validation.md deleted file mode 100644 index 3b941f79..00000000 --- a/docs/research/beta-validation.md +++ /dev/null @@ -1,123 +0,0 @@ -# Beta validation - -Validated October 28, 2026 against the integrated beta branch. The live run used -`a628508ae`; UI corrections were verified on `7b4c429e7`. The application scope is -selected drawn-to-schedule count coordination, as described in the -[implementation record](beta-implementation.md). - -## Automated proof - -- `DISPLAY=:0 bun run verify`: 177 tests, 1,210 assertions, type and Svelte checks, - lint, formatting, all builds, and both Electron flows passed. -- `bun run build:concepts` passed. -- `bun --bun electron-builder --publish never --linux dir` and - `DISPLAY=:0 bun run test:packaged` passed, including both packaged desktop flows. -- All fourteen embedded prompt assets matched their recorded SHA-256 hashes. -- The Svelte autofixer reported no issues. The design detector reported no findings. - -Astra removed unused notes scaffolding, dead exports and wrappers, a redundant -rotation test, and twenty-five upstream SDK conformance cases. The cleanup alone -removed 356 net lines. Application-owned SQLite facade tests, original-PDF pixel -and transform tests, real Pi execution, RPC, interruption, and recovery tests remain. -All twenty-one original lint warnings were eliminated. Immutable migrations and -legacy notes data were preserved. - -Bun 1.4.4 was already the current stable release. The app keeps that exact pin, -Bun's formatter and checker, `svelte-check --tsgo`, and Effect SQL. Backend cleanup -uses native hashing, HTML escaping, and bounded file reads where appropriate; -Electron retains its Node runtime boundary. - -## Live desktop proof - -GPT-6.1 Terra with xhigh reasoning and priority service drove the packaged app -through Electron Playwright on display `:0`, using isolated authored PDF fixtures. -The initial session did not expose the Windows CUA driver. This first run used -Electron interaction and screenshot evidence; the native CUA recheck is recorded -below. - -A custom Electrical request sent through the composer completed in seven Meta -Muse calls and zero SAM calls, within the twelve-call limit. It proposed two -drawn Type A luminaires against three scheduled in Room 101. The reply returned -to the requesting conversation. Proposals retained the prompt version, skills -revision, source version, run, and reviewer-message identity, and stayed proposed -until the reviewer explicitly accepted them. - -Terra verified the supported mismatch, false alarm and correction, unchanged -independent control, project/conversation switching with drafts, source-panel -Escape behavior, complete-package export, and reopen persistence. No renderer -errors were recorded. The focused recheck also monitored main-process stderr, -which contained only Electron debugger shutdown text. - -Terra found two defects, both corrected and rechecked: - -| Finding | Correction and observed result | -| -------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Call-limit selector blank | Number-valued Svelte binding displays and retains 4, 8, and 12 across settings close/reopen. A configured parent run also verified matching composer text for every choice. | -| Accepted proposals still awaiting acceptance | Pending proposals exclude versions already superseded by accepted inputs. Room 101 retains its supported issue, accepted counts, evaluation history, and both original proposals in storage. | - -The final configured check made no new inference calls: before and after it, -the isolated database held one run, seven model attempts, and zero SAM attempts. -Temporary session credentials were private, excluded from artifacts, and removed -after testing. Test applications were closed. - -## Native Windows CUA recheck - -After locating the Windows installation, GPT-6.1 Terra with xhigh reasoning and -priority service drove the native Windows build through CUA 0.35.4. The checkout -was `7d7bd5e79`; the executable came from the reviewed tree's -`desktop-Windows-X64` artifact in CI run `40645996427`, extracted without installing. - -The Windows executable is under -`%LOCALAPPDATA%\Programs\Cua\cua-driver\bin\cua-driver.exe` and can be invoked from -WSL through `/mnt/c/Users//AppData/Local/Programs/Cua/cua-driver/bin/cua-driver.exe`. -The existing Windows daemon could not enumerate the Linux X11 app window. A -separate persistent `cua-driver mcp --direct` connection launched the native app -with an explicit OS environment allowlist, `PROJECT_KIT_DATA_DIRECTORY` pointing -at a copied synthetic fixture, and a scratch Chromium profile. The existing -daemon and driver permission policies were unchanged. - -CUA verified settings open/close and each 4/8/12-call selection, accepted Room 101 -counts of two drawn and three scheduled, the supported issue, and the absence of -superseded pending proposals. After previewing page two in Sheets, Evidence -restored the finding's page-one source. Screenshots were inspected. Selection -keystrokes used foreground delivery only after background delivery was refused. -The database retained both historical proposals, passed its integrity check, and -remained at one run, seven attempts, and zero SAM attempts. No provider was -configured or inspection started. The app and CUA session were closed afterward. - -The recheck found a stale completion instruction asking reviewers to accept -counts that were already accepted. Completion copy now directs reviewers to -the results without asserting that acceptance is pending; the updated sentence -was verified through Electron Playwright against the follow-up desktop build. -Historical model messages remain unchanged. Renderer errors, conversation drafts, and mobile -resizing were not monitored or tested in this focused native run. - -## Review regressions - -Sol and Grok identified four additional defects. Each was reproduced against the -earlier behavior and fixed through the existing source loader or durable journal: - -| Finding | Fix and regression proof | -| ----------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Earlier-revision proposals could target a newer sheet | Only proposals for the finding's current operand sheet remain actionable. A captured pre-acceptance fixture showed Revision A proposals opening Revision B before the fix; afterward neither proposal was actionable, and both records remained stored. | -| Evidence could retain an unrelated sheet preview | Entering Evidence reloads the selected finding's current source and invalidates older render requests. The desktop regression switches from another sheet preview back to the finding and verifies its source page. | -| Resume could duplicate a committed tool reply | The terminal fallback checks persisted application messages for that run. A simulated interruption between Pi completion and domain completion resumes with no extra model request or application message. | -| Empty terminal output created misleading prose | The fallback saves only nonempty terminal text. The real SDK/SQLite integration covers both an actual answer and an empty response, without inventing a summary. | - -The repeated full suite and packaged desktop flows passed after these changes. -The review added one empty-response case and extended existing integration and -desktop tests for the reproduced failures. It did not restore vendor conformance -tests or add another recovery mechanism. - -The parent inspected dark and light captures at 1440×900 and 1100×733, plus the -findings interaction at 390×844. Source images were loaded before final capture. -The [desktop capture](screenshots/beta-workbench.png) and -[narrow findings capture](screenshots/beta-mobile-findings.png) use authored -synthetic project data. - -## Limits - -Governing-code verification, a complete permit review/report, and Modal handoff -remain outside this beta. Baseten crop acceptance remains unverified. Actual -image-token usage and cost remain unknown. Platform CI and council review are -recorded in the pull request before merge. diff --git a/docs/research/cloud-auto-sync-validation-result.json b/docs/research/cloud-auto-sync-validation-result.json deleted file mode 100644 index 00fa5538..00000000 --- a/docs/research/cloud-auto-sync-validation-result.json +++ /dev/null @@ -1,38 +0,0 @@ -{ - "cloudId": "12866eda-26f1-4b95-8c8b-085fe01da2fc", - "execution": { - "runId": "b08de3be-8ec5-4048-8af7-ca7aadc8a1c4", - "projectId": "18f51af6-1878-421d-b4d0-3aba8ae0df4f", - "findingId": "bcff0605-743c-4806-8646-e7a657bedef8", - "state": "uncertain", - "version": 1, - "route": "live", - "provider": "meta", - "model": "muse-spark-1.3", - "admittedCalls": 1, - "reservedInputTokens": 64000, - "reservedOutputTokens": 2048, - "unresolvedAttempts": 1, - "uncertaintyAcknowledged": false, - "costUsd": null, - "imageInputTokens": null - }, - "sourceHashes": ["5fcb96c2fd1893b714218bb931ac50697db19e30f19735f0f58d49a81ceace91"], - "automaticResultReturn": true, - "progressTransport": "rpc-stream", - "observedPhases": ["dispatching", "ready", "importing", "imported"], - "reopenedWithoutRedispatch": true, - "localWritesBlockedDuringCloud": true, - "localWritesAvailableAfterImport": true, - "allProposed": true, - "factCount": 0, - "artifactCount": 1, - "completedReview": false, - "modelOutcome": "uncertain after one admitted request; not replayed", - "attempts": [ - { - "status": "uncertain", - "sent": 1 - } - ] -} diff --git a/docs/research/cloud-sync.md b/docs/research/cloud-sync.md deleted file mode 100644 index 83ecb93a..00000000 --- a/docs/research/cloud-sync.md +++ /dev/null @@ -1,87 +0,0 @@ -# Automatic cloud result synchronization - -The reviewer pushes a new cloud review once. Progress appears live, and verified -results return automatically. A WebSocket by itself would not guarantee durable -delivery or safe workspace replacement, so transport is separated from recovery. - -## Selected design - -Reuse the existing trusted Effect RPC stream through Bun's private loopback -NDJSON endpoint and Electron MessagePorts. Modal's JavaScript SDK supports -`sandbox.filesystem.watch`, including directory events for atomic file replacement. -The backend watches `/moke/progress.json` through its parent directory and reads -the durable, schema-validated progress record after notifications. It also reads -on attachment and every ten seconds, since notifications can be missed and -watching can be unsupported. No public tunnel, incoming webhook service, or -renderer cloud credential is needed. - -The backend, rather than a mounted component, owns one synchronizer. Observation -only fans out its latest snapshot; opening two windows cannot admit two imports -or dispatch more model requests. Once the result snapshot exists it runs the -existing download, confirmed stop, complete validation, and journaled restore -under application maintenance. The imported receipt reaches the UI only after -maintenance ends. Proposed facts remain proposed. - -Reopen reconstructs ownership from the sidecar and performs the same reconciliation. -No renderer event cursor or old socket is needed. Persistent receipt revisions -reject out-of-order RPC responses. An unavailable service retries later; invalid -or conflicting input requires an explicit sync retry and does not repeatedly -download the same invalid result. Cancellation still requires acknowledged -possible charges and confirmed stop or explicit provider absence. -Paused errors are persisted in the receipt across restart. UI delivery is -acknowledged only after its local snapshot and execution status refresh succeed; -failed refreshes retry separately from the package import, with concurrent -deliveries coalesced for the same receipt. -Hydration captures the selected project, check, and receipt before awaiting a -snapshot or execution-status request. It rechecks those identities before -applying the response; switching checks cannot attach the old run's controls to -the new selection. Deferred-response tests cover check, project, and receipt changes. - -The streaming payload is review progress and synchronization state. Final messages -and evidence artifacts appear after a complete validated return. This change does -not implement token streaming, partial SQLite replication, persistent PDF caching, -or background PDF pre-upload. - -## Alternatives considered - -- Direct browser-to-sandbox WebSocket: adds a public connection/authentication - boundary and still needs durable reconnect reconciliation. -- A background renderer timer: stops with the view and duplicates controllers - across windows; it cannot own package replacement. -- Watching only filesystem events: misses already-completed offline work and - notifications lost during disconnection. -- Shared live SQLite on a Volume: conflicts with the existing single-owner and - consistent-package recovery rules. The Volume/PDF-cache proposal is separate. - -## Primary references - -- [Modal JavaScript Sandbox filesystem watch](https://modal.com/docs/sdk/js/latest/Sandbox#watch) -- [Modal networking boundaries](https://modal.com/docs/guide/sandbox-networking) -- Installed Effect `SubscriptionRef`, scoped fibers, Streams, and TestClock, - plus `repos/effect/LLMS.md` and matching read-only references. -- Existing `ObserveExecution`, authenticated backend RPC, and renderer bridge. - -Regression coverage includes automatic return without a renderer, completed-offline -reopen without redispatch, missed notifications, interrupted downloads, invalid -packages requiring retry, and real package restore with original hashes preserved. - -## Observed validation - -The live native-watch probe received an atomic `progress.json` replacement through -Modal's SDK stream with zero model calls. Disconnect cleanup took 2,004 ms with a -two-second local cleanup budget. Its sandbox was confirmed stopped. - -The compiled-backend live check closed and reopened the backend, then subscribed -to `ObserveCloudReview`. It observed `ready`, `importing`, and `imported` without -calling `ImportCloudReview` or `RefreshCloudReview`. Source hashes were preserved, -and a subsequent local message succeeded. See the [recorded result](cloud-auto-sync-validation-result.json). - -That live review did **not** complete its inspection: one request was marked sent -and remained uncertain, with no facts produced. Its cause was not established; -the request was not replayed or acknowledged automatically. The ordinary -completed-review assertion failed for that reason. Read-only receipt inspection -and a local write separately verified automatic return and preserved uncertainty. -This is transport/recovery evidence, not evidence of a newly completed model review. -The earlier completed seven-call review remains recorded in [cloud validation](cloud-validation.md). - -Final verification passed 156 smoke tests and two Electron desktop tests. diff --git a/docs/research/core-council.md b/docs/research/core-council.md deleted file mode 100644 index 49ca976b..00000000 --- a/docs/research/core-council.md +++ /dev/null @@ -1,49 +0,0 @@ -# Core implementation council - -Base: `a1d4bbc21` (includes merged SAM implementation, PR #5). Round 1 reviewed -`b73424d9c`, including the original local units 0–6 implementation and live evidence. -Optional Modal/cloud remains outside scope. All reviews were read-only, without -paid calls, nested agents, commits or PR watches. - -| Delegate | Round 1 | Round 2 | Final verdict | -| ----------------- | ------- | ------- | ------------- | -| GPT-6.1 Sol, high | 7/10 | 9/10 | Approve | -| Grok 4.7, high | 7/10 | 9/10 | Approve | -| Composer 2.5 | 8.5/10 | 8.5/10 | Approve | - -## Findings ledger - -| ID | Severity | Resolution and proof | -| --- | -------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | -| S1 | blocker | A durable replacement journal blocks public writes and maintenance until startup recovery. Notes check the barrier inside the SQL permit, including unreadable journals. The application test leaves the incoming DB installed, rejects new notes/projects, reopens to the original, and proves later commits survive another reopen. Maintenance finalizers run while holding their own maintenance permit. | -| S2 | should | Owner generation is excluded from semantic tool receipt identity and remains mandatory for new mutations. The real parent-port test replays a committed proposal with a new generation after correction; changed inputs still conflict. | -| S3 | should | Rollback persists `reverting` plus the saved original's hash and size before renames. It recognizes an original already returned to its live path by that identity. Tests cut before discard, after discard and after original return; a wrong identity blocks without changing the original. | -| G1 | should | Check inputs/dependencies now contain only facts passed to evaluation. Changed/unconfirmed correspondence explicitly explains rejected accepted inputs. The backend RPC test checks null rejected operand and a single current dependency. | -| G2 | should | Confirmed links are immutable and one-to-one in both command policy and migration 0006 indexes/triggers. Unconfirmed proposals have separate append-only history, allowing later confirmation without rewriting a link. Reassociation/ambiguous successor commands are rejected. Current-source traversal also rejects ambiguous graphs. | -| G3 | should | A confirmed destination cannot be unconfirmed later in the same association transaction. The RPC test creates a finding already on Rev B before association and verifies that destination stays confirmed. | -| G4 | nit | Transition checks `changes()=1`; a missed version rolls back its transaction and appends no phantom event. Focused persistence test covers the miss. | -| C1 | should | Unix session files reject group/other permissions before reading credentials, using the same open handle for stat/read. Tests cover private files, shared permissions and canonical-route rejection. Windows retains its ACL-based permissions model. | -| C2 | nit | Deferred environment allowlisting for the PDF worker. It is a trusted same-user subprocess with required OS/font/native-loader environment; it is not an OS security sandbox. Provider configuration uses the private startup channel. A narrower environment should be validated on all three packaging targets as separate hardening. | -| C3 | nit | Retained initial execution replay. The workbench's domain-event cursor is a different sequence and cannot replace the execution cursor. Initial replay reconstructs the latest finding run after reopening; reads paginate at 200. A dedicated execution snapshot/cursor would be a separate scale improvement. | -| C4 | doc nit | Updated the stale NotesRpc-only comment to the restricted review/chooser boundary. | - -Sol-loop used the single Sol member in two rounds and ended approved. Round 2 -checked changed code and the open findings, all of which were resolved or accepted -as deferred nits. Sol independently passed 16 focused tests /152 assertions in -round 2; reviewer provider usage was not exposed. The behavior diff grew from -86 files /18,649 insertions at the first freeze to 91 files /19,414 insertions at -the second, primarily regression tests, recovery fixes, provenance and evidence. - -Composer C5 (nit) proposed rejecting credential-file symlinks. Deferred: stat and -read use the same opened handle and enforce the target's permissions. An alias -to a private target does not bypass that check; the caller explicitly selects -the session path. No should/blocker findings remain. Final review freeze was -`a81b45226`; subsequent edits only record these review results. - -## CI filesystem follow-up - -Windows job 122586407781 on `cf89fe062` failed package tests because O_NOFOLLOW was -undefined. The follow-up preserves link rejection through opened-file identity -checks, exclusive output creation, portable manifest/restore separators, and -platform-specific directory flushing. No provider or engineering-rule behavior -changed. A third, bounded council pass reviews this filesystem diff before push. diff --git a/docs/research/core-implementation-sources.md b/docs/research/core-implementation-sources.md deleted file mode 100644 index 2462ed67..00000000 --- a/docs/research/core-implementation-sources.md +++ /dev/null @@ -1,101 +0,0 @@ -# Core implementation research - -Evidence checked 27 October 2026 for the [core implementation plan](../core-implementation-plan.md). The plan's sequencing, starting limits and architecture details are recommendations. Compatibility and performance remain implementation gates unless a concrete observation is listed below. - -## Project evidence and precedence - -- [Review loop inference harness](t3-thread://v1/3fcb26c2-f0ef-48b8-909a-e1b8759ea113) records the SDK comparison, Grok critique, and the user's selection of local Pi Durable with an explicit Modal Sandbox handoff. The research and decision were merged before this planning task. -- [Review Meta AI Hackathon Guide](t3-thread://v1/f84a1315-e1dd-43f5-8278-3e17a1b3d3e3) contains the original planning packet. The recovered `Moke-Planning-Packet-2026-10-27.zip` has SHA256 `d6d5381817f4c6f47def33e5524c991cde06dc368a74026b93c17b38186e1f89`. Its architecture, weekly plan and operations handbook define the four demonstration outcomes, evidence model, geometry tests, narrow case selection and evaluation target. The packet's toolchain/process/UI decisions were superseded by later work. -- [Runtime decision](../review-runtime.md), [SDK comparison](agent-harness-comparison.md), [model reference](../models.md), [data lifecycle](../data.md), [deployment](../deployment.md), [product brief](../../PRODUCT.md), [workbench design](../../design/DESIGN.md) and `.project-kit/manifest.json` establish the current project boundaries. -- The current backend has a 64 KiB request limit, rejects browser-origin calls, and requires the private launch credential. `backend/application.ts` serializes operations and recovery with one semaphore and scopes SQL resources per operation. `desktop/main.ts` restricts renderer RPC to the trusted top-level frame. Adding large-media transfer must account for those existing boundaries. -- The review engine is selected but unimplemented. There is no installed Pi, PDF engine or Modal client in `package.json`. Do not confuse the current notes smoke screen or sample UI study with implemented drawing review. - -## Exa investigation - -Discovery and follow-up retrieval used Exa through Executor, with Firecrawl used to retrieve Bun's documentation entry point. Searches covered Bun primitives, Pi's storage and recovery contracts, PDF.js rendering, Effect 4 worker/process APIs, Meta tool/response protocols, Baseten vision/reasoning limits, Modal lifecycle/persistence, and public drawing-evidence implementation ideas. - -Older Pi search hits described a synchronous `prepare`-based facade and a proposed conservative recovery mode. Current Pi Durable documentation instead describes an asynchronous executor and recoverable built-in generation tasks. Use current package source and published exports, not an old design document or a main-branch package version, to select the installed integration. - -Public drawing-review repository searches suggested useful patterns such as explicit coverage states and preserving reviewer corrections. Those are design references only. Their README claims are not evidence of engineering accuracy, and no application code is proposed for reuse. First-party runtime documentation and the supplied packet remain the implementation basis. - -## Bun primitives - -| Evidence | Consequence for the plan | -| ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| [Bun native API catalog](https://bun.com/docs/runtime/bun-apis) and [Web APIs](https://bun.com/docs/runtime/web-apis) document file I/O, hashing, workers, fetch, abort signals and streams. | Prefer built-ins behind application-owned adapters rather than adding HTTP, queue or file convenience libraries. | -| [SQLite](https://bun.com/docs/runtime/sqlite) documents synchronous query execution and synchronous transaction callbacks. | An async Pi transaction cannot be implemented by passing an async function to `Database.transaction`. Use a verified Effect SQL bridge or an explicitly serialized native facade. | -| [Image](https://bun.com/docs/runtime/image) documents a native decode/resize/encode pipeline. Local feature detection on pinned Bun 1.4.4 returned functions for `Bun.Image` and `Bun.file(...).image`. | Avoid a separate general image-processing package. API availability alone does not verify output fidelity or packaged behavior. This API does not establish PDF rasterization support. | -| [Workers](https://bun.com/docs/runtime/workers) marks worker support experimental, especially termination. | Validate cancellation, memory limits and failure isolation; keep a subprocess fallback behind the same processing contract. | -| [Compiled executables](https://bun.com/docs/bundler/executables) requires worker entrypoints to be included explicitly and describes asset/native-addon packaging. Embedded SQLite changes do not survive process exit. | Compile worker entrypoints and package fonts/CMaps/WASM/native addons explicitly. Keep live project databases on writable disk. | -| [File I/O](https://bun.com/docs/runtime/file-io) describes `Bun.write` and buffered `FileSink` flushing. | Stage, verify, close/sync and rename artifacts. Do not equate a buffered flush with an established power-loss guarantee. | -| [Subprocesses](https://bun.com/docs/runtime/child-process) documents `Bun.spawn`, abort signals, timeouts and exit handles. | A Bun subprocess is a credible isolation alternative if workers fail the PDF experiment. It still needs scoped cleanup and bounded I/O. | - -## Matching Effect source - -The repository pins Effect 4.0.4 and reference revision `e17ecdbe902aa7108f775c8ab9c0ddf9f38801ab`. `repos/effect/LLMS.md` and the coding-standards skill were read. All reference files remain read-only; implementation imports installed packages. - -| Reference under `repos/effect/` | Relevant observation | -| ---------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| `packages/sql/sqlite-bun/src/SqliteClient.ts` | Scoped Bun SQLite connection acquisition, WAL and busy timeout defaults, and explicit transactions. Domain queries should continue through Effect SQL. | -| `packages/effect/src/sql/SqlClient.ts` | `withTransaction` and a transaction-specific service bind queries to the acquired connection. A Promise facade must preserve that context rather than launching unrelated effects. | -| `packages/platform/bun/src/BunWorker.ts` and `BunWorkerRunner.ts` | Bun worker platform integration and scope finalization, with graceful shutdown followed by termination on timeout. | -| `packages/effect/src/rpc/RpcServer.ts`, `RpcClient.ts`, `RpcWorker.ts` | Typed streaming RPC and worker protocol integration are available; avoid a second untyped messaging system. | -| `packages/effect/src/http/FetchHttpClient.ts` | The platform fetch receives an abort signal; Bun's fetch can be supplied without an additional HTTP client library. | -| `packages/platform/bun/src/BunChildProcessSpawner.ts` | The current Bun module re-exports the shared Node child-process implementation. A Bun platform import alone does not prove use of `Bun.spawn`. | -| `packages/platform/bun/package.json` | `@effect/platform-bun` matches version 4.0.4 and is a candidate dependency when worker/platform services are used. | - -Public counterparts include [BunWorker](https://effect.website/docs/v4/api/platform-bun/BunWorker), [BunServices](https://effect.website/docs/v4/api/platform-bun/BunServices), [ChildProcessSpawner](https://effect.website/docs/v4/api/effect/process/ChildProcessSpawner), and [RpcWorker](https://effect.website/docs/v4/api/effect/rpc/RpcWorker). The matching local revision takes precedence if public documentation moves ahead. - -## Pi execution and storage - -The current [Pi Durable README](https://github.com/earendil-works/pi/blob/main/packages/durable/README.md) describes persistent submissions keyed by `requestId`, model/tool tasks, SQLite storage, extension hooks, interrupted-tool replay declarations, and a shared storage conformance suite. The API remains experimental. - -Its portable SQLite facade uses Promise-based `exec`, `run`, `get`, `all`, `transaction` and `close`. Transaction callbacks receive an exclusive transaction executor that expires after settlement. Unrelated operations queue behind the transaction. Re-entering the database object from that callback can deadlock. The [async-facade change](https://github.com/earendil-works/pi/pull/10232) explains the contract change, but the selected published package must be inspected directly. - -Council follow-up retrieval of the current [Chord Node SQLite adapter](https://github.com/earendil-works/pi/blob/main/packages/chord/src/sqlite/node.ts) confirms that it serializes asynchronous transaction callbacks and directly issues transaction-control SQL. Since Moke already exercises Bun's `node:sqlite` compatibility layer for migrations/backup, the plan now tests Pi's existing adapter before authoring an Effect SQL facade. This is a sequencing choice supported by existing code, not a new claim that the adapter has passed Bun conformance. Domain SQL remains owned by Effect. The documented Node journal guarantee is WAL with `synchronous=NORMAL`; a stronger setting is a separate experiment. - -Pi storage expects one owning process; it does not supply cross-process ownership fencing. Its documented Node SQLite adapter uses WAL with `synchronous=NORMAL`, which differs from a stronger power-loss target. Domain and Pi journal databases are not one atomic transaction. Stable operation receipts must reconcile that boundary. - -Pi's `beforeRequest` hook adjusts messages. Hooks are not a reliable enforcement boundary because hook errors can be reported and skipped. The implementation must prove that a provider wrapper can block automatic paid-call reissue, retries and compaction until the attempt ledger permits dispatch. This remains a go/no-go question, not a verified integration feature. - -Follow-up retrieval of Pi's [generation task](https://github.com/earendil-works/pi/blob/main/packages/durable/src/harness/generation.ts), [Responses adapter](https://github.com/earendil-works/pi/blob/main/packages/ai/src/api/openai-responses.ts), and [provider retry helper](https://github.com/earendil-works/pi/blob/main/packages/ai/src/utils/provider-retry.ts) distinguishes conversation `sessionId` from durable generation identity and identifies a possible inner HTTP retry layer. Set retries explicitly to zero and measure actual transport sends. Disable automatic compaction in the first bounded core until its request identity can satisfy the same policy. - -The generation source checks `toolExecution === "sequential"`; the plan uses that actual setting rather than a reviewer's shorthand `serial`. Pi's interrupted-tool recovery requires an explicit `replay: "safe"` declaration. Operation receipts alone do not cause replay, and declaring replay safe does not make an uncertain paid SAM request safe to resend. - -Pi's README also distinguishes detaching a wait, aborting a conversation and closing the harness. A close can wait on uncooperative task code. Moke's existing `desktop/backend.ts` grants the backend three seconds before forced termination, so graceful drain is bounded and forced termination must preserve uncertainty rather than promise every receipt was committed. - -The existing [Pi AI providers](https://github.com/earendil-works/pi/blob/main/packages/ai/README.md) are the first adapter candidates. Meta's credential naming differs from Moke's configuration, and the organizer Baseten model ID must be passed explicitly. No paid model probe was run during planning. - -## SQLite connection and ownership checks - -SQLite's [foreign-key documentation](https://sqlite.org/foreignkeys.html) requires runtime enforcement per connection and notes that changing it inside a transaction has no effect. The matching Effect Bun driver initializes WAL and busy timeout but does not explicitly initialize foreign keys. The planned connection initializer must set and verify enforcement before a transaction and apply its selected connection-local durability settings. - -The [SQLite locking protocol](https://sqlite.org/lockingv3.html) documents exclusive writer reservation and failure of a competing writer. This supports a dedicated local ownership-lock database as a candidate backend process guard. The plan deliberately separates that lifetime-scoped lock resource from short application transactions, excludes it from restore, and requires a two-process/crash test on each supported local filesystem. It is not a distributed lease or cloud fencing mechanism. A process-local registry and Electron's single-instance lock do not protect an independently launched backend. - -## PDF rendering - -Mozilla's [PDF-to-PNG example](https://github.com/mozilla/pdf.js/blob/master/examples/node/pdf2png/pdf2png.mjs) demonstrates `getDocument`, a viewport, the document's canvas factory, page rendering, PNG output and cleanup. It also configures CMaps and standard fonts. Its [canvas replacement](https://github.com/mozilla/pdf.js/pull/19015) moved the Node path to `@napi-rs/canvas`. - -This supports PDF.js as the first rendering candidate. It does not establish Bun compatibility, region-only rendering memory use, native-addon packaging or cancellation behavior. Test those with the installed versions, representative drawings and the compiled backend. Preserve original PDF coordinates rather than relying on a reduced overview for measurements. - -Council follow-up searches found a current [Mozilla packaging update](https://github.com/mozilla/pdf.js/commit/bc38b633eefc6d005f54ad921ab36f8f4d2efc19) that already lists `@napi-rs/canvas` as the distribution's optional dependency. An old `pdfjs-dist` GitHub master snapshot still describes version 3.2.146 with `canvas`; it cannot establish today's published dependency requirements. Inspect the exact selected npm package before considering an override. A blanket `canvas` alias is not a required first step. Bun's own [native-addon compilation tests](https://github.com/oven-sh/bun/blob/88417471/test/napi/napi.test.ts) show general compiled addon support, not proof that Moke's PDF package works. - -## Provider capability boundaries - -- [Meta Responses](https://dev.meta.ai/docs/protocols/responses) and [tool calling](https://dev.meta.ai/docs/tool-calling) document image inputs, tool-result continuation, response identity and background work. The actual Pi adapter must preserve the chosen context protocol and permit attempt admission. Background request creation has a documented 1 MiB body limit and cannot be combined with initial response streaming; file references and later stream retrieval are separate capabilities. -- [Meta protocol selection](https://dev.meta.ai/docs/protocols) distinguishes Responses reasoning continuity from Chat Completions. Opaque state needed for continuation is different from displaying or logging private reasoning. -- [Baseten Model APIs](https://docs.baseten.co/inference/model-apis/overview) currently states that Muse Spark Open does not support structured outputs or JSON mode. Tool calling and application-side Schema validation are the starting path. Do not send unsupported strict-output options simply because the route is OpenAI-compatible. -- [Baseten vision](https://docs.baseten.co/inference/model-apis/vision) and [reasoning](https://docs.baseten.co/inference/model-apis/reasoning) describe model-dependent capabilities. Public examples use a different Spark Open slug from the organizer's `meta-models/muse-spark-open-1026`. Test the exact granted route; public catalog behavior does not establish that grant's support or quota. -- [Meta segmentation](https://dev.meta.ai/docs/media-segmentation) remains a separate adapter investigation. A successful text or image-and-tool turn does not verify mask decoding or physical measurement. - -Exa follow-up search and retrieval of [SAM prompting](https://dev.meta.ai/docs/sam/segmenting), [output grammar](https://dev.meta.ai/docs/sam/reading-segmentation) and [client libraries](https://dev.meta.ai/docs/sam/client-libraries) confirms pixel-coordinate boxes with inclusive maximum edges and media width/height after display orientation. Mask raster dimensions can differ from the submitted image dimensions. Pin and use the official `@meta-sam/parser`; its version must support the chosen confidence-bearing output and encoding. Preserve the full transform to PDF coordinates. - -The matching Effect source states that `Schema.toJsonSchemaDocument` emits draft 2020-12 for the canonical JSON codec. Its default excess-property behavior and codec transformations can differ from naive decoding of the domain schema. The plan requires a supported tool subset and comparison cases before descriptor conversion; schema generation cannot replace application-side decoding. - -## Modal limits and handoff implications - -The official [Sandbox guide](https://modal.com/docs/guide/sandboxes), [filesystem guide](https://modal.com/docs/guide/sandbox-files), [snapshot guide](https://modal.com/docs/guide/sandbox-snapshots), [exit snapshots](https://modal.com/docs/guide/sandbox-exit-snapshots), [networking](https://modal.com/docs/guide/sandbox-networking), [JavaScript SDK](https://modal.com/docs/sdk/js/latest) and [Dict guide](https://modal.com/docs/guide/dicts) were investigated through Exa. - -Sandbox lifetimes are finite and must be configured. Filesystem snapshots exclude mounted Volumes. Volume commit/reload behavior is explicit; a best-effort exit snapshot is not the sole durable checkpoint strategy. Authenticated Sandbox access exists, but a tunnel or connect token does not define application authorization. SDK parity must be checked for the exact selected features. - -The checked sources do not establish a transactionally fenced owner ledger for Moke. A shared Volume or Dict is therefore not assumed to be a lease/CAS coordinator. The first plan uses a conservative one-way handoff with durable local relinquishment and one bound Sandbox; automatic takeover requires a separate design. No cloud resources were created during this investigation. diff --git a/docs/research/fast-initial-review.md b/docs/research/fast-initial-review.md deleted file mode 100644 index 9285d615..00000000 --- a/docs/research/fast-initial-review.md +++ /dev/null @@ -1,443 +0,0 @@ -# Fast initial review: upload, storage, parallel calls and preseeding - -Checked 2026-10-29. Question: why does **Push to cloud** spend so long uploading -before review starts, what would make reviews finish in minutes rather than 40+, -and what should a one-click **Initial Review** start? Sources are Modal, Meta Model -API and Baseten documentation fetched with Exa through Executor, the installed -`modal@0.11.1` and Pi sources, and one bounded Modal benchmark. The benchmark sent -random bytes only; it made no model requests and uploaded no drawings or credentials. - -Parallel per-sheet review is being built separately in -`feat/parallel-sheet-review`: one setup conversation, one Pi child per sheet, one -cross-sheet conversation, inside one sandbox on Meta. This note does not repeat that -design. It covers what surrounds it: transfer, storage, provider settings and the -initial-review request. - -## Decisions - -Recorded 2026-10-29 from the project owner: - -- The proposed changes are accepted. Work starts only after the current review - threads finish; nothing has been sent to them. -- **Cached-input target: at least 90%** of prompt tokens per review, measured as - `cacheRead / (input + cacheRead)` across all model calls. -- **Seed every project.** The drawing sets are sanitized, so uploading source files - to Modal and Meta at import time is approved for all projects. -- **Initial Review button**: one button that sends a high-quality, preset initial - QA request instead of a typed prompt. It is visible only during the initial - review phase. Once a revised set is uploaded it disappears, and the reviewer - works from typed prompts and suggested prompts. - -## Implementation status - -Implemented 2026-10-30 (cloud worker protocol 4; publish a new worker image before use): - -- **Storage cache.** A per-workspace Modal Volume v2 holds immutable package files by - SHA-256. Source PDFs and renders seed in the background after import. Send uploads the - database and files the cache lacks; the worker fills the rest from a read-only mount - and checks each hash. Uploaded files are copied into the cache from the input - snapshot, and result downloads reuse matching local artifacts. See - [cloud reviews](../cloud-reviews.md#workspace-cache). -- **Prompt cache.** Shared `prompt_cache_key` per prefix with 24-hour retention on Meta, - `x-session-affinity` on Baseten, shared fields first in sheet inputs, a warmed prefix - before fan-out, a stepped image window, and per-review and per-sheet cached share in - diagnostics. An offline test asserts each request extends the previous one byte for - byte. See [inference](../inference.md#prompt-caching-and-request-images). -- **Meta Files API.** Live Meta requests reference uploaded images by `file_id`. -- **Render concurrency.** PDF jobs run up to four at a time (one fewer than the cores). -- **Progressive findings** in the cloud card, and the **Initial Review** button - (`initial-review-v1`). - -Live validation on 2026-10-30 is in [fast-initial-review-validation.md](fast-initial-review-validation.md): -the 29-sheet Brewery set took 14.3 minutes on the Windows app at 91.2% cached and 15.4 -minutes in the cloud at 94.9% cached, and a cloud Send dispatched 5 s after the click. -It also added two fixes: Meta's intermittent 404 `model_not_found` is resent like a rate -limit, and result files download eight at a time. - -A live probe on 2026-10-30 (one synthetic image, six minimal-reasoning calls, file -deleted afterwards) answered two open questions below: Meta accepts `file_id` images in -`function_call_output`, and `prompt_cache_retention: "24h"` is accepted, with the second -request 99.1% cached. Still open: Meta's cache share on a real packet review, background -responses, an adaptive rate limiter, and lower per-sheet reasoning effort. - -## Summary - -1. **Upload time is set by the uplink, not by Modal.** Sandboxes start in about a - second. The review package uploads at the reviewer's measured uplink speed - (about 1.8 MiB/s), so the observed 506 MiB package costs about 4.5–5 minutes - before review can begin. Warm sandboxes and parallel copies cannot remove that. - The fix is to send fewer bytes. -2. **Cache source files by content hash in Modal storage and fill the cache when a - PDF is imported**, not when the reviewer presses Send. Package artifacts are - already named `artifacts/.pdf|png`, so a push only needs the database, - journals and artifacts missing from the cache. A set with no new revisions - uploads no PDFs. -3. **Engineer for a 90%+ cache hit rate.** Recorded Baseten reviews reached 79.8% - with no cache routing at all. Pi provides a session ID, but - `backend/inference/provider.ts` drops it when it rebuilds request options, so - no cache key or session-affinity header is sent today. The rolling image window - and compaction also rewrite earlier history, which discards the cached prefix. - See [section 3](#3-cache-hit-target-90). -4. **Upload sheet images to Meta's Files API once and reference them by `file_id`.** - Every turn currently resends every retained image as base64. This also unlocks - background responses, which avoid long-request timeouts but cap the request - body at 1 MiB. -5. **Initial Review should be a versioned request, not typed text**, started from - one button with the whole set, all detected disciplines, Meta Spark 1.3 and - cloud execution when configured. Show findings as each sheet finishes, before - the final package returns. - -## 1. Where Push to cloud spends its time - -### Current path - -`backend/cloud/modal.ts` resolves the image and secrets, runs a network-blocked -identity probe (2.1 s first time, 0.4 s cached, per `audit-3-cloud.md`), and the -application exports a complete workspace package. `upload` then calls -`sandbox.filesystem.copyFromLocal` once per file, **sequentially**, then -`snapshotDirectory("/moke")` before dispatch. In `modal@0.11.1`, -`copyFromLocal` runs a remote fs-tools exec and streams stdin in 256 KiB chunks -(`dist/index.js:75583`, `STREAMING_STDIN_CHUNK_SIZE`). - -The package is the **whole workspace**. The Audit 3 native run uploaded 506.1 MiB -in 297 files spanning 7 projects and downloaded a 506.3 MiB result -(`audit-3-validation.md`). Only one project was under review. - -### Benchmark - -Run from WSL2 on the development Windows host, assumed to share the Windows app's -network uplink, against `moke-review-engine:d7ba4f874`, 4 CPU / 8 GiB, network blocked. -Data: one 40 MiB file plus forty 1 MiB files of random bytes (80 MiB). - -| Operation | Time | Rate | -| ------------------------------------------ | ----------: | --------------- | -| `sandboxes.create` returns | 202 ms | | -| First exec completes | 952 ms | | -| Upload, sequential `copyFromLocal` | 51,319 ms | 1.56 MiB/s | -| Upload, 8 parallel `copyFromLocal` | 45,424 ms | 1.76 MiB/s | -| Upload, single tar stream over exec stdin | 42,424 ms | 1.89 MiB/s | -| `snapshotDirectory` of the 80 MiB folder | 3,755 ms | | -| Download 48 MiB, sequential `copyToLocal` | 6,261 ms | 7.7 MiB/s | -| Download 48 MiB, 8 parallel `copyToLocal` | 3,759 ms | 12.8 MiB/s | -| New 1 CPU sandbox: create + first exec | 1,279 ms | | -| `mountImage` of that directory snapshot | 301 ms | | -| Read all 80 MiB from the mounted snapshot | 1,165 ms | ~69 MiB/s | -| Raw uplink baseline, 20 MiB HTTP POST (×2) | 10.2–11.8 s | 1.69–1.96 MiB/s | - -Both sandboxes were terminated (`sb-01M64EKAQXN0GXYNRZS7C9PCH3`, -`sb-01M64EQP7ZQ6B0E56X1G1RSWPS`); the snapshot had a one-hour TTL. - -**Conclusions.** Modal's upload path matches the raw uplink, and parallelism adds -only about 12%. At that rate, 506 MiB takes about 4.6 minutes. Snapshotting the -same 506 MiB would take about 24 s if it scales linearly from the measured 80 MiB -(not measured). Returning the full package takes another 40–65 s. Sandbox start, -which a warm pool would save, is about one second. Remounting data that is -already in Modal costs well under a second. - -**What the bytes are.** The Windows development workspace (not the 506 MiB audit -workspace) contains a 3 MB `app.sqlite`, 7 source PDFs totaling 47.3 MiB, 124 -rendered PNG artifacts totaling 118.1 MiB, and 16 closed execution journals -totaling 92 MiB. Everything except the database and journals written since the -last push is immutable and content-addressed. Copied journals are historical only; -a cloud run never resumes them. - -Not measured: local package export time on Windows. - -## 2. Storage design for "use the cached PDF" - -The goal is to upload a source file once, ideally in the background right after -import, and have each review sandbox read it from Modal storage. - -| Option | How it fits | Limits | -| ---------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| **Named Volume v2, content-addressed** (recommended) | `client.volumes.fromName(name, { createIfMissing: true, version: 2 })`; mount with `volumes` in `sandboxes.create`, read-only in review sandboxes via `withMountOptions({ readOnly: true })`. Files never change once written under their hash. `sync /cache` commits explicitly (v2 only). | The JS SDK has **no Volume upload API** (Python's `batch_upload` only), so writes go through a short-lived uploader sandbox using `copyFromLocal`. Volumes are mounted at create time. | -| Directory-snapshot chain | Already used for checkpoints. `mountImage` measured at 0.3 s. Mount the previous cache snapshot, upload only missing hashes, snapshot again. | Each round re-snapshots the whole cache. Directory snapshots default to a 30-day TTL (configurable, including no expiry). | -| R2 bucket + `CloudBucketMount` | Desktop uploads with S3 multipart; sandboxes mount the bucket read-only. Uploads need no running sandbox and can resume. | Needs separate R2 credentials in a Modal Secret and in the desktop. Uplink speed is unchanged. Mountpoint semantics (no append or seek-write). | - -**Recommended flow.** - -1. On PDF import or revision import, the backend queues a background "seed" job: - list cached hashes, then upload missing artifacts to the Volume through an - uploader sandbox (0.25 CPU, a short `idleTimeoutMs`, no network, no provider - secrets). The same job seeds new renders and closed journals by hash as they - appear. A 47 MiB drawing set takes about 25 s at the measured uplink, while the - reviewer is still setting up sheets. -2. On Send, the package keeps today's complete-package manifest, but the upload - step skips any artifact whose hash is already in the cache. The worker - hard-links or copies cached files into `/moke/input` and verifies each SHA-256 - against the manifest. It is the same integrity check, fed from a different - place, so the current whole-workspace replacement semantics stay intact. - Unrelated projects stop costing upload time without needing the unresolved - project-scoped merge protocol. -3. The pre-dispatch checkpoint then covers only the small, changed part of - `/moke`. Cached artifacts are immutable and live on in the Volume. -4. On return, seed the download directory from local artifacts with matching - hashes before `downloadCloudPackage`. `downloadedCloudFileMatches` already - skips verified files, so only the database, journals and new crops come back. - -With seeding, a typical push uploads the 3 MB database plus any new journal, about -2 s at the measured uplink. Seeding is approved for every project. It still needs -a cache retention rule and a delete path for removed projects. The cloud -composer's current disclosure says files leave the device at Send; update it to -say source files are synced to cloud storage after import. - -**Warm sandboxes.** The current single-use warm pool (`bun run modal:warm`) saves -about a second per review and keeps paying until it expires. It is not worth -running as standing capacity. What is worth doing is starting the review sandbox -speculatively when the reviewer opens the cloud composer, and terminating it if -they leave. That overlaps the cache delta upload with the reviewer choosing -options. - -## 3. Cache-hit target: 90%+ - -### Baseline - -Read from a copy of the Windows development database (`packet_review_attempts`): -16 recorded packet reviews, all on Baseten, 128 model calls and 5.16M prompt -tokens. pi-ai stores `usage.input` as uncached tokens only -(`openai-completions.js:1186`), so the hit rate is `cacheRead / (input + -cacheRead)`. - -| Measure | Result | -| ---------------------------- | -----: | -| All prompt tokens cached | 79.8% | -| First call of each review | 65.7% | -| Later calls | 80.5% | -| Later calls below 50% cached | 19/112 | - -Per-turn patterns show where the misses come from. Some reviews fall steadily -(99% → 51% over eight turns). Others drop suddenly mid-review to 0–34% and then -recover. No Meta packet reviews were recorded, so Meta's baseline is unmeasured. - -### What breaks the cache today - -1. **No routing key.** `requestOptions` drops Pi's `sessionId`, so Meta gets no - `prompt_cache_key`. Baseten gets no `x-session-affinity` either. The fallback - Baseten model also lacks `compat.sendSessionAffinityHeaders`, so pi-ai would - not send the header even if the session ID were passed - (`openai-completions.js:544`). Requests can land on replicas without the - prefix, which fits the sudden drops. -2. **History rewrites.** `boundedImageContext` replaces images that fall out of - its window with a text notice. That applies to Baseten packet reviews (8 - images) and every non-packet call (6 images, 2 MiB). Each slide changes an - earlier position in the prompt, so everything after it is recomputed, which - fits the steady declines. Pi compaction (131,072-token reserve) also rewrites - history once the context nears the window. -3. **Cold starts.** Each review's first call finds no prefix unless an earlier - request sent the same one. Starting 29 sheet children at the same instant - would make all 29 cold. -4. **Eviction.** The default cache lives in memory and can be evicted between - turns. - -### Rules for 90%+ - -- **Append-only transcripts.** Never edit earlier turns. On Meta, keep images - rather than sliding a window. Per-sheet children stay well under the 50-image - limit, and reaching it should end the child or trigger one planned checkpoint. - On Baseten, when the 8-image window fills, start a fresh turn from a text - summary: one planned miss instead of one per turn. -- **One stable prefix, in a fixed order.** Versioned system prompt with no dates, - counters or request text; tool definitions in a fixed order; the project setup - record; the sheet inventory; then the child's sheet assignment last. Host notes - (progress, budgets, warnings) are appended as new messages. The packet prompt - already ends with its one variable line, `Requested scope`. -- **One routing key per shared prefix**, such as - `moke:::`. Send it as - `prompt_cache_key` on Meta and `x-session-affinity` on Baseten, for the setup - conversation, every sheet child and the cross-sheet pass. Meta warns that - per-session keys lower hit rates. -- **`prompt_cache_retention: "24h"`** on Meta, so same-day reruns and revision - re-reviews still hit. -- **Warm the prefix before fanning out.** Start sheet children after the setup - conversation's first response, or after one minimal request - (`max_output_tokens: 16`) carrying the shared prefix. -- **Byte-identical media.** Reference images by Files API `file_id`, or replay the - stored artifact bytes exactly. Never re-render or re-encode an image already in - the history. - -Illustrative arithmetic, not measured: a sheet child with a 21K-token shared -prefix that adds about 4K tokens per turn over 30 turns sends about 2.37M prompt -tokens, of which about 120K are new. That is about 95% cached if every turn hits. -One mid-child miss recomputes the whole context at that point, so a few misses -push a child below 90%. History rewrites matter most. - -At Spark 1.3 rates ($0.15 cached, $1.25 uncached per million), 80% cached costs -$0.37 per million input tokens and 95% costs $0.21, about 45% less. If Meta does -not count cached tokens toward its 4M TPM limit (undocumented), caching also -allows more sheets to run at once. - -### Measure and enforce - -- Report each review's cached share, and each sheet child's, in **Copy review - diagnostics**. The data is already stored as `cacheRead` on every response - receipt. -- Offline test, no paid calls: drive consecutive turns through the real provider - adapter with the recorded transport. Assert that each request's serialized - `input` starts with the previous request's `input` byte for byte. This catches - history rewrites, unstable ordering and changing system text in CI. -- Live acceptance on a sanitized set: a bounded Meta review must report at least - 90% weighted cached share. Treat anything under 85% as a regression. - -## 4. Provider levers for parallel review - -### Meta Model API (selected for parallel review) - -- **Rate limits** are per team, per model: `muse-spark-1.3` allows 3,000 RPM and - 4,000,000 TPM. Background-response creation is separately capped at 600 per - minute. Read `x-ratelimit-remaining-tokens` and back off on 429 with - `Retry-After`. Prefer an adaptive limiter driven by those headers over a fixed - sheet concurrency. The docs do not say whether cached tokens count toward TPM. - Baseten's docs say they do on Baseten. -- **Prompt caching** is automatic prefix caching. `prompt_cache_key` routes - requests that share a prefix to the same backend. Meta advises one stable key - per shared prefix, not per session. `prompt_cache_retention: "24h"` asks for - longer retention on Responses. Cached input costs $0.15 vs $1.25 per million. - Section 3 gives the prefix and key rules. -- **Current gap.** Pi Durable sets `sessionId` for every generation - (`pi-durable/dist/harness/generation.js:129`), and pi-ai maps it to - `prompt_cache_key` (`pi-ai/dist/api/openai-responses.js:267`). Our - `requestOptions` in `backend/inference/provider.ts` returns a new object without - `sessionId` or `cacheRetention`, so neither field is sent. Per-session keys - would also be too fine for children that share a prefix; pass an explicit - shared key instead. -- **Files API.** `POST /v1/files` with `purpose=user_data` returns a `file-` ID - usable as `input_image.file_id` or `input_file.file_id` in later requests. Files - can be up to 1 GiB, count toward a 100 GiB team storage limit, and never expire - unless `expires_after` (1 hour to 30 days) is set. Today pi-ai serializes every - retained image as a base64 data URL inside `function_call_output` - (`openai-responses-shared.js:216`), and stateless replay resends all of them on - every turn. Over the measured 1.8 MiB/s uplink, a local turn carrying 6 MiB of - images spends about 3 s just uploading. In the cloud sandbox this matters less. - Preseeding sheet renders through the Files API makes requests small in both - places. Untested: whether Meta accepts `file_id` images inside - `function_call_output`. Meta's docs say images belong in user messages, but the - parallel-review thread confirmed that base64 images in tool results are read. -- **Background responses** (`background: true`) are exempt from the server-side - time limit that returns HTTP 504 on long non-streaming requests. Responses can - be polled, stream-retrieved by ID with `starting_after`, and cancelled. The - creation body is capped at **1 MiB**, so image-heavy turns need Files API IDs - first. This is Meta's documented route for long reasoning turns. The 300 s - client timeout recorded in Audit 3 was on Baseten, which offers no equivalent - for Model APIs. -- **Reasoning effort** runs `minimal`, `low`, `medium`, `high`, `xhigh`, and `max` - (Spark 1.3 and Wonder only). Higher effort costs more tokens and time. Packet - review currently fixes `high`. An initial review could use `medium` or `high` - per sheet and `xhigh` for the cross-sheet pass. That is a hypothesis to test - against known findings, not a documented accuracy claim. -- **Parallel tool calls** default to on. `render_sheet` is declared - `executionMode: "sequential"`, so several crops requested in one turn render one - after another. Allowing them to run concurrently would use the sandbox's 4 CPUs. -- **Data use.** Content sent on free credits may be used for product improvement, - and Files API uploads stay on Meta's side until deleted or expired. The - drawing sets are sanitized and seeding is approved; still set `expires_after` - and delete files with the project to stay under the 100 GiB team limit. - -### Baseten (Muse Spark Open) - -Default Model API limits are 15 RPM / 100K TPM (unverified Basic), 120 / 500K -(verified Basic) and 120 / 1M (Pro). Cached and uncached input count equally -toward TPM, so on Baseten caching lowers cost but does not raise concurrency. -Caching is automatic. `x-session-affinity` routes related requests to the same -replica to improve cache hits. A 529 means no serving capacity even -within limits. Async inference exists only for dedicated deployments, not Model -APIs. Baseten's 8-image request window remains in force. At these limits Baseten -supports only a few parallel sheet children, so Meta is the right default for -parallel initial reviews. Baseten remains useful as the open-weight option. - -## 5. Initial Review button - -A new feature; no equivalent exists in the code. The composer has no suggested -prompts yet (the only suggestions are sheet labels). -`backend/review/prompts/assets/REVIEW_METHOD.md` defines the four passes, and -`docs/inference.md` says Auto routes a whole-set request. - -Specified behavior: - -- **What it sends.** One click sends a preset, high-quality user prompt through - the normal message path, so it appears in the conversation as the reviewer's - message. The text is a versioned template (`initial-review-v1`) whose identity - is recorded with the prompt hashes, so runs stay comparable. -- **When it is visible.** Only during the initial review phase: the project has - its first drawing set and no revised set yet. Revised sets are separate - `Revision` imports linked through `RevisionCorrespondence`. Once a revised set - is imported, the button is gone for good and the reviewer uses typed prompts - and suggested prompts. Before it ships, confirm how an initial set made of - several PDFs is modeled, so a multi-file first set doesn't count as a revision. -- **When it is enabled.** After sheet setup has a frozen inventory. Disabled with - status while an initial review is running. During the initial phase it can be - run again, for example after sheet corrections. -- **What it runs.** Whole set, disciplines inferred from the sheet index, all four - passes, Meta `muse-spark-1.3`, cloud when configured and local otherwise. Show - this as one summary line beside the button, not a form. -- **Template content.** Record project facts first (jurisdiction, occupancy, code - editions, areas, service size). Review each sheet for completeness and - discipline items. Run cross-sheet coordination last. Rank findings by severity, - state uncertainty explicitly, and never mark coverage complete without - evidence. Keep the text static: everything project-specific comes from tools - and the setup record, which keeps the prompt cacheable (section 3). -- **Preseeding at import**, before anyone clicks: render sheet overviews, upload - source files to the Modal cache, and upload sheet overviews to Meta Files with - an expiry. Paid model calls start only on click. -- **Progressive results.** As each sheet child finishes, publish its findings - through `progress.json` as a read-only preview, so the reviewer sees findings - within minutes and can start correcting. The authoritative import stays the - complete-package return, which avoids a live database merge. -- **After revisions.** Re-reviews run from typed or suggested prompts. Cached - source files and a warm 24-hour prompt cache mean unchanged sheets cost little. - Suggested prompts are a separate feature to design; candidates are "Re-review - changed sheets" and "Check that prior comments are closed". - -## Expected timeline, not measured end to end - -| Stage | Today (Brewery, 29 sheets) | With these changes | -| --------------------------- | ---------------------------- | ------------------------------------------------------- | -| Source upload | ~4.6 min at Send (506 MiB) | ~0 at Send; seeded in the background after import | -| Database and journal upload | included above | about 2 s for a 3 MB database, plus any new journal | -| Sandbox start | ~1 s | ~1 s, overlapped with option choice | -| Pre-dispatch checkpoint | ~24 s (extrapolated) | about a second for the changed portion | -| Review | 33+ min, 220 of 1,276 passes | parallel-review thread estimates 20–25 min at 16 sheets | -| First findings visible | at the end | as the first sheets finish | -| Result return | ~40–65 s download | database, journals and new crops only | - -## Unresolved - -- Local package export time on Windows. -- Volume v2 read throughput from a sandbox, and uploader-sandbox write speed into - a mounted Volume (expected to match the uplink). -- Meta's cache hit rate for packet reviews (no Meta reviews recorded yet), and - whether Meta replays reasoning items into the prefix unchanged across turns. - The first live Meta run should log per-turn cached share. -- Whether Meta accepts `file_id` images in `function_call_output`, and whether - cached tokens count toward Meta TPM. -- Accuracy of lower reasoning effort per sheet. This needs a small evaluation set - with known findings before changing defaults. -- How a multi-PDF initial set is modeled, for the Initial Review visibility rule. - -## Sources - -- Modal: [Sandbox snapshots](https://modal.com/docs/guide/sandbox-snapshots), - [directory snapshots blog](https://modal.com/blog/directory-snapshots-resumable-project-state-for-sandboxes), - [Filesystem access](https://modal.com/docs/guide/sandbox-files), - [Volumes](https://modal.com/docs/guide/volumes), - [Cloud bucket mounts](https://modal.com/docs/guide/cloud-bucket-mounts), - [Sandboxes: idle timeouts and readiness probes](https://modal.com/docs/guide/sandboxes), - [Networking and connect tokens](https://modal.com/docs/guide/sandbox-networking), - [Warm sandbox pool](https://modal.com/docs/examples/sandbox_pool), - [JS Sandbox API](https://modal.com/docs/sdk/js/latest/Sandbox). Installed - `modal@0.11.1` type definitions for `VolumeFromNameParams.version`, - `SandboxCreateParams.volumes` and `cloudBucketMounts`. -- Meta: [Prompt caching](https://dev.meta.ai/docs/features/prompt-caching), - [Pricing and rate limits](https://dev.meta.ai/docs/pricing-rate-limits), - [File handling](https://dev.meta.ai/docs/file-handling), - [Image understanding](https://dev.meta.ai/docs/image-understanding), - [Responses API](https://dev.meta.ai/docs/protocols/responses), - [Reasoning](https://dev.meta.ai/docs/reasoning), - [Tool calling](https://dev.meta.ai/docs/tool-calling), - [Token counting](https://dev.meta.ai/docs/token-counting). -- Baseten: [Pricing and limits](https://docs.baseten.co/inference/model-apis/pricing-and-limits), - [Async inference](https://docs.baseten.co/inference/async), - [Muse Spark Open](https://www.baseten.co/library/muse-spark-open/). -- Repository: `backend/cloud/modal.ts`, `backend/cloud/transfer.ts`, - `backend/inference/provider.ts`, `backend/packet-review/service.ts`, - `docs/cloud-reviews.md`, `docs/research/audit-3-cloud.md`, - `docs/research/audit-3-validation.md`. diff --git a/docs/research/hackathon-product-audit.md b/docs/research/hackathon-product-audit.md deleted file mode 100644 index 05592432..00000000 --- a/docs/research/hackathon-product-audit.md +++ /dev/null @@ -1,167 +0,0 @@ -# Hackathon product audit - -Checked 2026-10-30 against `main` at `9bf12bda1`, after PRs #15 and #16 merged. -Sol, gpt-6.1-sol on xhigh, audited the same code and data independently and -then checked this document's claims. Its corrections are included below. - -Evidence comes from 15 recorded reviews in five app databases, copied and read -without changes: - -- the Audit 3 recheck workspace from October 29, 11 reviews; -- the fast initial review runs from October 30, four reviews of the brewery and - civil sets, on this computer and in the cloud. - -## Where things stand - -Since this audit, PR #17 added reviewer decisions on findings, a firm checklist -and jurisdiction leads, and PR #20 added the Florida code index, code search and -PDF source text. The gaps below describe main before those PRs. - -Moke is a strong drawing coordination checker built on solid infrastructure. It -does not yet check drawings against requirements, and it does not yet track -its own findings from one revision to the next. Those are the two things the -product is supposed to be about. - -The speed problem is mostly solved. On October 30 the 29-sheet brewery set -finished in about 15 minutes for about $11, and the 5-sheet civil set in 6 to 7 -minutes. Over 90% of prompt tokens came from the cache. - -The strongest impact evidence is a real permit. The owner ran the same Moke -review instructions by hand in Codex for about two weeks of internal revisions -on a 13,000 square foot custom home worth several million dollars. The county -returned 3 comments after one day, where a job like that typically goes through -3 or more rounds of county revisions, and approved the permit one week after -submittal. This is one project, reported by the owner, using the method that -predates the event rather than the Moke application. - -The biggest risk to the submission is the video. Raw full-screen recordings of -the four October 30 runs exist, but there is no script, edit or voiceover yet. - -## Scorecard - -| Criterion | Weight | Standing | Why | What would move it | -| ------------------- | -----: | ----------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------- | -| Impact | 30% | Adequate, close to strong | The problem is real and expensive, and one real permit came back with 3 comments after a day. The app's own findings have not been checked for accuracy. | Show the permit dates and comment letter, check a sample of findings, and show one requirements example | -| Meta tooling | 25% | Strong | Muse Spark 1.3 and Muse Spark Open both run full reviews. The model chooses which sheets to open, what to zoom into, what to calculate and what to search. | Show SAM doing useful work once, or say plainly that the review does not need it | -| Technical execution | 20% | Strong | Durable runs, recovery, offline reports, local and cloud execution, and passing CI on Windows, macOS and Linux. | Little; protect stability through the freeze | -| Demo | 25% | Weak today | Good raw footage and a polished workbench. No finished video. | A narrated three-minute cut with one clear engineering outcome | -| Eligibility | Gate | Pass once the video is posted | The repository is public and identifies the work built during the event. You are the only Git author. | Post the video, list contributors on the form, and use a drawing set you are allowed to show publicly | - -The Meta tooling and execution scores do not depend on the new features. Impact -and demo do. - -## Submission requirements - -| Requirement | Status | -| -------------------------------------- | ----------------------------------------------------------------------------------------------- | -| Deadline | November 4, 9 PM Eastern | -| Public repository | Done | -| Work built during the event identified | Done in the README | -| Meta models used | Done: Muse Spark 1.3, Muse Spark Open on Baseten, and SAM 3.1 in the count check | -| Three-minute video | Raw footage only | -| Contributors listed | Fill in on the submission form | -| Shareable sample drawings | Only the authored lighting schematic. The sanitized client sets are not cleared for public use. | -| Installer for judges | No GitHub release yet. CI builds all three platforms. | - -## Requirements review - -Moke is asked to check code, jurisdiction and firm requirements, but in -practice it does not verify any of them. - -- **No verified citations.** None of the 15 recorded reviews contains a - verified citation. The four October 30 runs recorded 226 findings and no - citations at all. Their 13 findings labeled Code all remain Needs input. -- **The code text is out of reach.** Every code source the model tried was - unavailable: ICC's Florida codes, Municode, and the Pasco County land - development code page. The only pages it could read were the Florida Building - Commission's list of links and the State Fire Marshal's code page. - `read_source` also rejects remote PDFs. -- **Code claims slip through as drawing conflicts.** The host requires a - verified citation only for findings labeled Code, Permit condition or Agency - request, at `backend/packet-review/service.ts:1137`. In the October 30 brewery - run, 24 confirmed findings labeled Drawing conflict or Recommendation cite - NEC, FBC or NFPA without a citation. One says a note on E-1 references the - wrong NEC table. Sol found this; the first version of this audit missed it. -- **There is no place for supplied requirements.** A reviewer can import a - jurisdiction checklist or firm standard as a PDF, but Moke treats its pages as - drawing sheets. It cannot read them as governing text or cite them. -- **Firm standards are not supported.** The firm memory plan, uncommitted on - branch `research/aec-firm-memory-system`, recommends waiting on the full memory system - and starting with a firm checklist and a per-jurisdiction requirements record. -- **The instructions are written for Florida only.** - -### Built on October 30 - -Branch `feat/florida-code-index` addresses the access problem: - -- **Code search.** A new `search_code` tool searches a Florida code index of 307 - entries: the owner's 2023 FBC plan review reference, plus Florida Statutes, - Florida Administrative Code rules, the Fire Prevention Code adoption and the - 9th Edition volumes. A citation names an index entry, and the host attaches - the section's link to the citation, the review panel and the HTML report. -- **PDF text.** `read_source` now extracts embedded text from PDFs, starting at - a `#page=N` fragment. A live check read pages 2 to 17 of an 85-page Florida - Building Commission PDF. -- **Readable statutes.** Statute links resolve to the Florida Legislature's - official text, which `read_source` can retrieve and verify against. - `flrules.org` replaces a misspelled domain in the official-source list. -- **Still unverified:** ICC section text, which the engineer opens from the - citation's link, and Florida rule text, which is published only as Word - documents. Sol's labeling gap above is not fixed yet. - -## Revisions - -The count check handles revisions properly: the engineer confirms which new -sheet replaces which old one, and affected results are rechecked with history -kept. The main packet review does not use any of that. - -- Packet review counts a sheet as unchanged only when it comes from the same PDF - file, at `src/lib/review/packet-coverage.ts:35`. Any revised PDF makes every - sheet count as new. -- In the civil Rev 2 follow-up, review `29364835`, all 7 findings carried over - from Rev 1. Six still point at Rev 1 sheets, and one refers to sheet C-4, - which Rev 2 removed. The review had four confirmed unchanged sheet links and - used none of them. -- A finding is either Confirmed or Needs input. There is no way to record that - a revision fixed it, partly fixed it or left it open. -- The engineer cannot mark a finding as wrong. The only control is the - Completed checkbox, which means the drafter made the correction. -- Closure does exist for earlier agency comments, gated on inspecting the - revised sheet. No recorded review has used it. -- A reissue of 3 sheets cannot be reviewed together with the 26 unchanged - sheets from the earlier set. - -## What remains - -Ranked by effect on the score. Sol's effort estimates add up to more than the -four working days left, so the order matters. - -1. **Video.** Script it around one engineering outcome, cut the existing - footage, record the final build, add the voiceover and captions, and post it. - About 1 to 1.5 days. This gates eligibility. -2. **Check 10 to 20 findings by hand.** Mark each as correct, wrong or - overstated against the drawings. This gives the impact claim a real number. - About half a day of engineering time, and it can run in parallel with the - build work. -3. **Finish the requirements path.** The code index and PDF text are built. - Still to do: a "confirmed by engineer" step for ICC citations, a citation - requirement for any confirmed finding that names a code section, and the - firm checklist from `Discipline_Review_Checklists.md`. Then run one real - review to measure how many citations it records. About half a day to a day. -4. **Revision closure.** Pass the confirmed sheet links into packet review. - Each open finding from the earlier review is checked on its replacement - sheet and marked fixed, partly fixed, still open or no longer applicable. - Add a way for the engineer to mark a finding as wrong and say why. The report - then reads like "Rev 1 had 7 issues; Rev 2 fixed 4." About 1 to 1.5 days. -5. **Judge package.** Publish a release build with a shareable sample set, and - reuse the video for the open-weight, agentic, viral and community bonuses. - A few hours. - -Freeze the build early enough to record the version you submit. - -## Decisions needed - -- Which of items 3 and 4 to build, or both with a hard freeze date. -- Which drawing set can appear in the public video and repository. -- Who scripts and voices the video, and the date for the final recording. -- Whether you will check the 10 to 20 findings yourself. diff --git a/docs/research/memory-sota-notes.md b/docs/research/memory-sota-notes.md deleted file mode 100644 index a5f54dfe..00000000 --- a/docs/research/memory-sota-notes.md +++ /dev/null @@ -1,558 +0,0 @@ -# Long-term memory for LLM agents: state of the art (October 2026) - -Research date: October 30, 2026. Status: literature and product survey. This note informs a later MOKE memory design; it proposes no design itself. - -Question: what does the field know, with evidence, about agent memory that learns an organization's practices and resurfaces prior work, and which of the evidence transfers to firm-scoped review of engineering drawing PDFs? - -Evidence labels: - -- **[measured]**: an experiment with a stated protocol. The authors may still be the vendor. -- **[vendor claim]**: a self-reported product number or a capability statement without an independent protocol. -- **[inference]**: my reading of the evidence. - -Dates are publication or last-revision dates. Undated documentation was read on 2026-10-30. arXiv identifiers encode the submission month (`2605.x` means May 2026). - -Repository context: Bun 1.4.4 (`.bun-version`), `effect` 4.0.4 and `@effect/sql-sqlite-bun` 4.0.4 (`package.json`). - ---- - -## 1. Memory architectures and products - -### Mem0 - -**2025 paper** (arXiv 2504.19413, 2025-04-28): https://arxiv.org/abs/2504.19413 - -- **Write path.** An LLM (GPT-4o-mini in the paper) extracts candidate facts from a new exchange. It sees a rolling summary plus the last m=10 messages. Each candidate is then compared with the s=10 most similar stored memories, and the LLM chooses one of four operations: `ADD`, `UPDATE` (augment), `DELETE` (contradicted) or `NOOP`. -- **Graph variant (Mem0g).** It extracts entity and relation triplets. A conflict resolver _marks relations invalid rather than deleting them_. -- **Retrieval.** Mem0 uses vector similarity. Mem0g adds entity-anchored subgraph expansion and triplet semantic matching. -- **LoCoMo results** (J = LLM-as-judge accuracy) [measured, vendor-run]: - - | System | Overall J | p95 total latency | Notes | - | -------------------------- | --------- | ----------------- | ---------------------- | - | Full context (~26k tokens) | 72.90 | 17.1 s | | - | Mem0g | 68.44 | 2.59 s | | - | Mem0 | 66.88 | 1.44 s | ~1.8k retrieved tokens | - | Zep | 65.99 | 2.93 s | | - | Best RAG | ~61 | | | - | LangMem | 58.10 | 60 s | | - | OpenAI memory | 52.90 | | | - | A-Mem | 48.38 | | | - -- **Store size.** Mem0 stores about 7k tokens per conversation and Mem0g about 14k. The paper puts Zep's graph above 600k tokens. -- **Takeaway.** Even in the vendor's own run, full context beat every memory system on accuracy. Memory won on latency and cost. - -**Dispute.** Zep re-ran the benchmark and reported a corrected Zep score of **75.14 ±0.17 J** (blog, 2025-05-06): https://www.getzep.com/blog/lies-damn-lies-statistics-is-mem0-really-sota-in-agent-memory/. Mem0 disputed the setup in an undated GitHub issue that claims 58.44%: https://github.com/getzep/zep-papers/issues/5. [vendor claims both ways] - -**April 2026 algorithm** (blog 2026-04-16, updated 2026-09-28): https://mem0.ai/blog/mem0-the-token-efficient-memory-algorithm - -- **Extraction.** Mem0 abandoned two-pass `ADD/UPDATE/DELETE` reconciliation for **single-pass, ADD-only extraction**: "overwrites sometimes erased key information … deletes sometimes removed information that would be relevant later"; new facts now live alongside old ones. Mem0 says this roughly halved extraction latency. -- **Agent facts.** Facts generated by the agent, such as "I booked X", are now stored as first-class memories. -- **Retrieval.** Semantic, keyword (with verb normalization) and entity scores are computed in parallel and fused by rank. -- **Reported results** [vendor claim]: - - | Benchmark | Old algorithm | New algorithm | - | ----------- | ------------- | ------------- | - | LoCoMo | 71.4 | 92.5 | - | LongMemEval | 67.8 | 94.4 | - | BEAM-1M | n/a | 64.1 | - | BEAM-10M | n/a | 48.6 | - - Each query uses about 7k tokens. - -- **Survey and pricing.** Mem0's own survey of the field: https://mem0.ai/blog/state-of-ai-agent-memory-2026 (2026-04-01, updated 2026-10-10). The pricing page lists graph memory, user profiles and "Dream (memory consolidation)" only on custom plans: https://mem0.ai/pricing. - -**Independent check.** Under a matched gpt-4.1-mini answer model, another group measured LoCoMo at **61.4% for Mem0**, 65.4% for Supermemory and about 71% for Zep (arXiv 2605.04897, 2026-05-06): https://arxiv.org/abs/2605.04897 [measured, competitor-run] - -### Zep / Graphiti - -Zep paper, arXiv 2501.13956 (2025-01-20): https://arxiv.org/abs/2501.13956 - -**Graph structure.** Graphiti builds a three-tier graph: an episode subgraph (raw messages and documents, non-lossy), a semantic entity subgraph (entities and fact edges) and a community subgraph (label-propagation clusters with summaries). - -**Bi-temporal edges.** Each edge carries `t_valid`/`t_invalid`, when the fact held in the world, and `t'_created`/`t'_expired`, when the system recorded or retired it. - -**Contradictions.** An LLM compares a new edge with semantically related edges. When it finds an overlapping contradiction, it sets the old edge's `t_invalid` to the new edge's `t_valid`. The old fact is invalidated, not deleted. - -**Retrieval.** Cosine similarity, BM25 and breadth-first graph search, followed by rerankers: RRF, MMR, episode-mentions, node-distance or a cross-encoder. - -**Results** [measured, vendor-run]: - -- Deep Memory Retrieval: 94.8% vs MemGPT 93.4%. The authors call DMR too small: 60-message conversations that fit in context. -- LongMemEval_s (~115k-token histories): - - | Answer model | Full context | Zep | Latency (full → Zep) | Prompt tokens with Zep | - | ------------ | ------------ | ----- | -------------------- | ---------------------- | - | gpt-4o | 60.2% | 71.2% | 28.9 s → 2.58 s | ~1.6k | - | gpt-4o-mini | 55.4% | 63.8% | not extracted | ~1.6k | - -**Status and cost.** - -- Graphiti is open source. A 2026-06-08 commit brought its MCP server to "core-parity" for bi-temporal queries, sagas, communities and custom types: https://github.com/getzep/graphiti/commit/29a756595e3a00a7511b8e82d2593d6a0a587e61 -- Zep Cloud bills 1 credit per Episode up to 350 bytes. Retrieval and storage are unmetered. https://www.getzep.com/pricing -- [inference] Writes are LLM-heavy: extraction, deduplication and invalidation run on every episode. The 600k-token graph figure comes from Mem0's paper, not from Zep. - -### Letta / MemGPT - -- **MemGPT (2023)** treats the context window as virtual memory. Core memory blocks are labelled, size-limited text that the agent edits through tools. Recall memory is searchable conversation history. Archival memory is a vector store the agent writes passages into. Paper: https://arxiv.org/abs/2310.08560. Memory blocks are explained at https://www.letta.com/blog/memory-blocks/ (2025-05-14). -- **Sleep-time compute** (arXiv 2504.13171, 2025-04-17; https://arxiv.org/abs/2504.13171; blog https://www.letta.com/blog/sleep-time-compute/ 2025-04-21). A background agent rewrites shared memory between tasks. Measured: about **5x less test-time compute** for equal accuracy on Stateful GSM-Symbolic and Stateful AIME; up to +13% and +18% accuracy when sleep-time compute is scaled; 2.5x lower cost per query when amortized over related queries; gains track how predictable the queries are. [measured] -- **Filesystem baseline** (blog 2025-08-12; https://www.letta.com/blog/benchmarking-ai-agent-memory/). A Letta agent on gpt-4o-mini reached **74.0% on LoCoMo** with conversation history stored in plain files and searched by the agent, without specialized memory tooling. Letta argues the result says more about how well agents use search tools than about specialized memory mechanisms. [measured, vendor-run] -- **Context Repositories and MemFS** (blog 2026-02-12; https://www.letta.com/blog/context-repositories/; docs https://docs.letta.com/concepts/memfs/index.md). Letta Code memory is now a **git-backed file tree**. Every memory change is a commit; subagents learn in isolated worktrees and merge through git conflict resolution, so several reflection agents can write at once; `/doctor` audits duplication and token use. The older sleep-time docs now redirect to MemFS: https://docs.letta.com/guides/agents/architectures/sleeptime. - -### LangMem (LangChain) - -Source: https://langchain-ai.github.io/langmem/concepts/conceptual_guide/ - -- **Memory types.** Semantic (a schema'd profile or an unbounded fact collection), episodic (past experiences as few-shot examples or summaries) and procedural (prompt rules updated by a prompt optimizer). -- **Writes.** Memories form "in the hot path" through agent tools, or in the background through a debounced reflection executor. `create_memory_store_manager` defaults to `enable_deletes=False`. -- **Measured result.** In Mem0's LoCoMo run, LangMem had the highest search latency of any system (p95 ~60 s). [measured, competitor-run] - -### A-MEM - -A-Mem (arXiv 2502.12110, NeurIPS 2025): https://arxiv.org/abs/2502.12110 - -- **Write path.** Zettelkasten-style notes. On insert, the LLM generates context, keywords and tags, links the note to similar historical notes, and can rewrite their attributes ("memory evolution"). -- **Measured result.** A-Mem scored 48.38 J in Mem0's LoCoMo run. [measured, competitor-run] -- [inference] Rewriting old notes on insert is the same context-collapse risk that ACE describes (section 2). - -### MemOS (MemTensor) - -Papers: arXiv 2505.22101 (2025-05-28) and 2507.03724 (v4 2025-12-03). https://arxiv.org/abs/2507.03724 - -- **Units.** The MemCube unit wraps three kinds of memory: plaintext, activation (KV-cache) and parameter (LoRA). Each carries provenance and version metadata. It is scheduled like an operating-system resource. -- **Claimed results.** MemOS claims first place on LoCoMo, LongMemEval, PreFEval and PersonaMem against MIRIX, Mem0, Zep, Memobase, MemU and Supermemory. [vendor claim] -- **Related benchmark.** MemTensor authors also built HaluMem (section 3), where MemOS ranks first. That is a conflict of interest worth noting. - -### Cognee - -Docs: https://docs.cognee.ai/python-api - -- **Pipeline.** `add` → `cognify` (chunk, LLM-extract a knowledge graph and embed) → `search`. It stores relational, vector and graph data together. `memify` is deprecated in favour of `improve()` (https://docs.cognee.ai/core-concepts/main-operations/legacy-operations/memify). -- **Paper** (arXiv 2505.24478, 2025-05-30; https://arxiv.org/abs/2505.24478). Systematic hyperparameter tuning of chunking, graph construction, retrieval and prompts on HotPotQA, TwoWikiMultiHop and MuSiQue gave "consistent but not uniform" gains. The authors stress that the evaluation metrics are unreliable. [measured, vendor-run] - -### HippoRAG 2 - -HippoRAG 2 (arXiv 2502.14802, ICML 2025): https://arxiv.org/abs/2502.14802 - -- **Method.** It builds an open knowledge graph from passages and retrieves with Personalized PageRank over a combined passage and phrase graph, with an LLM filtering recognition. -- **Results.** About **+7% on associative (multi-hop) memory** tasks over the best embedding retriever, while staying at or above dense RAG on simple factual recall. Earlier graph RAG methods had fallen below dense RAG on factual recall. [measured] - -### GraphRAG, LazyGraphRAG and LightRAG - -- **LazyGraphRAG** (Microsoft Research blog, 2024-11-25; https://www.microsoft.com/en-us/research/blog/lazygraphrag-setting-a-new-standard-for-quality-and-cost/). Full GraphRAG indexing with community summaries is expensive; LazyGraphRAG's indexing cost equals vector RAG's and is **0.1% of GraphRAG's**, and it matches GraphRAG Global Search quality at **more than 700x lower query cost**. [measured, vendor-run] -- **LightRAG** (arXiv 2410.05779; EMNLP Findings 2025; https://arxiv.org/abs/2410.05779). Dual-level (entity and theme) graph plus vector retrieval, with incremental graph updates. -- **Independent comparison** (arXiv 2502.11371, v3 2026-03-04; https://arxiv.org/html/2502.11371v3). RAG and GraphRAG are complementary, with no consistent winner: RAG wins on single-hop, detail-oriented questions and GraphRAG on multi-hop questions. Community-summary search loses query-specific detail, LLM-judge comparisons of summaries show strong position bias, and GraphRAG costs more to build, has higher latency and a larger footprint, and is sensitive to graph quality. [measured] - -### Supermemory - -- **Data model** (https://supermemory.ai/docs/concepts/graph-memory). Raw documents are the source of truth for RAG, and extracted "memories" are a derived fact graph. Edges are typed as _updates_, _extends_ or _derives_ (blog 2026-07-23; https://supermemory.ai/blog/memory-graph-relationship-semantics/). -- **Consolidation.** An asynchronous "dreaming" step builds the graph from documents (https://github.com/supermemoryai/supermemory/blob/main/apps/docs/concepts/how-it-works.mdx). -- **Claim.** LongMemEval-S **Recall@20 = 97%** (report May 2026; https://supermemory.ai/research/longmembench/). [vendor claim] That figure is retrieval recall, not QA accuracy. An independent matched run measured 65.4% LoCoMo QA (arXiv 2605.04897), and HaluMem measured a 16.4% correct-update rate. - -### Notable 2025–2026 entrants (all self-reported) - -- **Hindsight** (Vectorize; arXiv 2512.12818, Dec 2025; ACL 2026 demo; https://arxiv.org/abs/2512.12818). Memory is split into four networks (world facts, agent experiences, synthesized entity observations, evolving opinions or beliefs) behind `retain`, `recall` and `reflect`, because existing systems "blur the line between evidence and inference". An open 20B backbone rose from 39% (full context) to 83.6%; larger backbones reached 91.4% on LongMemEval and 89.61% on LoCoMo. The product site claims LongMemEval-S 94.6% and BEAM-10M 64.1% (https://hindsight.vectorize.io/). -- **EverMemOS** (arXiv 2601.02163, ACL 2026; https://arxiv.org/abs/2601.02163). A self-organizing memory OS. A third party reports 94.5% on LoCoMo with a GPU embedder and Neo4j (arXiv 2605.04897). -- **True Memory / "Storage Is Not Memory"** (Sauron Labs, arXiv 2605.04897, 2026-05-06). Argues that "extraction at ingestion is the wrong primitive" because content discarded before the query is known cannot be recovered. It keeps events verbatim and puts the work into a multi-stage retrieval pipeline in **a single SQLite file on CPU**, with no vector or graph database. Results: LoCoMo 93.0% (semantic-match judge, more lenient than strict scoring), LongMemEval 87.8%, BEAM-1M 76.6%; a 56-configuration ablation spanned only 1.3 points. -- **Agent Zero Memory** (arXiv 2608.29606, Aug 2026; https://arxiv.org/abs/2608.29606). Three parallel views of one history: an episodic event timeline, an entity–event graph, and a curated, **citation-locked** hierarchy of durable facts that abstains when no citation exists. It reports 95.60% LongMemEval and 93.60% LoCoMo; across eight backbones, accuracy varied 3.4 points while cost per query varied about 30x. -- Others seen but not evaluated here: MAGMA multi-graph memory (arXiv 2601.03236, 2026-01-06), MIRIX, Memobase, MemU, ZenBrain. - -### Cross-cutting pattern (2025 → 2026) - -[inference, supported by the sources above] The field is moving away from destructive, extract-and-overwrite memory and toward three practices: - -- **Keep raw episodes immutable.** Zep's episode subgraph, Mem0's 2026 ADD-only change, True Memory's verbatim events, Supermemory's documents as source of truth, and Letta's git history all do this. -- **Derive views that can be rebuilt.** Facts, entities, summaries and playbooks are derived from those episodes and carry links back to them. -- **Retrieve with several signals fused.** Lexical, semantic, entity, temporal and graph signals are combined, then reranked. - -Consolidation increasingly runs asynchronously: Letta sleep-time agents, Mem0 "Dream" and Supermemory "dreaming". - ---- - -## 2. Learning from experience - -### Measured gains - -| Method | What is stored | Source | Benchmark and measured effect | -| ---------------------------------- | ---------------------------------------------------------------------------------------------------------------------- | -------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------- | -| Reflexion | Verbal self-reflections in an episodic buffer | arXiv 2303.11366 (Mar 2023) | HumanEval pass@1 91% vs GPT-4 80% | -| Voyager | Executable code skills, retrieved by embedding | arXiv 2305.16291 (May 2023) | Minecraft: 3.3x unique items; tech-tree milestones up to 15.3x faster | -| Agent Workflow Memory | Workflows (reusable sub-routines) induced from trajectories, offline or online | arXiv 2409.07429 (Sep 2024; ICML 2025) | +24.6% relative success on Mind2Web, +51.1% on WebArena, fewer steps | -| Dynamic Cheatsheet | Self-curated "cheatsheet" of strategies and code, rewritten after each query | arXiv 2504.07952 (Apr 2025) | Claude 3.5 Sonnet AIME accuracy more than doubled; GPT-4o Game of 24 10% → 99%; GPQA-Diamond +9%, MMLU-Pro +8%; no labels | -| ReasoningBank (Google) | Title/description/content strategy items distilled from **self-judged successes and failures** (LLM-as-judge) | arXiv 2509.25140 (Sep 2025) | Up to 20% relative success and 16% fewer steps on WebArena/Mind2Web; SWE-Bench-Verified 34.2 → 38.8 (Gemini 2.5 Flash), 54.0 → 57.4 (Pro) | -| ACE (Agentic Context Engineering) | Itemized "playbook" bullets with counters, edited through **incremental deltas** by a generator, reflector and curator | arXiv 2510.04618 (Oct 2025; ICLR 2026) | +10.6% on agent tasks (AppWorld), +8.6% on finance (FiNER/Formula), up to +17.1% on AppWorld from execution feedback alone; 86.9% lower adaptation latency | -| Memory-R1 | RL-trained memory manager (ADD/UPDATE/DELETE/NOOP) plus answer agent | arXiv 2508.19828 (Aug 2025) | Beats Mem0-style baselines on LoCoMo, MSC and LongMemEval with only **152 training QA pairs**, at 3B–14B | -| Mem-α | RL-trained manager for core, episodic and semantic memory | arXiv 2509.25911 (Sep 2025) | Trained on ≤30k-token streams; generalizes to over 400k tokens (13x) | -| Procedural memory, AFTER benchmark | Evolved skills for enterprise tasks | arXiv 2606.23127 (Jun 2026) | One refinement round +3.7 to +6.7 points; skills from diverse multi-model traces 73.1% cross-model | - -Each row's arXiv ID resolves at `https://arxiv.org/abs/` (for example https://arxiv.org/abs/2510.04618 for ACE). - -Every row except AFTER is measured by the method's own authors. Most gains are on web, coding, maths or games tasks with an automatic success signal. - -### Failure modes with evidence - -- **Context collapse** (ACE, §2). Monolithic LLM rewriting of an accumulated context can erase it. In ACE's Dynamic Cheatsheet run, the context went from 18,282 tokens (66.7% accuracy) to **122 tokens (57.1%)** in one step, below the no-memory baseline of 63.7%. ACE's countermeasure is localized delta edits plus deterministic merge and deduplication ("grow-and-refine"); Mem0's 2026 move to ADD-only writes addresses the same failure. [measured] -- **Brevity bias** (ACE). Optimizers that favour concise prompts drop the domain-specific detail that made them work. [measured, qualitative] -- **Experience-following and error propagation** (arXiv 2505.16067, May 2025; ACL 2026; https://arxiv.org/abs/2505.16067). When a retrieved memory's input is similar to the current task, the agent's output closely copies it, so wrong stored experiences compound, and "misaligned experience replay" means even seemingly correct records can mislead. Gating additions on evaluator quality, and using later task evaluations as free quality labels, mitigates both. [measured] -- **Raw versus distilled memory.** The evidence conflicts: - - **Distilled wins:** ReasoningBank's distilled strategies beat raw trajectories and AWM. AWM _degrades_ on WebArena's cross-website subset. Abstract procedural memories transfer more reliably than detailed trajectories (arXiv 2604.27003, Apr 2026). - - **Raw wins:** SkillEvolBench (arXiv 2605.24117, May 2026; https://arxiv.org/abs/2605.24117) finds that agents "rarely form robust reusable skills". Raw-trajectory reuse often beats distilled skills, and writing more skills adds "episode-specific drift and procedural clutter". - - **Narrow sources overfit:** AFTER finds that skills evolved from narrow experience overfit their source context. - - [measured] The disagreement is unresolved. -- **Interference and forgetting** (arXiv 2604.27003; https://arxiv.org/html/2604.27003). External memory "does not resolve the continual-learning problem; it reshapes it". Old and new experiences compete for retrieval, negative transfer hurts the hard cases most, and finer-grained organization can cause severe forgetting. [measured, ALFWorld and BabyAI] -- **Hallucinated writes.** HaluMem (section 3) shows that errors come mostly from extraction and updating, then propagate into answers. -- **Poisoning.** Covered in section 6. Self-generated memories are the attack surface in MINJA and MemoryGraft. - -### Anthropic and OpenAI first-party features - -- **Anthropic memory tool and context editing** (2025-09-29; https://www.anthropic.com/news/context-management; docs https://platform.claude.com/docs/en/agents-and-tools/tool-use/memory-tool). The memory tool is client-side file operations under `/memories`; the application owns storage and must prevent path traversal. On an internal agentic-search evaluation, memory plus context editing scored +39% over baseline and context editing alone +29%; over 100 turns of web search, tokens fell 84%. [vendor claim] -- **Agent Skills** (2025-10-16; open standard 2025-12-18; https://www.anthropic.com/engineering/equipping-agents-for-the-real-world-with-agent-skills). A skill is procedural memory as a folder holding `SKILL.md`, scripts and resources. Progressive disclosure keeps only the name and description in the prompt and loads the body on demand. Anthropic warns that untrusted skills can exfiltrate data. -- **Claude Managed Agents memory stores** (beta `agent-memory-2026-07-22`; https://platform.claude.com/docs/en/managed-agents/memory). Stores are file-mounted under `/mnt/memory/`, and **every change creates an immutable memory version** (an audit trail with point-in-time recovery). Access defaults to `read_write`; `read_only` is recommended for reference stores because prompt injection can write malicious memory. Limits: 10,000 memories per store, 8 stores per session. -- **Claude Code memory** (https://code.claude.com/docs/en/memory). Human-written `CLAUDE.md` rules are kept apart from agent-written "auto memory", which is typed (`user`, `feedback` for corrections, `project`, `reference`) and reviewed with `/memory`. -- **Claude app memory** (https://www.anthropic.com/news/memory): Team/Enterprise 2025-09-11, Pro/Max 2025-10-23, free plan with import from other assistants 2026-03-02 (https://www.theverge.com/ai-artificial-intelligence/887885/anthropic-claude-memory-upgrades-importing). Memory is project-scoped, with view/edit controls and incognito chats. Anthropic tested for "over-accommodation" and for reinforcing harmful patterns before launch. -- **ChatGPT memory** (https://openai.com/index/memory-and-new-controls-for-chatgpt/): saved memories 2024-02-13, "reference chat history" 2025-04-10, free tier 2025-06-03. Enterprise/Edu got improved memory with a reviewable memory summary (release notes 2026-09-29; https://help.openai.com/en/articles/10128477-chatgpt-enterprise-edu-release-notes). -- Neither vendor publishes accuracy evaluations of its consumer memory. - ---- - -## 3. Benchmarks and evaluation - -### What the main benchmarks measure - -- **LoCoMo** (arXiv 2402.17753; ACL 2024; https://arxiv.org/abs/2402.17753). Synthetic persona dialogues of about 300 turns and 9k tokens over up to 35 sessions; the released set has 10 conversations and 1,540 scored questions (single-hop, multi-hop, temporal, open-domain). - - **Audit** (Penfield, 2026-04-08; https://penfieldlabs.substack.com/p/we-audited-locomo-64-of-the-answer; https://github.com/dial481/locomo-audit): 99 of 1,540 golden answers (6.4%) are wrong, which caps the achievable score at **93.57%**; the standard LLM judge accepts up to 63% of intentionally wrong answers; the open-domain category (n=96) needs a difference above 15 points to be significant. [measured] -- **LongMemEval** (arXiv 2410.10813; ICLR 2025; https://arxiv.org/abs/2410.10813). 500 questions over five abilities: information extraction, multi-session reasoning, temporal reasoning, **knowledge updates**, abstention. The _S setting uses ~115k-token histories; _M is much longer. Commercial assistants dropped about 30% in accuracy over sustained interactions. - - Design findings: decompose sessions into rounds or facts; expand index keys with extracted facts; expand queries with time-aware ranges. Penfield notes that _S fits in modern context windows. -- **MemoryAgentBench** (arXiv 2507.05257, Jul 2025; https://arxiv.org/abs/2507.05257). Four competencies fed incrementally: accurate retrieval, test-time learning, long-range understanding, selective forgetting. RAG wins on retrieval; long-context models win on learning and global understanding. **Every method scores at most 28% on multi-hop selective forgetting.** [measured] -- **BEAM** (arXiv 2510.27246, Oct 2025, v2 2026-02-21; https://arxiv.org/abs/2510.27246). 100 coherent conversations of up to **10M tokens** with 2,000 validated questions. Models with 1M-token context still degrade as conversations lengthen; the authors' LIGHT architecture (episodic + working memory + scratchpad) gains 3.5–12.7%. -- **HaluMem** (arXiv 2511.03506; EMNLP 2026; https://arxiv.org/abs/2511.03506; by MemTensor). It scores each operation separately: extraction, update and QA. [measured] - - | System | Extraction recall | Correct-update rate | - | ----------- | ----------------- | ------------------- | - | MemOS | 74.1% | 62.1% | - | Zep | not reported | 47.3% | - | Mem0 | 42.9% | 25.5% | - | Supermemory | 41.5% | 16.4% | - | Memobase | 14.6% | 5.2% | - - Update omission rates reached 74% for Mem0. QA correctness ranged 35–67%. Errors start at extraction and update, then propagate to answers. - -- **Evo-Memory** (arXiv 2511.20857, Nov 2025; https://arxiv.org/abs/2511.20857). A streaming test of test-time learning across 10 datasets with more than ten memory modules. It provides an ExpRAG baseline (retrieve prior task experiences) and ReMem. -- **MemoryArena** (arXiv 2602.16313, 2026-02-18; https://arxiv.org/abs/2602.16313). Multi-session agentic tasks whose subtasks depend on each other. Agents "with near-saturated performance on existing long-context memory benchmarks like LoCoMo perform poorly" here. [measured] -- **Procedural and skill benchmarks:** AFTER (382 enterprise tasks, 22 skills; arXiv 2606.23127), SkillEvolBench (arXiv 2605.24117), SWE Context Bench (arXiv 2602.08316). - -### Self-reported results, October 2026 - -| System | LoCoMo | LongMemEval(-S) | BEAM | Source | -| -------------------------------- | ------------------------------- | --------------------------------- | ------------------------------------------------ | ---------------------------- | -| Full context, gpt-4o-mini (2025) | 72.9 | 55.4 (gpt-4o-mini), 60.2 (gpt-4o) | n/a | Mem0 paper; Zep paper | -| Zep (2025) | 75.1 (self), ~71 (third party) | 71.2 (gpt-4o) | n/a | Zep blog; 2605.04897 | -| Mem0, April 2026 | 92.5 (self), 61.4 (third party) | 94.4 | 64.1 (1M), 48.6 (10M) | Mem0 blog; 2605.04897 | -| Hindsight | 89.6 (paper) / 92.0 (site) | 91.4 / 94.6 | 73.9 (1M, cited in 2605.04897); 64.1 (10M, site) | 2512.12818; site; 2605.04897 | -| True Memory Pro | 93.0 (lenient judge) | 87.8 | 76.6 (1M) | 2605.04897 | -| Agent Zero Memory | 93.6 | 95.6 | n/a | 2608.29606 | -| EverMemOS | 94.5 (reported by third party) | n/a | n/a | 2605.04897 | - -[inference] These numbers are not comparable. Several LoCoMo scores sit at or above the audited 93.57% ceiling, which implies lenient judging; answer models and judges differ from row to row; the same Mem0 system scores 92.5 self-reported and 61.4 in a matched third-party run. No benchmark involves images, documents as the primary memory, or organizational (multi-user) knowledge. - -### Building a domain evaluation (what the literature supports) - -These are evaluation methods with published precedent, not a MOKE design: - -- **Score each operation separately** (HaluMem): extraction precision and recall, update correctness and omission, retrieval recall@k, then end-task accuracy. Black-box QA alone hides where errors start. -- **Replay time-ordered streams** (MemoryAgentBench, Evo-Memory): ingest chronologically and query between steps. That measures both forward transfer and forgetting of earlier knowledge (arXiv 2604.27003). -- **Include superseded facts** (LongMemEval knowledge updates, MemoryAgentBench selective forgetting) and multi-hop versions of them. This is the weakest capability measured. -- **Include abstention cases** (LongMemEval), where the correct behaviour is to suggest nothing. -- **Measure downstream task success** (MemoryArena, AFTER), not only recall: does a surfaced prior detail change a review outcome? Test cross-project and cross-reviewer transfer, as AFTER does across roles. -- **Validate the judge** (Penfield audit): seed known-wrong answers to measure judge false acceptance, hand-audit gold labels, and report Wilson confidence intervals per category. Use exact or structured checks where possible (Enginuity found token-overlap metrics under-report by 2–6x; section 4). -- **Hold the answer model fixed** across memory variants, as Agent Zero Memory and True Memory did, and report tokens and latency per query. - ---- - -## 4. Multimodal and visual document memory for drawings - -### Page-level visual retrievers - -- **ColPali** (arXiv 2407.01449, v6 2025-02-28; https://arxiv.org/html/2407.01449). - - Stores one 128-dim vector per image patch: 1,024 patches plus prompt tokens, about **257.5 KB per page in fp16**. For comparison, BGE-M3 needs about 8.6 KB per page. - - Halving the patch count cost −24.8 nDCG@5. [measured] - - Projection [inference]: 10k sheets is about 2.6 GB uncompressed. -- **ColQwen2** has dynamic resolution, capped at 768 patches: https://huggingface.co/vidore/colqwen2-v1.0 -- **ViDoRe V3** (released 2025-11-05; paper arXiv 2601.08620, 2026-01; https://arxiv.org/html/2601.08620). 26k enterprise pages and 3,099 human-verified queries. - - NDCG@10 [measured]: - - | Type | Model | NDCG@10 | - | ------ | ------------------------------ | ------- | - | Visual | ColEmbed-3B-v2 | 59.8 | - | Visual | jina-v4 | 57.6 | - | Visual | ColNomic-7B | 57.4 | - | Visual | ColQwen2.5 | 51.9 | - | Visual | ColPali | 43.1 | - | Text | Qwen3-8B on extracted Markdown | 51.0 | - | Text | BGE-M3 | 41.3 | - | Text | BM25 | 20.3 | - - - The industrial-maintenance subset (USAF tech orders) peaks at 51.7; the authors say models "struggle significantly… interpreting dense schematics". - - Late interaction and textual reranking help substantially. - - **Grounding is poor**: evidence bounding-box F1 is 0.065–0.089 for frontier VLMs, against 0.602 between humans. - - In answer generation, hybrid image + text context beat image alone by only 0.2 points. -- **Leaderboard top, early 2026.** nemotron-colembed-vl-8b-v2 scored 63.54 on ViDoRe V3 (2026-01-26; https://huggingface.co/nvidia/nemotron-colembed-vl-8b-v2). Its licence is non-commercial or research-only. [vendor claim] - -### Model options - -Open-weight models (all [vendor claim] unless marked): - -- **ColModernVBERT**: 250M parameters, MIT licence. ViDoRe v1 81.2 and v2 56.0; the authors claim parity with models 10x larger and CPU-friendly latency (arXiv 2510.01149; https://huggingface.co/ModernVBERT/colmodernvbert). [measured, author-run] -- **Qwen3-VL-Embedding and Qwen3-VL-Reranker**: 2B and 8B, Apache-2.0. Any-to-any input, Matryoshka dimensions, 32K context; MMEB-V2 77.8 for the 8B embedder (arXiv 2601.04720, Jan 2026). Upstream llama.cpp did not support it as of Feb 2026 (https://github.com/ggml-org/llama.cpp/discussions/19516). -- **jina-embeddings-v4**: 3.8B, single-vector and multi-vector output, Qwen Research License (arXiv 2506.18902, 2025-06-23). -- **ColQwen2.5**: Qwen Research License on the backbone (https://huggingface.co/vidore/colqwen2.5-v0.2). -- **ColNomic**: the 7B is Apache-2.0; the 3B is under the Qwen Research License. -- **ONNX exports**: one exists for ColPali v1.3 (Qdrant/FastEmbed, https://huggingface.co/Qdrant/colpali-v1.3-fp16). I found none for ColModernVBERT, ColQwen or Qwen3-VL-Embedding. - -API-only models: - -- **voyage-multimodal-3.5** (2026-01-15; https://blog.voyageai.com/2026/01/15/voyage-multimodal-3-5/): $0.12 per M text tokens and $0.60 per B pixels (OpenRouter listing). - - [inference] A 200-DPI ARCH D sheet is about $0.02 if billed at full resolution. Whether the service downsamples images is undocumented. -- **Cohere Embed 4** (2025-04-15; https://cohere.com/blog/embed-4). It beat every open model on IRPAPERS page images (R@1/5/20 = 58/87/97; arXiv 2602.17687). [measured, third party] -- **Cohere Embed 5** (2026-09-30; https://cohere.com/blog/embed-5): $0.08–0.12 per M tokens, with fused text and image input. [vendor claim] - -### Reducing multi-vector storage - -- **Light-ColPali token merging**: 98.2% of retrieval quality at 11.8% of the memory, and 94.6% at 2.8%. Query-agnostic pruning works poorly (arXiv 2506.04997, 2025-06; https://arxiv.org/abs/2506.04997). [measured] -- **Token pooling** at index time: factor 2 keeps about 100% of quality and halves the vectors; factor 3 keeps about 99% and cuts 66% (https://www.answer.ai/posts/colbert-pooling.html, 2024-06-27). [measured, text ColBERT] -- **Binarized MaxSim**: 32x less memory and about 4x faster scoring, with a float rerank afterwards (Vespa, 2024-09-20; https://blog.vespa.ai/scaling-colpali-to-billions/). [vendor measurement] -- **MUVERA** fixed-dimensional encodings let a single-vector index serve as the first stage for multi-vector retrieval (arXiv 2405.19504; NeurIPS 2024). [measured, text] - - [inference] One route onto a single-vector SQLite index: MUVERA first stage, then exact MaxSim rerank of the top-k. - -### Region, crop and drawing-specific evidence - -- **PlanSightRAG** (state DOT standard plans; 1,898 pages and 4,056 query–page pairs; arXiv 2608.26091, 2026-08-26; https://arxiv.org/html/2608.26091). [measured] - - R@5 by retriever: - - | Retriever | R@5 | - | -------------------- | ------------------------------------------------- | - | Nemotron-ColEmbed-8B | 95.28% | - | ColNomic-3B | 91.47% (91.40% zero-shot on a held-out state set) | - | ColPali | 76.89% | - | OCR + BGE-M3 | **36.79%** | - - - LoRA fine-tuning added nothing. - - Tiling into 400-DPI 1024 px tiles with 256 px overlap (15–30 per sheet) raised R@5 by 5.2 points and judge accuracy by 4.5 points. It also lost cross-references when a note and the detail it refers to fell in different tiles, so the authors index full pages and tiles together. - - Failure modes: binding a dimension to the wrong component, and borrowing values from another view. -- **Blueprint** (ORNL archive of 770k engineering drawings; arXiv 2602.13345, 2026-02-12; https://arxiv.org/html/2602.13345). [measured] - - Pipeline: detect the title block, revision table and parts list; OCR them with a VLM; normalize identifiers; fuse BM25 with dense retrieval; rerank at region level. - - Results: +10.1 points Success@3 and +18.9% relative nDCG@3 over VLM-only retrieval, at about 9.7 s per file. -- **RegionRAG** (arXiv 2510.27261, Oct 2025): region-level retrieval gave +10.0% R@1 and +3.6% QA accuracy while using 71% of the visual tokens. **GARRET** (arXiv 2610.27829, Oct 2026): region retrieval without region labels, with 47.8% fewer vectors per page at page-retrieval parity with ColQwen2.5. [measured] -- **AECV-Bench** (floor plans; arXiv 2601.04819, 2026-01-08). VLMs reach about 0.95 on OCR-style questions, but **symbol counting is unsolved at 0.40–0.55**. The authors recommend tool-augmented, human-in-the-loop workflows. [measured] -- **MechVQA** (mechanical drawings; arXiv 2605.30794; ICML 2026). [measured] - - Frontier VLMs score 71–77 overall; a 4B model fine-tuned with RL reaches 84.85. - - Weakest areas: localization and multi-view consistency. - - Typical errors: "numerically plausible but constraint-inconsistent" answers. -- **Enginuity** (parts manuals; arXiv 2606.03410): token-overlap metrics under-report capability by 2–6x compared with semantic similarity. [measured] -- **Image-to-image drawing similarity remains weak.** DeepPatent (WACV 2022) on design-patent drawings: mAP 0.29 with ImageNet features and 0.37 with patent training. A small PCB pilot measured CLIP R@10 at 0.13 against ColPali's 0.93 (https://github.com/yangchiehkang/PCB-VisualRAG; 30 queries). [measured, weak] - -### Hybrid image + text retrieval - -- **IRPAPERS** (arXiv 2602.17687, Feb 2026). R@1/5/20 [measured]: - - | Method | R@1/5/20 | - | -------------------------- | ------------ | - | Text hybrid (BM25 + dense) | 46/78/91 | - | ColModernVBERT on images | 43/78/93 | - | Fusion of both | **49/81/95** | - - The two modalities fail on different queries. - -- [inference] Identifier and text queries (sheet numbers, title blocks, notes) favour OCR plus BM25 by region (Blueprint). Geometry and detail queries favour patch-level visual retrieval (PlanSightRAG, where OCR collapsed). -- **Commercial AEC products** advertise sheet segmentation (plan, schedule, legend, title block) and citations to bounding boxes: Bedrock (https://docs.bedrock.cv/guides/the-drawing-index) and VitruAI (https://vitruai.com/use-cases/construction-plan-archive-search/). [vendor claim, unevaluated] - ---- - -## 5. Local-first implementation options (Bun, TypeScript, SQLite) - -### What the repository already supports - -- **Effect SQL can load extensions.** `@effect/sql-sqlite-bun` 4.0.4 exposes `loadExtension(path)` and `export` on `SqliteClient`. The module doc says "Database export and extension loading are supported; streaming queries and `updateValues` are not." - - Source: `repos/effect/packages/sql/sqlite-bun/src/SqliteClient.ts` (lines 11, 61–71, 256–260), checked locally. - - The client wraps one `bun:sqlite` connection behind `Semaphore.make(1)`, so an extension loaded once applies to every later query on that connection. The raw `Database` is not exposed. -- **No code uses vectors or full-text search yet.** No backend code or migration uses sqlite-vec, FTS5 or `loadExtension`. - -### bun:sqlite constraints - -- **macOS.** Apple's system SQLite disables extensions. Bun requires `Database.setCustomSQLite()` before the first `Database` is created; on other OSes "SQLite is statically linked: the call does nothing and returns false" (https://bun.com/docs/runtime/sqlite, read 2026-10-30). The macOS failure message is reported in https://github.com/asg017/sqlite-vec/issues/78 (2024-08-09). -- **Compiled executables.** `bun build --compile` can embed N-API addons and assets under `/$bunfs/` (https://bun.com/docs/bundler/executables). - - [inference, unverified] `loadExtension` calls dlopen on a real OS path, so a sqlite-vec `.dll/.so/.dylib` likely has to ship beside the executable (Electron `extraResources`) rather than inside `$bunfs`. -- **Windows locking.** The `bun:sqlite` "database is locked" issue on Windows is open (https://github.com/oven-sh/bun/issues/16119, updated 2025-11-25). -- **Bun 1.4.** The 2026-08-20 release rewrote Bun's core in Rust and fixed more than 2,900 issues (https://bun.com/blog/bun-v1.4). Older native-addon crash reports may or may not still apply. - -### Vector storage options - -- **sqlite-vec** (https://github.com/asg017/sqlite-vec/releases). - - Status [vendor]: - - Stable v0.1.9 (2026-03-31) is **brute-force KNN only**. - - ANN (rescore, IVF, DiskANN) exists only in v0.1.10 alphas (alpha.4 2026-05-18), and alpha.2 notes that DiskANN deletes are expensive. - - Features [vendor]: - - `vec0` tables support metadata columns usable in KNN `WHERE` clauses. - - **`partition key` columns** shard the index internally, which suits firm and project scoping. - - Auxiliary columns are also supported (v0.1.6, 2024-11-20; https://alexgarcia.xyz/blog/2024/sqlite-vec-metadata-release/). - - float32, int8 and bit vectors; `vec_quantize_binary` with Hamming distance (32x compression); Matryoshka truncation. - - Author benchmarks (2024-08-01; https://alexgarcia.xyz/blog/2024/sqlite-vec-stable-release/index.html) [measured]: At 1M vectors, brute force takes 192 ms at 192 dims and 8.5 s at 3072 dims; binary quantization is about 10x faster with roughly 95% anecdotal recall. [inference] Corpora of 10k–100k items per firm are well within brute-force range. - - Bun support [vendor]: - - The JS helper `load()` supports `bun:sqlite` (https://alexgarcia.xyz/sqlite-vec/js.html). - - Prebuilt binaries cover Windows, macOS and Linux, including the PhotoStructure fork (https://github.com/photostructure/sqlite-vec/releases, 2026-01-04). -- **FTS5 + vector hybrid in one SQL query** (Alex Garcia, 2024-10-02; https://alexgarcia.xyz/blog/2024/sqlite-vec-hybrid-search/index.html; commentary https://simonwillison.net/2024/Oct/4/hybrid-full-text-search-and-vector-search-with-sqlite/). - - Two CTEs rank `vec0 MATCH … k=?` results by distance and FTS5 results by `rank` (negative BM25), each with `row_number()`. - - A `FULL OUTER JOIN` on rowid combines them, scored as `Σ weight/(rrf_k + rank)` with `rrf_k = 60` (reciprocal rank fusion). - - The post also covers keyword-first and semantic-rerank variants. - - [inference] Check FTS5 and `FULL OUTER JOIN` (SQLite ≥3.39) in Bun's static build with `PRAGMA compile_options` and `sqlite_version()` on each target OS. -- **libSQL / Turso.** - - libSQL has native `F32_BLOB(n)` columns, `libsql_vector_idx` and `vector_top_k()` with LM-DiskANN (2024-06-05 and 2024-07-26; https://turso.tech/blog/approximate-nearest-neighbor-search-with-diskann-in-libsql). - - It needs `@libsql/client`, a separate native engine, not `bun:sqlite`. The PR to add a bun:sqlite backend is still open (https://github.com/tursodatabase/libsql-client-ts/pull/71, updated 2026-01-07). - - [inference] Adopting libSQL would replace the Effect Bun client, not extend it. - - Turso's Rust rewrite (pre-1.0) has vectors with linear scan and experimental FTS; DiskANN is an open issue (https://github.com/tursodatabase/turso/issues/832). -- **LanceDB.** - - `@lancedb/lancedb` 0.41.0 (2026-10-29) ships native builds for Windows, macOS and Linux (https://www.npmjs.com/package/@lancedb/lancedb). - - Multivector (ColBERT/ColPali MaxSim) columns are documented with Python examples; I did not confirm JS support (https://docs.lancedb.com/search/multivector-search). - - Indirect Bun evidence: it runs inside a Bun single-file executable after a `createRequire` workaround (https://github.com/prosperitypirate/codexfi/issues/141, 2026-04-05). - - It is a second storage engine beside SQLite. -- **usearch** (npm 2.25.1, 2026-04-16) offers HNSW through N-API or WASM. Its SQLite extension provides only distance functions, with no index. - -### Embeddings - -- **Local ONNX under Bun has open defects** [measured, issue reports, all on Bun 1.3.x]: - - transformers.js 4.x fails inside `bun build --compile` binaries (onnxruntime-node's `libonnxruntime` is not bundled, `sharp` is a static import, and the WASM fallback passes paths instead of buffers; https://github.com/huggingface/transformers.js/issues/1672, 2026-05-04, open). One project worked around it with onnxruntime-web (WASM) and a bundled model. - - Bun 1.3.x segfaults inside `onnxruntime.dll` on Windows (https://github.com/oven-sh/bun/issues/28008, 2026-03-11). - - `bun test` crashes with onnxruntime-node (https://github.com/oven-sh/bun/issues/30431, 2026-05-09). - - No evidence either way for Bun 1.4.x. fastembed-js (https://github.com/Anush008/fastembed-js) has the same onnxruntime-node dependency. -- **Local text models.** MTEB English v2 scores from a secondary aggregator (https://selfhost.directory/models/benchmarks/mteb-eng-v2): - - | Model | Size | MTEB eng v2 | Licence | - | ----------------------------- | ---- | ----------- | ---------------------- | - | Qwen3-Embedding-0.6B | 596M | 70.5 | Apache-2.0 | - | EmbeddingGemma-300m | 303M | 69.7 | Gemma terms | - | nomic-embed-text-v2-moe | 475M | 63.6 | Commercial use allowed | - | snowflake-arctic-embed-l-v2.0 | 568M | 63.6 | Commercial use allowed | - | granite-embedding-english-r2 | 149M | 62.8 | Commercial use allowed | - - Qwen3-Embedding-0.6B: 32k context, Matryoshka 32–1024 dims; https://huggingface.co/Qwen/Qwen3-Embedding-0.6B (2025-06). - - EmbeddingGemma-300m: transformers.js ONNX q8/q4 builds exist (https://huggingface.co/onnx-community/embeddinggemma-300m-ONNX; model released 2025-09-04). - - Avoid jina-v5-nano (non-commercial licence). - -- **Hosted embeddings** (prices read 2026-10-30): - - Voyage: voyage-4-lite $0.02/M tokens, voyage-4 $0.06/M, voyage-4-large and voyage-context-4 $0.12/M; first 200M tokens free (https://docs.voyageai.com/docs/pricing). - - OpenAI text-embedding-3-small: $0.02/M. - - Gemini: gemini-embedding-001 $0.15/M; gemini-embedding-2 (text, image and PDF) $0.20/M text and $0.45/M image (https://aistudio.google.com/docs/pricing). - - Cohere has moved Embed 5 to dedicated Model Vault instances (https://cohere.com/pricing). - - [inference] Hosted embedding sends firm drawing content to a third party; that is a governance question as well as a cost one. - -### Rerankers - -- **Qwen3-Reranker-0.6B** reports MTEB-R 65.8, against 57.0 for bge-reranker-v2-m3 (568M, Apache-2.0) (https://huggingface.co/Qwen/Qwen3-Reranker-0.6B, 2025-06). [vendor claim] -- **jina-reranker-m0** reranks visual documents (2.4B, ViDoRe v1 91.0) but is CC-BY-NC (https://jina.ai/en-US/models/jina-reranker-m0/, 2025-04-08). **Qwen3-VL-Reranker** is the Apache-2.0 multimodal option (section 4). -- **Voyage hosted rerank:** rerank-3 $0.05/M tokens and rerank-3-lite $0.02/M. -- [inference, unmeasured] A 0.6B cross-encoder on desktop CPU through WASM is likely slow for 50–100 candidates. - ---- - -## 6. Governance - -### Memory poisoning and injection - -- **AgentPoison** (NeurIPS 2024; arXiv 2407.12784; https://arxiv.org/abs/2407.12784): backdoor triggers in memory or RAG. Attack success above 80% at a poison rate below 0.1%, with under 1% loss on benign tasks. [measured] -- **MINJA** (arXiv 2503.03704, 2025-03; NeurIPS 2025; https://arxiv.org/abs/2503.03704). Ordinary queries trick the agent into writing malicious reasoning records itself. Injection succeeds 98.2% of the time, with 76.8% average attack success. [measured] -- **MINJA under realistic conditions** (arXiv 2601.05504, 2026-01-09). A memory bank already holding legitimate records "dramatically reduce[s]" attack effectiveness. Trust-scored sanitization helps, but its threshold needs calibration to avoid rejecting real memories. [measured] -- **MemoryGraft** (arXiv 2512.16962, 2025-12-18). Implanted fake "successful experiences" are imitated. A few poisoned records can account for a large share of retrieved experiences under lexical + embedding retrieval. [measured] This applies directly to any "prior similar work" suggestion feature. -- **Zombie Agents** (arXiv 2602.15654, 2026-03) and **WhisperBench / MemGhost** (arXiv 2607.05189, 2026-07). - - One email payload achieved stealthy memory injection end-to-end in 87.5% of cases on OpenClaw with GPT-5.4 and 71.4% on the Claude Code SDK with Sonnet 4.6. - - Per-session prompt filtering is insufficient. [measured, preprints] -- **Provenance laundering** (arXiv 2607.29167, 2026-07-31). LLM consolidation rewrote external observations as apparent user history, with attack success up to 1.0. [measured] [inference] Summarization and merging are trust boundaries. -- **Field reports:** - - ChatGPT "SpAIware" persistent exfiltration via planted memories (Rehberger; https://arstechnica.com/security/2024/09/false-memories-planted-in-chatgpt-give-hacker-persistent-exfiltration-channel/, 2024-09-24) - - Gemini delayed memory-tool invocation (https://arstechnica.com/security/2025/02/new-hack-uses-prompt-injection-to-corrupt-geminis-long-term-memory/, 2025-02-11) - - Microsoft's "AI Recommendation Poisoning": more than 50 "remember X as a trusted source" prompts from 31 companies (https://www.microsoft.com/en-us/security/blog/2026/02/10/ai-recommendation-poisoning/, 2026-02-10) -- **MEXTRA** (ACL 2025; arXiv 2502.13172). Black-box prompts extracted more than 10–30% of stored private queries from agent memory. [measured] - -### Defenses - -- **Spotlighting** (arXiv 2403.14720): delimiting or datamarking untrusted text cut attack success from above 50% to below 2%. It was tested on single-turn injection, not persistent memory. [measured] -- **A-MemGuard** (arXiv 2510.02373; PMLR 2026): consensus across related memories plus a separate "lessons" memory. Attack success fell by more than 95% at minimal utility cost. [measured, author-run] -- **Platform-held provenance:** - - PPMF (arXiv 2607.29167) authorizes a tool call only when the action's risk matches the authority of the memories behind it. - - MemLineage (arXiv 2605.14421, 2026-05-14) uses signed entries, a Merkle log, a derivation DAG, and refusal of actions descending from external sources; it was the only configuration with 0 attack success on all three workloads. [measured, author harnesses, preprints, no replication] -- **OWASP.** - - Agentic AI Threats and Mitigations, T1 Memory Poisoning (2025-02-17; https://genai.owasp.org/resource/agentic-ai-threats-and-mitigations/). - - Top 10 for Agentic Applications 2026, **ASI06 Memory & Context Poisoning** (2025-12-09; https://genai.owasp.org/resource/owasp-top-10-for-agentic-applications-for-2026/). - - Recommended controls: validate writes before commit; require source attribution; expire unverified entries; weight retrieval by trust and tenancy; "prevent automatic re-ingestion of an agent's own generated outputs into trusted memory"; keep snapshots, rollback and versioning; use per-tenant namespaces; put humans in the loop for high-risk actions. -- **NIST and CSA.** NIST CAISI published agent-hijacking evaluation work (2025-01-17) and an RFI (2026-01-12). CSA MAESTRO (2025-02-06) gives threat layers. None has controls specific to memory. - -### Bi-temporal validity and provenance - -- **Zep/Graphiti** keeps four timestamps per edge (section 1) and invalidates contradicted edges instead of deleting them (https://arxiv.org/abs/2501.13956). Episodes stay as the lossless source. The edge-resolution code path was read from a secondary source (https://deepwiki.com/getzep/graphiti/5.3-edge-extraction-and-resolution). -- **Other systems with citations or versions:** - - Agent Zero Memory: "citation-locked" facts that abstain without a citation. - - MemOS: MemCube provenance and version fields. - - Claude Managed Agents: immutable memory versions. - - Letta: git commits. -- **W3C PROV-O** (2013-04-30; https://www.w3.org/TR/prov-o/) provides `wasDerivedFrom`, `wasAttributedTo` and related relations. [inference] No memory product checked exports PROV. - -### Human approval - -- **Products.** - - Claude apps let users view and edit memory, and each project has its own (https://www.anthropic.com/news/memory). - - Claude Code separates human rules (`CLAUDE.md`) from agent-written auto memory, which users audit with `/memory`. - - ChatGPT has separate toggles for saved memories and chat history; deleted memories may stay in logs for up to 30 days (https://help.openai.com/en/articles/8590148-memory-in-chatgpt). - - LangMem has no built-in approval step. -- **Evidence.** - - A CHI 2026 interview study (n=20; https://doi.org/10.1145/3772318.3791635) found users see memory as "unforgetting" and feel privacy strain. - - Gating memory additions on strict evaluation, including human evaluation, mitigated error propagation (arXiv 2505.16067). - - I found **no controlled study** showing that human-approved learned rules improve downstream accuracy over unreviewed ones. [gap] - -### Multi-tenant isolation - -- **Mem0** scopes calls by `user_id`, `agent_id`, `app_id` and `run_id`, and filters support a `*` wildcard (https://docs.mem0.ai/platform/features/entity-scoped-memory). [inference] Isolation depends on every call carrying the right tags. -- **Zep** gives each user a graph, plus separate shared graphs (https://help.getzep.com/users-and-user-graphs). -- **Letta** documents three patterns: one agent per user, a shared agent with identities, or org-owned shared memory repositories (https://docs.letta.com/concepts/shared-memory/). -- **Incident.** Asana's MCP server exposed data across organizations (disclosed 2025-06-18; https://www.bleepingcomputer.com/news/security/asana-warns-mcp-ai-feature-exposed-customer-data-to-other-orgs/). This was tenant scoping failing in an AI access layer, not in a memory product. - -### Forgetting and right to delete - -- **Mem0** deletes by ID, in batches of up to 1,000, or by filter; `delete_all` with no filter is rejected (https://docs.mem0.ai/api-reference/memory/delete-memories). -- **Zep** cascades a user delete as its right-to-be-forgotten path. But deleting an episode **does not regenerate** node summaries shared with other episodes, and facts it invalidated stay invalidated (https://help.getzep.com/deleting-data-from-the-graph). [vendor-documented gap] Derived summaries can keep content from a deleted source. -- **Embeddings leak text.** vec2text recovers 92% of 32-token inputs exactly (arXiv 2310.06816, EMNLP 2023), and ZSInvert (arXiv 2504.00147, 2025) works zero-shot on any encoder. [measured] Vectors need the same deletion and access rules as their source text. -- **GDPR.** EDPB Opinion 28/2024 (2024-12-17; https://www.edpb.europa.eu/system/files/2024-12/edpb_opinion_202428_ai-models_en.pdf) decides anonymity case by case. [inference] Treat derived memories as personal data where their source is. - ---- - -## What the evidence supports - -### Measured with a disclosed protocol - -These are often author-run but can be reproduced: - -1. **Accuracy.** Even in vendor-run studies, memory layers beat full context on cost and latency (90%+ fewer tokens; seconds instead of tens of seconds), but not consistently on accuracy. Full context won Mem0's own 2025 LoCoMo run. Zep beat full context on LongMemEval_s, where histories of ~115k tokens degrade long-context reading. -2. **Destructive rewriting loses information.** ACE's collapse measurement, HaluMem's 5–62% correct-update rates, MemoryAgentBench's ≤28% on multi-hop selective forgetting, and Mem0's own retreat to ADD-only writes all point the same way. -3. **Lexical + dense hybrid retrieval with reranking reliably helps.** Evidence comes from Zep, Mem0 2026, LongMemEval's key and query expansion, IRPAPERS fusion, Blueprint, and the RAG-vs-GraphRAG comparison. Graph structure helps multi-hop questions and hurts detail questions. -4. **Experience memory improves agents when there is a success signal** (AWM, ReasoningBank, ACE, Dynamic Cheatsheet, AFTER). The gains are typically single- to low-double-digit points. - - Stored errors propagate: agents copy similar past records (experience following). - - Whether distilled rules or raw examples transfer better is contested (ReasoningBank and arXiv 2604.27003 versus SkillEvolBench). -5. **Drawing retrieval.** On drawing sheets, patch-level visual retrievers far exceed OCR-text retrieval (PlanSightRAG: 91–95% vs 37% R@5). Region and title-block OCR with BM25 wins for identifier queries (Blueprint), and fusion beats either alone (IRPAPERS). VLM grounding and symbol counting remain weak (ViDoRe V3 bounding-box F1 below 0.09; AECV-Bench counting 0.40–0.55). -6. **Memory poisoning is practical** against agents that write their own memories (MINJA 98% injection; MemoryGraft; WhisperBench 71–88% end-to-end). Defenses that rely on platform-held provenance, not LLM judgment, performed best in author harnesses. - -### Vendor claims to treat as unverified - -- Benchmark leadership claims: Mem0 92.5 / 94.4, Supermemory "#1" and 97% Recall@20, Hindsight 94.6, Agent Zero 95.6, MemOS "first in all categories", EverMemOS 94.5. - - LoCoMo scores above the audited 93.57% ceiling indicate lenient judges. - - A matched third-party run put Mem0 at 61.4% and Supermemory at 65.4%. -- Anthropic's +39% memory-plus-context-editing figure comes from an undisclosed internal evaluation. -- Graph memory "excels at temporal reasoning": the 2025 Mem0g gain was about 1.5 J overall. -- Model-card retrieval scores and AEC product capabilities (Bedrock, VitruAI) have not been independently evaluated. - ---- - -## Open uncertainties - -1. **No benchmark covers organizational, multi-user, visual or document-centric memory.** Every number in section 3 comes from personal chat history or agent tasks, so transfer to firm drawing-review practice is untested. -2. **Raw episodes versus distilled rules** for procedural knowledge such as reviewer corrections: the evidence conflicts, and nobody has measured corrections made by human experts rather than self-judged outcomes. -3. **Whether human approval of learned rules improves outcomes** has no controlled study. Trust thresholds and consensus filters may suppress rare but correct firm-specific practices; false-rejection rates are unreported. -4. **Image-to-image detail similarity** (a crop as the query, finding similar details from prior projects) has no benchmark. Generic encoders do poorly on drawing similarity, and how to tile and rasterize large sheets rests on one paper. -5. **Deletion through derived memories.** Zep explicitly does not regenerate shared summaries, and no vendor documents rebuilding consolidated rules after a source is deleted. -6. **2026 preprints.** Many results cited here are preprints without peer review or replication: True Memory, Agent Zero Memory, PlanSightRAG, GARRET, MemLineage, PPMF, WhisperBench, SkillEvolBench. -7. **Running the strongest open visual retrievers locally under Bun** is unverified: no official ONNX export exists for ColModernVBERT, ColQwen or Qwen3-VL-Embedding. Section 5 covers the text-embedding side. diff --git a/docs/research/meta-vs-sonnet-unslop-findings.md b/docs/research/meta-vs-sonnet-unslop-findings.md deleted file mode 100644 index a18fcb91..00000000 --- a/docs/research/meta-vs-sonnet-unslop-findings.md +++ /dev/null @@ -1,60 +0,0 @@ -# Meta review findings versus Sonnet 5.5 unslop rewrites - -Generated 2026-10-29. Source: the Moke app databases on Windows, mainly `AppData/Roaming/moke-meta-hackathon/app.sqlite`, plus the Audit-3 restore workspace, retained package restores, and cloud output copies. The Linux `/tmp` databases are test fixtures with no packet reviews. - -## What was compared - -- 44 unique findings from 6 completed packet reviews. 55 findings exist in total, and 11 findings in review `7d250ad7` repeat review `c3502d59` word for word, so they appear once. -- The left column is Meta's text from the `observed` field, which is the issue, and the `correction` field, which is the fix. Only line breaks and table escaping were changed. Title, severity, status, and discipline come from the same finding. -- The right column is a Sonnet 5.5 rewrite following the unslop rules, cut to one short sentence for the issue and one for the fix. It keeps each finding's main point and drops supporting detail such as counts, quoted notes, and code section numbers. Those details remain in the left column. It does not add findings or correct Meta's technical claims. -- Meta's issue and fix text totals 5,504 words. The rewrite totals 1,499 words, about 27 percent of the original. -- Models behind the left column: meta-models/muse-spark-open-1026 on Baseten, muse-spark-1.3 on Meta. - -## Comparison - -| # | Meta response | Sonnet 5.5 unslop | -| --- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | **Packet contains three unrelated projects in one PDF - not a single permit**
_Major, Confirmed, general_
**Issue:** Sheet footers/labels identify at least three distinct scopes: Sample A bagel shop tenant build-out (B-1, M-2, E-3), Sample B residential unpermitted-bedroom demo (B-1/B-2), Sample C medical-to-ophthalmology conversion (P-2, E-3). All were combined into quick-start-selected-sheets.pdf (8 pages). No cover tying them to one address/parcel/permit.
**Fix:** Separate submittals per address/permit. If this was a training compilation, clarify intent and provide one complete permit packet at a time with cover, application, and index. Resubmit with correct sheet set and title blocks matching the permit address. | **Three projects in one PDF**
**Issue:** The 8-page PDF mixes a bagel shop, a residential bedroom demolition, and a clinic conversion, with no shared address or permit.
**Fix:** Submit each project separately with its own cover sheet, index, and title blocks. | -| 2 | **No permit identity, code edition, or sealed code analysis**
_Major, Needs input, general_
**Issue:** No cover sheet, permit application, index, code analysis, building area/height, occupant load, construction type, flood zone, or design professional seal/signature on any of 8 pages. Code citations appear piecemeal (FBC-Accessibility 904 etc. on B-1, 2023 FBC-M Table M1507.4 on M-2, FBC-Plumbing 403.1 on B-1) with no governing edition (8th edition 2023? 7th? 8th supplement?) declared.
**Fix:** Provide cover with jurisdiction, address/parcel, scope (existing/new/demo), use/occupancy per FBC Ch.3, construction type, areas, height, stories, sprinkler/standpipe, flood, code editions and amendments, index, and sealed code summary. Make all sheets consistent. | **No permit identity or code basis**
**Issue:** No cover sheet, code analysis, seal, or governing code edition appears on any of the 8 pages.
**Fix:** Add a cover sheet with jurisdiction, address, occupancy, construction type, area, and code editions, plus a sealed code summary. | -| 3 | **Life safety: no occupant load, egress analysis, or exit signage/lighting**
_Major, Needs input, life_safety_
**Issue:** No occupant load calculation, occupant load factor, number of exits, common path/travel distance, exit separation, door hardware/swing, or emergency lighting/exit signage shown for Bagel shop, residential, or clinic. Bagel shop fixture table implies Business but area and load not stated to justify 1 WC/1 lav. Clinic has multiple exam/procedure rooms but no exiting plan.
**Fix:** Provide code analysis per FBC Ch.3/10: use/classification, occupant load (area/factor), exits required/provided, travel/common path/dead-end with dimensions on plan, exit separation, door swings, hardware, and emergency lighting/exit signage coordinated with electrical/lighting. | **No life safety information**
**Issue:** No occupant load, exit count, travel distance, emergency lighting, or exit signs are shown for the shop, residence, or clinic.
**Fix:** Add an FBC Chapter 3 and 10 analysis, and show exits, travel distance, and exit lighting and signs on the plans. | -| 4 | **Plumbing fixture count vs. FBC 403 cannot be verified - OUT OF SCOPE restrooms**
_Major, Needs input, plumbing_
**Issue:** Table shows BUSINESS/BAGEL SHOP: 1 WC required/provided (1 per 25 for first 50 and 1 per 50 remainder), 1 lav (1 per 40 for first 80), 0 urinals/bathtubs, drinking fountain 1 per 100 but note says not required as water in container free per FBC 410.4, plus 1 service sink. Plan marks restroom area OUT OF SCOPE gray hatch. No occupant load or building area shown to verify factors. Food-service fixtures (3-comp sink, mop sink 1-grate floor drain, 2x6 plumbing wall, hand sinks 12x16, service sink 20x29-1/2) shown on Proposed plan but not reconciled to table counts; many are food-prep not plumbing-fixture count.
**Fix:** Show occupant load calculation, gross/net area, and FBC-Plumbing 403.1/410.1 compliance. Clarify if existing common restrooms serve tenant and provide accessibility compliance for doors/clearances. List all fixtures (water closets, lavs, service/mop, hand sinks, dish sinks) and indicate which count toward 403. | **Fixture count cannot be checked**
**Issue:** The fixture table has no occupant load or area behind it, the restrooms are marked out of scope, and the food service fixtures are not reconciled with the table.
**Fix:** Show occupant load and area, say whether existing restrooms serve the tenant, and list which fixtures count under FBC 403. | -| 5 | **Plumbing routing: no DWV isometrics, slopes, cleanouts, vents, materials, or calculations (Sample C P-2 - primary QA request)**
_Major, Confirmed, plumbing_
**Issue:** Plan shows horizontal routing with pipe sizes (1/2, 3/4, 1, 1-1/2, 2, 3, 4 inch) tagged at fixtures (LAV, WC, KS, BS, REF, DF, SS, etc.) and along trunk: e.g., 3 corridor waste, 1-1/4 lobby branch, 4 main to existing sanitary. However: no drainage/waste/vent isometrics or riser diagrams; no material/slope callouts; no cleanout spacing/details (CO shown only at Hall 110); no trap-seal primer or wet-vent details; no DFU/WFU or water supply demand calculations to justify sizes; water-heater note says heaters in upper floor above Pre-Test 118 (not shown) route CW up/HW down - no heater/gage/T&P, expansion tank, recirculation or tempering detail; sanitary and water noted to 'connect to existing' with no invert/connection point verified.
**Fix:** Provide complete plumbing submittal: floor plans at scale + DWV isometrics/risers with DFUs, vents to roof terminations, cleanouts at base of stacks and changes of direction per code, slopes, materials, hangers, and firestop at rated penetrations; water riser with WFUs, pressures, backflow/meter/shutoff locations, heater capacities/T&P/expansion/tempered water. Coordinate utility verification letters. Do not scale off NTS plan - provide dimensioned plans. | **P-2 plumbing missing isometrics and calculations**
**Issue:** P-2 has no drain, waste and vent isometrics, slopes, materials, cleanouts, or sizing calculations, and the existing connections are unverified.
**Fix:** Provide scaled plans, DWV isometrics, sizing calculations, and a water heater detail, and verify the existing connection points. | -| 6 | **Mechanical: no ventilation/outside-air, load, or energy compliance substantiation**
_Major, Needs input, mechanical_
**Issue:** Mechanical plan shows supply diffusers each labeled 135/159/135 CFM (~8 SA diffusers), 12x26 RA plenum, 20x20 SA plenum to 16 duct, ERV with 38 CFM outside air and 20 CFM exhaust, 24x24/338 CFM grilles, 4 dryer exhaust to roof, bagel oven/combustion/steam exhaust detailed for LFR 1-rack ovens (51-1/4, 61, 99-1/4 dims). No Manual N/J load, no ASHRAE 62.1 ventilation calc, no IMC exhaust/make-up air balance, no energy forms/envelope/lighting. ERV 38 CFM OSA appears low for food prep occupancy. No roof plan showing equipment/curb/pitch/clearances.
**Fix:** Provide heating/cooling load calcs, outdoor-air ventilation calc per FBC-M/ASHRAE 62.1 for each room use, kitchen/bathroom exhaust rates and make-up air, equipment schedules with capacity/efficiency, controls/thermostat locations, roof plan and details for ERV/CU/exhaust through roof, and energy code compliance forms. | **Mechanical support missing**
**Issue:** There are no load calculations, ventilation calculations, or energy forms, and the 38 CFM of outside air looks low for food preparation.
**Fix:** Add load and ASHRAE 62.1 ventilation calculations, equipment schedules, a roof plan, and energy compliance forms. | -| 7 | **Mechanical legend/version conflict: Round vs Rectangular duct and duplicate M-2**
_Minor, Confirmed, mechanical_
**Issue:** Two M-2 sheets are essentially identical plans but legends differ: Revision 1 lists 8x8 Round rigid duct for SA/RA; Baseline lists 8x8 Rectangular rigid duct for same colors/tags. Plan overlays use rectangular duct graphics (blue SA trunk with transitions) - neither round nor 8x8 consistent with 20x20 plenum and 16 SA duct transitions noted. Indicates copy/paste or version control error.
**Fix:** Reconcile duct type/shape/size on legend, plan tags, and transitions. Provide duct schedule with actual free-area velocity and pressure drop. Remove superseded version from permit set and cloud changes on revision. | **M-2 sheets disagree on duct type**
**Issue:** One M-2 legend says round duct and the other says rectangular for the same runs, and the plan matches neither.
**Fix:** Reconcile duct type and size, and remove the superseded sheet. | -| 8 | **Electrical: no panel schedules, load calculations, service sizing, or lighting/emergency**
_Major, Needs input, electrical_
**Issue:** Both electrical sets show device symbols and homeruns labeled to panels (CKT #) but no panel schedules, connected load/VA per circuit, demand factors, service/feeder one-line, grounding, available fault current or AIC ratings, or working-clearance dimensions. Bagel shop has complex loads: bagel oven/proofer, 2 bakery trolleys, mixers, conveyor toaster, walk-in freezer/chiller, dryer, plus two rotary phase converters (idler motors for PL-7/PL-15) - no feeder sizes, disconnects, or phase-converter specs. No lighting plan or emergency/egress lighting shown. Low-voltage/data symbols present but not coordinated to dedicated circuits.
**Fix:** Provide panel schedules (panel, ckt, description, load VA, wire/OCP), load calc per NEC 220 for general/lighting/receptacle/appliance/HVAC, riser/one-line with service size, grounding/bonding, and fault-current/AIC verification; provide lighting/emergency plan. Show working clearances per code and label dedicated circuits. Provide rotary converter cut sheets and feeder calc. | **Electrical data missing**
**Issue:** There are no panel schedules, load calculations, one-line, fault current, or lighting plan, and the phase converters have no feeder sizes.
**Fix:** Add panel schedules, an NEC 220 load calculation, a one-line, a lighting and emergency plan, and converter cut sheets. | -| 9 | **Accessibility sales/service counter: code notes without supporting detail/section**
_Minor, Confirmed, building_design_
**Issue:** Note requires counter height 38 max, check writing surfaces 28-34, parallel/forward approach clear floor space with knee/toe space, etc. per FBC-Accessibility 904/902.3. Serving counter, point-of-sale, and knee walls (48 high wall, 60 high plumbing wall) are called to detail A/S-1 but detail/section not included; counter length in plan not dimensioned to verify 36 min accessible portion.
**Fix:** Provide enlarged plan/section/detail A/S-1 at scale with heights, depths, clear floor spaces, and material; dimension accessible portion length and location; coordinate with accessibility legend (4 clear floor, 5 turning). Provide door detail for 403.5.1/404.2.4 compliance where called. | **Accessible counter has no detail**
**Issue:** The counter notes cite detail A/S-1, which is not in the set, and the accessible length is not dimensioned.
**Fix:** Add detail A/S-1 and dimension the accessible portion. | -| 10 | **Plumbing: water heaters off-plan, tempering, expansion, and backflow not detailed**
_Major, Confirmed, plumbing_
**Issue:** Hot/cold mains run full corridor length to serve LAV/WC/KS on both wings with 1/2 branches, but heat source is remote upper floor not shown. No heater type/capacity, T&P relief, drain pan, expansion tank, hot-water recirculation for long runs, or ASSE 1070 tempering/thermostatic mixing for public lavs shown. No reduced pressure/ backflow preventer at service entrance (note says connect to existing water service/meter/backflow/shutoff not shown for clarity, existing to remain - not verified). Long hot-water runs (~100+ ft) will have delivery delay and potential Legionella/energy issues.
**Fix:** Provide water-heater plan/detail on its actual floor with WH type, storage/recovery, T&P, pan/drain, expansion control, recirc pump/loop and balancing, and trap-primed floor drains. Show service entry with backflow, meter, shutoffs, and pressure. Provide hot-water temperature control and circulation design. | **Water heater, tempering, and backflow not shown**
**Issue:** The water heaters are on an upper floor that is not shown, and the set has no relief, expansion, recirculation, tempering, or backflow detail.
**Fix:** Show the heater room and add heater, recirculation, tempering, and service entry backflow details. | -| 11 | **Mechanical/electrical/plumbing coordination: roof/exterior penetrations not shown together**
_Minor, Confirmed, mechanical_
**Issue:** MEP scopes each call for roof/wall penetrations: bagel oven Class B double-wall vents + windproof cap, dryer 4 hard-piped, plumbing vents to roof, ERV/exhaust, electrical service conduit/penetrations. No composite roof plan or exterior elevation to verify separation from operable windows (odor/noxious), roof structure, or plumbing vent to air intake distances. Condensate to 6 off grade needs lawful discharge not to walkway/foundation - civil/MEP coordination missing.
**Fix:** Provide roof plan indicating all penetrations, equipment, exhaust terminations, intakes, and plumbing vents with dimensions and heights above roof. Detail exhaust caps and clearance to property lines/intakes per mechanical code. Coordinate condensate termination to grade with site/plumbing. | **Roof penetrations not coordinated**
**Issue:** Each trade adds roof or wall penetrations, but no roof plan shows them together or checks their distances.
**Fix:** Add a roof plan with every penetration, intake, and vent, with dimensions, and a condensate discharge point. | -| 12 | **P-2 deep dive: venting layout not shown - trap protection cannot be verified**
_Major, Needs input, plumbing_
**Issue:** Legend defines VENT TO ROOF as black dot plus METER CLEAN-OUT and CONNECT symbols. Plan shows only brown dashed waste red HW blue CW and CO symbols. No black-dot vents no vent size tags no vent-through-roof locations and no isometric showing how 6 WCs 6 LAVs 3 KSs 8 BSs and 2 DFs are vented.
**Fix:** Show venting for every trap: individual, wet-vented or circuit-vented layout with vent sizes, grades, connections above trap weirs, and roof terminations on a roof plan with heights/clearances. Provide DWV isometric/riser so trap-arm lengths, developed vent lengths and aggregate vent sizing can be checked against the declared FBC-Plumbing edition. Do not rely on color-coded horizontal plan alone. | **Venting not shown**
**Issue:** The plan has no vents, vent sizes, or roof terminations, so trap protection cannot be checked.
**Fix:** Show venting for every trap with sizes and roof terminations, and add a DWV isometric. | -| 13 | **P-2 deep dive: cleanout coverage facially deficient**
_Major, Needs input, plumbing_
**Issue:** Only one interior CO shown at Hall 110 on the 3-in. corridor waste plus exterior COs at the south sanitary connection. Long 3-in. and 4-in. horizontal runs serving 6 WCs and 11 sinks across two wings show bends, junctions and fixture branches with no intervening cleanouts, no base-of-stack COs, and no spacing/direction-change compliance demonstration.
**Fix:** Show all cleanouts: base of each soil/waste stack, each change of direction greater than 45 degrees, each 100 ft or less of developed horizontal length per governing edition, and building-drain terminus. Provide cleanout size, cover, access clearance, and detail. Verify existing building-drain COs to remain or add new. | **Too few cleanouts**
**Issue:** One interior cleanout serves long 3 in. and 4 in. runs with many bends and branches.
**Fix:** Add cleanouts at stack bases, direction changes over 45 degrees, and the required run lengths. | -| 14 | **P-2 deep dive: DWV sizing/grade/material not verifiable - no calcs or riser**
_Major, Confirmed, plumbing_
**Issue:** Waste tags read 2-in. at individual traps, 3-in. corridor collectors, 4-in. building drain to existing sanitary south of Lobby 100. No DFU schedule, no pipe material, no slope percent or inverts, no trap-arm dimensions, no direction-change fittings, no riser. Visual count is 6 WC 6 LAV 3 KS 8 BS 2 DF totaling 25 fixtures for screening approx 53 DFU assuming WC4 LAV1 KS2 BS2 DF0.5 - 4-in. main is facially plausible for that load but capacity depends on unverified slope/venting and public vs private fixture values, so sizing cannot be verified.
**Fix:** Provide DWV isometric/riser with DFU per fixture and per section, pipe material, grade/slope callouts with invert elevations, trap sizes with seal protection, hanger and firestop details at rated penetrations, and lawful discharge point. Declare slope basis so 2-in. 3-in. and 4-in. capacities can be checked. Do not scale NTS plan for grade. | **Drain sizing cannot be verified**
**Issue:** There is no DFU schedule, slope, material, or riser, so the 4 in. main cannot be confirmed, though it looks plausible at about 53 DFU.
**Fix:** Provide a DWV isometric with DFU, slopes, inverts, and materials. | -| 15 | **P-2 deep dive: water distribution sizing/pressure/backflow not verifiable**
_Major, Needs input, plumbing_
**Issue:** Cold water enters as 1-1/4-in. at Lobby 100 from existing service then runs full building length stepping to 1-in. in Hall 110 and 3/4-in. in upper Hall 203 with 1/2-in. branches to each fixture; hot mirrors it from remote heaters. No static/residual pressure, no meter/backflow size/type, no developed-length pressure-loss calc, no WSFU schedule, no WC flush type. Screening is 25 fixtures approx 34.5 WSFU only if tank-type WCs assumed - flushometer/sensor WCs would be much higher and could make 1-1/4-in. service and 3/4-in. hall mains undersized. Adequacy cannot be verified.
**Fix:** Provide water riser with WSFU per fixture and per section, meter size with utility pressures/flows, backflow type/location, main and branch sizes with developed lengths, shutoffs/hammer arrestors, and fixture flow rates. State WC flush type (tank vs flushometer/sensor) because it controls main size. Verify 1-1/4-in. service and 1-in. to 3/4-in. hall reductions by calculation. | **Water sizing cannot be verified**
**Issue:** There is no pressure, backflow, or fixture unit data, and flushometer water closets could leave the 1-1/4 in. service undersized.
**Fix:** Add a water riser with fixture units, pressures, backflow, and WC flush type, and verify the main sizes. | -| 16 | **P-2 deep dive: remote water heaters off-plan - no safety or recirc design**
_Major, Confirmed, plumbing_
**Issue:** Hot water is generated on an upper floor not in the packet above Pre-Test 118 with cold piped up and hot piped down to serve both east Sterile Processing wing and west Kitchen/Exam wing. No heater count type capacity fuel T and P discharge pan expansion tank recirculation loop tempering/mixing or access/pan drain shown. Red HW mains run the full corridor with no recirc branch, so far lavs/KSs face long dead-leg delivery delay and tempering control cannot be verified.
**Fix:** Show water-heater room on its actual upper floor to scale with heater type fuel storage/recovery ratings T and P relief with lawful drain pan/drain expansion control shutoffs thermostatic/ASSE 1070 tempering and recirculation pump/loop with balancing for the long runs. Provide upper-floor to lower-floor riser with sizes and insulation. Evaluate Legionella/scald/delivery-time for remote central heaters serving clinic lavs. | **Remote water heaters lack safety and recirculation**
**Issue:** The heaters sit on an upper floor that is not shown, with no recirculation or tempering, so far fixtures face long delays and tempering cannot be checked.
**Fix:** Show the heater room with relief, expansion, recirculation, and tempering. | -| 17 | **P-2 deep dive: fixture/equipment types and indirect/backflow needs undefined**
_Major, Confirmed, plumbing_
**Issue:** Plan uses BS KS LAV WC DF REF SS tags with pipe sizes only. No schedule defines BS vs lavatory vs clinical sink, no wall/floor mount, no flushometer vs tank, no sensor vs manual, no DF hi-lo or accessibility compliance, no REF ice-maker indirect drain/air-gap, no sterilizer or optical-lab equipment drainage/backflow demands. Trap sizes/seal protection and any required indirect waste or RPZ/vacuum-breaker protection for clinical/lab/sterile equipment cannot be verified.
**Fix:** Provide fixture/equipment schedule keyed to plan tags: WC type/mount/flush, LAV public vs staff with tempering, KS/BS clinical use with trap/vent and air-gap/indirect waste where required, DF hi-lo/accessibility with trap/vent, REF/ice-maker supply plus indirect drain, sterilizer/optical-lab equipment loads with backflow protection. Show trap primers where required and coordinate medical equipment cut sheets. | **Fixture and equipment types undefined**
**Issue:** No schedule defines the tagged fixtures, and indirect waste and backflow needs for the ice maker, sterilizer, and lab equipment are not shown.
**Fix:** Add a fixture and equipment schedule with indirect waste and backflow protection. | -| 18 | **P-2 deep dive: NTS illegible leaders and unverified existing ties**
_Major, Needs input, plumbing_
**Issue:** Sheet is marked SCALE NTS with small overlapping size bubbles where brown/blue/red leaders cross, so lengths/grades cannot be scaled and attribution is ambiguous at Private RR 106 LAV/WC waste, Procedure 208 HW/CW/waste stubs, Lobby DF 2-in. branch vs 4-in. main, and matchline joints. Both sanitary and water are noted connect to existing not shown for clarity existing to remain with no invert/size/condition/capacity, meter/backflow verification, or utility letter.
**Fix:** Resubmit P-2 to scale with legible leaders/tags, flow arrows, invert/slope callouts, building section/riser, and verified existing-system tie data: sanitary invert/size/material/condition/capacity letter and water meter size/pressure/backflow verification. Cloud and list any assumed existing-to-remain versus new work. Do not approve connections to unknown existing without utility/plumbing verification. | **Unscaled drawing and unverified existing ties**
**Issue:** P-2 is not to scale, leaders overlap, and the existing sanitary and water connections have no invert, capacity, or pressure data.
**Fix:** Resubmit to scale and verify the existing connections. | -| 19 | **Incoherent permit packet - three unrelated sample projects combined into one PDF**
_Major, Confirmed, general_
**Issue:** Packet contains Sample A (commercial bagel shop B-1/M-2/E-3 Revision 1), Sample A Baseline M-2 duplicate, Sample B (residential B-1/B-2), Sample C (medical clinic P-2/E-3). Same PDF name quick-start-selected-sheets.pdf used for all 8 pages with no cover/index reconciling them. Sheet numbers reused: B-1 appears for both Sample A and Sample B; E-3 appears for both Sample A and Sample C; M-2 appears twice with different content. Footer tags inconsistent: REVISION 1 vs BASELINE vs COMPARISON vs DEVELOPMENT. This is a hackathon anonymized sample set, not a submittable permit set.
**Fix:** Separate into three standalone submittals with unique project name/address/parcel, cover sheet, sheet index, revision tracking, and seal block. Do not co-mingle unrelated scopes. Provide a single north arrow orientation and consistent scale/datum across disciplines for each project. | **Three unrelated projects combined**
**Issue:** Samples A, B, and C share one file with repeated sheet numbers B-1, E-3, and M-2 and inconsistent footers. The review calls it a sample set, not a permit set.
**Fix:** Split it into three submittals, each with its own cover, index, revision tracking, and seal. | -| 20 | **Missing permit identity, code basis, seals, and required code analysis block**
_Major, Confirmed, general_
**Issue:** No sheet provides jurisdiction, site address/parcel, permit number, occupancy/use, construction type, building area, story/height, sprinklered assumption, alteration level/path (FBC-Existing Building Ch. 5-10), or governing code editions. Only isolated 2023 FBC references appear in mechanical symbols (FBC-Table M1507.4) and FBC-Accessibility 305.3/304.3.1/904 and FBC-Plumbing 403.1. No design professional seal/signature/date per F.S. 481/471 & FBC 107. No energy, accessibility, or life-safety code analysis. Sample A fixture table lists BUSINESS/BAGEL SHOP but no occupant load or egress table; Sample B bedroom count change has no area or egress window data; Sample C has no occupancy update for clinic change of use.
**Fix:** Add cover sheet with project data, applicable FBC 8th Edition (or governing edition per AHJ adoption), FBC-Accessibility, NEC, FFPC/NFPA editions, local amendments, scope narrative, area/height, occupancy, construction type, work area/alteration level, flood zone/BFE/FFE, and sealed code analysis. Provide required separate approvals/letters. Coordinate data across all disciplines. | **No permit identity, code basis, or seals**
**Issue:** No sheet gives the jurisdiction, address, occupancy, area, alteration level, code editions, or a design professional seal.
**Fix:** Add a cover sheet with project data and code editions, and a sealed code analysis. | -| 21 | **Sample A - Unresolved detail references and Out of Scope coordination**
_Major, Confirmed, building_design_
**Issue:** Proposed plan calls out 48 kneewall per detail A/S-1 (Regency 12x16 hand sink) and 60 kneewall per detail A/S-1, plus 2x6 plumbing wall, but no A/S-1 sheet is in packet. Large gray OUT OF SCOPE areas (restrooms/showers at top right and top left) remain on both existing and proposed but still contain accessibility callouts: CLEAR WIDTH IAW FBC-Accessibility 403.5.1 and DOOR TO BE IAW FBC-Accessibility 404.2.4.1. Conflicts with NOTE: All exterior doors/windows are existing to remain, yet mechanical adds exterior louvered vent and condensing unit penetration. Interior door tag 3680 = 36x80 verified but no door schedule.
**Fix:** Provide A/S-1 kneewall/plumbing wall details, verify 36 min counter and knee/toe clearance per FBC-Accessibility 904 (referenced notes 4-6). Clarify scope boundary: either include restroom work and show compliant turning space/clear floor space or remove accessibility tags from OUT OF SCOPE. Add door/window schedule and reconcile exterior wall penetrations with structural/envelope. | **Missing A/S-1 and conflicting scope**
**Issue:** The plan cites detail A/S-1, which is missing. Out-of-scope restrooms still carry accessibility callouts, and the added exterior vent conflicts with the note that exterior openings remain.
**Fix:** Provide A/S-1, settle the scope boundary, and add door and window schedules. | -| 22 | **Sample A mechanical - duplicate sheets, duct labeling drift, missing calculations and controls**
_Major, Confirmed, mechanical_
**Issue:** Two M-2 sheets both titled MECHANICAL PLAN 1/4=1' (Sample A Revision 1 and Baseline) appear to be revision alternatives of same design. Legend drift: Revision 1 lists ROUND RIGID DUCT (SA/RA), Baseline lists RECTANGULAR RIGID DUCT (SA/RA) for identical blue/pink duct runs (20x20 SA plenum riser to 16 SA duct, 12x26 RA plenum, 38 CFM and 20 CFM branches, 135-159 CFM diffusers). No Outdoor Air, load, or ventilation calculations provided; only generic Mechanical Scope note. ERV suspended from roof, condensing unit mounted per detail C/S-1 (missing), condensate lift pump to 6 from grade - route shown but no trap/overflow per FBC-M. Suggested LFR 1 rack oven vent diagram (manufacturer cut sheet) lacks engineered vent sizing, clearances, Class B double wall termination, and combustion air verification. Tstat symbol shown in legend but not located on plan (Tp missing).
**Fix:** Remove duplicate sheet, maintain single current revision with revision cloud/delta. Align duct symbol, size, and CFM tags with calculations per FBC-M 403/603 and energy code. Provide detail C/S-1, structural support for suspended ERV, condensate disposal with trap/vent, and engineered oven/burner steam & exhaust sizing/location with roof penetration details and clearances to combustibles (51-1/4, 61 dimensions shown need code verification). Show thermostat and control sequence. | **Mechanical sheets duplicated and incomplete**
**Issue:** Two M-2 sheets differ on duct shape, and there are no load, ventilation, oven vent, condensate, or thermostat details.
**Fix:** Keep one revision, and add detail C/S-1, calculations, oven vent sizing, a condensate trap, and the thermostat. | -| 23 | **Sample A electrical power plan - no service/panel data, missing GFCI/AFCI and disconnect labeling**
_Major, Confirmed, electrical_
**Issue:** E-3 shows extensive branch circuits (CKT 16,18,20,35,38,40,42,44-48,59-61, etc.), GFCI tags, 3-phase power via rotary phase converter (PL-7/PL-15) for bagel machine, idler motor notes, existing WH/AC disconnects to remain, and conductors capped for future. No one-line/riser, panel schedule, service size, available fault current, or load calculation. Circuit callouts partially illegible, some doubled (CKT 39, CKT 60/62). GFCI noted for kitchen but no AFCI, dedicated, or disconnect locations to scale. Contractor Notes require typed panel directory yet no panel shown. Mechanical equipment electrical needs (ERV 38 CFM, condensing unit, rack oven) not cross-referenced with loads.
**Fix:** Provide complete electrical service: utility point, meter/disconnect, panel location/working clearance, one-line/riser, panel schedules, load calc per NEC 220, GFCI/AFCI per NEC 210.8/210.12 as adopted by FBC, and equipment disconnects with FLA/MCA/MOCP coordination to mechanical nameplates. Clarify 3-phase derivation and code compliance of rotary converters. Make all CKT tags legible and tie to panel. | **Electrical service and panel data missing**
**Issue:** E-3 has no one-line, panel schedule, service size, fault current, or load calculation, and some circuit tags are illegible or doubled.
**Fix:** Add service, panel, and load data, GFCI and AFCI coverage, disconnects, and legible circuit tags. | -| 24 | **Sample A life safety/plumbing fixture coordination - fixture table incomplete, no egress documentation**
_Major, Confirmed, life_safety_
**Issue:** Fixture table lists BUSINESS/BAGEL SHOP with WATER CLOSETS 1 per 25 first 50/1 per 50 remainder → Required 1 Provided 1, URINALS Not required, LAVATORIES 1 per 40 first 80/1 per 80 remainder → 1/1, BATHTUBS 0, DRINKING FOUNTAIN 1 per 100 → 1/0 but footnote a. says drinking fountain not required as drinking water will be provided in container free of charge per FBC-Plumbing 410.6, SERVICE SINK 1 per occupancy → 1/1. No occupant load calc shown to justify factor application; no travel distance, exit count, common path, exit signs, emergency lighting, or sprinkler/alarm assumptions. Proposed plan shows single main entrance/exit with serving counter creating egress constriction (27-3 serving counter, 8 egg oven island) but no clear egress width dimension. No rated walls/corridors identified.
**Fix:** Provide occupant load per FBC Table 1004.5, egress analysis with travel/common path, number/width of exits, door hardware/swing, exit signage/emergency lighting per FBC Ch.10, and coordinate sprinkler trigger. Verify fixture count with calculated occupant load and provide plumbing plan/riser for mop/hand/dish sinks shown on proposed plan; document drinking water alternative per AHJ acceptance. | **Life safety and fixture table incomplete**
**Issue:** The shop has no occupant load, exit, travel distance, signage, or lighting data, and the counter and oven island may constrict the single entrance.
**Fix:** Add an occupant load and egress analysis, check the fixture count against it, and document the drinking water alternative. | -| 25 | **Sample B residential - unpermitted bedroom remediation lacks structural/demo detail and building code info**
_Major, Confirmed, building_design_
**Issue:** Existing plan identifies ~10x14 hatched UNPERMITTED BEDROOM created from screened porch without permit, with notes: BEDROOM WALLS ARE NON LOAD BEARING / ORIGINAL POST & BEAM LEFT IN PLACE TO REMAIN FOR SCREENED PORCH, EXISTING POST TO REMAIN (2x), WALL TO BE REMOVED, DOOR TO BE REMOVED. Proposed plan restores screened porch with no new construction detail, no wall/roof/connection, screening, slab, drainage, or threshold detail. NOTES on both sheets say contractor to field verify all dimensions & notify EOR of discrepancies but no EOR listed, no building section, no egress, no smoke/CO, no energy. North arrow inconsistent: Existing has S at top, E left; Proposed has S at top as well but orientation not tied to survey. No flood/wind/seismic criteria.
**Fix:** Provide demo plan with sequence, debris protection, structural letter confirming non-load-bearing walls and post/beam adequacy for restored screen enclosure, plus screened porch detail: post/base, beam, screen frame attachment, wind load, slab edge, ceiling height, electrical. Identify permit history resolution and required inspections. Add code data (occupancy, construction type, wind speed/exposure) and coordinate with property survey. | **Bedroom remediation lacks detail**
**Issue:** The proposed screened porch has no structural, demolition, or construction detail, no EOR, and no code data.
**Fix:** Add a demolition plan, a structural letter, porch details, and code data. | -| 26 | **Sample C plumbing/electrical - extensive clinic layout with no code analysis, no risers, no life safety**
_Major, Confirmed, plumbing_
**Issue:** Sample C shows full medical office conversion: lower level Waiting 100/101/102, Exam #1-5, DR Library, Staff areas; upper level Pre-Op, Procedure, Sterile Supply. Plumbing plan colors sanitary (brown dashed), HW (red), CW (blue), vent to roof, with fixture vents, cleanouts, 1/2-4 pipe tags, water heaters located in upper floor above Pre-Test (not shown), routing CW up and HW down from HWH per plan. No plumbing riser/isometric, no vent stack termination detail, no backflow/medical gas, no interceptors, no storm. Electrical plan shows NP-1/NP-2 panels serving exam lights, receptacles, dedicated USB, sterile processing equipment (CKT 8/10/11), but no lighting plan, no emergency/nurse call/fire alarm, no panel schedules, no load calc, no grounding. Both rely on MATCHLINE A-A split and existing to remain walls; no accessibility, egress, or rated partition info. Footers say Anonymized hackathon sample with no jurisdiction/edition.
**Fix:** Provide for each project: code path for existing building change of use, occupant load/egress, rated barriers, accessibility upgrades, plumbing matrix with fixture units, HW/CW demand, sanitary DFU, vent sizing, riser/isometric, backflow, and point of connection verified to civil/utility. Provide electrical load, distribution, lighting, life safety and low-voltage systems with panel schedules, fault current, and equipment elevations. Coordinate plumbing/electrical with mechanical and structural. | **Clinic plans lack code analysis and risers**
**Issue:** The clinic plumbing and electrical plans have no riser, panel schedules, load calculations, egress, or rated partition information.
**Fix:** Add the code path, plumbing and electrical calculations, risers, and life safety information. | -| 27 | **QA - Graphic quality, legibility, and cross-discipline coordination gaps**
_Minor, Confirmed, general_
**Issue:** All disciplines suffer legibility/cohesion issues: B-1 dimensions overlap accessibility circles (2-6 min clear floor, 5 turning space) and SALES & SERVICE COUNTER ACCESSIBILITY NOTE (items 1-8) and WALL COVERINGS NOTE; freezer on wheels, bakery trolley, mop sink callouts crowded. Mechanical 135 CFM diffuser chain at walk-in freezer/chiller is tightly spaced with 6-11 turning circle dashed over POS. Electrical CKT tags and GFCI AFF 60 / 42 tags overlap, some unreadable (AFF with degree symbol, NP-1 CKT 12 duplicates). Compass inconsistency: Sample A W at top (N at right), Sample B S at top, Sample C N typical - causes coordination errors. Scale 1/4=1 for B-1 but NTS for P-2 creates cross-reference mismatch. No revision cloud or date.
**Fix:** Clean drawing: increase text/leader clarity, ensure minimum 3/32 text, separate crowded tags, use consistent north, scale bar, and reference datum. Add revision date/cloud and reconcile compass across architectural/structural/MEP. Provide larger-scale enlarged kitchen/mechanical room plans and equipment schedules. | **Poor legibility and coordination**
**Issue:** Overlapping tags, inconsistent north arrows and scales, and no revision cloud or date make the sheets hard to read.
**Fix:** Clean up text and tags, use one north and scale convention, and add revision dates and clouds. | -| 28 | **C1 porch-post count cannot be compared - Column Schedule provides no quantity**
_Minor, Needs input, structural_
**Issue:** Foundation Plan 1/S-2 shows one (1) new C1 post callout: a square pier symbol labeled FD-1 / C1 at the angled porch wall (dimensioned 10'-4 3/4\" and 5'-3 1/4\" from adjacent piers). Other new posts on same plan are labeled C2 (two instances: FD-1/C2 at top diagonal and at east wall) and C3 (two instances: FD-2/C3 at south wall and east wall). Column Schedule top-center lists C1 as 1/8\" 6061-T6 ALUM POST, 3-1/2\" x 3-1/2\", COL. HEIGHT 9'-4\", BASE SEE E/S-3, CAP SEE G/S-3. The schedule columns are MARK, DESCRIPTION, COLUMN SIZE, COL. HEIGHT, BASE, CAP - there is no QTY/QUANTITY column and no numeric count value; note below schedule only requires contractor to field verify dimensions prior to fabrication.
**Fix:** Do not invent a scheduled count. Add a QTY column or explicit quantity note to the Column Schedule for C1 (and C2/C3) indicating the total number of new posts required, or clarify that the schedule is a type specification only and reference the plan quantity. Re-verify that the single FD-1/C1 shown on the Foundation Plan is the intended total for new C1 posts; confirm no other new C1 posts are hidden in details or delegated packages. | **C1 post count cannot be compared**
**Issue:** The plan shows one new C1 post, but the Column Schedule has no quantity column, so no count can be compared.
**Fix:** Add a quantity to the schedule, or say it is type only, and confirm one C1 post is intended. | -| 29 | **Tie-in to existing FACU / existing notification zones ambiguous**
_Major, Confirmed, fire_alarm_
**Issue:** Riser shows SLC extending from SK5808 with F-CR-CR-DD-HD-HD-CR-CR-F chain; NAC side shows 3 devices but arrows labeled EXISTING TO REMAIN are ambiguous as to which NAC (1-4) serves addition vs remains. Plan has dual home-run tags TO EXISTING FIRE ALARM CONTROL UNIT for SLC and NAC without conduit/pathway separation shown. No note on reprogramming, firmware compatibility of SD505 devices with SK5808, or spare SLC addresses; existing load schedule not provided to prove capacity.
**Fix:** Clarify single-line: assign new SLC loop/zone identifier and NAC number (or new NAC booster), list existing zone loads vs spare, and confirm programming. Provide existing panel zone list/schedule showing before/after loads. Coordinate raceway separation per NEC 760. Add note that contractor shall test and re-accept entire system and central station. | **Tie-in to the existing panel is unclear**
**Issue:** It is unclear which NAC serves the addition, and there is no existing load schedule or compatibility note.
**Fix:** Identify the NAC and SLC, show existing against spare capacity, and add a test and re-acceptance note. | -| 30 | **Missing secondary power / battery calculations for addition to existing FACU**
_Critical, Needs input, fire_alarm_
**Issue:** Existing SK5808 shown with NAC #1-4 EXISTING TO REMAIN and new branch adding 2 HD +1 DD +2 F +4 CR (SD505) and NAC devices HV 115cd + V 115cd + V 75cd on 14AWG. No battery load table, standby/alarm currents, or verification that existing batteries/charger can carry added load. No calc sheets in packet.
**Fix:** Provide NFPA 72-2022 10.6.7 battery calculations for combined existing+new loads (24h standby + alarm), confirm SK5808 SLC/NAC capacity, and size replacement batteries/charger if needed; add schedule to riser. | **Battery calculations missing**
**Issue:** Nothing shows that the existing batteries can carry the added devices.
**Fix:** Provide NFPA 72 battery calculations for existing plus new loads. | -| 31 | **No sequence of operation / I-O matrix; DH and TD delegated to owner**
_Critical, Needs input, fire_alarm_
**Issue:** Legend lists DH qty1 and TD qty2 as TBD by Owner; riser shows CR→TD, CR→DH, CR/CR→TD/F chains with no input/output matrix or sequence defining door release, HVAC shutdown, delays, or central station signals. General Notes 6-7 reference FL 61G15-32.008 submittal but no matrix is included.
**Fix:** Provide NFPA72 Ch.21/23 sequence / I-O matrix: each input (F, HD, DD) vs outputs (CR/TD/DH actions, notification, central station). Specify DH model, fail-safe release, supervised power, and TD time setting with code basis. Engineer must specify, not owner. | **No sequence of operation**
**Issue:** No input and output matrix defines door release, HVAC shutdown, or central station signals, and the door holder and time delay devices are left to the owner.
**Fix:** Add an I/O matrix and specify the DH and TD devices. | -| 32 | **No NAC voltage drop calc for 14AWG notification circuit**
_Major, Needs input, fire_alarm_
**Issue:** Wire Legend calls new NAC 14AWG FPL/FPLP 1-pair feeding PC2R HV 115cd + SCR 115cd + SCR 75cd. No wire length, load current, or voltage-drop calc verifies end-of-line voltage at device minimum.
**Fix:** Submit NAC voltage drop calc per NFPA72 for longest run (measure from FACU through Tool Rental Center to Maint Room), include device currents/voltages, and show compliant margin. Upsize conductor or add booster if needed; indicate EOL resistor. | **No voltage drop calculation**
**Issue:** The 14AWG notification circuit has no length or voltage drop calculation.
**Fix:** Submit a voltage drop calculation for the longest run, and upsize the wire or add a booster if needed. | -| 33 | **Notification and initiating device mounting heights / coverage not documented**
_Major, Needs input, fire_alarm_
**Issue:** Plan locates HV/V devices without room dimensions, ceiling heights, or spacing analysis; legend lists makes (System Sensor SCR/PC2R, SD505-AHS, SD505-PS) but no detail at 48in AFF for pulls, heat detector spacing per ceiling, or candela coverage to NFPA72 Ch.18. Wire Routing Note defers exact locations to contractor.
**Fix:** Provide mounting heights (pulls 42-48in, strobes per listing 80-96in wall / ceiling), heat detector spacing per ceiling type/height, and visual coverage calculation (room size vs 75/115cd). Clarify ceiling vs wall mounting and provide installation details. | **Device mounting and coverage not documented**
**Issue:** The plan gives no room dimensions, mounting heights, heat detector spacing, or candela coverage.
**Fix:** Add mounting heights, detector spacing, and a visual coverage calculation. | -| 34 | **Duct detector supply-side without HVAC shutdown interface**
_Major, Needs input, fire_alarm_
**Issue:** DD qty1 on SLC (SD505-DUCT) paired to CR on plan/riser but no identified air handler tag, fan-shutdown relay interface, or mechanical coordination. No HVAC sheets in this fire-alarm-only packet to confirm unit serving addition.
**Fix:** Designate controlled HVAC unit(s) (tag/location/CFM), show CR-to-control wiring and shutdown logic, add mechanical coordination note; if shutdown not required cite exception or remove DD. | **Duct detector has no shutdown interface**
**Issue:** The duct detector has no air handler tag or fan shutdown wiring, and no HVAC sheets are included.
**Fix:** Identify the unit and show relay wiring and shutdown logic, or cite an exception. | -| 35 | **Notification color and wiring deferrals incomplete**
_Minor, Needs input, fire_alarm_
**Issue:** Color note requires devices to match existing color without stating red/white and listing; Wire Routing Note declares all routing diagrammatic with exact locations to be determined in field by GC/contractor. No circuit Class (A/B), survivability, or fire rating identified though checklist items (b)-(d) refer to Note 15 class of circuits.
**Fix:** Specify device color/finish (e.g., red or white per existing and listing) and state Class B (or A) per NFPA72 Table on riser; limit field-routing disclaimer and require listed raceway types (FPL/FPLP, EMT) and separation per NEC 760. | **Color and wiring deferrals incomplete**
**Issue:** Device color is not named, routing is left to the contractor, and circuit class is not stated.
**Fix:** Name the color, state Class A or B, and require listed raceways. | -| 36 | **Battery & voltage-drop calc incomplete for existing FACP extension**
_Major, Needs input, fire_alarm_
**Issue:** Calc shows Standby 24h / Alarm 5 min, derating 1.2 → 8.90 AH min. NAC #2 new (0.597A alarm, #14 300 ft, 1.51 ohm, 19.50V EOL, 4.42% drop) is calculated. NAC #1 (#12 Existing To Remain, 0.025A alarm), NAC #3 and NAC #4 Existing To Remain list 0.00V/0.00% with blank length/one-way footage (0 ft). SLC device loads show dashed “added devices” but total standby/alarm columns are redacted; existing panel load not fully summed. Max NAC 3.0A / Max Panel 5.0A limits are listed but not compared to summed added load.
**Fix:** Provide complete battery calc per NFPA 72 10.6.7 including: (a) all existing SLC/NAC/aux loads on Silent Knight 5808 with wire lengths for NAC #1, #3, #4, (b) summed standby/alarm current (existing + new), (c) voltage-drop for every NAC with actual one-way length, (d) confirmation panel and NAC currents <80% per Note 22 and < max ratings. Show battery size installed vs 8.90 AH required. | **Battery and voltage drop calculation incomplete**
**Issue:** The calculation covers only the new NAC. Existing NAC data is blank, totals are redacted, and the limits are not compared with the summed load.
**Fix:** Complete the calculation for all NACs and show the installed battery against the 8.90 AH required. | -| 37 | **Notification coverage not demonstrated for Mercantile Group M addition**
_Major, Confirmed, fire_alarm_
**Issue:** Legend specifies: wall strobe SCR 75cd (1), ceiling strobe SCR 115cd (1), ceiling horn/strobe PC2R 115cd (1). Plan shows ~3 notification appliances covering ~2,500+ sf Tool Rental Center plus Maint. Room. No room dimensions, ceiling height, sound pressure (dB), candela spacing analysis per NFPA 72 Ch.18, or isobars. Note 23 states 115cd ceiling strobe to be placed at bottom of bar joist centered over aisle, but does not give aisle width, mounting height, or obstruction analysis. Existing NAC #2 load 0.597A confirms only 3 devices - insufficient evidence of full coverage.
**Fix:** Provide NFPA 72 18.4/18.5 analysis: effective candela vs room size, ceiling height, mounting height, and horn dB (15 dB above ambient per Note 14, verify ambient). Show spacing on plan with candela rating and indicate synchronization. If relying on existing building notification, delineate existing vs new coverage with existing candela ratings. | **Notification coverage not shown**
**Issue:** Three appliances cover over 2,500 sf with no spacing, candela, or sound level analysis.
**Fix:** Provide an NFPA 72 coverage analysis and separate existing from new devices. | -| 38 | **Egress time-delay hardware / magnetically held doors - undefined devices and sequence**
_Critical, Confirmed, fire_alarm_
**Issue:** Note 26 requires time-delay egress hardware to unlock on sprinkler/fire detection, loss of power, etc., with approved control devices. Legend lists DH (Door Holder) Qty 1 TBD by Owner, TD (Time Delay) Qty 2 TBD by Owner, CR (Control Relay) Qty 4 Silent Knight SD500-ARM. Plan shows 2 DH symbols near Maint. Room doors and 2 TD+CR pairs at exits. I/O matrix Col M (Release Magnetically Held Doors) shows X for rows 4-6 only (clouded Rev 1), Col N (Activates Delay Egress Release) shows X for rows 5 only. No power supply, fail-safe/fail-secure, or UL listing for TD/DH; no detail showing relay contact rating or wiring to door hardware. 61G15-32.008 FA contractor certification required per Note 6 but device compatibility not yet shown.
**Fix:** Specify listed TD and DH models compatible with Silent Knight 5808, voltage/current, fail-safe operation, and show wiring via CR contacts. Clarify sequence: any fire alarm initiation, power loss to egress control device, and building power loss shall release doors per FBC/FFPC and NFPA 101. Update I/O matrix to match notes 26.1/26.2 and show which rows correspond to which initiating devices. Provide cut sheets and door hardware schedule coordination with architectural. | **Door release devices and sequence undefined**
**Issue:** The door holder and time delay devices have no listing, power, or wiring detail, and the I/O matrix does not clearly map initiating devices to door release.
**Fix:** Specify listed, compatible devices, wiring, and the release sequence, and update the matrix. | -| 39 | **Duct detector RTU shutdown interface not detailed**
_Major, Confirmed, fire_alarm_
**Issue:** FA-2 shows one SD505-DUCT detector on supply side with sampling tube and one CR relay at RTU. Wire legend shows SLC 16 AWG. Riser positions DD between CRs on SLC. Sequence matrix Col O Activates RTU Shutdown shows X only for row 6 (presumably duct alarm). No mechanical plan reference, RTU identifier, shutdown contact (normally open/closed), auxiliary power, or interlock to HVAC control per FA-1 Fire Sprinkler Note building is fully sprinklered but HVAC not in fire-alarm scope. Note 15 Class B circuits does not explain supervisory vs alarm action for duct detector.
**Fix:** Show duct detector relay wiring to RTU control circuit, identify RTU tag, state supervisory vs alarm per NFPA 72, and coordinate with mechanical to shut down on duct smoke. Indicate remote test station if required and that sampling tube extends full duct width. Include mechanical sheet reference. | **Duct detector RTU shutdown not detailed**
**Issue:** The set gives no RTU tag, relay wiring, or supervisory or alarm action for the duct detector.
**Fix:** Show the relay wiring, RTU tag, and action type, and coordinate with mechanical. | -| 40 | **Listed device models deferred to Owner - not shown**
_Major, Confirmed, fire_alarm_
**Issue:** Legend: HD Silent Knight SD505-AHS + SD505-6AB base, CR SD500-ARM, F SD500-PS, DD SD505-DUCT, V/HV System Sensor SCR/PC2R - all specified. Exceptions: DH Quantity 1 To Be Determined By Owner, TD Quantity 2 To Be Determined By Owner. Per Note 6, fire alarm contractor shall provide 61G15-32.008 final design confirmation and Note 7 confirmation report, but permit set leaves door holder and time-delay hardware as TBD without UL/listing or compatibility with existing Silent Knight 5808.
**Fix:** Provide listed, compatible DH and TD model numbers, voltage, current draw (include in battery calc), and confirm with 61G15-32.008 letter that added devices are compatible with existing panel and central station. Do not leave as TBD. | **Device models left to the owner**
**Issue:** The door holder and time delay hardware are left as TBD, with no listing or proof they work with the existing panel.
**Fix:** Provide listed, compatible models and include their current in the battery calculation. | -| 41 | **Audible notification note references speakers - system uses horns**
_Minor, Confirmed, fire_alarm_
**Issue:** Note 14 states Speaker Output: 15 dB Above Ambient Noise Floor, Contractor to Verify. No speakers shown; notification is via System Sensor PC2R horn/strobe 115cd and SCR strobes. Note 15 correctly requires SLC/IDC/NAC Class B. Horn audibility should be per NFPA 72 18.4.3 (15 dB above ambient or 5 dB above max 30-sec average). Speaker language is boilerplate inconsistent with horn design and no ambient dB measurement is referenced.
**Fix:** Revise Note 14 to Horn Audibility (or remove if using only visual plus existing audible), state design ambient level, required dB, and verification method. Ensure total alarm current reflects horn draw (0.229A included for PC2R). | **Note refers to speakers, but the system uses horns**
**Issue:** Note 14 says speaker output, but the design uses horns and strobes.
**Fix:** Change Note 14 to horn audibility and state the ambient level and verification method. | -| 42 | **Notification device color and synchronization not specified on plans**
_Minor, Confirmed, fire_alarm_
**Issue:** Note states notification device color shall match color of existing notification devices currently installed in other portions of store - no color named (red/white) and no confirmation of strobe synchronization per NFPA 72 18.5.4.4. System Sensor SCR/PC2R are addressable but existing NACs are Existing To Remain; riser shows 3 new notification appliances on NAC #2 plus existing NAC loads - synchronization across existing + new not addressed.
**Fix:** State actual color (e.g., white or red) to match existing, confirm all strobes on same NAC/circuit are synchronized per manufacturer and NFPA 72, and that synchronization module/protocol is compatible with Silent Knight 5808. | **Color and synchronization not specified**
**Issue:** No device color is named, and synchronization of new and existing strobes is not addressed.
**Fix:** State the color and confirm the strobes are synchronized and compatible with the 5808. | -| 43 | **Exposed FPLP wiring method requires AHJ acceptance - protection not detailed**
_Minor, Needs input, fire_alarm_
**Issue:** FA-1 Note 24: All fire alarm cabling shall be routed through conduit from devices to bottom of ceiling pan. At that point cabling shall be run exposed along the ceiling if acceptable by AHU. All exposed wiring shall be Type FPLP. FA-2 Wire Routing Note states routing is diagrammatic and final locations to be determined in field. No conduit type/size, support, or protection detail where cable transitions from pan to open ceiling. FPLP is plenum-rated but not inherently protected from physical damage.
**Fix:** Confirm with Manatee County AHJ in writing that exposed FPLP on open bar-joist ceiling is permitted; otherwise require conduit/raceway per NEC 760.24. Provide detail at ceiling-pan transition and support per NEC 760.32. | **Exposed FPLP needs AHJ acceptance**
**Issue:** Exposed FPLP cable is allowed only if the AHJ accepts it, and no protection detail is given.
**Fix:** Get written AHJ approval or require conduit, and add a detail at the pan transition. | -| 44 | **Fire alarm plan lacks FACP/power location and survivability details**
_Minor, Confirmed, fire_alarm_
**Issue:** Riser shows Existing FACP Silent Knight 5808 with NAC #1-4 and Central Station tie; Plan annotates To Existing Fire Alarm Control Unit with leaders but does not show FACP room location, dedicated branch circuit/disconnect labeling per NEC 760.41, or pathway survivability for this Level per NFPA 72. Note 6/13 reference NEC 760 fire-rated cable unless AHJ requires otherwise but no pathway class (NAC is Class B per Note 15) is coordinated with riser.
**Fix:** Identify FACP location on plan (outside addition is acceptable with reference), show primary power disconnect location and labeling, confirm NAC/SLC pathway classification meets design intent, and reference central station monitoring arrangement. | **FACP location and survivability not shown**
**Issue:** The plan does not show the FACP room, its power disconnect, or pathway survivability.
**Fix:** Show the FACP location, disconnect labeling, and pathway class. | diff --git a/docs/research/pdf-packaging-probe-result.json b/docs/research/pdf-packaging-probe-result.json deleted file mode 100644 index 9be76a59..00000000 --- a/docs/research/pdf-packaging-probe-result.json +++ /dev/null @@ -1,11 +0,0 @@ -{ - "layoutDir": "/tmp/moke-pdf-packaged-l49pXp", - "probeBinary": "/home/scfred/.t3/worktrees/moke-meta-hackathon/t3-9e930b33/dist/backend/pdf-packaging-probe", - "exitCode": 0, - "stdout": "ok 512 663", - "stderr": "", - "overviewBytes": 2988, - "elapsedMs": 204, - "platform": "linux", - "arch": "x64" -} diff --git a/docs/research/pdf-packaging.md b/docs/research/pdf-packaging.md deleted file mode 100644 index 394046bd..00000000 --- a/docs/research/pdf-packaging.md +++ /dev/null @@ -1,84 +0,0 @@ -# PDF packaging - -The early standalone probe is retired. Current packaging verification uses -`bun run package:desktop` followed by `bun run test:packaged`, which imports and -renders the demonstration PDF through the shipped worker. Measurements below -are retained as historical evidence. - -Primary demo target: **Linux x64**. Packages pin `pdfjs-dist@6.4.299` and `@napi-rs/canvas@1.0.13`. - -## Installed layout - -After `bun run prepare:pdf-assets`, the backend bundle includes: - -| Path | Role | -| ----------------------------------------------------- | --------------------------------------------------------------- | -| `dist/backend/pdf-assets/pdfjs/` | `legacy/build/pdf.worker.mjs`, `cmaps/`, `standard_fonts/` | -| `dist/backend/pdf-assets/native/skia..node` | Platform `@napi-rs/canvas` binary | -| `dist/backend/pdf-assets/modules/@napi-rs/canvas/` | JavaScript support for PDF.js runtime canvas imports | -| `dist/backend/pdf-assets/layout.json` | Selected native package for `process.platform` / `process.arch` | -| `dist/backend/pdf-assets/licenses.json` | Bundled license text | -| `dist/backend/pdf-worker` | NDJSON worker (import, bounded region render) | - -Runtime environment for workers: - -- `MOKE_PDFJS_ASSETS` → pdfjs tree (no developer `node_modules`) -- `NAPI_RS_NATIVE_LIBRARY_PATH` → absolute path to the staged `.node` file -- `NODE_PATH` → the staged `modules/` directory, replacing inherited search paths - -PDF.js uses a runtime `createRequire` for canvas globals and auxiliary canvases. -The executable's static canvas import does not satisfy that lookup. Stage the -JavaScript support files as well as the native library. Packaged Electron tests -launch from an isolated data directory so a developer checkout cannot supply a -missing runtime dependency. The shell rejects assets for a different OS or CPU -before starting the backend. - -Platform selection lives in `scripts/prepare-pdf-assets.ts` (`canvasNativePackage` / `canvasNativeFileName`). - -## Measured probe evidence - -### Preflight scratch (`/tmp/moke-pdf-probe`, 2026-10-27) - -- Dependencies: `pdfjs-dist@6.4.299`, `@napi-rs/canvas@1.0.13` -- Dev render (`render-probe.mjs`): page 1 MediaBox view `[0, 0, 612, 792]`; overview **512×372 px**, **112 KB** PNG; 300 DPI crop **2500×1667 px**, **95 KB** PNG -- Packaged asset smoke (`MOKE_PDFJS_ASSETS=/tmp/moke-packaged-layout/pdfjs`, native **37 MB** `skia.linux-x64-gnu.node`): overview **512×372**, stdout `ok 512 372` - -### Historical worktree packaging probe - -Recorded run (generated fixture `tests/fixtures/generated/original.pdf`, temp layout mirroring installed resources, **no** repo `node_modules` on `PATH` for the probe binary): - -```json -{ - "exitCode": 0, - "stdout": "ok 512 663", - "overviewBytes": 2850, - "elapsedMs": 130, - "platform": "linux", - "arch": "x64" -} -``` - -Full machine-readable output: `docs/research/pdf-packaging-probe-result.json`. - -Compiled artifact sizes (Linux x64, Bun 1.4.4 compile): - -- `dist/backend/pdf-worker` — bundled worker (~147 MB class, same order as preflight `pdf-render-bin`) - -Electron Builder copies `app-backend`, `pdf-worker`, and `pdf-assets` under `resources/backend/` (`electron-builder.json`). - -## Application API (backend) - -Import from the owning modules under `backend/documents/`: - -- `Documents` (`Context.Service`) with `Documents.layer(config)` -- `DocumentsConfig`: `{ dataDirectory, workerCommand, assetsDirectory, nativeLibraryPath }` -- `importPdf(source)` → `ImportedPdf` -- `render(originalSha256, sheet, region, dpi)` → `RenderedPdf` -- `read(sha256, mediaType, offset, limit)` → `{ base64, totalBytes }` -- Shared errors: `DocumentError`, `ReviewStorageError` in `#lib/review/contracts.ts` (unchanged) - -Parent wiring should set `workerCommand` to the packaged `pdf-worker` binary and asset paths under the installed `resources/backend/pdf-assets` directory. - -## Tests - -`tests/smoke/documents.test.ts` covers import/hash storage, rotation round-trip (≤1 px), malformed PDF rejection, pixel-limit rejection (`userUnit` 2 full page @ 300 DPI), chunked read, and interruptible import (worker delay hook). diff --git a/docs/research/protocol-3-deployment-result.json b/docs/research/protocol-3-deployment-result.json deleted file mode 100644 index 3c82c366..00000000 --- a/docs/research/protocol-3-deployment-result.json +++ /dev/null @@ -1,94 +0,0 @@ -{ - "sourceCommit": "d7ba4f87415b1715189b1ee364108bef96ed87f0", - "appName": "moke-reviews", - "environment": "main", - "imageId": "im-01M630XTWWQWB3FM0K4NY8NR0D", - "immutableTag": "moke-review-engine:d7ba4f874", - "latestAlias": "moke-review-engine:latest", - "selectedImage": "moke-review-engine:d7ba4f874", - "identity": { - "protocol": 3, - "engine": "review-pi-1.4.0-v1", - "prompt": "moke-selected-coordination-v2", - "instructions": "8a0e2648738018631eaa6d24ee0b6ebaa239afb1", - "packetPrompt": "moke-packet-review-v1" - }, - "publishedAt": "2026-10-29T05:08:07.883Z", - "promotedAt": "2026-10-29T05:08:37.406Z", - "finalVerifiedAt": "2026-10-29T05:10:29.540Z", - "readiness": { - "passed": true, - "modelCalls": 0, - "networkBlocked": true, - "migrationHashesMatched": 9, - "pdfPages": 2, - "renderWidth": 612, - "renderHeight": 792, - "pngBytes": 49988 - }, - "liveValidation": { - "startedAt": "2026-10-29T05:08:42.067Z", - "inspectedAt": "2026-10-29T05:09:42.115Z", - "cloudId": "1773f02d-546f-4363-9009-ced31ca284cb", - "workerProtocol": 3, - "sandboxId": "sb-01M630YXGCYGG5K4N1J4MBAMQ2", - "inputSnapshotId": "im-01M630Z26DRAVRV31PTJAEXQV0", - "resultSnapshotId": "im-01M630Z6W3T6MJE33STE9DPB56", - "provider": "meta", - "model": "muse-spark-1.3", - "maxCalls": 12, - "maxInputTokensPerCall": 64000, - "maxOutputTokensPerCall": 2048, - "timeoutMinutes": 15, - "admittedCalls": 1, - "sentAttempts": 1, - "unresolvedAttempts": 1, - "successfulResponseReceipts": 0, - "samCalls": 0, - "retries": 0, - "executionState": "uncertain", - "uncertaintyAcknowledged": false, - "phase": "imported", - "strictValidatorExitCode": 1, - "modelCompleted": false, - "drawn": [], - "scheduled": [], - "costUsd": null, - "imageInputTokens": null, - "sourceHash": "5fcb96c2fd1893b714218bb931ac50697db19e30f19735f0f58d49a81ceace91", - "sourceHashesUnchanged": true, - "localWritesBlockedDuringCloud": true, - "localWritesAvailableAfterImport": true, - "backendClosedAndReopened": true, - "reopenedWithoutRedispatch": true, - "automaticResultReturn": true, - "packageFilesHashVerified": 3, - "progressTransport": "rpc-stream", - "observedPhases": ["dispatching", "ready", "importing", "imported"], - "remoteStopped": true, - "originalA08TriggerReproduced": false - }, - "rollback": { - "sourceCommit": "c359e522f06f1a73888057233b4854193e2da9b2", - "immutableTag": "moke-review-engine:c359e522", - "imageId": "im-01M62TGRYMR5FNNF3BJVSXR2PX", - "protocol": 2, - "retained": true - }, - "operator": { - "privateConfigurationMode": "600", - "oldWorktreeConfigurationPreserved": true, - "providerCopyUnchanged": true, - "unrelatedEnvContentPreserved": true, - "poolQueue": null - }, - "cleanup": { - "ownedRecordedSandboxesStopped": 5, - "ownedTaggedSandboxesRunning": 0, - "newWarmCapacity": 0, - "imagesDeleted": 0, - "durableEvidenceRetained": true - }, - "evidenceDirectory": ".local/protocol-3-deployment", - "fixtureDirectory": ".local/cloud-validation/2026-10-29T05-08-42.067Z" -} diff --git a/docs/research/protocol-3-deployment.md b/docs/research/protocol-3-deployment.md deleted file mode 100644 index fddb627c..00000000 --- a/docs/research/protocol-3-deployment.md +++ /dev/null @@ -1,160 +0,0 @@ -# Protocol 3 deployment and operator synchronization - -Protocol 3 is deployed to Modal app `moke-reviews`, environment `main`. -The deployment and this checkout's private operator setup are synchronized. -The one live validation returned its package safely, but did not complete model -inference. Its single sent request remains uncertain and was not retried. - -## Published image - -| Field | Verified value | -| ------------------------------------------------ | ------------------------------------------ | -| Source commit | `d7ba4f87415b1715189b1ee364108bef96ed87f0` | -| Immutable tag | `moke-review-engine:d7ba4f874` | -| Latest alias | `moke-review-engine:latest` | -| Image ID for both names | `im-01M630XTWWQWB3FM0K4NY8NR0D` | -| Selected private operator image | `moke-review-engine:d7ba4f874` | -| Protocol | `3` | -| Engine | `review-pi-1.4.0-v1` | -| Count prompt | `moke-selected-coordination-v2` | -| Packet prompt | `moke-packet-review-v1` | -| Instructions | `8a0e2648738018631eaa6d24ee0b6ebaa239afb1` | -| Published | `2026-10-29T05:08:07.883Z` | -| Latest alias and operator selection synchronized | `2026-10-29T05:08:37.406Z` | -| Final alias and cleanup verification | `2026-10-29T05:10:29.540Z` | - -The SDK resolved the previous immutable tag and `latest` before publication. -Both named `im-01M62TGRYMR5FNNF3BJVSXR2PX`. A bounded, network-blocked -probe verified its protocol-2 identity. The old tag and image remain available. - -An ignored operator script followed `scripts/modal.ts image`: Bun 1.4.4 built -the current worker and PDF worker, the Modal base image installed exact canvas -1.0.13 and PDF.js 6.4.299 dependencies, and preparation copied only those two -workloads and the nine existing SQL migrations. The script additionally retained -image and owned-sandbox receipts. It checked the current identity, took a -non-expiring filesystem snapshot and published the immutable tag first. - -A fresh sandbox resolved from that tag passed the worker readiness probe and -strict protocol/engine/prompt/instruction decoding. All nine remote migration -SHA256 values matched this checkout. The real PDF worker inspected the authored -two-page fixture and rendered its first page to a 612 x 792 PNG of 49,988 bytes. -This readiness sandbox blocked networking and made zero model calls. The first -readiness attempt stopped on a local probe-script schema construction error; -the corrected probe passed. Both attempts were confirmed stopped. No application -implementation changed. - -Only after readiness passed did the operator publish `latest` and change this -checkout's selected image. Final SDK reads confirmed both names resolve to the -same new image ID. The existing private connection files were copied into this -checkout's ignored `.local/modal/session.json` and `.local/provider/session.json` -with permissions 600. The ignored `.env` now selects those files, preserving -unrelated content. The provider copy is unchanged; the Modal copy differs only -in its selected image. The old worktree's private configuration remains unchanged. -The timeout remains 15 minutes, `poolQueue` remains null, and no warm controller -or capacity was started. The named Secret `moke-reviews-meta-api` was reused -through outbound header injection. No model key was retrieved from Modal. - -## One bounded live validation - -The unmodified `bun scripts/validate-cloud.ts --live --calls 12` ran against the -freshly built headless `BackendApi` at `2026-10-29T05:08:42.067Z`. It used only -the repository-authored Rev A two-sheet schematic in a new isolated data -directory. It did not access the live desktop or user workspace. - -- Cloud job: `1773f02d-546f-4363-9009-ced31ca284cb`. -- Sandbox: `sb-01M630YXGCYGG5K4N1J4MBAMQ2`. -- Input snapshot: `im-01M630Z26DRAVRV31PTJAEXQV0`. -- Result snapshot: `im-01M630Z6W3T6MJE33STE9DPB56`. -- Provider/model: Meta `muse-spark-1.3`, without provider fallback. -- Limits: 12 calls, 64,000 input tokens and 2,048 output tokens per admitted - call, 15-minute cloud deadline, zero automatic provider retries. -- Actual accounting: one admitted and sent model request, one unresolved - attempt, no successful response receipt, zero SAM calls. Cost and image input - tokens are unknown. Reserved text/output tokens are 64,000 / 2,048. - -The durable receipt records `workerProtocol: 3`. The backend rejected a local -write while the cloud owned the fixture, closed completely, reopened with the -same job and observed progress over RPC. The observed phases were `dispatching`, -`ready`, `importing` and `imported`. It automatically restored the returned -package through the existing verification and maintenance boundary. A separate -read of all three manifest-authorized output files confirmed their byte counts -and SHA256 values. The original PDF hash remains -`5fcb96c2fd1893b714218bb931ac50697db19e30f19735f0f58d49a81ceace91`. - -The model execution became `uncertain` after its first sent request. The normal -validator therefore exited 1 after import because it requires completed -inference, proposed drawn 2 / scheduled 3, and a final conversation answer. -This run produced no count facts or final model answer. No successful provider -receipt or reliable HTTP failure classification is retained, so the cause is -unknown; it is not evidence that credentials are invalid or inference succeeded. - -A subsequent headless reopen inspected the same saved job and snapshot, confirmed -there was still only one run and one admitted call, and successfully saved a -fixture-only local message. The unresolved attempt remains unacknowledged and -was never resumed or replayed. Findings remain open. Source hashes are unchanged. -The original review sandbox was confirmed stopped with exit 137, the result of -the orchestrator's explicit termination after checkpointing. - -This proves protocol-3 admission and the safe cloud transfer/return path even -when inference is unresolved. It does not establish successful protocol-3 model -counts, a live packet/scope review, or the original A08 transport disconnect -trigger. Existing offline scope and lifecycle recovery evidence remains as -recorded in [Audit 2 validation](audit-2-validation.md). - -## Rollback, retained evidence and cleanup - -Rollback reference: `moke-review-engine:c359e522`, image -`im-01M62TGRYMR5FNNF3BJVSXR2PX`, source -`c359e522f06f1a73888057233b4854193e2da9b2`. It was verified as protocol 2 before -this deployment. An authorized rollback can republish that image under `latest` -and select its immutable tag in the private operator configuration. New -protocol-3 clients would refuse it for fresh reviews, so an application rollback -must be coordinated. Recovery of existing jobs retains its separate compatibility -rules. No image was deleted or rollback performed. - -Sanitized tracked evidence is in -[protocol-3-deployment-result.json](protocol-3-deployment-result.json). -Local operator artifacts are under `.local/protocol-3-deployment/`: - -- `before.json`, `previous-identity.json`, `published.json`, `readiness.json`, - `promoted.json` and `final-check.json` record identities and resource state. -- `inspect.log`, `publish.log`, `readiness-attempt-1.log`, `readiness.log`, - `promote.log` and `backend-build.log` retain deployment/build output. -- `live-validation.log`, `inspect-live.log` and `live-evidence.json` retain the - nonzero validation outcome and inspection without retry. -- `*-owned.json` and `*-cleanup.json` record the previous-image probe, - preparation sandbox and both readiness attempts. `final-check.json` also - records the live-review sandbox and confirms all five are stopped. - -The fixture, admitted request, complete input/output packages, ownership receipt -and execution journal are preserved under -`.local/cloud-validation/2026-10-29T05-08-42.067Z/`. Durable snapshots are retained -under the existing seven-day checkpoint policy. Final SDK lists found zero -running sandboxes with this operation's probe/preparation tags or live job tag. -Only owned resources were stopped. No queue, warm capacity or user dataset was -changed. No GUI, preview, development server, desktop restart, push, PR or merge -was performed. - -## Repository validation - -The deployment record introduces no application changes. The deployed -application source remains the already verified `d7ba4f874`. Its existing local -evidence is 321 smoke tests with 2,152 assertions, seven isolated Electron tests -and seven packaged tests in `.local/audit-2/ci-11-verify.log` and -`.local/audit-2/ci-11-packaged.log`. The parent supplied passing Linux, macOS and -Windows hosted checks, including packaged tests, for merged PR #11. Those checks -were not rerun here. The backend build, repository formatting and diff checks -passed for this deployment. No application behavior or isolation guard changed. - -SDK operations were checked against the installed `modal@0.11.1` source and the -official [Image API](https://modal.com/docs/sdk/js/latest/Image) and -[Sandbox API](https://modal.com/docs/sdk/js/latest/Sandbox). - -The first Windows CI run exceeded Bun's default five-second deadline in three -integration tests; a single rerun still timed out the continuation case at about -seven seconds. Those three cases now use explicit 15-second test limits, matching -adjacent integration tests. Their assertions, application deadlines and isolation -controls are unchanged. Focused validation passed three tests with 114 assertions; -full verification passed 321 smoke tests with 2,152 assertions and seven isolated -Electron tests. Evidence is retained in `ci-timeout-focused.log` and -`ci-timeout-verify.log` under the local deployment evidence directory. diff --git a/docs/research/screenshots/beta-mobile-findings.png b/docs/research/screenshots/beta-mobile-findings.png deleted file mode 100644 index 1eb2cab3..00000000 Binary files a/docs/research/screenshots/beta-mobile-findings.png and /dev/null differ diff --git a/docs/research/screenshots/beta-workbench.png b/docs/research/screenshots/beta-workbench.png deleted file mode 100644 index 774ffc14..00000000 Binary files a/docs/research/screenshots/beta-workbench.png and /dev/null differ diff --git a/docs/review-runtime.md b/docs/review-runtime.md index 826d953f..c0a70db3 100644 --- a/docs/review-runtime.md +++ b/docs/review-runtime.md @@ -2,7 +2,7 @@ Decision recorded October 27, 2026: use **Pi Durable for agent execution**, behind an Effect application service. Reviews execute locally by default. **Push to cloud**, implemented October 28, starts a new review in a Modal Sandbox running the matching Bun engine. Effect owns application contracts, project operations, PDF and SAM tools, accepted facts, calculations and dependency invalidation. -The local engine, persistent journal, provider integration, and packaged desktop are implemented. The [beta workflow](research/beta-implementation.md) describes the connected prompts and UI; [core validation](research/core-validation.md) records earlier runtime proof. [Cloud reviews](cloud-reviews.md) documents the implemented whole-workspace transfer, single-owner fence, snapshots, setup, tests, and outstanding live-account validation. It starts fresh runs and preserves the prohibition on replaying copied journals. The sections below describe the broader future contract for continuing an existing run; that continuation is not implemented. The [SDK comparison](research/agent-harness-comparison.md) records the alternatives and independent critique. +The local engine, persistent journal, provider integration, and packaged desktop are implemented. [Core validation](validation/core-validation.md) records earlier runtime proof. [Cloud reviews](cloud-reviews.md) documents the implemented whole-workspace transfer, single-owner fence, snapshots, setup, and tests. It starts fresh runs and preserves the prohibition on replaying copied journals. The sections below also describe a future contract for continuing an existing run; that continuation is not implemented. ## Local and cloud responsibilities @@ -14,7 +14,7 @@ The local engine, persistent journal, provider integration, and packaged desktop | Source evidence | Original PDFs, selected sheets, crops, transforms and content hashes | Explicitly transferred source package sufficient for subsequent crop requests | | Model calls | Meta Model API or eligible partner inference | Same selected model route unless the user explicitly changes it | -Local execution describes the engine and project state. Meta/Baseten model calls still send selected evidence to hosted inference. Moving inference weights onto the user's hardware is a separate choice. The planned Modal path hosts the review engine; moving the engine does not itself require moving model serving. +Local execution describes the engine and project state. Meta/Baseten model calls still send selected evidence to hosted inference. Moving inference weights onto the user's hardware is a separate choice. The Modal path hosts the review engine; moving the engine does not itself require moving model serving. Cloudflare PiHarness is an alternative considered during research, not the selected cloud path. Modal is the selected provider for this handoff. A local review does not require a running Modal Sandbox or a permanent cloud copy of the project. @@ -105,8 +105,6 @@ evidence: the current drawings and governing sources still establish every findi state or county/city, in other conversations. The result labels them as leads; the model reads a source again before citing it. The review records which earlier reviews were offered. -The [firm memory decision](firm-memory-plan.md) explains why MOKE stops here for now. - ## Future continuation of an existing run Transfer application state at a safe boundary rather than migrating live process memory: @@ -129,39 +127,3 @@ Modal documents Sandboxes that continue running after a client detaches, configu A Pi commit to a Sandbox filesystem is not automatically a recoverable checkpoint after the Sandbox itself is lost. The storage integration must define when a checkpoint has reached durable storage outside that Sandbox. Until verified, distinguish process-restart recovery from Sandbox-loss recovery. Use a single owner for the active SQLite database. Persist consistent closed or backed-up checkpoint files with explicit acknowledgement. Modal Volumes have commit/reload semantics and document last-writer-wins behavior for conflicting file modifications; they do not provide distributed file locking. Do not use a shared live SQLite file as the ownership protocol. Snapshots have retention limits, so retention must be configured for the review rather than assumed indefinite. [Volume consistency](https://modal.com/docs/guide/volumes), [snapshot retention](https://modal.com/docs/guide/sandbox-snapshots). - -## Hackathon eligibility - -The current [Official Rules](https://cerebralvalley.ai/e/meta-ai-developer-hackathon/waiver) explicitly permit open-source components when licenses are followed and the submission builds on the underlying product. Pi Durable's [package manifest](https://github.com/earendil-works/pi/blob/main/packages/durable/package.json) declares MIT. Keep applicable notices, identify Pi as a dependency, and clearly distinguish Moke's new engineering workflow from the underlying harness. - -The [participant guide](https://cerebralvalley.ai/e/meta-ai-developer-hackathon/details), re-read in the authenticated browser on October 27, requires meaningful Meta-model use and explicitly lists Modal resources including Sandboxes. The [public event page](https://cerebralvalley.ai/e/meta-ai-developer-hackathon) also lists Modal as a partner. The reviewed rules do not require an exclusively Meta-owned SDK or infrastructure stack, and do not specify a scoring deduction for Pi or Modal. This supports the proposed combination; it does not guarantee any judging score. - -Keep the repository public, distinguish the reused Moke name and review prompts from new event work, use source drawings and assets with appropriate rights, and make the submitted three-minute demo show the new implementation. Do not present Pi's existing harness features as original Moke work. The original contribution is the drawing-review application, evidence handling, corrections, revision behavior and portable review execution. - -## Competing for first place - -The Official Rules and current guide assign 30% to impact, 25% to Meta AI tooling, 20% to technical depth and execution, and 25% to the demo. The following is the proposed evidence for those criteria, not a prediction of awarded points. - -| Criterion | Evidence to demonstrate | -| ------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------- | -| Impact, 30% | A practitioner-relevant finding, the verification effort it saves, a corrected false alarm and a useful revision recheck | -| Meta tooling, 25% | Muse selects a missing fact and inspection action, requests a precise PDF crop, uses SAM evidence, and changes the supported outcome | -| Technical execution, 20% | Accurate source transforms, typed tool results, fact-version invalidation, honest uncertainty, recovery and one-owner local/cloud handoff | -| Demo, 25% | Actual recorded model/tool activity, legible evidence, polished narration, a working live interaction and truthful elapsed-time transitions | - -Keep Meta models responsible for meaningful perception and inspection decisions. Keep calculations explicit and testable. A deterministic script that merely displays a model summary would provide weaker evidence for the Meta-tooling and agentic criteria. - -Prioritize Best Agentic Use-Case and retain a genuine evaluated Spark Open review mode for the open-weight category. Reuse the main demo for the viral and community opportunities; creative-build features remain outside the planned scope. The [bonus assessment](hackathon-bonuses.md) records all five categories, amounts, eligibility evidence and launch actions, subject to the organizer discretion in the Official Rules. - -Prioritize the main track over adding nominal integrations for bonus categories. Build the complete local review and start collecting real footage early. Keep the engine portable from the start, then add one narrow checkpoint handoff. The cloud scene earns its place by showing a useful review survive a desktop disconnect and reopen with the same evidence, rather than by adding infrastructure logos to the demo. - -## Validation gates - -- Complete a real Muse image-and-tool round and SAM operation through the selected providers, preserving reasoning continuity and response identities where supported. -- Kill the compiled Bun engine around model submission, response persistence, tool execution and domain-result acknowledgement; reconcile uncertain calls and prevent duplicate findings. -- Transfer a checkpoint to the Linux Sandbox engine and prove that the local executor cannot continue the transferred run. -- Disconnect and reopen the desktop, then verify the remote review and source-linked results. -- Destroy the Sandbox and recover from the last acknowledged external checkpoint, with any possible lost progress stated precisely. -- Apply a reviewer correction or new PDF revision and verify that stale remote output cannot overwrite current facts. - -No resources were provisioned and no paid model probes were run while recording this decision. diff --git a/docs/research/baseten-rate-limits.md b/docs/validation/baseten-rate-limits.md similarity index 97% rename from docs/research/baseten-rate-limits.md rename to docs/validation/baseten-rate-limits.md index eb2cdab8..0bb81ca7 100644 --- a/docs/research/baseten-rate-limits.md +++ b/docs/validation/baseten-rate-limits.md @@ -98,7 +98,7 @@ ask only for project configuration and sources. Sheet fan-out began about 25 min costs more. The best alternative gains one point and removes a tenth of each request's images, so the window stays as it is. Baseten's 8-image limit keeps Spark Open packet reviews near 88 to 90% cached; Meta, which keeps every image, measured 91 to 95% on the same packets - ([fast initial review validation](fast-initial-review-validation.md)). + ([fast initial review validation](fast-initial-review.md)). ## Decisions @@ -122,11 +122,11 @@ ask only for project configuration and sources. Sheet fan-out began about 25 min [Configure HTTP clients](https://docs.baseten.co/inference/http-client-configuration#implement-retries), [Request lifecycle](https://docs.baseten.co/deployment/autoscaling/request-lifecycle), [Muse Spark Open](https://www.baseten.co/library/muse-spark-open/). -- Client-side limiter designs found with Exa: +- Client-side limiter designs: [Anvil's per-provider RPM and TPM buckets](https://github.com/esanmohammad/Anvil/blob/main/packages/agent-core/src/router/rate-limiter.ts), [dsh-llm-rate-limit](https://github.com/Asong6824/dsh-llm-rate-limit) (FIFO admission, shared cooldown, usage reconciliation) and [Nucleo's admission accounting](https://docs.redouble.ai/nucleo/current/harness-llm-internals.html) (reserve before the call, roll back only when the call never happened). - Repository: `backend/inference/pacing.ts`, `backend/inference/provider.ts`, - `backend/packet-review/service.ts`, `docs/research/audit-3-provider-timeout.md`. + `backend/packet-review/service.ts`. diff --git a/docs/research/citation-links-real-set.md b/docs/validation/citation-links-real-set.md similarity index 98% rename from docs/research/citation-links-real-set.md rename to docs/validation/citation-links-real-set.md index e802993f..40a8cc2d 100644 --- a/docs/research/citation-links-real-set.md +++ b/docs/validation/citation-links-real-set.md @@ -112,5 +112,3 @@ want to correct the reference itself. request and receipts do not keep the field. The console shows the real figure. - **A full set on OpenCode Go.** Only six sheets have run there. At run B's token counts a 46-sheet review would cost about $1.26 at Go's rates, but tokens vary from run to run. -- **OpenCode Go's other models.** MiMo-V2.6-Flash, GLM-5.3-Flash and GPT-6 Luna are offered and have - not been tried on drawings. diff --git a/docs/research/cloud-validation-result.json b/docs/validation/cloud-validation-result.json similarity index 100% rename from docs/research/cloud-validation-result.json rename to docs/validation/cloud-validation-result.json diff --git a/docs/research/cloud-validation.md b/docs/validation/cloud-validation.md similarity index 100% rename from docs/research/cloud-validation.md rename to docs/validation/cloud-validation.md diff --git a/docs/research/core-validation-result.json b/docs/validation/core-validation-result.json similarity index 100% rename from docs/research/core-validation-result.json rename to docs/validation/core-validation-result.json diff --git a/docs/research/core-validation.md b/docs/validation/core-validation.md similarity index 95% rename from docs/research/core-validation.md rename to docs/validation/core-validation.md index 6763f22e..b1423ac7 100644 --- a/docs/research/core-validation.md +++ b/docs/validation/core-validation.md @@ -46,7 +46,7 @@ the exported package, receipts and the closed test data. The actual image files are `packaged-live-muse.png` and `review-workbench.png`. They contain only the authored schematic. This directory and credentials are excluded from git. -![Packaged Meta/SAM review with completed live status](screenshots/packaged-live-review.png) +![Packaged Meta/SAM review with completed live status](packaged-live-review.png) ## Repeatable checks @@ -58,7 +58,7 @@ only the authored schematic. This directory and credentials are excluded from gi compiled SIGKILL/ownership recovery, zero additional sends after uncertainty, pause/cancel, command/response receipts, image blocks, bounded image windows, journal incompatibility and persisted progress pagination. -- `tests/smoke/review-fences.test.ts`: the real parent port and SQLite permit correction +- `tests/smoke/review-fences.test.ts`: the real application port and SQLite permit correction while a response waits, reject its stale promotion and replay a committed receipt. - `tests/smoke/backend.test.ts`: auth/origin/process failures plus streaming HTTP delivery before a subscription closes. @@ -111,7 +111,7 @@ Provider retrieval and guaranteed remote cancellation are unavailable here; unknown outcomes remain explicit, with no automatic resend. A recorded possible- charge acknowledgement permits only a separate new run and budget. -The council-fix rerun also passed all four hero outcomes. Its revision needs-evidence +A later rerun after review fixes also passed all four hero outcomes. Its revision needs-evidence row cites only the still-current schedule operand; the rejected baseline plan operand is null. Full verification passed 198 tests and 1,309 assertions before -that live rerun. See [the council ledger](core-council.md) for recovery and history fixes. +that live rerun. diff --git a/docs/research/fast-initial-review-validation.md b/docs/validation/fast-initial-review.md similarity index 96% rename from docs/research/fast-initial-review-validation.md rename to docs/validation/fast-initial-review.md index 90d597d0..3c9c69fc 100644 --- a/docs/research/fast-initial-review-validation.md +++ b/docs/validation/fast-initial-review.md @@ -1,7 +1,6 @@ # Fast initial review: live validation -Measured 2026-10-30 on branch `feat/fast-initial-review`, against the plan in -[fast-initial-review.md](fast-initial-review.md). Every run used Muse Spark 1.3 at +Measured 2026-10-30 on branch `feat/fast-initial-review`. Every run used Muse Spark 1.3 at `xhigh` reasoning with Meta web search, started from the **Initial Review** button (`initial-review-v1`) in a fresh workspace. Costs use Meta's list prices ($1.25 input, $0.15 cached input, $4.25 output per million tokens); the keys were on free credits. @@ -30,7 +29,7 @@ tokens cached"), and **Copy review diagnostics** reports it per sheet. ## Where cloud time goes (Brewery) -| Stage | Before (Audit 3) | Now | +| Stage | Before | Now | | ----------------------------------- | ------------------------------ | -------------------------------------------- | | Source sync | at Send | 13 files, 22.5 MB, cached ~16 s after import | | Click to dispatch | ~5 min (506 MiB at 1.8 MiB/s) | 5 s; 262 KB uploaded, 13 files reused | diff --git a/docs/validation/muse-spark-open.md b/docs/validation/muse-spark-open.md new file mode 100644 index 00000000..c5b2ea72 --- /dev/null +++ b/docs/validation/muse-spark-open.md @@ -0,0 +1,24 @@ +# Muse Spark Open reviews + +Muse Spark Open is one of Meta's open-weight models. Moke runs it through Baseten as +`meta-models/muse-spark-open-1026`. + +## Completed reviews + +Two reviews of a supplied three-sheet fire-alarm set ran to completion in the cloud on +Muse Spark Open during desktop acceptance runs on October 29, 2026. The earlier run +covered the fire-alarm discipline. The later run recorded all 132 coverage rows, some of +which can be marked not applicable: 11 disciplines, 4 passes and 3 sheets. + +| Run | Calls | Passes recorded | Findings | Elapsed | Notes | +| ------------------------- | ----: | --------------: | -------: | -------- | ------------------------------------------------------------------------------------------------------------------------------- | +| Earlier, fire-alarm scope | 30 | 12 / 12 | 7 | Not kept | Recognized the supplied 8.90 Ah battery requirement, NAC tables and matrix. | +| Later, every discipline | 36 | 132 / 132 | 11 | 13:04 | The app restarted while the job ran. It uploaded 311.7 MiB (191 files) and synced back 340.5 MiB, with no sandbox left running. | + +## Limits + +These reviews cover a small set. A 29-sheet Initial Review on Spark Open paused at its +budget instead of finishing; see +[Baseten rate limits and caching](baseten-rate-limits.md). The raw receipts stayed +local. These runs predate later prompt versions and are not a benchmark for unseen +drawings. diff --git a/docs/research/screenshots/packaged-live-review.png b/docs/validation/packaged-live-review.png similarity index 100% rename from docs/research/screenshots/packaged-live-review.png rename to docs/validation/packaged-live-review.png diff --git a/docs/research/parallel-sheet-review.md b/docs/validation/parallel-sheet-review.md similarity index 80% rename from docs/research/parallel-sheet-review.md rename to docs/validation/parallel-sheet-review.md index 2c30bfa5..351a8c6d 100644 --- a/docs/research/parallel-sheet-review.md +++ b/docs/validation/parallel-sheet-review.md @@ -8,11 +8,11 @@ on free credits. ## Results -| Set | Sheets | Time | Checks recorded | Settled | Findings | Calls | Cost | -| ------------------------------------------- | -----: | --------- | --------------: | ------: | -------: | ----: | -----: | -| Civil grading (project-04) | 5 | 9.0 min | 220 / 220 | 211 | 29 | 115 | $1.98 | -| Brewery (project-01), 16 at a time | 29 | 43.4 min | 1,276 / 1,276 | 1,249 | 69 | 941 | $16.55 | -| Brewery, one conversation (Audit 3 recheck) | 29 | 33.6 min¹ | 220 / 1,276 | — | — | — | — | +| Set | Sheets | Time | Checks recorded | Settled | Findings | Calls | Cost | +| --------------------------------------- | -----: | --------- | --------------: | ------: | -------: | ----: | -----: | +| Civil grading (project-04) | 5 | 9.0 min | 220 / 220 | 211 | 29 | 115 | $1.98 | +| Brewery (project-01), 16 at a time | 29 | 43.4 min | 1,276 / 1,276 | 1,249 | 69 | 941 | $16.55 | +| Brewery, one conversation (earlier run) | 29 | 33.6 min¹ | 220 / 1,276 | — | — | — | — | ¹ Muse Spark Open on Baseten; the review stopped before finishing. diff --git a/electron-builder.json b/electron-builder.json index 212706f4..205db088 100644 --- a/electron-builder.json +++ b/electron-builder.json @@ -16,6 +16,10 @@ "from": "migrations", "to": "migrations" }, + { + "from": "src/lib/review/TABLER-LICENSE", + "to": "licenses/TABLER-LICENSE" + }, { "from": "dist/backend", "to": "backend", diff --git a/package.json b/package.json index 64de7e9a..5e454190 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "moke-meta-hackathon", - "version": "0.2.0", + "version": "0.2.1", "private": true, "description": "A desktop workbench for evidence-backed engineering drawing review", "homepage": "https://github.com/sc-frederick/moke-meta-hackathon", diff --git a/src/lib/forge.css b/src/lib/forge.css index c22b3565..f37550e4 100644 --- a/src/lib/forge.css +++ b/src/lib/forge.css @@ -1,5 +1,5 @@ /* Forge design system tokens, base elements and shared controls. - Adapted from the licensed reference in design/reference/forge. See design/DESIGN.md. */ + Adapted from Uiverse Design's Forge under its license. See docs/design.md. */ :root { --color-ink: #0b0b0c; diff --git a/design/TABLER-LICENSE b/src/lib/review/TABLER-LICENSE similarity index 100% rename from design/TABLER-LICENSE rename to src/lib/review/TABLER-LICENSE diff --git a/src/lib/review/components/OpenCodeGoSettings.svelte b/src/lib/review/components/OpenCodeGoSettings.svelte index ad1c1f54..d87e8437 100644 --- a/src/lib/review/components/OpenCodeGoSettings.svelte +++ b/src/lib/review/components/OpenCodeGoSettings.svelte @@ -1,6 +1,6 @@