diff --git a/bun.lock b/bun.lock index b13867cad..dd495f373 100644 --- a/bun.lock +++ b/bun.lock @@ -179,6 +179,7 @@ "version": "0.2.2", "dependencies": { "@intx/crypto": "0.3.0", + "@intx/log": "workspace:*", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "arktype": "catalog:", diff --git a/docs/VENDORING.md b/docs/VENDORING.md index 617ee9500..e3040b37e 100644 --- a/docs/VENDORING.md +++ b/docs/VENDORING.md @@ -24,17 +24,17 @@ points straight at `./src/*.ts` files rather than a `dist/` build. | Package | Vendor path | License | Synced from upstream commit | Retrieved | Local patches | | ------------------------------------------------------------------------------- | --------------------------------------------------------------- | ------------- | ------------------------------------------ | ---------- | ------------------------------------------------- | -| `@intx/inference` | `vendor/intx-inference/` | LGPL-2.1-only | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | 2026-10-14 | Yes — see `vendor/intx-inference/PATCHES.md` | -| `@intx/types` | `vendor/intx-types/` | LGPL-2.1-only | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | 2026-10-14 | Yes — see `vendor/intx-types/PATCHES.md` | -| `@intx/storage-isogit` | `vendor/intx-storage-isogit/` | LGPL-2.1-only | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | 2026-10-14 | Yes — see `vendor/intx-storage-isogit/PATCHES.md` | -| `@intx/agent` | `vendor/intx-agent/` | LGPL-2.1-only | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | 2026-10-14 | Yes — see `vendor/intx-agent/PATCHES.md` | -| `@intx/authz` | `vendor/intx-authz/` | LGPL-2.1-only | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | 2026-10-14 | None — verbatim | -| `@intx/log` | `vendor/intx-log/` | LGPL-2.1-only | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | 2026-10-14 | None — verbatim | -| `@intx/tools-posix` | `vendor/intx-tools-posix/` | LGPL-2.1-only | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | 2026-10-14 | Yes — see `vendor/intx-tools-posix/PATCHES.md` | -| `@intx/mailbox` | `vendor/intx-mailbox/` | LGPL-2.1-only | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | 2026-10-14 | None — verbatim | -| `@intx/harness` | `vendor/intx-harness/` | LGPL-2.1-only | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | 2026-10-14 | None — verbatim | -| `@intx/mime` | `vendor/intx-mime/` | LGPL-2.1-only | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | 2026-10-14 | None — verbatim | -| `@intx/workflow-host` (partial — `adapters/` + `workflow-definition-loader.ts`) | `vendor/intx-workflow-host/` (two paths, not the whole package) | LGPL-2.1-only | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | 2026-10-14 | None — verbatim | +| `@intx/inference` | `vendor/intx-inference/` | LGPL-2.1-only | `4cc697241fa037244a9164a50aec507c112bc9fb` | 2026-10-14 | Yes — see `vendor/intx-inference/PATCHES.md` | +| `@intx/types` | `vendor/intx-types/` | LGPL-2.1-only | `4cc697241fa037244a9164a50aec507c112bc9fb` | 2026-10-14 | Yes — see `vendor/intx-types/PATCHES.md` | +| `@intx/storage-isogit` | `vendor/intx-storage-isogit/` | LGPL-2.1-only | `4cc697241fa037244a9164a50aec507c112bc9fb` | 2026-10-14 | Yes — see `vendor/intx-storage-isogit/PATCHES.md` | +| `@intx/agent` | `vendor/intx-agent/` | LGPL-2.1-only | `4cc697241fa037244a9164a50aec507c112bc9fb` | 2026-10-14 | Yes — see `vendor/intx-agent/PATCHES.md` | +| `@intx/authz` | `vendor/intx-authz/` | LGPL-2.1-only | `4cc697241fa037244a9164a50aec507c112bc9fb` | 2026-10-14 | None — verbatim | +| `@intx/log` | `vendor/intx-log/` | LGPL-2.1-only | `4cc697241fa037244a9164a50aec507c112bc9fb` | 2026-10-14 | None — verbatim | +| `@intx/tools-posix` | `vendor/intx-tools-posix/` | LGPL-2.1-only | `4cc697241fa037244a9164a50aec507c112bc9fb` | 2026-10-14 | Yes — see `vendor/intx-tools-posix/PATCHES.md` | +| `@intx/mailbox` | `vendor/intx-mailbox/` | LGPL-2.1-only | `4cc697241fa037244a9164a50aec507c112bc9fb` | 2026-10-14 | None — verbatim | +| `@intx/harness` | `vendor/intx-harness/` | LGPL-2.1-only | `4cc697241fa037244a9164a50aec507c112bc9fb` | 2026-10-14 | None — verbatim | +| `@intx/mime` | `vendor/intx-mime/` | LGPL-2.1-only | `4cc697241fa037244a9164a50aec507c112bc9fb` | 2026-10-14 | None — verbatim | +| `@intx/workflow-host` (partial — `adapters/` + `workflow-definition-loader.ts`) | `vendor/intx-workflow-host/` (two paths, not the whole package) | LGPL-2.1-only | `4cc697241fa037244a9164a50aec507c112bc9fb` | 2026-10-14 | None — verbatim | ## Provenance, ownership, and kill dates @@ -47,18 +47,18 @@ package; the date is the deadline even if it is not. | Vendor path | Upstream repo | Upstream commit | Patched | Why not the published package | Owner | Proposed kill date | | --------------------------------------------------------- | ----------------------- | ------------------------------------------ | ------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------- | ------------------------------------------------------------------------------------------------------ | -| `vendor/intx-inference/` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | Yes — `PATCHES.md` | Local fixes not yet upstream | runtime | 2027-03-14 or when patches land upstream and publish | -| `vendor/intx-types/` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | Yes — `PATCHES.md` | Cross-package coupling with `@intx/inference` | runtime | 2027-03-14 or when the coupled trio publishes past `0.3.0` | -| `vendor/intx-storage-isogit/` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | Yes — `PATCHES.md` | Cross-package coupling with `@intx/inference` | runtime | 2027-03-14 or when the coupled trio publishes past `0.3.0` | -| `vendor/intx-agent/` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | Yes — `PATCHES.md` | Local patches (`vendor/intx-agent/PATCHES.md`); published `@intx/agent@0.4.0` does not include them | runtime | 2027-03-14 or when those patches land upstream and publish | -| `vendor/intx-authz/` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | No | Published at npm `0.4.0` (verbatim). Kept so the vendored set stays one workspace pin with the patched packages | runtime | 2027-03-14 or when patched siblings no longer need the shared vendored pin | -| `vendor/intx-log/` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | No | Published at npm `0.4.0` (verbatim). Kept so the vendored set stays one workspace pin with the patched packages | runtime | 2027-03-14 or when patched siblings no longer need the shared vendored pin | -| `vendor/intx-tools-posix/` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | Yes — `PATCHES.md` | Local `registry.ts` paging patch (`vendor/intx-tools-posix/PATCHES.md`); `@intx/tools-posix@0.4.0` does not include it | runtime | 2027-03-14 or when that patch lands upstream and publishes | -| `vendor/intx-mailbox/` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | No | Published at npm `0.4.0` (verbatim). Kept so the vendored set stays one workspace pin with the patched packages | step-1 | 2027-03-14 or when patched siblings no longer need the shared vendored pin | -| `vendor/intx-harness/` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | No | Published `@intx/harness@0.4.0` exports `driveConnectorReplies` (verbatim). Kept on the shared vendored pin | step-1 | 2027-03-14 or when patched siblings no longer need the shared vendored pin | -| `vendor/intx-mime/` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | No | `buildMessageHeaders` is past npm `0.3.0` (verified 2026-09-07: absent from the published tarball); `@intx/mailbox` re-exports it | step-1 | 2027-03-14 or when a published `@intx/mime` exports `buildMessageHeaders` | -| `vendor/intx-workflow-host/adapters/` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | No | App-internal: `substrate-mailbox-store.ts` has never been published in any `@intx/workflow-host` release (verified 2026-09-07: absent from the `0.3.0` tarball) | step-1 | 2027-03-14 or when a published `@intx/workflow-host` exports `createSubstrateMailboxStore` | -| `vendor/intx-workflow-host/workflow-definition-loader.ts` | `faremeter/interchange` | `5453d0b01da9d749aa446c4ba34fcf2aeb4549c0` | No | Director loading from a workflow closure (`loadWorkflowDirectorRegistryFromClosure`); never published in any `@intx/workflow-host` release (verified 2026-09-14: absent from the `0.3.0` tarball) | step-1 | 2027-03-14 or when a published `@intx/workflow-host` exports `loadWorkflowDirectorRegistryFromClosure` | +| `vendor/intx-inference/` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | Yes — `PATCHES.md` | Local fixes not yet upstream | runtime | 2027-03-14 or when patches land upstream and publish | +| `vendor/intx-types/` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | Yes — `PATCHES.md` | Cross-package coupling with `@intx/inference` | runtime | 2027-03-14 or when the coupled trio publishes past `0.3.0` | +| `vendor/intx-storage-isogit/` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | Yes — `PATCHES.md` | Cross-package coupling with `@intx/inference` | runtime | 2027-03-14 or when the coupled trio publishes past `0.3.0` | +| `vendor/intx-agent/` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | Yes — `PATCHES.md` | Local patches (`vendor/intx-agent/PATCHES.md`); published `@intx/agent@0.4.0` does not include them | runtime | 2027-03-14 or when those patches land upstream and publish | +| `vendor/intx-authz/` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | No | Published at npm `0.4.0` (verbatim). Kept so the vendored set stays one workspace pin with the patched packages | runtime | 2027-03-14 or when patched siblings no longer need the shared vendored pin | +| `vendor/intx-log/` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | No | Published at npm `0.4.0` (verbatim). Kept so the vendored set stays one workspace pin with the patched packages | runtime | 2027-03-14 or when patched siblings no longer need the shared vendored pin | +| `vendor/intx-tools-posix/` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | Yes — `PATCHES.md` | Local `registry.ts` paging patch (`vendor/intx-tools-posix/PATCHES.md`); `@intx/tools-posix@0.4.0` does not include it | runtime | 2027-03-14 or when that patch lands upstream and publishes | +| `vendor/intx-mailbox/` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | No | Published at npm `0.4.0` (verbatim). Kept so the vendored set stays one workspace pin with the patched packages | step-1 | 2027-03-14 or when patched siblings no longer need the shared vendored pin | +| `vendor/intx-harness/` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | No | Published `@intx/harness@0.4.0` exports `driveConnectorReplies` (verbatim). Kept on the shared vendored pin | step-1 | 2027-03-14 or when patched siblings no longer need the shared vendored pin | +| `vendor/intx-mime/` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | No | `buildMessageHeaders` is past npm `0.3.0` (verified 2026-09-07: absent from the published tarball); `@intx/mailbox` re-exports it | step-1 | 2027-03-14 or when a published `@intx/mime` exports `buildMessageHeaders` | +| `vendor/intx-workflow-host/adapters/` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | No | App-internal: `substrate-mailbox-store.ts` has never been published in any `@intx/workflow-host` release (verified 2026-09-07: absent from the `0.3.0` tarball) | step-1 | 2027-03-14 or when a published `@intx/workflow-host` exports `createSubstrateMailboxStore` | +| `vendor/intx-workflow-host/workflow-definition-loader.ts` | `faremeter/interchange` | `4cc697241fa037244a9164a50aec507c112bc9fb` | No | Director loading from a workflow closure (`loadWorkflowDirectorRegistryFromClosure`); never published in any `@intx/workflow-host` release (verified 2026-09-14: absent from the `0.3.0` tarball) | step-1 | 2027-03-14 or when a published `@intx/workflow-host` exports `loadWorkflowDirectorRegistryFromClosure` | ### The 2026-09-07 step-1 vendor pass @@ -204,7 +204,7 @@ pin — all line-grep evidence below is against pristine `5453d0b0`): (`fcdbc8a5`, "Compose signed outbound mail in one place") carried natively by the verbatim `@intx/mime`/`@intx/mailbox` trees — no local patch exists to adjudicate. "Adapter manifest" corresponds to upstream - `4cc6972` ("Refuse sidecar reuse across an adapter manifest change"), + `4cc697241fa037244a9164a50aec507c112bc9fb` ("Refuse sidecar reuse across an adapter manifest change"), past this pin and touching no vendored path. - `expectedFrom` reconciliation: upstream still carries the field at `0.4.0` (reactor marker spread; `runtime.ts`/`store.ts` validators). @@ -288,6 +288,99 @@ table above is the only thing that reflects actual content; the `version` field exists only because some tooling expects `package.json` to declare one. +### The 2026-10-14 re-sync to `4cc697241fa037244a9164a50aec507c112bc9fb` (CL-10656, HEAD-drift triage) + +Second sync of the day: + +`5453d0b0` → `4cc697241fa037244a9164a50aec507c112bc9fb` (upstream, retrieved 2026-10-14). The range holds +~196 upstream commits and 154,000+ inserted lines; ~13,900 of those sit +in the vendored-package surface this repo adopted (the bulk belongs to +trees this repo does not vendor — `workflow-deploy`, `workflow-host` +supervisor, `workflow`, `db`, `hub-*`). The sync is a **partial-tree +triage**, not a +blind adopt: only the vendored paths moved, all eleven trees moved to the +same pin (single-commit coherence), and correctness fixes went first. + +Triage per area (commit-hash evidence from `git log 5453d0b0..4cc697241fa037244a9164a50aec507c112bc9fb -- +packages/`): + +| Area | Commits | Vendored surface touched | Decision | Why | +| ---------------------------------- | ------- | ---------------------------- | ------------------ | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| `mime` | 12 | whole `src/` | **Adopt** | CRLF/shape validation (`9313a743`, `02abb4e0`), relay-safe encoding (`a6804df4`), header folding; prerequisite for the mailbox adoption. Fully verbatim re-carried. | +| `mailbox` | 10 | whole `src/` | **Adopt** | Signed outbound composition (`fcdbc8a5`), fetch-failure preservation (`faad83aa`), no-date sorting (`8c243d6f`). Depends on adopted `mime` + `types` subpaths. | +| `harness` | 7 | whole `src/` | **Adopt** | Supervisor-mailbox threading (`c7a7bf89`, `58e5689b`), step credential assembly (`7b8cfbbc`). Depends on adopted `types`/`agent`. | +| `types` | 50 | whole `src/` (runtime split) | **Adopt** | Required by mailbox/harness/inference/workflow-host (new `runtime-core`, `credential-delivery`, `workflow-definition`, `attachments`, `format-safety-rating` subpaths). Carries `adapter-manifest` (`2d973a4`), `PerCallInferenceOptions` (`d1835b`), identity/authz tightening (`0f72f95`, `1ff8805`). Four patched trees make this the biggest re-carry; no clean way to pin lower without breaking the adopted consumers, and the trio coherence rule forbids a types-only move. | +| `inference` | 9 | whole `src/` | **Adopt** | Reasoning-effort mapping (`c8be7e9`) and auth-claim fixes (`0f72f957`, `1ff88053`). 9 patched files three-way merged, 2 conflict regions resolved by hand (see `PATCHES.md`). | +| `storage-isogit` | 5 | whole `src/` | **Adopt** | `e75eee2` `sync.request`/state-push removal, verified safe (below). Coupled-trio rule forces the pin to move with `types`/`inference` anyway. | +| `agent` | 3 | whole `src/` | **Adopt** | Per-send inference options (`d1835be`), runtime-contract imports (`660ef81`), and the `./authoring` subpath (`d1e05e9`) consumed by the adopted workflow-host loader. | +| `workflow-host` (partial: 3 files) | 68 | 3 vendored files only | **Adopt** (narrow) | The other ~100 drift files live in supervisor/workflow trees we do not vendor and are excluded by design. `workflow-definition-loader.ts` dropped the `@intx/hub-sessions/substrate` import for `@intx/types/workflow-definition`, in the direction CL-9021 already wants. | +| `authz`, `log`, `tools-posix` | 0 | nothing | **No change** | `git diff 5453d0b0..4cc697241fa037244a9164a50aec507c112bc9fb -- packages/

` is empty for all three; pin bumped with the set for single-commit coherence only. | +| `crypto` | 2 | not vendored | **No action** | Published npm dep stays at `0.3.0`; its adopted consumers (`generateKeyPair`, `createEd25519Crypto`, `importPublicKeyBytes`) all already export from the installed `0.3.0` dist (verified against `node_modules/@intx/crypto/dist/index.d.ts`). | + +Nothing was adopted from `packages/workflow-deploy`, `packages/workflow`, +`packages/db`, `packages/hub-*`, or any other non-vendored tree. + +`e75eee2` ("Remove sync.request and the undeploy's agent-state push") +safety evidence: in this repo, `grep` for `sync.request`, `statePushed`, +`createStatePack`, and `undeploy` across `src/` and `e2e/` returns zero +matches (only `vendor/intx-types/src/sidecar.ts` carried the union — +dropped by the adopted `4cc697241fa037244a9164a50aec507c112bc9fb` tree); upstream-side, the commit's +`packages/storage-isogit` portion is comment-only (`pack-send.ts`, +`repo-lock.ts` doc blocks) plus the `hub-agent` caller removal, and the +`undeploy.ack` `statePushed` validators live only in non-vendored +hub-side schemas at this pin. Adoption is verified safe. + +`c8be7e9` ("Map reasoning effort onto each provider's request") verdict: +**adopted**, riding the `4cc697241fa037244a9164a50aec507c112bc9fb` `@intx/inference` pin (`effort.ts`, the +three provider adapters, `classifyRequestBuildError`). Adoption follows +the whole-package decision above; first-party rollers that never set +`effort` see no wire change (the mapping only rewrites requests that +carry an effort name, and `thinking` semantics are unchanged). + +Ledger adjudication (ticket CL-10656): **all ledger entries KEEP** — +`isStreamTerminal`, `ephemeralTurns`, `isPollOnlyPendingBatch`, `loadErrors`, +`turnsRevision`/`deepFreeze`, `isCommitting`, `MAX_LINE_LENGTH`, and the +extended `classifyAbortError(reason)` plumbing are all still absent from the +upstream `4cc697241fa037244a9164a50aec507c112bc9fb` vendored-area trees (grep per package, evidence in each +`PATCHES.md`). No entry was dropped; no disposition changed. The +divergence-surface proof after the merge: `bin/vendor-patch-diff` against +the recorded SHA prints exactly the pre-sync +/− line multiset plus three +new `runtime-ts-auth-recovery-context` marker comment lines in +`@intx/types` (`+/−` totals: inference 1324+/230−, agent 382+/2−, +storage-isogit 73+/1−, types 33→36+, tools-posix 13+/1−). Resume audit +(same day): a `bun run build` failure exposed two half-wired adoptions — +the vendored `@intx/types` export map was missing fifteen upstream +subpath exports (fixed; export-map completions are outside the src +SHA-diff so the totals above are unaffected) — and the runtime-split +dropped `formatSafetyRatingText` from the `./runtime` subpath, breaking +the installed `@corbits/openai-responses@0.2.1` dist. The compat +re-export (ledgered `vendor-types-ts-runtime-format-safety-rating-compat` +in `vendor/intx-types/PATCHES.md`, +7 line change) moves the `@intx/types` +divergence surface to its final measured 43+/6−. + +First-party adaptation required by adopted APIs (only imported-symbol +moves, no behavior change): `src/provider/inference-dependencies.ts` +imports `AdapterManifest` from `@intx/types` (upstream dropped the +inference barrel re-export); optional `messageId` on inbound headers +(`1ff8805`, "no identity invented for a message that names none") surfaced +at `src/tui/runner/submit.ts` (use the narrowed local), two +`src/session/runtime-assembly.test.ts` assertions, and one +`src/tui/queued-input-chrome.test.ts` guard. No vendored-tree API the +first-party copy consumed was removed otherwise. + +Procedure: this ticket lands as two commits. The first adopts upstream +`4cc697241fa037244a9164a50aec507c112bc9fb` content into the vendored trees and is not a pristine +unmodified snapshot of that commit — local lines remain, including +`ephemeralTurns`, `loadErrors`, and the tools-posix read_file paging +description. The second re-carries the patch ledgers and adapts first-party +`src`. Verbatim trees (`mime`, `mailbox`, `harness`, and the narrow +`workflow-host` subset) were copied from upstream `4cc697241fa037244a9164a50aec507c112bc9fb`. +Patched trees were merged per-file via `git merge-file` (ours = vendored, +base = `5453d0b0`, theirs = upstream `4cc697241fa037244a9164a50aec507c112bc9fb`) with two +hand-resolved conflict regions in `intx-inference`. `package.json` changes +were applied by hand onto the vendored rewritten files (new subpath exports +for `types` / `agent` / `inference`, mailbox's `@intx/log` dependency). + ## How a vendored package resolves Root `package.json`: diff --git a/src/provider/inference-dependencies.ts b/src/provider/inference-dependencies.ts index 892101e34..65181b79d 100644 --- a/src/provider/inference-dependencies.ts +++ b/src/provider/inference-dependencies.ts @@ -1,9 +1,8 @@ -import { - createDependencies, - type Dependencies, - type AdapterManifest, -} from "@intx/inference"; +import { createDependencies, type Dependencies } from "@intx/inference"; import { loadAdapterRegistry } from "@intx/inference/providers"; +// AdapterManifest moved from the @intx/inference barrel to @intx/types at +// upstream 4cc6972 (the inference barrel no longer re-exports it). +import type { AdapterManifest } from "@intx/types"; import * as openaiCompatible from "./openai-compatible-adapter.js"; import * as opencodeGo from "./opencode-go-adapter.js"; import * as codexResponses from "./codex-responses.js"; diff --git a/src/session/runtime-assembly.test.ts b/src/session/runtime-assembly.test.ts index 2f5584fdc..3cd888eeb 100644 --- a/src/session/runtime-assembly.test.ts +++ b/src/session/runtime-assembly.test.ts @@ -1218,7 +1218,7 @@ describe("buildCompactionContinuationMessage", () => { expect(message.ref).toEqual({ uid: 0, mailbox: "system" }); expect(message.headers.from).toBe("user@local"); expect(message.headers.to).toEqual(["agent@local"]); - expect(message.headers.messageId.startsWith("compact-continue-")).toBe( + expect(message.headers.messageId?.startsWith("compact-continue-")).toBe( true, ); }); @@ -1251,6 +1251,6 @@ describe("buildMailboxMailMessage", () => { expect(message.ref).toEqual({ uid: 0, mailbox: "system" }); expect(message.headers.from).toBe("user@local"); expect(message.headers.to).toEqual(["agent@local"]); - expect(message.headers.messageId.startsWith("mailbox-mail-")).toBe(true); + expect(message.headers.messageId?.startsWith("mailbox-mail-")).toBe(true); }); }); diff --git a/src/tui/queued-input-chrome.test.ts b/src/tui/queued-input-chrome.test.ts index 12b3830d3..6d8ffcef3 100644 --- a/src/tui/queued-input-chrome.test.ts +++ b/src/tui/queued-input-chrome.test.ts @@ -718,6 +718,8 @@ test("mid-run steer paints QUEUED INPUT until the real director reads it", async deliver: (text, pending, settle) => { const message = userInboundMessage(text, pending); const messageId = message.headers.messageId; + if (messageId === undefined) + throw new Error("userInboundMessage set no messageId"); steerMessageId.current = messageId; bridge.beginQueuedInputDelivery(messageId); void deliverAgentMessage({ diff --git a/src/tui/runner/submit.ts b/src/tui/runner/submit.ts index 401853400..04d184f47 100644 --- a/src/tui/runner/submit.ts +++ b/src/tui/runner/submit.ts @@ -581,7 +581,7 @@ export function createDeliverRouting( () => targetAgent.deliver(message), (result) => { if (result.status === "accepted" && isOperatorMessage) { - state.acceptedInputObserver?.(message.headers.messageId); + state.acceptedInputObserver?.(messageId); } else if (isOperatorMessage) { state.discardAcceptedInputObserver?.(messageId); } diff --git a/vendor/intx-agent/PATCHES.md b/vendor/intx-agent/PATCHES.md index 2b786fcf4..e85c5057f 100644 --- a/vendor/intx-agent/PATCHES.md +++ b/vendor/intx-agent/PATCHES.md @@ -26,6 +26,20 @@ test files into the new `testing/reactor-waiters.ts` (exported from `loadErrors` hooks and the FORBIDDEN_DEPS credential-failure companion suite re-carried onto the new shapes. +### 2026-10-14 re-sync to `4cc6972` (CL-10656, HEAD drift 0.4.0 → base) + +All three entries re-carried; none dropped (`loadErrors` and error-seq +resumption still absent upstream in this range). Upstream touched +`agent.ts` (`d1835b` per-send `deliver` options adopted upstream, `660ef8` +signature reflow); `agent.ts` three-way merged cleanly with zero +adaptation — no upstream change intersects the two patch sites. New +upstream files `authoring.ts` (authoring-only subpath barrel) and the +`canonicalize.ts` helpers carried verbatim, with the `./authoring` and +`./canonicalize` subpath exports wired into the vendored `package.json` +(the inert `vendor/intx-workflow-host/workflow-definition-loader.ts` copy +imports `./authoring` at its new pin). Upstream also deleted no vendored +files and changed `testing/` not at all. + ## agent-ts-resume-error-seq `agent.ts` — `createAgent` resumes `errorSeq` from `auditStore.loadErrors` diff --git a/vendor/intx-agent/README.md b/vendor/intx-agent/README.md index b8420bb3e..4fe13964c 100644 --- a/vendor/intx-agent/README.md +++ b/vendor/intx-agent/README.md @@ -4,6 +4,14 @@ In-process agent runtime built on `createReactorAssembly` from `@intx/inference`. Construct an agent, `send()` it a message, get a reply. +Workflow packages can import `defineAgent`, tool and director declarations, +and director registries from `@intx/agent/authoring`. That entry point keeps +the reactor and agent execution runtime out of definition loading. + +`@intx/agent/canonicalize` exports `canonicalizeForHash` and +`CanonicalizationError` for deterministic serialization of deploy-hash inputs +without loading the agent execution runtime. + Use this package when you want an agent you can drive from inside your own program — a CLI, a worker, a test, an embedded assistant. If you want an agent that lives behind a mailbox instead, see diff --git a/vendor/intx-agent/package.json b/vendor/intx-agent/package.json index 48cd36613..8ab46ef1c 100644 --- a/vendor/intx-agent/package.json +++ b/vendor/intx-agent/package.json @@ -11,6 +11,14 @@ "./testing": { "types": "./src/testing/index.ts", "default": "./src/testing/index.ts" + }, + "./authoring": { + "types": "./src/authoring.ts", + "default": "./src/authoring.ts" + }, + "./canonicalize": { + "types": "./src/canonicalize.ts", + "default": "./src/canonicalize.ts" } }, "dependencies": { diff --git a/vendor/intx-agent/src/agent.ts b/vendor/intx-agent/src/agent.ts index 8ae10f11b..20cfb917b 100644 --- a/vendor/intx-agent/src/agent.ts +++ b/vendor/intx-agent/src/agent.ts @@ -61,6 +61,7 @@ import type { ConversationTurn, InboundMessage, InferenceSource, + PerCallInferenceOptions, ReactorDirector, ToolCall, ToolDefinition, @@ -114,6 +115,16 @@ export type SendOptions = { signal?: AbortSignal; /** Override the default "from" header on the synthetic inbound message. */ from?: string; + /** + * Inference options for the model calls this send drives. Applies to this + * send alone; a key the director names outright still wins. + */ + inference?: PerCallInferenceOptions; +}; + +type QueuedSend = { + message: InboundMessage; + inference: PerCallInferenceOptions | undefined; }; export type SendResult = @@ -477,7 +488,7 @@ export async function createAgent( // the language allows; the comment block above is what makes // the "assigned before any reachable read" invariant explicit. // eslint-disable-next-line prefer-const -- forward declaration; const cannot express this ordering - let sendQueue: SendQueue; + let sendQueue: SendQueue; // shutdownComplete resolves from the assembly's onShutdown hook // (composed after audit flush by the assembly) or, as a fallback, from @@ -770,11 +781,11 @@ export async function createAgent( ...(env.compactors !== undefined ? { compactors: env.compactors } : {}), }); - sendQueue = createSendQueue({ + sendQueue = createSendQueue({ maxDepth: env.sendQueueMax ?? DEFAULT_SEND_QUEUE_MAX, - start: (message) => { + start: ({ message, inference }) => { activeCycle = { lastAssistantTurn: undefined }; - reactor.deliver(message); + reactor.deliver(message, inference !== undefined ? { inference } : {}); }, }); @@ -813,7 +824,10 @@ export async function createAgent( // and per the design must fail loud. if (closed) return Promise.reject(new AgentClosedError()); const message = buildInboundMessage(content, opts); - return sendQueue.enqueue(message, opts?.signal); + return sendQueue.enqueue( + { message, inference: opts?.inference }, + opts?.signal, + ); } function stream(): AsyncIterable { diff --git a/vendor/intx-agent/src/authoring.ts b/vendor/intx-agent/src/authoring.ts new file mode 100644 index 000000000..74d53a88e --- /dev/null +++ b/vendor/intx-agent/src/authoring.ts @@ -0,0 +1,40 @@ +/** Agent definitions and declarations, without the agent execution runtime. */ +export { + defineAgent, + type AgentDefinition, + type DefineAgentConfig, + type EnvRequiredByAll, + type InferencePreference, +} from "./definition"; +export { + defineTool, + definePlugin, + isAnnotatedPluginFactory, + toolApprovalEffect, + type AnnotatedPluginFactory, + type AnnotatedToolFactory, + type ToolDeclaration, +} from "./tool"; +export { + defineDirector, + isAnnotatedDirectorFactory, + type DefinedDirector, +} from "./director"; +export { + createDefaultDirectorRegistry, + createDirectorRegistry, + createWorkflowDirectorRegistry, + UnknownDirectorIdError, +} from "./director-registry"; +export { + buildDefaultDirectorRef, + defaultDirectorFactory, + type DefaultDirectorConfig, +} from "./default-director"; +export { effectiveDirectorRef } from "./env-validation"; +export type { + AnnotatedDirectorFactory, + DirectorRef, + DirectorRegistry, +} from "./director-types"; +export type { BaseEnv } from "./env"; diff --git a/vendor/intx-agent/src/default-director.ts b/vendor/intx-agent/src/default-director.ts index cce673977..783f6d689 100644 --- a/vendor/intx-agent/src/default-director.ts +++ b/vendor/intx-agent/src/default-director.ts @@ -18,7 +18,7 @@ import { type } from "arktype"; import { createDefaultDirector, type DefaultDirectorPolicy, -} from "@intx/inference"; +} from "@intx/inference/default-director"; import { defineDirector } from "./director"; diff --git a/vendor/intx-agent/src/send-queue.ts b/vendor/intx-agent/src/send-queue.ts index 60774dff0..ad7f96dac 100644 --- a/vendor/intx-agent/src/send-queue.ts +++ b/vendor/intx-agent/src/send-queue.ts @@ -15,6 +15,9 @@ // queue ordered against actual reactor cycles — two send() promises // cannot interleave at the reactor level. // +// A `start` that throws synchronously rejects that item and pumps the +// next, so a rejected item never occupies the active slot. +// // Queue depth (active + pending) is bounded by `maxDepth`; exceeding it // throws `SendQueueFullError` synchronously from enqueue() so a buggy // caller flooding sends fails loud instead of silently buffering. @@ -106,7 +109,13 @@ export function createSendQueue( continue; } active = next; - opts.start(next.item); + try { + opts.start(next.item); + } catch (cause) { + active = null; + settle(next, "reject", cause); + continue; + } return; } } diff --git a/vendor/intx-agent/src/source.ts b/vendor/intx-agent/src/source.ts index 518f33a12..4d9f5a451 100644 --- a/vendor/intx-agent/src/source.ts +++ b/vendor/intx-agent/src/source.ts @@ -18,7 +18,7 @@ import { InferenceSource as InferenceSourceValidator, applyInferenceSourceFields, type InferenceSource, -} from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; export class InvalidInferenceSourceError extends Error { constructor(message: string) { diff --git a/vendor/intx-agent/src/tool.ts b/vendor/intx-agent/src/tool.ts index d4467b7c2..ea7b4821b 100644 --- a/vendor/intx-agent/src/tool.ts +++ b/vendor/intx-agent/src/tool.ts @@ -18,7 +18,7 @@ // handler throws. // // `createToolRunner(tools)` builds a `ToolRunner` that dispatches by tool -// name. Per the ToolRunner contract (packages/types/src/runtime.ts), `run` +// name. Per the ToolRunner contract (packages/types/src/runtime-core.ts), `run` // must not throw -- unknown tool names and handler exceptions are surfaced // as `ToolResult` with `isError: true` so the model sees them and can // recover. diff --git a/vendor/intx-harness/src/connector-router.test.ts b/vendor/intx-harness/src/connector-router.test.ts index 0e8d8abc1..e2cb42f19 100644 --- a/vendor/intx-harness/src/connector-router.test.ts +++ b/vendor/intx-harness/src/connector-router.test.ts @@ -520,6 +520,278 @@ describe("createConnectorRouter", () => { }); }); + describe("message carrying no originator", () => { + // A message with no `From` at all reaches the router on the production + // fetch path: buildMessageHeaders omits the field when the wire message + // carried none. Such a message names no address a reply could go to, so + // the router must derive none. Naming a stand-in instead would record it + // as a thread participant and address the thread's replies to it. The + // decision is the router's own -- `passthrough`, not a throw a caller has + // to translate -- so each test below pins the returned kind and drives the + // harness's own route-then-commit sequence to pin the resulting state. + function inboundWithNoFrom( + messageId: string, + opts?: { references?: string[] }, + ): InboundMessage { + return { + ref: { uid: 7, mailbox: "INBOX" }, + headers: { + to: ["agent@example.com"], + date: new Date().toISOString(), + messageId, + ...(opts?.references !== undefined + ? { references: opts.references } + : {}), + }, + flags: [], + content: "x", + signatureStatus: "missing", + }; + } + + test("route() passes through on start and opens no thread", () => { + const router = createConnectorRouter(); + const message = inboundWithNoFrom(""); + + const decision = router.route(message); + expect(decision.kind).toBe("passthrough"); + + router.commit(decision); + + // No thread exists to reply on, rather than one whose reply address + // the router chose. + expect(router.snapshot()).toBeNull(); + expect(() => router.composeReply()).toThrow(NoActiveConnectorThreadError); + }); + + test("route() passes through on continue and leaves the thread untouched", () => { + const router = createConnectorRouter(); + router.commit(router.route(startMessage({ subject: "Hello" }))); + const before = router.snapshot(); + + const followup = inboundWithNoFrom("", { + references: [""], + }); + const decision = router.route(followup); + expect(decision.kind).toBe("passthrough"); + + router.commit(decision); + + // The thread still names its real speaker: the message that could not + // be attributed neither became replyTo nor joined cc. + expect(router.snapshot()).toEqual(before); + expect(router.composeReply()).toEqual({ + to: "user@example.com", + cc: [], + inReplyTo: "", + subject: "Hello", + }); + }); + + test("an absent originator passes through where an unparseable one throws", () => { + // The two share an outcome at the caller -- the harness delivers the + // message and leaves the thread unadvanced either way -- but they reach + // it differently, and only the throw is a failure. Pinning all three + // shapes in one closure keeps a change that collapses them together + // from passing. + const absent = createConnectorRouter(); + expect(absent.route(inboundWithNoFrom("")).kind).toBe( + "passthrough", + ); + + const unparseable = createConnectorRouter(); + const malformed: InboundMessage = { + ref: { uid: 8, mailbox: "INBOX" }, + headers: { + from: "not-an-address", + to: ["agent@example.com"], + date: new Date().toISOString(), + messageId: "", + }, + flags: [], + content: "x", + signatureStatus: "missing", + }; + expect(() => unparseable.route(malformed)).toThrow(); + + const named = createConnectorRouter(); + const decision = named.route(startMessage({ subject: "Hello" })); + expect(decision.kind).toBe("start"); + named.commit(decision); + expect(named.snapshot()).toEqual({ + threadRoot: "", + lastMessageId: "", + replyTo: "user@example.com", + cc: [], + subject: "Hello", + }); + }); + }); + + describe("message carrying no message id", () => { + // A message with no `Message-ID` still names an address to reply to, so + // the thread opens. What it denies is an anchor: no later message can + // match it as a continuation, so the next one re-anchors the thread. + function inboundWithNoMessageId(opts?: { + from?: string; + inReplyTo?: string; + references?: string[]; + }): InboundMessage { + return { + ref: { uid: 11, mailbox: "INBOX" }, + headers: { + from: opts?.from ?? "user@example.com", + to: ["agent@example.com"], + date: new Date().toISOString(), + ...(opts?.inReplyTo !== undefined + ? { inReplyTo: opts.inReplyTo } + : {}), + ...(opts?.references !== undefined + ? { references: opts.references } + : {}), + }, + flags: [], + content: "x", + signatureStatus: "missing", + }; + } + + test("route() opens a thread so the reply is not dropped", () => { + const router = createConnectorRouter(); + const decision = router.route(inboundWithNoMessageId()); + + expect(decision.kind).toBe("start"); + router.commit(decision); + + const state = router.snapshot(); + expect(state).not.toBeNull(); + expect(state?.replyTo).toBe("user@example.com"); + expect(state?.threadRoot).toBeUndefined(); + expect(state?.lastMessageId).toBeUndefined(); + }); + + test("composeReply() omits In-Reply-To rather than emitting an empty one", () => { + const router = createConnectorRouter(); + router.commit(router.route(inboundWithNoMessageId())); + + const parts = router.composeReply(); + expect(parts.to).toBe("user@example.com"); + expect(parts.inReplyTo).toBeUndefined(); + expect("inReplyTo" in parts).toBe(false); + }); + + test("a continuation naming no id still threads against the root", () => { + // The reply drain only resolves a References chain when `inReplyTo` is + // present, so dropping it here costs the outbound reply both threading + // headers and starts a new thread in every participant's client. + const router = createConnectorRouter(); + router.commit(router.route(startMessage({ subject: "Invoice" }))); + + const decision = router.route( + inboundWithNoMessageId({ + from: "bob@example.com", + references: [""], + }), + ); + expect(decision.kind).toBe("continue"); + router.commit(decision); + expect(router.snapshot()?.lastMessageId).toBeUndefined(); + + expect(router.composeReply()).toEqual({ + to: "bob@example.com", + cc: ["user@example.com"], + inReplyTo: "", + subject: "Invoice", + }); + }); + + test("an arriving message re-anchors the thread rather than continuing it", () => { + const router = createConnectorRouter(); + router.commit(router.route(inboundWithNoMessageId())); + + const next = continuationByInReplyTo("", { + from: "other@example.com", + }); + const decision = router.route(next); + + expect(decision.kind).toBe("start"); + router.commit(decision); + expect(router.snapshot()?.replyTo).toBe("other@example.com"); + expect(router.snapshot()?.cc).toEqual([]); + expect(router.snapshot()?.threadRoot).toBe(""); + }); + + test("a blank In-Reply-To does not continue an anchorless thread", () => { + // The empty id is the case that has to hold, not merely a non-matching + // one: an anchorless thread holds no `lastMessageId`, and a message + // whose `In-Reply-To` header was written blank carries an empty id. + // Compared as values the two are equal, so a continue would push the + // real correspondent into `cc`. The value reaches the router through + // the wire, not through `createInboundMessage`, which rejects an empty + // `inReplyTo`. + const router = createConnectorRouter(); + router.commit(router.route(inboundWithNoMessageId())); + + const stranger = inboundWithNoMessageId({ + from: "stranger@example.com", + inReplyTo: "", + }); + const decision = router.route(stranger); + + expect(decision.kind).toBe("start"); + + router.commit(decision); + expect(router.snapshot()?.replyTo).toBe("stranger@example.com"); + expect(router.snapshot()?.cc).toEqual([]); + }); + + test("an anchorless thread is not an absorbing state", () => { + // Before the re-anchor rule an anchorless thread could never match + // `isContinuation`, so every later message routed `passthrough` -- the + // INBOX never drained and the first sender kept the reply address for + // the life of the agent. + const router = createConnectorRouter(); + router.commit(router.route(inboundWithNoMessageId())); + + const anchored = startMessage(); + const decision = router.route(anchored); + expect(decision.kind).toBe("start"); + router.commit(decision); + + expect(router.snapshot()?.threadRoot).toBe(""); + expect(router.snapshot()?.lastMessageId).toBe(""); + + // The re-anchored thread continues normally from here. + const follow = router.route( + continuationByReferences("", { + from: "other@example.com", + }), + ); + expect(follow.kind).toBe("continue"); + router.commit(follow); + expect(router.snapshot()?.replyTo).toBe("other@example.com"); + expect(router.snapshot()?.cc).toEqual(["user@example.com"]); + }); + + test("onReplySent anchors the thread so later replies continue it", () => { + // The agent replying is the ordinary way an anchorless thread acquires + // an anchor: the reply's own id becomes `lastMessageId`. + const router = createConnectorRouter(); + router.commit(router.route(inboundWithNoMessageId())); + router.onReplySent({ + messageId: "", + status: "delivered", + }); + + const decision = router.route( + continuationByInReplyTo(""), + ); + expect(decision.kind).toBe("continue"); + router.commit(decision); + expect(router.snapshot()?.replyTo).toBe("user@example.com"); + }); + }); + describe("onStateChanged callback", () => { test("fires after a start decision commits", () => { const events: (ConnectorThreadState | null)[] = []; diff --git a/vendor/intx-harness/src/connector-router.ts b/vendor/intx-harness/src/connector-router.ts index b7f8009a2..db4f32087 100644 --- a/vendor/intx-harness/src/connector-router.ts +++ b/vendor/intx-harness/src/connector-router.ts @@ -1,10 +1,13 @@ // Connector-thread routing for the agent harness. // -// The connector is one durable thread per agent. Participants accumulate -// as they speak; no one is displaced. `replyTo` tracks the most recent +// The connector is one durable thread per agent. On a thread holding an +// anchor -- a thread root or a last message id -- participants accumulate +// as they speak and no one is displaced: `replyTo` tracks the most recent // speaker (the primary recipient on the next outbound reply) and `cc` // tracks every other participant who has spoken (carried on outbound so -// everyone stays in the loop). +// everyone stays in the loop). A thread holding neither anchor offers +// nothing for a later message to continue, so the next arrival opens a +// fresh thread and the accumulated list starts over. // // Two-phase decision: route() is pure and returns a discriminated kind // plus an opaque carrier of the next state; commit() advances router @@ -30,7 +33,11 @@ export type RouteDecision = export type ConnectorReplyParts = { to: string; cc: string[]; - inReplyTo: string; + /** + * The nearest identified ancestor: the last message id, or the thread + * root when the last message named no id. Absent when neither exists. + */ + inReplyTo?: string; subject?: string; }; @@ -67,13 +74,16 @@ export interface ConnectorRouter { * does not mutate router state. The returned decision must be passed to * `commit()` to take effect. * - * Throws when `message.headers.from` is not a parseable bare addr-spec - * (per `extractAddrSpec` from `@intx/mime`). The production fetch path - * copies the wire `From:` header verbatim, so a malformed sender is a - * normal-shape runtime concern, not a programmer error. Callers should - * treat the throw as passthrough — deliver the message to the reactor - * but do not advance router state or consume the message from the - * INBOX. + * Returns `passthrough` when `message.headers.from` is absent: the + * message names nobody to reply to. + * + * Throws when it is present but is not a parseable bare addr-spec. + * Callers should treat the throw as passthrough — deliver the message + * but do not advance router state or consume it from the INBOX. + * + * Does not read `Interchange-Type`. A structured payload that names a + * `From` starts or continues the thread exactly as a conversation + * message does. */ route(message: InboundMessage): RouteDecision; @@ -170,7 +180,11 @@ export function createConnectorRouter( const { inReplyTo, references } = message.headers; - if (references !== undefined && references.includes(state.threadRoot)) { + if ( + state.threadRoot !== undefined && + references !== undefined && + references.includes(state.threadRoot) + ) { return true; } @@ -189,12 +203,39 @@ export function createConnectorRouter( return [...existing, value]; } + function speakerOf(message: InboundMessage): string | null { + const { from } = message.headers; + if (from === undefined) return null; + return extractAddrSpec(from); + } + + function passthroughWithNoReplyAddress( + message: InboundMessage, + ): RouteDecision { + logger.debug`Message ${message.headers.messageId} carries no From header, so it has no reply address; routing it as passthrough and leaving the connector thread unadvanced`; + return { kind: "passthrough" }; + } + + // A thread opened by a message that named no id holds neither anchor, so + // `isContinuation` can never match it again. The next message starts a + // fresh thread rather than being absorbed by it forever. + function isAnchored(s: ConnectorThreadState): boolean { + return s.threadRoot !== undefined || s.lastMessageId !== undefined; + } + function route(message: InboundMessage): RouteDecision { - if (state === null) { + if (state === null || !isAnchored(state)) { + const replyTo = speakerOf(message); + if (replyTo === null) return passthroughWithNoReplyAddress(message); + const nextState: ConnectorThreadState = { - threadRoot: message.headers.messageId, - lastMessageId: message.headers.messageId, - replyTo: extractAddrSpec(message.headers.from), + ...(message.headers.messageId !== undefined + ? { + threadRoot: message.headers.messageId, + lastMessageId: message.headers.messageId, + } + : {}), + replyTo, cc: [], ...(message.headers.subject !== undefined ? { subject: message.headers.subject } @@ -206,7 +247,9 @@ export function createConnectorRouter( } if (isContinuation(message)) { - const nextSpeaker = extractAddrSpec(message.headers.from); + const nextSpeaker = speakerOf(message); + if (nextSpeaker === null) return passthroughWithNoReplyAddress(message); + // The previous most-recent speaker moves into the cc list; the // new speaker becomes replyTo. Dedup so a sender returning after // others have spoken doesn't appear twice. @@ -214,8 +257,12 @@ export function createConnectorRouter( (addr) => addr !== nextSpeaker, ); const nextState: ConnectorThreadState = { - threadRoot: state.threadRoot, - lastMessageId: message.headers.messageId, + ...(state.threadRoot !== undefined + ? { threadRoot: state.threadRoot } + : {}), + ...(message.headers.messageId !== undefined + ? { lastMessageId: message.headers.messageId } + : {}), replyTo: nextSpeaker, cc: carriedCc, ...(state.subject !== undefined ? { subject: state.subject } : {}), @@ -247,10 +294,15 @@ export function createConnectorRouter( throw new NoActiveConnectorThreadError(); } + // RFC 5322 3.6.4: a parent that named no Message-ID leaves the root as + // the nearest identified ancestor, and the References chain built from + // it is what keeps the reply on the thread. + const parent = state.lastMessageId ?? state.threadRoot; + return { to: state.replyTo, cc: [...state.cc], - inReplyTo: state.lastMessageId, + ...(parent !== undefined ? { inReplyTo: parent } : {}), ...(state.subject !== undefined ? { subject: state.subject } : {}), }; } @@ -260,7 +312,9 @@ export function createConnectorRouter( throw new NoActiveConnectorThreadError(); } applyState({ - threadRoot: state.threadRoot, + ...(state.threadRoot !== undefined + ? { threadRoot: state.threadRoot } + : {}), lastMessageId: receipt.messageId, replyTo: state.replyTo, cc: [...state.cc], @@ -271,8 +325,12 @@ export function createConnectorRouter( function snapshot(): ConnectorThreadState | null { if (state === null) return null; return { - threadRoot: state.threadRoot, - lastMessageId: state.lastMessageId, + ...(state.threadRoot !== undefined + ? { threadRoot: state.threadRoot } + : {}), + ...(state.lastMessageId !== undefined + ? { lastMessageId: state.lastMessageId } + : {}), replyTo: state.replyTo, cc: [...state.cc], ...(state.subject !== undefined ? { subject: state.subject } : {}), @@ -284,8 +342,12 @@ export function createConnectorRouter( next === null ? null : { - threadRoot: next.threadRoot, - lastMessageId: next.lastMessageId, + ...(next.threadRoot !== undefined + ? { threadRoot: next.threadRoot } + : {}), + ...(next.lastMessageId !== undefined + ? { lastMessageId: next.lastMessageId } + : {}), replyTo: next.replyTo, cc: [...next.cc], ...(next.subject !== undefined ? { subject: next.subject } : {}), diff --git a/vendor/intx-harness/src/harness.test.ts b/vendor/intx-harness/src/harness.test.ts index b40916447..0276f0cb9 100644 --- a/vendor/intx-harness/src/harness.test.ts +++ b/vendor/intx-harness/src/harness.test.ts @@ -38,6 +38,7 @@ import type { ReactorCapabilities, ReactorInboundEvent, ReactorState, + SearchQuery, } from "@intx/types/runtime"; import { createConnectorRouter } from "./connector-router"; @@ -45,6 +46,7 @@ import { createHarness, createWrappedStorageOverrides, defineMailTools, + MAIL_FETCH_FAILED_FLAG, type MailEnv, } from "./harness"; import { waitForReactorDone } from "@intx/agent/testing"; @@ -64,9 +66,11 @@ interface MockTransportShape { fire(event: { type: string; uid: number }): void; fireExists(uid: number): void; enqueue(uid: number, message: InboundMessage): void; + enqueueUnfetchable(uid: number): void; watchCount(): number; unsubscribeCount(): number; - getDeletedRefs(): MessageRef[]; + getFlags(uid: number): string[]; + isPresent(uid: number): boolean; getFetchedUids(): number[]; getSent(): unknown[]; } @@ -100,19 +104,36 @@ function makeMockTransport(): { headers?: unknown; }) => void; const callbacks: WatchCallback[] = []; - const deletedRefs: MessageRef[] = []; const fetchedUids: number[] = []; const sent: unknown[] = []; const messages = new Map(); + const unfetchable = new Set(); + // The uids the mailbox holds, and the flags each carries. A uid leaves + // `present` only through an expunge, so a test can tell "the harness left + // the message alone" from "the harness consumed it". + const present = new Set(); + const flagsByUid = new Map>(); let unsubscribes = 0; + function flagsOf(uid: number): Set { + let flags = flagsByUid.get(uid); + if (flags === undefined) { + flags = new Set(); + flagsByUid.set(uid, flags); + } + return flags; + } + // The harness reads `transport.watch`, `transport.fetchFull`, - // `transport.setFlags`, `transport.expunge`, and `transport.send`. - // The mock provides those; the rest of the `MessageTransport` - // surface is satisfied via the double-cast pattern, which the - // project conventions sanction for library-type test stubs. + // `transport.search`, `transport.setFlags`, `transport.expunge`, and + // `transport.send`. The mock provides those; the rest of the + // `MessageTransport` surface is satisfied via the double-cast pattern, + // which the project conventions sanction for library-type test stubs. const stub = { - watch(_mailbox: unknown, callback: WatchCallback): () => void { + async watch( + _mailbox: unknown, + callback: WatchCallback, + ): Promise<() => void> { callbacks.push(callback); return () => { unsubscribes += 1; @@ -120,18 +141,43 @@ function makeMockTransport(): { }, async fetchFull(ref: MessageRef): Promise { fetchedUids.push(ref.uid); + if (unfetchable.has(ref.uid)) { + throw new Error(`invalid message payload on uid ${String(ref.uid)}`); + } const message = messages.get(ref.uid); if (message === undefined) { throw new Error(`no message for uid ${String(ref.uid)}`); } return message; }, - async setFlags(ref: MessageRef): Promise { - deletedRefs.push(ref); + async search(_mailbox: string, query: SearchQuery): Promise { + // The harness only ever queries by keyword. Any other query shape is a + // change the mock has not been taught, so it says so rather than + // answering with an empty result the caller would read as "no match". + const wanted = query.hasFlags; + if (wanted === undefined || Object.keys(query).length !== 1) { + throw new Error( + `the mock implements only a hasFlags search; got ${JSON.stringify(query)}`, + ); + } + return [...present] + .filter((uid) => wanted.every((flag) => flagsOf(uid).has(flag))) + .map((uid) => ({ uid, mailbox: "INBOX" })); + }, + async setFlags(ref: MessageRef, flags: string[]): Promise { + for (const flag of flags) flagsOf(ref.uid).add(flag); }, async expunge(): Promise<{ expungedUids: number[] }> { - // No-op for the mock; the test asserts via deletedRefs. - return { expungedUids: [] }; + const expungedUids = [...present].filter((uid) => + flagsOf(uid).has("\\Deleted"), + ); + for (const uid of expungedUids) { + present.delete(uid); + flagsByUid.delete(uid); + messages.delete(uid); + unfetchable.delete(uid); + } + return { expungedUids }; }, async send(message: unknown): Promise<{ messageId: string }> { sent.push(message); @@ -155,6 +201,14 @@ function makeMockTransport(): { }, enqueue(uid: number, message: InboundMessage) { messages.set(uid, message); + present.add(uid); + }, + // An envelope `fetchFull` cannot assemble, a uid a concurrent expunge + // removed, or a faulting read: the transport throws rather than + // returning a message with its content omitted. + enqueueUnfetchable(uid: number) { + unfetchable.add(uid); + present.add(uid); }, watchCount(): number { return callbacks.length; @@ -162,8 +216,11 @@ function makeMockTransport(): { unsubscribeCount(): number { return unsubscribes; }, - getDeletedRefs(): MessageRef[] { - return deletedRefs; + getFlags(uid: number): string[] { + return [...flagsOf(uid)]; + }, + isPresent(uid: number): boolean { + return present.has(uid); }, getFetchedUids(): number[] { return fetchedUids; @@ -692,6 +749,98 @@ describe("createHarness message delivery", () => { } }); + test("a message that cannot be fetched is flagged and left in the INBOX", async () => { + // A fetch throws on an envelope the sender chose, and also on a uid a + // concurrent expunge removed and on a faulting read. Consuming the message + // here would hand a peer the power to delete its own mail out of the INBOX + // by malforming a header, so the message stays and carries the reason. + const { transport, control } = makeMockTransport(); + const storage = await createIsogitStore(workDir); + const received = { count: 0 }; + const env: MailEnv = { + ...mailEnv({ workdir: workDir, storage, transport }), + directors: recordingDirectorRegistry(received), + }; + const harness = await createHarness(recordingDef(), env); + + try { + control.enqueueUnfetchable(13); + control.fireExists(13); + + await waitUntil(() => + control.getFlags(13).includes(MAIL_FETCH_FAILED_FLAG), + ); + + // The flag write is the last act of the failure path, so once it lands + // the decision about the message is made: it was not marked `\Deleted` + // and it is still in the mailbox. + expect(control.getFlags(13)).not.toContain("\\Deleted"); + expect(control.isPresent(13)).toBe(true); + + // The delivery of a message the harness can handle is the fence for + // the negative: `agent.deliver` runs before `consumeFromInbox` on the + // delivered path, so a count of one after uid 14 lands proves uid 13 + // reached the reactor not at all. + const message = createInboundMessage({ + from: "alice@example.com", + to: AGENT_ADDRESS, + content: "Hello", + interchangeType: "conversation.message", + }); + control.enqueue(14, { ...message, ref: { uid: 14, mailbox: "INBOX" } }); + control.fireExists(14); + + await waitUntil(() => received.count >= 1); + expect(received.count).toBe(1); + } finally { + await harness.close(); + } + }); + + test("a flagged message is not fetched again when its arrival replays", async () => { + // Preserving the message cannot mean retrying it forever. A transport that + // replays `exists` -- on reconnect, or on a fresh watch over a mailbox that + // still holds the message -- would otherwise drive the same failing fetch + // for as long as the message sits there. + const { transport, control } = makeMockTransport(); + const storage = await createIsogitStore(workDir); + const received = { count: 0 }; + const env: MailEnv = { + ...mailEnv({ workdir: workDir, storage, transport }), + directors: recordingDirectorRegistry(received), + }; + const harness = await createHarness(recordingDef(), env); + + try { + control.enqueueUnfetchable(21); + control.fireExists(21); + await waitUntil(() => + control.getFlags(21).includes(MAIL_FETCH_FAILED_FLAG), + ); + + const message = createInboundMessage({ + from: "alice@example.com", + to: AGENT_ADDRESS, + content: "Hello", + interchangeType: "conversation.message", + }); + control.enqueue(22, { ...message, ref: { uid: 22, mailbox: "INBOX" } }); + + // The mock invokes its callbacks in order and each arrival consults the + // keyword before anything else, so uid 21's replay has finished its + // flag query by the time uid 22's fetch is recorded. Uid 21 appearing + // once in the record is therefore the skip. + control.fireExists(21); + control.fireExists(22); + + await waitUntil(() => control.getFetchedUids().includes(22)); + expect(control.getFetchedUids()).toEqual([21, 22]); + expect(control.isPresent(21)).toBe(true); + } finally { + await harness.close(); + } + }); + test("deliver() injects a message directly into the reactor", async () => { const { transport } = makeMockTransport(); const storage = await createIsogitStore(workDir); diff --git a/vendor/intx-harness/src/harness.ts b/vendor/intx-harness/src/harness.ts index 773dfa3cf..4eb117712 100644 --- a/vendor/intx-harness/src/harness.ts +++ b/vendor/intx-harness/src/harness.ts @@ -29,8 +29,8 @@ import type { ContextStore, InboundMessage, InferenceSource, + MessageRef, MessageTransport, - Unsubscribe, } from "@intx/types/runtime"; import { createConnectorRouter, type RouteDecision } from "./connector-router"; @@ -38,6 +38,23 @@ import { driveConnectorReplies } from "./reply-drain"; const logger = getLogger(["interchange", "harness"]); +/** + * Interchange keyword flag the INBOX watch puts on a message whose + * `transport.fetchFull` threw. The message stays in the INBOX rather than being + * consumed, because part of what makes a fetch throw is an envelope the sender + * chose and the rest is local (a uid a concurrent expunge removed, a faulting + * read), so deleting on it would put a peer in charge of whether its own mail + * survives. + * + * The keyword is what keeps the watch from attempting the same uid again in the + * run that flagged it, on the premise that a transport delivers a second + * arrival event for a uid already resident in the mailbox -- which no transport + * here does. Nothing in this package re-attempts a flagged message either, so + * the flag preserves the mail and surfaces it to a search for + * `hasFlags: [MAIL_FETCH_FAILED_FLAG]`, but gives it no path back to delivery. + */ +export const MAIL_FETCH_FAILED_FLAG = "$FetchFailed"; + /** * Env extension the composition layer requires beyond `BaseEnv`. Tools * shipped by this package declare the matching `requires` so @@ -294,8 +311,8 @@ export async function createHarness( // From here through the final `return`, the agent is constructed // and the workdir lock is held. Anything that throws -- the - // `driveConnectorReplies` setup, `transport.watch()`, - // anything in the watch callback's synchronous registration -- has + // `driveConnectorReplies` setup, `transport.watch()` rejecting, + // a throw from the watch callback's registration -- has // to release the lock by closing the agent before re-raising; the // caller never sees the agent and cannot do it themselves. // `createAgent` covers its own internal failure paths via its @@ -327,35 +344,49 @@ export async function createHarness( : {}), }); - // Delete a message from the INBOX after it has been delivered to the - // reactor. - // - // A failure here is logged and swallowed: the router state has - // already been committed and `agent.deliver` has accepted the - // message, so re-raising would unwind a half-applied delivery. The - // message stays in the INBOX and a future startup (or watch firing) - // re-fetches it, re-routes it, and re-delivers it. The router's - // persisted state makes that benign on the routing side: the sender - // is already a thread participant, so `route()` returns either a - // `continue` (which is a no-op state mutation since the sender is - // unchanged) or a `passthrough` (no headers match). The agent's - // director sees a duplicate `message.received`; idempotent - // directors are unaffected, and the audit trail records the - // duplicate for post-hoc reconciliation. - async function consumeFromInbox(message: InboundMessage): Promise { + // A failure here is logged and swallowed: the router state is already + // committed and `agent.deliver` has accepted the message, so re-raising + // would unwind a half-applied delivery. A redelivery shows the director + // a duplicate `message.received`. + async function consumeFromInbox(ref: MessageRef): Promise { try { - await transport.setFlags(message.ref, ["\\Deleted"]); + await transport.setFlags(ref, ["\\Deleted"]); await transport.expunge("INBOX"); } catch (cause) { - logger.warn`Failed to consume message uid=${message.ref.uid} from INBOX: ${cause}`; + logger.warn`Failed to consume message uid=${ref.uid} from INBOX: ${cause}`; + } + } + + // Whether an earlier attempt on this uid already failed its fetch and + // said so on the message. The transport is asked rather than an + // in-process uid set because that is where the record lives: on the + // message, which outlives the process that wrote it. That record is + // what a search on `MAIL_FETCH_FAILED_FLAG` surfaces to an operator. + async function fetchAlreadyFailed(ref: MessageRef): Promise { + const flagged = await transport.search("INBOX", { + hasFlags: [MAIL_FETCH_FAILED_FLAG], + }); + return flagged.some((candidate) => candidate.uid === ref.uid); + } + + // A flag write that fails leaves the message unflagged, so a later + // arrival event for the same uid attempts the fetch again. That is the + // safe direction to fail in: the message is still in the INBOX, and the + // attempt that comes after a transient fault may well deliver it. + async function markFetchFailed(ref: MessageRef): Promise { + try { + await transport.setFlags(ref, [MAIL_FETCH_FAILED_FLAG]); + } catch (cause) { + logger.warn`Failed to flag unfetchable message uid=${ref.uid} in INBOX: ${cause}`; } } - // INBOX watch loop. Subscribe before the agent's reactor is fully - // settled so no message is missed in the window between subscription - // and the first watch callback. + // INBOX watch loop. The transport registers the callback before its + // promise resolves, so a message that arrives during acceptance is + // not missed. A refusal rejects inside this try, and the finally + // closes the agent. let stopped = false; - const unsubscribe: Unsubscribe = transport.watch("INBOX", (event) => { + const unsubscribe = await transport.watch("INBOX", (event) => { if (stopped) return; if (event.type !== "exists") return; @@ -363,11 +394,33 @@ export async function createHarness( void (async () => { try { + if (await fetchAlreadyFailed(ref)) { + // An earlier attempt on this uid failed and said so on the + // message. Part of what makes a fetch fail is permanent, and the + // watch cannot tell that part from the rest, so it leaves the + // message flagged in the INBOX and delivers nothing. + logger.debug`Skipping message uid=${event.uid}: an earlier fetch failed and flagged it ${MAIL_FETCH_FAILED_FLAG}`; + return; + } + if (stopped) return; + let message: InboundMessage; try { message = await transport.fetchFull(ref); } catch (cause) { - logger.error`Failed to fetch message uid=${event.uid}: ${cause}`; + // What reaches here is not only a message the transport could not + // assemble. The same throw covers a uid a concurrent expunge + // already removed, where there is nothing left to consume, and a + // faulting read of sound bytes, where a later attempt could still + // succeed. Only the assembly case is input a peer chooses, and + // consuming on it would let a peer delete its own mail out of the + // INBOX by malforming one header -- so the message is flagged and + // left where it is. Inventing the flags and signature status the + // transport never read would vouch for a message it did not, so + // the message is not delivered either. + logger.error`Failed to fetch message uid=${event.uid}; flagging it ${MAIL_FETCH_FAILED_FLAG} and leaving it in the INBOX: ${cause}`; + if (stopped) return; + await markFetchFailed(ref); return; } @@ -403,7 +456,7 @@ export async function createHarness( connectorRouter.commit(decision); if (stopped) return; agent.deliver(message); - await consumeFromInbox(message); + await consumeFromInbox(message.ref); } catch (cause) { // `agent.deliver` throws `AgentClosedError` synchronously when // called after the agent has closed. The `if (stopped) return` @@ -449,7 +502,7 @@ export async function createHarness( } finally { if (!harnessSucceeded) { // Close the agent without waiting on its shutdown timeout so a - // synchronous post-`createAgent` throw does not stall the + // throw after `createAgent` does not stall the // caller's failure path. The `.catch` swallows any rejection // from the close: the caller is already receiving the original // throw, and a noisier-than-original close failure here would diff --git a/vendor/intx-harness/src/index.ts b/vendor/intx-harness/src/index.ts index 6ea215ab4..9aec19a24 100644 --- a/vendor/intx-harness/src/index.ts +++ b/vendor/intx-harness/src/index.ts @@ -1,6 +1,7 @@ export { createHarness, defineMailTools, + MAIL_FETCH_FAILED_FLAG, type Harness, type MailEnv, type MailToolWrapper, @@ -30,6 +31,15 @@ export type { ResolvedCredentialBinding, } from "./credential-capability"; +export { + buildCredentialCapabilities, + createInferenceCredentialResolver, +} from "./step-credential-capabilities"; +export type { + CredentialMaterialCell, + StepCredentialWiring, +} from "./step-credential-capabilities"; + export { createConnectorRouter, NoActiveConnectorThreadError, diff --git a/vendor/intx-harness/src/reply-drain.ts b/vendor/intx-harness/src/reply-drain.ts index 8c604a064..234f2c7c6 100644 --- a/vendor/intx-harness/src/reply-drain.ts +++ b/vendor/intx-harness/src/reply-drain.ts @@ -61,9 +61,8 @@ export interface ConnectorReplyDrainOpts { * located (the very first reply on a fresh thread, or a malformed id); the * drain then omits `references` and the transport derives `[inReplyTo]`. * - * Optional: a caller with no mailbox to consult (`createHarness`) omits it, - * leaving the pre-existing single-element threading unchanged. The warm - * workflow-host wiring supplies it from the deployment's committed mailbox. + * Optional. When it is omitted, the drain leaves `references` unset. + * `createHarness` omits it, and the transport then derives `[inReplyTo]`. */ resolveReferences?: (inReplyTo: string) => Promise; /** @@ -245,7 +244,8 @@ export function driveConnectorReplies( // (parent absent, malformed id) yields `undefined`, and the // transport derives `[inReplyTo]` as before. const references = - opts.resolveReferences !== undefined + opts.resolveReferences !== undefined && + parts.inReplyTo !== undefined ? await opts.resolveReferences(parts.inReplyTo) : undefined; const receipt = await opts.send({ diff --git a/vendor/intx-harness/src/step-credential-capabilities.test.ts b/vendor/intx-harness/src/step-credential-capabilities.test.ts new file mode 100644 index 000000000..2c35efbfb --- /dev/null +++ b/vendor/intx-harness/src/step-credential-capabilities.test.ts @@ -0,0 +1,351 @@ +import { describe, test, expect } from "bun:test"; +import { toolConsumer, type GrantRule } from "@intx/authz"; +import type { CredentialProvider, CredentialShapeContext } from "@intx/types"; +import type { CredentialDelivery } from "@intx/types/credential-delivery"; +import type { ToolCredentialDeclaration } from "@intx/types/package-json"; + +import { createCredentialProviderRegistry } from "./credential-providers"; +import { buildCredentialCapabilities } from "./step-credential-capabilities"; + +// A provider that records every shape context it is handed, so a test can +// reach the `readCurrentMaterial` closure a shaped handle reads through. +function trackingProvider(): { + provider: CredentialProvider; + shapes: CredentialShapeContext[]; +} { + const shapes: CredentialShapeContext[] = []; + const provider: CredentialProvider = { + key: "fake", + shape(ctx) { + shapes.push(ctx); + return { + kind: "http", + fetch: async () => new Response(), + dispose() { + /* no-op: the fake http handle holds nothing to release */ + }, + }; + }, + }; + return { provider, shapes }; +} + +function grant( + overrides: Partial & + Pick, +): GrantRule { + return { + id: "grt_test", + origin: "system", + conditions: null, + expiresAt: null, + roleId: null, + principalId: null, + ...overrides, + }; +} + +// The assembly reads only `packageName` and `declaredCredentials`. +function fac( + packageName: string, + declaredCredentials: ToolCredentialDeclaration[], +) { + return { packageName, declaredCredentials }; +} + +const ORIGIN = "https://api.example.com"; + +describe("buildCredentialCapabilities", () => { + test("a credential granted to one package cannot be resolved by another (confused deputy)", async () => { + const track = trackingProvider(); + // The same credential is bound for both packages, but only pkg-a holds the + // credential:c1/use grant scoped to its consumer. + const cell = { + current: { + bindings: [ + { + handle: "shared", + credentialId: "c1", + consumer: toolConsumer("@intx/pkg-a"), + }, + { + handle: "shared", + credentialId: "c1", + consumer: toolConsumer("@intx/pkg-b"), + }, + ], + materials: [ + { + credentialId: "c1", + providerKey: track.provider.key, + origin: ORIGIN, + secret: "sk-1", + }, + ], + }, + }; + const caps = buildCredentialCapabilities( + [fac("@intx/pkg-a", []), fac("@intx/pkg-b", [])], + { + materialCell: cell, + resolveGrants: () => [ + grant({ + resource: "credential:c1", + action: "use", + effect: "allow", + conditions: { tool: toolConsumer("@intx/pkg-a") }, + }), + ], + providers: createCredentialProviderRegistry([track.provider]), + }, + ); + + const capA = caps.get("@intx/pkg-a"); + const capB = caps.get("@intx/pkg-b"); + if (capA === undefined || capB === undefined) { + throw new Error("expected a capability for both packages"); + } + + // pkg-a's grant authorizes it; pkg-b binds the same credential but holds no + // grant for it, so Gate 2 refuses -- the credential does not leak sideways. + await expect(capA.resolve("shared")).resolves.toMatchObject({ + kind: "http", + }); + await expect(capB.resolve("shared")).rejects.toThrow(/not authorized/); + }); + + test("a declared handle no binding resolves fails the build closed", () => { + const track = trackingProvider(); + const cell = { current: { bindings: [], materials: [] } }; + expect(() => + buildCredentialCapabilities( + [fac("@intx/pkg-a", [{ handle: "needed" }])], + { + materialCell: cell, + resolveGrants: () => [], + providers: createCredentialProviderRegistry([track.provider]), + }, + ), + ).toThrow( + /declares credential handle\(s\) that no binding resolves: needed/, + ); + }); + + test("dropping a credential's material starves an already-shaped handle", async () => { + const track = trackingProvider(); + const cell: { current: CredentialDelivery | null } = { + current: { + bindings: [ + { + handle: "cred", + credentialId: "c1", + consumer: toolConsumer("@intx/pkg-a"), + }, + ], + materials: [ + { + credentialId: "c1", + providerKey: track.provider.key, + origin: ORIGIN, + secret: "sk-1", + }, + ], + }, + }; + const caps = buildCredentialCapabilities([fac("@intx/pkg-a", [])], { + materialCell: cell, + resolveGrants: () => [ + grant({ + resource: "credential:c1", + action: "use", + effect: "allow", + conditions: { tool: toolConsumer("@intx/pkg-a") }, + }), + ], + providers: createCredentialProviderRegistry([track.provider]), + }); + const capA = caps.get("@intx/pkg-a"); + if (capA === undefined) throw new Error("expected a capability for pkg-a"); + + await capA.resolve("cred"); + const shaped = track.shapes[0]; + if (shaped === undefined) + throw new Error("expected the handle to be shaped"); + + // Live read works while the material is delivered. + expect(shaped.readCurrentMaterial()).toEqual({ secret: "sk-1" }); + + // A re-push that revokes pkg-a drops c1's material: the already-shaped + // handle starves on its next read rather than serving a stale secret. + cell.current = { bindings: [], materials: [] }; + expect(() => shaped.readCurrentMaterial()).toThrow(/no longer delivered/); + + // An emptied cell (no delivery at all) fails closed the same way. + cell.current = null; + expect(() => shaped.readCurrentMaterial()).toThrow(/cell is empty/); + }); + + test("a material whose provider or origin drifted under a shaped handle is refused", async () => { + const track = trackingProvider(); + const bindings = [ + { + handle: "cred", + credentialId: "c1", + consumer: toolConsumer("@intx/pkg-a"), + }, + ]; + const cell: { current: CredentialDelivery | null } = { + current: { + bindings, + materials: [ + { + credentialId: "c1", + providerKey: track.provider.key, + origin: ORIGIN, + secret: "sk-1", + }, + ], + }, + }; + const caps = buildCredentialCapabilities([fac("@intx/pkg-a", [])], { + materialCell: cell, + resolveGrants: () => [ + grant({ + resource: "credential:c1", + action: "use", + effect: "allow", + conditions: { tool: toolConsumer("@intx/pkg-a") }, + }), + ], + providers: createCredentialProviderRegistry([track.provider]), + }); + const capA = caps.get("@intx/pkg-a"); + if (capA === undefined) throw new Error("expected a capability for pkg-a"); + await capA.resolve("cred"); + const shaped = track.shapes[0]; + if (shaped === undefined) + throw new Error("expected the handle to be shaped"); + + // A rotation may change the secret, never the provider/origin. A live entry + // whose origin drifted is refused rather than followed. + cell.current = { + bindings, + materials: [ + { + credentialId: "c1", + providerKey: track.provider.key, + origin: "https://api.attacker.example", + secret: "sk-2", + }, + ], + }; + expect(() => shaped.readCurrentMaterial()).toThrow( + /changed provider\/origin/, + ); + }); + + test("a descriptor with no backing material fails the build closed", () => { + const track = trackingProvider(); + const cell = { + current: { + bindings: [ + { + handle: "cred", + credentialId: "c1", + consumer: toolConsumer("@intx/pkg-a"), + }, + ], + materials: [], + }, + }; + expect(() => + buildCredentialCapabilities([fac("@intx/pkg-a", [])], { + materialCell: cell, + resolveGrants: () => [], + providers: createCredentialProviderRegistry([track.provider]), + }), + ).toThrow(/delivery is malformed/); + }); + + test("a package that declares and binds no credential gets no capability", () => { + const track = trackingProvider(); + // A credential is delivered, but only for a different package's consumer. + const cell = { + current: { + bindings: [ + { + handle: "x", + credentialId: "c1", + consumer: toolConsumer("@intx/other"), + }, + ], + materials: [ + { + credentialId: "c1", + providerKey: track.provider.key, + origin: ORIGIN, + secret: "sk-1", + }, + ], + }, + }; + const caps = buildCredentialCapabilities([fac("@intx/pkg-a", [])], { + materialCell: cell, + resolveGrants: () => [], + providers: createCredentialProviderRegistry([track.provider]), + }); + expect(caps.has("@intx/pkg-a")).toBe(false); + }); + + // The grants thunk must be read only when a package actually needs a + // capability. This pins the self-discovery-resume fix: a toolless resume + // precedes the grants barrier, so resolving grants for it would fault on a + // snapshot that is not present yet. The "no capability" test above passes a + // benign `() => []`, so it stays green whether grants resolve lazily or + // eagerly -- these two use a throwing thunk to catch a laziness regression. + const throwingGrants = (): readonly GrantRule[] => { + throw new Error("resolveGrants must not be called for a step needing none"); + }; + + test("a step whose packages need no capability never resolves grants", () => { + const track = trackingProvider(); + const cell = { current: { bindings: [], materials: [] } }; + expect(() => + buildCredentialCapabilities([fac("@intx/pkg-a", [])], { + materialCell: cell, + resolveGrants: throwingGrants, + providers: createCredentialProviderRegistry([track.provider]), + }), + ).not.toThrow(); + }); + + test("grants are resolved once a package needs a capability", () => { + const track = trackingProvider(); + const cell = { + current: { + bindings: [ + { + handle: "cred", + credentialId: "c1", + consumer: toolConsumer("@intx/pkg-a"), + }, + ], + materials: [ + { + credentialId: "c1", + providerKey: track.provider.key, + origin: ORIGIN, + secret: "sk-1", + }, + ], + }, + }; + expect(() => + buildCredentialCapabilities([fac("@intx/pkg-a", [])], { + materialCell: cell, + resolveGrants: throwingGrants, + providers: createCredentialProviderRegistry([track.provider]), + }), + ).toThrow(/resolveGrants must not be called/); + }); +}); diff --git a/vendor/intx-harness/src/step-credential-capabilities.ts b/vendor/intx-harness/src/step-credential-capabilities.ts new file mode 100644 index 000000000..e0c0409fd --- /dev/null +++ b/vendor/intx-harness/src/step-credential-capabilities.ts @@ -0,0 +1,258 @@ +// Per-consumer assembly of the runtime `credentials` capability for a step's +// tool bundles. +// +// A step can carry tool factories from several packages, and each package is a +// distinct credential consumer (`toolConsumer(packageName)`). Gate 2 -- the +// `{ tool }` condition on a `credential:{id}` / `use` grant -- is baked into +// each capability at construction, so a single capability shared across a +// step's packages would be a confused-deputy hole (package A's grant would +// authorize package B's resolve). This module therefore builds ONE capability +// per package, keyed by package name, and the bundle-invocation site layers +// each package's capability onto only that package's bundles. +// +// Two invariants this module owns, both fail-closed: +// 1. Material is read from the LIVE delivery cell on every use, never +// snapshotted at shape time. A rotation updates the cell in place; a +// re-push that revokes a consumer's access DROPS the material, and an +// already-shaped handle must starve the instant that happens. This live +// read is the revocation-enforcement surface -- Gate 2 stays a +// launch/first-resolve gate (Seam A memoizes it), so mid-deploy +// revocation takes effect here, at the material, not at the gate. +// 2. The declared-vs-bound reconcile runs per consumer against that +// consumer's OWN bound set, so package A's declaration can never be +// satisfied by a handle bound for package B. + +import { toolConsumer } from "@intx/authz"; +import type { + CredentialMaterial, + CredentialMaterialResolver, + CredentialMaterialSource, +} from "@intx/types"; +import type { GrantRule } from "@intx/types/authz"; +import type { ToolCredentialDeclaration } from "@intx/types/package-json"; +import type { CredentialDelivery } from "@intx/types/credential-delivery"; + +import { + createCredentialCapability, + reconcileDeclaredCredentials, + type HostCredentialCapability, + type ResolvedCredentialBinding, +} from "./credential-capability"; +import type { CredentialProviderRegistry } from "./credential-providers"; + +/** + * The mutable cell the control channel writes each credential delivery into. + * Held structurally (rather than importing `CredentialMaterialRef` from + * `@intx/workflow-host`) so this module stays free of the transport package and + * a test can drive it with a plain object. + */ +export interface CredentialMaterialCell { + readonly current: CredentialDelivery | null; +} + +/** + * Everything the per-consumer assembly needs beyond the step's factories: the + * live material cell, the step's grants for Gate 2, and the provider registry + * that shapes a credential into a mediated handle. The sidecar assembles this + * at the invoke-step boundary -- the material cell and grants ride in from the + * run child, the providers are sidecar-static. + */ +export interface StepCredentialWiring { + readonly materialCell: CredentialMaterialCell; + /** + * Resolves the step's grants for Gate 2. A thunk rather than a resolved + * array, so the grants are read only when a capability is actually built: a + * step with no credential-consuming package never touches them. This matters + * on the resume path -- a run resumed by self-discovery does not pass through + * the pre-trigger grants barrier, so the credentials snapshot a Gate-2 read + * needs may not be present yet; a toolless resume must not fault on that. + */ + readonly resolveGrants: () => readonly GrantRule[]; + readonly providers: CredentialProviderRegistry; +} + +/** + * Build the credential capabilities for a step's tool factories, one per + * distinct package, keyed by package name. + * + * A package with neither a declared nor a bound credential gets no entry: its + * bundles keep the base capabilities bag untouched, so `resolve("credentials")` + * fails closed as "not provided by host" rather than handing back an empty + * sub-registry. Every returned capability owns a `dispose` the caller must run + * on teardown. + * + * The factory list is structural. The body reads only `packageName` and + * `declaredCredentials`. Importing `StepToolFactory` or `@intx/tool-packaging` + * is rejected because `bin/check-deps.ts` counts `import type`, and that + * dependency does not belong on this package. + */ +export function buildCredentialCapabilities( + factories: readonly { + readonly packageName: string; + readonly declaredCredentials: readonly ToolCredentialDeclaration[]; + }[], + wiring: StepCredentialWiring, +): Map { + const byPackage = new Map(); + const seenPackages = new Set(); + // Resolve the step's grants at most once, and only if some package actually + // needs a capability -- a step with no credential-consuming package never + // reads them (see StepCredentialWiring.resolveGrants). + let grants: readonly GrantRule[] | undefined; + + for (const stf of factories) { + if (seenPackages.has(stf.packageName)) continue; + seenPackages.add(stf.packageName); + + const consumer = toolConsumer(stf.packageName); + const bindings = buildConsumerBindings(consumer, wiring.materialCell); + + // Fail the launch closed if the package declares a handle no binding + // resolves for it -- the tool needs a credential the definition never + // bound. Checked against THIS consumer's bound set only (invariant 2). + reconcileDeclaredCredentials( + consumer, + stf.declaredCredentials, + new Set(bindings.keys()), + ); + + if (stf.declaredCredentials.length === 0 && bindings.size === 0) { + continue; + } + + grants ??= wiring.resolveGrants(); + byPackage.set( + stf.packageName, + createCredentialCapability({ + consumer, + bindings, + providers: wiring.providers, + grants: [...grants], + }), + ); + } + + return byPackage; +} + +/** + * Join the delivery's descriptors and materials into the binding map one + * consumer's capability is built from: every descriptor addressed to this + * consumer, resolved to the material that backs its credential. A descriptor + * whose material is absent is a malformed delivery and fails closed at build. + */ +function buildConsumerBindings( + consumer: string, + cell: CredentialMaterialCell, +): Map { + const bindings = new Map(); + const delivery = cell.current; + if (delivery === null) return bindings; + + for (const descriptor of delivery.bindings) { + if (descriptor.consumer !== consumer) continue; + + const material = delivery.materials.find( + (entry) => entry.credentialId === descriptor.credentialId, + ); + if (material === undefined) { + // The hub sends a material for every descriptor's credential; a + // descriptor with no material is a delivery bug. Refuse it at build + // rather than let the handle resolve to nothing at first use. + throw new Error( + `credential delivery is malformed: descriptor for handle "${descriptor.handle}" (consumer ${consumer}) references credential ${descriptor.credentialId} but the delivery carries no material for it`, + ); + } + + // Capture the provider key and origin at build time (they identify the + // credential and its pinned origin); the secret is read live per use. + bindings.set(descriptor.handle, { + credentialId: descriptor.credentialId, + providerKey: material.providerKey, + origin: material.origin, + readCurrentMaterial: makeReadCurrentMaterial({ + cell, + credentialId: descriptor.credentialId, + providerKey: material.providerKey, + origin: material.origin, + consumer, + }), + }); + } + + return bindings; +} + +/** + * The rotation/revocation indirection a shaped handle reads through: it looks + * the credential's material up in the LIVE cell on every call (invariant 1), + * failing closed when the material is gone (a re-push dropped it) and asserting + * the provider/origin have not drifted under the already-shaped handle. + */ +function makeReadCurrentMaterial(args: { + cell: CredentialMaterialCell; + credentialId: string; + providerKey: string; + origin: string; + consumer: string; +}): CredentialMaterialSource { + const { cell, credentialId, providerKey, origin, consumer } = args; + return () => { + const delivery = cell.current; + if (delivery === null) { + throw new Error( + `credential material for ${credentialId} (consumer ${consumer}) is not available: the delivery cell is empty`, + ); + } + const material = delivery.materials.find( + (entry) => entry.credentialId === credentialId, + ); + if (material === undefined) { + throw new Error( + `credential material for ${credentialId} (consumer ${consumer}) is no longer delivered: a re-push dropped it (rotated away or revoked)`, + ); + } + // A rotation changes the SECRET, not the provider or origin. A live entry + // whose provider/origin diverged would mean the shaped handle now + // authenticates somewhere it was not pinned to; surface it rather than + // silently follow the change. + if (material.providerKey !== providerKey || material.origin !== origin) { + throw new Error( + `credential ${credentialId} changed provider/origin under an already-shaped handle (${providerKey}@${origin} -> ${material.providerKey}@${material.origin}); a shaped handle cannot follow that change`, + ); + } + return { secret: material.secret }; + }; +} + +/** + * A generic inference credential resolver over the live cell: resolves a + * credential's current secret by `credentialId`, failing closed when the cell is + * empty or the credential is absent (a re-push dropped it -- rotated away or + * revoked). Unlike `makeReadCurrentMaterial` it is keyed by `credentialId` at + * call time -- an inference source's forward-only failover chain carries a + * distinct credential per entry -- and pins no provider/origin: an inference + * request authenticates to the source's own `baseURL` (resolved separately), + * so there is no shaped handle to protect against origin drift. + */ +export function createInferenceCredentialResolver( + cell: CredentialMaterialCell, +): CredentialMaterialResolver { + return (credentialId: string): CredentialMaterial => { + const delivery = cell.current; + if (delivery === null) { + throw new Error( + `inference credential material for ${credentialId} is not available: the delivery cell is empty`, + ); + } + const material = delivery.materials.find( + (entry) => entry.credentialId === credentialId, + ); + if (material === undefined) { + throw new Error( + `inference credential material for ${credentialId} is no longer delivered: a re-push dropped it (rotated away or revoked)`, + ); + } + return { secret: material.secret }; + }; +} diff --git a/vendor/intx-inference/PATCHES.md b/vendor/intx-inference/PATCHES.md index 6cb73fa35..6b3c3fb77 100644 --- a/vendor/intx-inference/PATCHES.md +++ b/vendor/intx-inference/PATCHES.md @@ -185,6 +185,51 @@ spot-checks at the recorded positions are unchanged for the same reason reason now gains an in-tree companion test (added this sync) and is exercised on the abort paths of `e2e/reactor-approval-acceptance.test.ts`. +### 2026-10-14 re-sync to `4cc6972` (CL-10656, HEAD drift 0.4.0 → base) + +Every entry re-carried against the pristine `4cc6972` tree; none dropped as +upstream-absorbed (grep of the pristine tree: `isStreamTerminal`, +`isPollOnlyPendingBatch`, `ephemeralTurns`, `loadErrors`, `turnsRevision`/ +`deepFreeze`, `isCommitting`, `MAX_LINE_LENGTH` — all still absent). +Nine upstream commits touch this package (`0f72f95`, `1ff8805`, `a0be30`, +`660ef8`, `d1e05e`, `cb61da`, `d1835b`, `c8be7e`, `2d973a`). Adaptation +surface: 22 upstream-changed files; 9 vendored files carry local content +and were three-way merged (`git merge-file`, ours/base/theirs), the rest +copied verbatim. + +- `errors.ts` conflict: upstream `c8be7e` added + `classifyRequestBuildError` between `classifyNetworkError` and the + patched `classifyAbortError(reason?)`; both kept (upstream function + adopted, patch re-carried with its `ClassifiedAbortRaw` type). +- `reactor.ts` two conflicts: (1) `d1835b` added + `mergeInferenceOptions`/`PerCallInferenceOptions` strip-copy beside the + patched `ExtendedInferenceOptions` (`ephemeralTurns`) — both kept; (2) + `d1835b`/`0f72f95` restructured `processDelivery` (per-send `deliver` + options, `deliveredInference` adoption, `expectedFrom` removal) inside + the atomic-acceptance critical section — the upstream adoption block + was re-slit above the patched `disposition` declaration, and the + `reactor-ts-atomic-approval-acceptance` `try/finally` boundary is + intact. +- Clean three-way merges on patched files: `assembly.ts`, + `correlation.ts`, `errors.test.ts`, `harness.ts`, `index.ts`, + `providers/anthropic.ts`, `providers/google-genai.ts`, + `providers/openai.ts`, `reactor.test.ts`; patch hunks carry with zero + adaptation (`providers-ts-openai-exclusive-cache-usage`, + `inference-ts-cl-7783-truncated-tool-call` stop-reason chain verified + against the new `@intx/types` `inference-events.ts` sites and + `providers/anthropic.ts`). The vendored `index.ts` merged with + upstream's barrel: upstream moved the `AdapterManifest` export to + `@intx/types` (`adapter-manifest.ts`); first-party + `src/provider/inference-dependencies.ts` imports it there now. +- `effort.ts`/`effort.test.ts` (`c8be7e` reasoning-effort mapping) and + `manifest.ts`'s new callers (`2d973a` adapter manifest) adopted + verbatim; the vendored `package.json` gains the upstream `./manifest` + and `./default-director` subpath exports. +- Post-merge divergence proof: `bin/vendor-patch-diff` against the + recorded SHA prints the same +/− line multiset as the pre-sync ledger + diff (1324 + / 230 − for this package), i.e. no patch content lost and + no unledgered divergence gained. + ## adapter-ts-stream-terminal-detector `adapter.ts` — Adds `StreamTerminalDetector`/`ProviderAdapter.isStreamTerminal`. diff --git a/vendor/intx-inference/README.md b/vendor/intx-inference/README.md index f7bc4f25c..6cd0e8f8b 100644 --- a/vendor/intx-inference/README.md +++ b/vendor/intx-inference/README.md @@ -9,6 +9,10 @@ prompt to settled tool calls. Consumed by `@intx/harness`, which composes the reactor with tool runners, directors, and runtime capabilities. +Hosts that only load adapter configuration can import `loadAdapterFactories` +from `@intx/inference/manifest` without loading the reactor and inference +assembly. The `AdapterManifest` schema lives in `@intx/types/adapter-manifest`. + ```ts import { createReactorAssembly, createDefaultDirector } from "@intx/inference"; diff --git a/vendor/intx-inference/package.json b/vendor/intx-inference/package.json index 812f7d27b..983c740aa 100644 --- a/vendor/intx-inference/package.json +++ b/vendor/intx-inference/package.json @@ -11,6 +11,14 @@ "./providers": { "types": "./src/providers/index.ts", "default": "./src/providers/index.ts" + }, + "./default-director": { + "types": "./src/default-director.ts", + "default": "./src/default-director.ts" + }, + "./manifest": { + "types": "./src/manifest.ts", + "default": "./src/manifest.ts" } }, "dependencies": { diff --git a/vendor/intx-inference/src/assembly.ts b/vendor/intx-inference/src/assembly.ts index a118fefa9..ae9ad334e 100644 --- a/vendor/intx-inference/src/assembly.ts +++ b/vendor/intx-inference/src/assembly.ts @@ -23,7 +23,7 @@ import { type ToolDefinition, type ToolResultTransform, type ToolRunner, -} from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; import { createAuditCollector, type AuditCollector } from "./audit-collector"; import { diff --git a/vendor/intx-inference/src/correlation.ts b/vendor/intx-inference/src/correlation.ts index 9ab5d95a5..185d40cb3 100644 --- a/vendor/intx-inference/src/correlation.ts +++ b/vendor/intx-inference/src/correlation.ts @@ -11,6 +11,10 @@ import type { InboundMessage, PendingOperation } from "@intx/types/runtime"; * Validates whether an inbound message is an authentic response to a * registered pending operation. Consumers provide this at reactor construction * time to enforce sender identity and signature checks. + * + * The reactor performs no sender check of its own, so a composition that + * supplies no validator resolves a pending operation for any message bearing a + * registered correlation ID. See MESSAGE.md (Correlation Security). */ export interface CorrelationValidator { /** diff --git a/vendor/intx-inference/src/default-director.ts b/vendor/intx-inference/src/default-director.ts index e9c2dbd57..41aec3a34 100644 --- a/vendor/intx-inference/src/default-director.ts +++ b/vendor/intx-inference/src/default-director.ts @@ -21,17 +21,17 @@ // reply so the problem is visible, and the agent remains alive for retries. import { getLogger } from "@intx/log"; -import { - formatSafetyRatingText, - type ReactorDirector, - type ReactorInboundEvent, - type ReactorState, - type ReactorCapabilities, - type ReactorAction, - type AssistantTurn, - type ToolCall, - type ToolDefinition, +import type { + ReactorDirector, + ReactorInboundEvent, + ReactorState, + ReactorCapabilities, + ReactorAction, + AssistantTurn, + ToolCall, + ToolDefinition, } from "@intx/types/runtime"; +import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; const logger = getLogger(["interchange", "inference", "default-director"]); diff --git a/vendor/intx-inference/src/effort.test.ts b/vendor/intx-inference/src/effort.test.ts new file mode 100644 index 000000000..30bc6e712 --- /dev/null +++ b/vendor/intx-inference/src/effort.test.ts @@ -0,0 +1,326 @@ +import { + afterEach, + beforeEach, + describe, + expect, + mock, + spyOn, + test, +} from "bun:test"; +import { type } from "arktype"; + +import { getLogger } from "@intx/log"; +import type { + ConversationTurn, + LastCycleSource, +} from "@intx/types/runtime-core"; + +import { + budgetSource, + rejectBudgetOverCap, + resetDroppedOptionReports, +} from "./effort"; +import { createAnthropicAdapter } from "./providers/anthropic"; +import { createGoogleGenAIAdapter } from "./providers/google-genai"; +import { createOpenAIAdapter } from "./providers/openai"; + +const logger = getLogger(["interchange", "inference", "adapter"]); + +const messages: ConversationTurn[] = [ + { + role: "user", + content: [{ type: "text", text: "hi" }], + timestamp: 1000, + }, +]; + +const anthropicSource: LastCycleSource = { + sourceId: "test-anthropic", + provider: "anthropic", + model: "test-anthropic-model", +}; + +const openaiSource: LastCycleSource = { + sourceId: "test-openai", + provider: "openai", + model: "test-openai-model", +}; + +const geminiSource: LastCycleSource = { + sourceId: "test-google-genai", + provider: "google-genai", + model: "test-google-genai-model", +}; + +const tool = { + name: "t", + description: "t", + inputSchema: {}, +}; + +beforeEach(resetDroppedOptionReports); + +afterEach(() => { + mock.restore(); +}); + +function warnSpy() { + return spyOn(logger, "warn"); +} + +const WireBody = type("Record"); + +function wireBody(body: string): Record { + const parsed = WireBody(JSON.parse(body)); + if (parsed instanceof type.errors) { + throw new Error(`unexpected request body shape: ${parsed.summary}`); + } + return parsed; +} + +function droppedCall(warn: ReturnType): { + option: unknown; + model: unknown; + reason: unknown; +} { + expect(warn).toHaveBeenCalledTimes(1); + const args = warn.mock.calls[0]; + if (args === undefined) throw new Error("expected a warn call"); + const values: unknown[] = []; + for (const arg of args) values.push(arg); + return { option: values[1], model: values[2], reason: values[3] }; +} + +describe("dropped option warn", () => { + test("classic Anthropic with thinking off drops a named effort", () => { + const warn = warnSpy(); + const req = createAnthropicAdapter(anthropicSource).buildRequest( + messages, + "claude-haiku-4-5-20251001", + { thinking: { enabled: false }, effort: "high" }, + ); + const call = droppedCall(warn); + expect(call.option).toBe('effort "high"'); + expect(call.model).toBe("claude-haiku-4-5-20251001"); + expect(call.reason).toBe( + "classic Anthropic carries effort only as a thinking budget, and thinking is off", + ); + const body = wireBody(req.body); + expect(body).not.toHaveProperty("thinking"); + expect(body).not.toHaveProperty("output_config"); + }); + + test("adaptive Anthropic drops a thinking budget", () => { + const warn = warnSpy(); + createAnthropicAdapter(anthropicSource).buildRequest( + messages, + "claude-opus-5", + { thinking: { enabled: true, budgetTokens: 2048 } }, + ); + const call = droppedCall(warn); + expect(call.option).toBe("thinking.budgetTokens 2048"); + expect(call.reason).toBe( + "adaptive Anthropic takes effort, not a thinking budget", + ); + }); + + test("adaptive Anthropic with thinking off keeps a named effort", () => { + const warn = warnSpy(); + createAnthropicAdapter(anthropicSource).buildRequest( + messages, + "claude-opus-5", + { thinking: { enabled: false }, effort: "low" }, + ); + expect(warn).not.toHaveBeenCalled(); + }); + + test("Gemini with thinking off drops a named effort", () => { + const warn = warnSpy(); + const req = createGoogleGenAIAdapter(geminiSource).buildRequest( + messages, + "gemini-2.5-flash", + { thinking: { enabled: false }, effort: "medium" }, + ); + const call = droppedCall(warn); + expect(call.option).toBe('effort "medium"'); + expect(call.reason).toBe( + "Gemini carries effort only as a thinking setting, and thinking is off", + ); + expect(wireBody(req.body)["generationConfig"]).toEqual({ + thinkingConfig: { thinkingBudget: 0 }, + }); + }); + + test.each([["gemini-2.5-flash-image"], ["gemini-3.1-flash-image"]])( + "the Gemini image model %p drops a named effort and nothing else", + (model) => { + const warn = warnSpy(); + const req = createGoogleGenAIAdapter(geminiSource).buildRequest( + messages, + model, + { effort: "high" }, + ); + const call = droppedCall(warn); + expect(call.option).toBe('effort "high"'); + expect(call.reason).toBe( + "this model has no thinking setting an effort maps to", + ); + expect(wireBody(req.body)).not.toHaveProperty("generationConfig"); + }, + ); + + test("gpt-5.6 with tools drops a named effort", () => { + const warn = warnSpy(); + const req = createOpenAIAdapter(openaiSource).buildRequest( + messages, + "gpt-5.6-sol", + { effort: "high", tools: [tool] }, + ); + const call = droppedCall(warn); + expect(call.option).toBe('effort "high"'); + expect(call.reason).toBe( + "gpt-5.6 Chat Completions tool calls require reasoning_effort none", + ); + expect(wireBody(req.body)["reasoning_effort"]).toBe("none"); + }); + + test("gpt-5.6 with tools and max reports only the tool override", () => { + const warn = warnSpy(); + createOpenAIAdapter(openaiSource).buildRequest(messages, "gpt-5.6-terra", { + effort: "max", + tools: [tool], + }); + const call = droppedCall(warn); + expect(call.reason).toBe( + "gpt-5.6 Chat Completions tool calls require reasoning_effort none", + ); + }); + + test("a Gemini level model with thinking off drops a named effort", () => { + const warn = warnSpy(); + const req = createGoogleGenAIAdapter(geminiSource).buildRequest( + messages, + "gemini-3.6-flash", + { thinking: { enabled: false }, effort: "high" }, + ); + expect(droppedCall(warn).option).toBe('effort "high"'); + expect(wireBody(req.body)["generationConfig"]).toEqual({ + thinkingConfig: { thinkingBudget: -1 }, + }); + }); + + test("the Gemini image models keep today's thinking budget", () => { + for (const model of ["gemini-2.5-flash-image", "gemini-3.1-flash-image"]) { + const req = createGoogleGenAIAdapter(geminiSource).buildRequest( + messages, + model, + { thinking: { enabled: true }, effort: "high" }, + ); + expect(wireBody(req.body)["generationConfig"]).toEqual({ + thinkingConfig: { thinkingBudget: 1024, includeThoughts: true }, + }); + } + }); + + test("gpt-5.6 with tools and no named effort does not warn", () => { + const warn = warnSpy(); + createOpenAIAdapter(openaiSource).buildRequest(messages, "gpt-5.6-sol", { + tools: [tool], + }); + expect(warn).not.toHaveBeenCalled(); + }); + + test("OpenAI max with no maxEffort quirk sends high and warns", () => { + const warn = warnSpy(); + const req = createOpenAIAdapter(openaiSource).buildRequest( + messages, + "gpt-5.5", + { effort: "max" }, + ); + const call = droppedCall(warn); + expect(call.option).toBe('effort "max"'); + expect(call.reason).toBe( + 'no maxEffort quirk on this source, so it sends "high"', + ); + expect(wireBody(req.body)["reasoning_effort"]).toBe("high"); + }); + + test("OpenAI max with a maxEffort quirk does not warn", () => { + const warn = warnSpy(); + createOpenAIAdapter(openaiSource, { maxEffort: "xhigh" }).buildRequest( + messages, + "gpt-5.5", + { effort: "max" }, + ); + expect(warn).not.toHaveBeenCalled(); + }); + + test("a repeated identical drop is reported once", () => { + const warn = warnSpy(); + const adapter = createGoogleGenAIAdapter(geminiSource); + const options = { thinking: { enabled: false }, effort: "low" } as const; + adapter.buildRequest(messages, "gemini-2.5-pro", options); + adapter.buildRequest(messages, "gemini-2.5-pro", options); + expect(warn).toHaveBeenCalledTimes(1); + }); + + test("drops of different values are reported separately", () => { + const warn = warnSpy(); + const adapter = createGoogleGenAIAdapter(geminiSource); + adapter.buildRequest(messages, "gemini-2.5-pro", { + thinking: { enabled: false }, + effort: "low", + }); + adapter.buildRequest(messages, "gemini-2.5-pro", { + thinking: { enabled: false }, + effort: "high", + }); + expect(warn).toHaveBeenCalledTimes(2); + }); + + test("unset effort never warns", () => { + const warn = warnSpy(); + createAnthropicAdapter(anthropicSource).buildRequest( + messages, + "claude-haiku-4-5-20251001", + { thinking: { enabled: true } }, + ); + createGoogleGenAIAdapter(geminiSource).buildRequest( + messages, + "gemini-2.5-flash", + { thinking: { enabled: true } }, + ); + createOpenAIAdapter(openaiSource).buildRequest(messages, "gpt-5.5", {}); + expect(warn).not.toHaveBeenCalled(); + }); +}); + +describe("budgetSource", () => { + test("names the effort when no budget is set", () => { + expect(budgetSource("medium", undefined)).toBe('effort "medium"'); + }); + + test("names the explicit budget when one is set", () => { + expect(budgetSource("low", 5000)).toBe("thinking.budgetTokens"); + }); +}); + +describe("rejectBudgetOverCap", () => { + test("accepts a cap above the budget", () => { + expect(() => + rejectBudgetOverCap("m", 'effort "low"', 1024, 1025), + ).not.toThrow(); + }); + + test.each([[8192], [4096]])( + "rejects a cap of %p under a budget of 8192", + (maxTokens) => { + expect(() => + rejectBudgetOverCap("m", 'effort "medium"', 8192, maxTokens), + ).toThrow( + `effort "medium" on model m needs a thinking budget of 8192 tokens, ` + + `but maxTokens is ${maxTokens}; set maxTokens above 8192.`, + ); + }, + ); +}); diff --git a/vendor/intx-inference/src/effort.ts b/vendor/intx-inference/src/effort.ts new file mode 100644 index 000000000..d7aecc229 --- /dev/null +++ b/vendor/intx-inference/src/effort.ts @@ -0,0 +1,76 @@ +import { getLogger } from "@intx/log"; +import type { InferenceEffort } from "@intx/types/runtime-core"; + +const logger = getLogger(["interchange", "inference", "adapter"]); + +/** + * The thinking budget an effort maps to on a model that takes a token + * budget rather than an effort name (classic Anthropic, Gemini 2.5). The + * low, medium, and high values follow Google's published mapping for its + * OpenAI-compatible endpoint. `max` repeats `high`: 24576 is the largest + * budget every Gemini 2.5 model accepts. + */ +export const EFFORT_THINKING_BUDGET: Readonly> = + { + low: 1024, + medium: 8192, + high: 24576, + max: 24576, + }; + +/** Name the setting a thinking budget came from, for an error message. */ +export function budgetSource( + effort: InferenceEffort, + budgetTokens: number | undefined, +): string { + return budgetTokens === undefined + ? `effort "${effort}"` + : "thinking.budgetTokens"; +} + +/** + * Reject a thinking budget that leaves no room under the output cap. The + * cap bounds thinking and answer together, and the caller owns the cap, so + * the adapter fails the call with a fix rather than raise the cap itself or + * let the provider reject the request. `source` names the setting the + * budget came from (see `budgetSource`). + */ +export function rejectBudgetOverCap( + model: string, + source: string, + budget: number, + maxTokens: number, +): void { + if (maxTokens > budget) return; + throw new Error( + `${source} on model ${model} needs a thinking budget of ` + + `${budget} tokens, but maxTokens is ${maxTokens}; set maxTokens ` + + `above ${budget}.`, + ); +} + +// A step routing to a model that cannot carry an option re-issues the same +// drop on every turn and every retry, so each distinct drop is reported once +// for the life of the process. +const reported = new Set(); + +/** Forget every reported drop so tests can assert on the next one. */ +export function resetDroppedOptionReports(): void { + reported.clear(); +} + +/** + * A per-call inference option the adapter cannot put on the wire for this + * model. The selector accepts the option with no model awareness, so the + * drop is warn-logged rather than a silent no-op. + */ +export function warnDroppedOption( + model: string, + option: string, + reason: string, +): void { + const key = JSON.stringify([model, option, reason]); + if (reported.has(key)) return; + reported.add(key); + logger.warn`Dropping ${option} for model ${model}: ${reason}`; +} diff --git a/vendor/intx-inference/src/errors.test.ts b/vendor/intx-inference/src/errors.test.ts index fdd05226a..e94f5337e 100644 --- a/vendor/intx-inference/src/errors.test.ts +++ b/vendor/intx-inference/src/errors.test.ts @@ -2,6 +2,7 @@ import { describe, test, expect } from "bun:test"; import { classifyHTTPError, classifyNetworkError, + classifyRequestBuildError, classifyAbortError, classifyStreamError, classifyProtocolMismatch, @@ -82,6 +83,16 @@ describe("classifyNetworkError", () => { }); }); +describe("classifyRequestBuildError", () => { + test("Error instance → fatal with message", () => { + const cause = new Error("set maxTokens above 8192."); + const err = classifyRequestBuildError(cause); + expect(err.category).toBe("fatal"); + expect(err.message).toBe("set maxTokens above 8192."); + expect(err.raw).toBe(cause); + }); +}); + describe("classifyAbortError", () => { test("always returns aborted category", () => { const err = classifyAbortError(); diff --git a/vendor/intx-inference/src/errors.ts b/vendor/intx-inference/src/errors.ts index 93cc78c86..003cdbabb 100644 --- a/vendor/intx-inference/src/errors.ts +++ b/vendor/intx-inference/src/errors.ts @@ -43,6 +43,14 @@ export function classifyNetworkError(cause: unknown): InferenceError { return { category: "retryable", message, raw: cause }; } +// An adapter that refuses to build a request has rejected the call's own +// options before any network call, so the identical request fails the same +// way on every attempt. +export function classifyRequestBuildError(cause: unknown): InferenceError { + const message = cause instanceof Error ? cause.message : String(cause); + return { category: "fatal", message, raw: cause }; +} + /** * `origin` mirrors AbortSignal.reason from the send path * (e.g. intercode `user-stop` / `internal-recovery` string literals). diff --git a/vendor/intx-inference/src/harness.test.ts b/vendor/intx-inference/src/harness.test.ts index 10794b93e..085b58d1a 100644 --- a/vendor/intx-inference/src/harness.test.ts +++ b/vendor/intx-inference/src/harness.test.ts @@ -753,6 +753,47 @@ describe("runInference — source quirks reach the adapter registry", () => { }); }); +describe("runInference — adapter request-build failure", () => { + test("is a fatal error and sends nothing", async () => { + let fetched = false; + const registry: AdapterRegistry = { + has: () => true, + resolve: () => ({ + buildRequest: () => { + throw new Error("set maxTokens above 8192."); + }, + parseResponse: () => [], + parseJSONResponse: () => [], + }), + }; + let seq = 0; + const events = await collect( + runInference({ + turns: [userTurn("hi")], + source: SOURCE, + nextSeq: () => ++seq, + deps: { + fetch: () => { + fetched = true; + return Promise.resolve(new Response("")); + }, + scheduler: createDefaultScheduler(), + adapters: registry, + }, + }), + ); + const error = events.find((e) => e.type === "inference.error"); + expect(error?.type === "inference.error" ? error.data.error : null).toEqual( + { + category: "fatal", + message: "set maxTokens above 8192.", + raw: expect.any(Error), + }, + ); + expect(fetched).toBe(false); + }); +}); + describe("runInference — non-streaming JSON responses", () => { const JSON_SOURCE: InferenceSource = { id: "test-json:model-x", diff --git a/vendor/intx-inference/src/harness.ts b/vendor/intx-inference/src/harness.ts index 5077d70a9..0ce5d300c 100644 --- a/vendor/intx-inference/src/harness.ts +++ b/vendor/intx-inference/src/harness.ts @@ -55,6 +55,7 @@ import { injectCredentials } from "./auth"; import { classifyHTTPError, classifyNetworkError, + classifyRequestBuildError, classifyAbortError, classifyStreamError, classifyTimeoutError, @@ -427,7 +428,7 @@ async function* runSingleAttempt( type: "inference.error", seq: nextSeq(), data: { - error: classifyNetworkError(cause), + error: classifyRequestBuildError(cause), partial: snapshotPartial(partial), }, }; diff --git a/vendor/intx-inference/src/index.ts b/vendor/intx-inference/src/index.ts index 2fba587bd..bc6ed9e9f 100644 --- a/vendor/intx-inference/src/index.ts +++ b/vendor/intx-inference/src/index.ts @@ -21,7 +21,6 @@ export type { AdapterRegistry, AdapterFactory, } from "./adapter"; -export { AdapterManifest, AdapterManifestEntry } from "./manifest"; export { CREDENTIAL_SENTINEL, BEARER_CREDENTIAL_SENTINEL } from "./auth"; export { classifyHTTPError, diff --git a/vendor/intx-inference/src/manifest.test.ts b/vendor/intx-inference/src/manifest.test.ts index 5b67a1966..b5251ae10 100644 --- a/vendor/intx-inference/src/manifest.test.ts +++ b/vendor/intx-inference/src/manifest.test.ts @@ -1,10 +1,7 @@ import { describe, test, expect } from "bun:test"; import type { AdapterFactory, ProviderAdapter } from "./adapter"; -import { - loadAdapterFactories, - type AdapterManifest, - type ModuleImporter, -} from "./manifest"; +import type { AdapterManifest } from "@intx/types"; +import { loadAdapterFactories, type ModuleImporter } from "./manifest"; function createStubAdapter(): ProviderAdapter { return { diff --git a/vendor/intx-inference/src/manifest.ts b/vendor/intx-inference/src/manifest.ts index 00666fc06..4fab2bad3 100644 --- a/vendor/intx-inference/src/manifest.ts +++ b/vendor/intx-inference/src/manifest.ts @@ -1,21 +1,5 @@ -import { type } from "arktype"; import type { AdapterFactory } from "./adapter"; - -// Describes one custom adapter: the provider identifier it serves, the module -// specifier to import, and the named export within that module to use as its -// factory. Specifiers are operator-config-only and resolve to arbitrary code -// via `import()`, so they must originate solely from trusted operator -// configuration, never from tenant or deploy data. The shape is validated at -// every deserialization boundary; the value is trusted operator input. -export const AdapterManifestEntry = type({ - provider: "string", - specifier: "string", - export: "string", -}); -export type AdapterManifestEntry = typeof AdapterManifestEntry.infer; - -export const AdapterManifest = AdapterManifestEntry.array(); -export type AdapterManifest = typeof AdapterManifest.infer; +import type { AdapterManifest } from "@intx/types"; // Imports a module by specifier. The production importer is `import()`; tests // inject a synthetic importer so they can exercise the loader without fixture diff --git a/vendor/intx-inference/src/providers/anthropic.test.ts b/vendor/intx-inference/src/providers/anthropic.test.ts index b96bcce39..0b1e32adc 100644 --- a/vendor/intx-inference/src/providers/anthropic.test.ts +++ b/vendor/intx-inference/src/providers/anthropic.test.ts @@ -635,7 +635,7 @@ describe("Anthropic parser — citations_delta to inference.citation", () => { test("citation missing cited_text throws ProtocolMismatchError", () => { // CitationBlock.citedText is required ("Both providers emit it; // required for inspection and for fallback offset reconstruction" - // — runtime.ts). Surfacing a missing wire field as a thrown error + // — runtime-core.ts). Surfacing a missing wire field as a thrown error // is the load-bearing alternative to coalescing to an empty // string and silently emitting a content-free citation. const adapter = createAnthropicAdapter(TEST_SOURCE); diff --git a/vendor/intx-inference/src/providers/anthropic.ts b/vendor/intx-inference/src/providers/anthropic.ts index 7888aa844..1b1f4694d 100644 --- a/vendor/intx-inference/src/providers/anthropic.ts +++ b/vendor/intx-inference/src/providers/anthropic.ts @@ -1,21 +1,25 @@ import { type } from "arktype"; +import type { InferenceEvent } from "@intx/types/inference-events"; import type { ConversationTurn, ContentBlock, - InferenceEvent, InferenceOptions, LastCycleSource, MediaSource, PartialMessage, TokenUsage, -} from "@intx/types/runtime"; -import { - CitationBlock as CitationBlockType, - formatSafetyRatingText, -} from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; +import { CitationBlock as CitationBlockType } from "@intx/types/runtime-core"; +import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; import type { ProviderAdapter, BuiltRequest } from "../adapter"; import { CREDENTIAL_SENTINEL } from "../auth"; +import { + EFFORT_THINKING_BUDGET, + budgetSource, + rejectBudgetOverCap, + warnDroppedOption, +} from "../effort"; import { ProtocolMismatchError } from "../errors"; import { decodeToolName, @@ -47,13 +51,16 @@ export const ADAPTIVE_THINKING_MODELS: ReadonlySet = new Set([ "claude-sonnet-4-6", ]); -// The effort this adapter sends on the adaptive-thinking wire in production. -// "high" is the Anthropic API default. The discovery capture rig deliberately -// sends "max" instead: only "max" reliably elicits a thinking block to capture, -// so the production default and the capture value are an intentional pair, not -// drift. ADAPTIVE_THINKING_MODELS above must match across the two layers; the -// effort values, by contrast, are meant to differ. A guard test in the -// discovery package checks both effort values. +// The effort this adapter sends on the adaptive-thinking wire when thinking +// is on and the call names no effort. "high" is the Anthropic API default for +// every model in ADAPTIVE_THINKING_MODELS; a model whose API default differs +// (some default to "medium") must not join that set without revisiting this. +// The discovery capture rig deliberately sends "max" instead: only "max" +// reliably elicits a thinking block to capture, so the production default and +// the capture value are an intentional pair, not drift. The model set above +// (ADAPTIVE_THINKING_MODELS) must match across the two layers; the effort +// values, by contrast, are meant to differ. A guard test in the discovery +// package checks both effort values. export const ADAPTIVE_THINKING_EFFORT = "high"; // --------------------------------------------------------------------------- @@ -81,9 +88,10 @@ function buildRequest( ? options.systemPrompt : systemText || undefined; + const maxTokens = options.maxTokens ?? 4096; const body: Record = { model, - max_tokens: options.maxTokens ?? 4096, + max_tokens: maxTokens, messages: conversationMessages.map((msg, i) => { // Place a cache breakpoint on the last user message so all prior // turns are cached on the next request. @@ -105,20 +113,7 @@ function buildRequest( ]; } - if (options.thinking?.enabled) { - // Adaptive models reject the classic budget_tokens shape with - // invalid_request_error and require thinking:{type:"adaptive"} - // plus output_config.effort. - if (ADAPTIVE_THINKING_MODELS.has(model)) { - body["thinking"] = { type: "adaptive" }; - body["output_config"] = { effort: ADAPTIVE_THINKING_EFFORT }; - } else { - body["thinking"] = { - type: "enabled", - budget_tokens: options.thinking.budgetTokens ?? 1024, - }; - } - } + applyReasoning(body, model, options, maxTokens); if (options.tools !== undefined && options.tools.length > 0) { const tools: Record[] = options.tools.map((t) => ({ @@ -148,6 +143,70 @@ function buildRequest( }; } +// Effort and thinking.budgetTokens are two spellings of one intent, so an +// effort with no thinking setting turns thinking on. Adaptive models take the +// effort name on output_config.effort, with or without thinking; classic +// models take it as a thinking budget. An explicit thinking.enabled: false +// wins over an effort on a classic model, where effort has no other field. +function applyReasoning( + body: Record, + model: string, + options: InferenceOptions, + maxTokens: number, +): void { + const { effort, thinking } = options; + const effortSet = effort !== undefined; + const thinkingOn = thinking?.enabled ?? effortSet; + + // Adaptive models reject the classic budget_tokens shape with + // invalid_request_error and require thinking:{type:"adaptive"}. + if (ADAPTIVE_THINKING_MODELS.has(model)) { + if (thinkingOn) { + body["thinking"] = { type: "adaptive" }; + } + if (effort !== undefined) { + body["output_config"] = { effort }; + } else if (thinkingOn) { + body["output_config"] = { effort: ADAPTIVE_THINKING_EFFORT }; + } + if (thinking?.budgetTokens !== undefined) { + warnDroppedOption( + model, + `thinking.budgetTokens ${thinking.budgetTokens}`, + "adaptive Anthropic takes effort, not a thinking budget", + ); + } + return; + } + + if (!thinkingOn) { + if (effort !== undefined) { + warnDroppedOption( + model, + `effort "${effort}"`, + "classic Anthropic carries effort only as a thinking budget, and thinking is off", + ); + } + return; + } + + let budget = 1024; + if (thinking?.budgetTokens !== undefined) { + budget = thinking.budgetTokens; + } else if (effort !== undefined) { + budget = EFFORT_THINKING_BUDGET[effort]; + } + if (effort !== undefined) { + rejectBudgetOverCap( + model, + budgetSource(effort, thinking?.budgetTokens), + budget, + maxTokens, + ); + } + body["thinking"] = { type: "enabled", budget_tokens: budget }; +} + // Anthropic's Messages API has no native structured-outputs surface. // `text` is the default and a no-op. `json` and `json-schema` raise // here at the marshaling boundary rather than silently dropping the diff --git a/vendor/intx-inference/src/providers/google-genai.ts b/vendor/intx-inference/src/providers/google-genai.ts index af61f329f..ba50e33fd 100644 --- a/vendor/intx-inference/src/providers/google-genai.ts +++ b/vendor/intx-inference/src/providers/google-genai.ts @@ -1,20 +1,26 @@ import { type } from "arktype"; +import type { InferenceEvent } from "@intx/types/inference-events"; import type { CodeExecutionRequestBlock, CodeExecutionResultBlock, ConversationTurn, ContentBlock, - InferenceEvent, InferenceOptions, LastCycleSource, MediaSource, PartialMessage, TokenUsage, -} from "@intx/types/runtime"; -import { formatSafetyRatingText } from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; +import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; import type { ProviderAdapter, BuiltRequest } from "../adapter"; import { CREDENTIAL_SENTINEL } from "../auth"; +import { + EFFORT_THINKING_BUDGET, + budgetSource, + rejectBudgetOverCap, + warnDroppedOption, +} from "../effort"; import { ProtocolMismatchError } from "../errors"; import { decodeToolName, @@ -37,6 +43,25 @@ const THINKING_MANDATORY_MODELS: ReadonlySet = new Set([ "gemini-3.6-flash", ]); +// Gemini 3 models that take thinkingConfig.thinkingLevel, whose low, medium, +// and high levels match the effort names. Google rejects a request that +// carries both thinkingLevel and thinkingBudget with HTTP 400, so a level +// model sends exactly one of them. +const THINKING_LEVEL_MODELS: ReadonlySet = new Set([ + "gemini-3-flash-preview", + "gemini-3.1-pro-preview", + "gemini-3.5-flash", + "gemini-3.6-flash", +]); + +// Image models with no thinking setting an effort name can map to: +// gemini-3.1-flash-image accepts only the "minimal" and "high" levels, and +// Google documents no thinking support on gemini-2.5-flash-image. +const NO_EFFORT_MODELS: ReadonlySet = new Set([ + "gemini-2.5-flash-image", + "gemini-3.1-flash-image", +]); + // Dynamic thinking budget sentinel: the model decides how much to // think. Used when suppressing thought parts on thinking-mandatory // models that reject a zero budget. @@ -502,6 +527,74 @@ function tryParseJSONObject(text: string): Record | null { // generationConfig // --------------------------------------------------------------------------- +// thinking.enabled === true -> ask Gemini to surface thought parts, with a +// level on a level model that names an +// effort, or a budget (from budgetTokens, +// else the effort's budget, else 1024) +// thinking.enabled === false -> suppress thoughts: budget 0 when the +// model allows it, or dynamic (-1) for +// thinking-mandatory models that reject +// a zero budget with HTTP 400 +// thinking absent -> an effort turns thinking on as above; +// with no effort, omit thinkingConfig and +// Gemini uses the model's default +function buildThinkingConfig( + model: string, + options: InferenceOptions, +): Record | undefined { + const { thinking } = options; + const effort = NO_EFFORT_MODELS.has(model) ? undefined : options.effort; + if (options.effort !== undefined && effort === undefined) { + warnDroppedOption( + model, + `effort "${options.effort}"`, + "this model has no thinking setting an effort maps to", + ); + } + const effortSet = effort !== undefined; + if (thinking === undefined && !effortSet) return undefined; + + const thinkingOn = thinking?.enabled ?? effortSet; + if (!thinkingOn) { + if (effort !== undefined) { + warnDroppedOption( + model, + `effort "${effort}"`, + "Gemini carries effort only as a thinking setting, and thinking is off", + ); + } + return { thinkingBudget: minimalThinkingBudget(model) }; + } + + if ( + effort !== undefined && + thinking?.budgetTokens === undefined && + THINKING_LEVEL_MODELS.has(model) + ) { + // "high" is the top level, so "max" maps onto it. + const thinkingLevel = effort === "max" ? "high" : effort; + return { thinkingLevel, includeThoughts: true }; + } + + let budget = 1024; + if (thinking?.budgetTokens !== undefined) { + budget = thinking.budgetTokens; + } else if (effort !== undefined) { + budget = EFFORT_THINKING_BUDGET[effort]; + } + // maxOutputTokens counts thought tokens and cuts the response off, so a + // budget at or above it leaves no room for the answer. + if (effort !== undefined && options.maxTokens !== undefined) { + rejectBudgetOverCap( + model, + budgetSource(effort, thinking?.budgetTokens), + budget, + options.maxTokens, + ); + } + return { thinkingBudget: budget, includeThoughts: true }; +} + function buildGenerationConfig( model: string, options: InferenceOptions, @@ -515,26 +608,9 @@ function buildGenerationConfig( config["temperature"] = options.temperature; } - // thinking.enabled === true -> include a budget (default 1024) and - // ask Gemini to surface thought parts - // thinking.enabled === false -> suppress thoughts: budget 0 when the - // model allows it, or dynamic (-1) for - // thinking-mandatory models that reject - // a zero budget with HTTP 400 - // thinking absent -> omit thinkingConfig entirely; Gemini - // uses the model's default - if (options.thinking !== undefined) { - if (options.thinking.enabled) { - const thinkingBudget = options.thinking.budgetTokens ?? 1024; - config["thinkingConfig"] = { - thinkingBudget, - includeThoughts: true, - }; - } else { - config["thinkingConfig"] = { - thinkingBudget: minimalThinkingBudget(model), - }; - } + const thinkingConfig = buildThinkingConfig(model, options); + if (thinkingConfig !== undefined) { + config["thinkingConfig"] = thinkingConfig; } if ( diff --git a/vendor/intx-inference/src/providers/index.test.ts b/vendor/intx-inference/src/providers/index.test.ts index 6c650d245..0ce776e71 100644 --- a/vendor/intx-inference/src/providers/index.test.ts +++ b/vendor/intx-inference/src/providers/index.test.ts @@ -1,7 +1,8 @@ import { describe, test, expect } from "bun:test"; import type { LastCycleSource } from "@intx/types/runtime"; import type { AdapterFactory, ProviderAdapter } from "../adapter"; -import type { AdapterManifest, ModuleImporter } from "../manifest"; +import type { ModuleImporter } from "../manifest"; +import type { AdapterManifest } from "@intx/types"; import { createBuiltinRegistry, loadAdapterRegistry } from "./index"; const builtinProviders = [ diff --git a/vendor/intx-inference/src/providers/index.ts b/vendor/intx-inference/src/providers/index.ts index c615b1b29..e2e5a0dbb 100644 --- a/vendor/intx-inference/src/providers/index.ts +++ b/vendor/intx-inference/src/providers/index.ts @@ -2,7 +2,8 @@ import { createAdapterRegistry } from "../adapter"; import type { AdapterFactory, AdapterRegistry } from "../adapter"; import { createDependencies, type Dependencies } from "../harness"; import { loadAdapterFactories } from "../manifest"; -import type { AdapterManifest, ModuleImporter } from "../manifest"; +import type { ModuleImporter } from "../manifest"; +import type { AdapterManifest } from "@intx/types"; import { createAnthropicAdapter } from "./anthropic"; import { createGoogleGenAIAdapter } from "./google-genai"; import { createOpenAIAdapter } from "./openai"; diff --git a/vendor/intx-inference/src/providers/openai.ts b/vendor/intx-inference/src/providers/openai.ts index 3974b99d2..cb0666722 100644 --- a/vendor/intx-inference/src/providers/openai.ts +++ b/vendor/intx-inference/src/providers/openai.ts @@ -1,17 +1,19 @@ import { type } from "arktype"; +import type { InferenceEvent } from "@intx/types/inference-events"; import type { ConversationTurn, ContentBlock, - InferenceEvent, + InferenceEffort, InferenceOptions, LastCycleSource, PartialMessage, TokenUsage, -} from "@intx/types/runtime"; -import { formatSafetyRatingText } from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; +import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; import type { ProviderAdapter, BuiltRequest } from "../adapter"; import { BEARER_CREDENTIAL_SENTINEL } from "../auth"; +import { warnDroppedOption } from "../effort"; import { ProtocolMismatchError } from "../errors"; import { decodeToolName, @@ -46,6 +48,11 @@ export const OpenAIQuirks = type({ // through the same adapter (e.g. OpenCode Zen) still take `max_tokens`. // Defaults to `max_tokens` so every existing deployment is unchanged. "maxTokensField?": "'max_tokens' | 'max_completion_tokens'", + // The reasoning_effort value that an effort of "max" sends. Models differ: + // gpt-5.2 through gpt-5.5 top out at "xhigh", gpt-5.6-sol accepts "max", + // and older models accept neither. Absent, "max" sends "high", the highest + // value every reasoning model accepts, and the adapter warns. + "maxEffort?": "'xhigh' | 'max'", // Reject unknown keys so a mistyped quirk name fails loudly at construction // rather than being silently ignored and running with default behavior. "+": "reject", @@ -65,12 +72,31 @@ type ResolvedOpenAIQuirks = { forceAssistantReasoningContent: boolean; reasoningFieldNames: readonly ReasoningField[]; maxTokensField: "max_tokens" | "max_completion_tokens"; + maxEffort: "xhigh" | "max" | null; }; // --------------------------------------------------------------------------- // Request building // --------------------------------------------------------------------------- +// low, medium, and high are reasoning_effort values every OpenAI reasoning +// model accepts, so they pass as given. "max" names the highest level, which +// is a per-model value the deployment supplies through the maxEffort quirk. +function toReasoningEffort( + model: string, + effort: InferenceEffort, + maxEffort: "xhigh" | "max" | null, +): string { + if (effort !== "max") return effort; + if (maxEffort !== null) return maxEffort; + warnDroppedOption( + model, + `effort "max"`, + 'no maxEffort quirk on this source, so it sends "high"', + ); + return "high"; +} + function buildRequest( messages: ConversationTurn[], model: string, @@ -92,6 +118,25 @@ function buildRequest( body["temperature"] = options.temperature; } + const hasTools = options.tools !== undefined && options.tools.length > 0; + // gpt-5.6 Chat Completions rejects function tools unless + // reasoning_effort is explicitly "none" (reasoned tool use is on + // the Responses API). Keep this list aligned with the discovery + // protocol builder's TOOL_CALL_REASONING_NONE_MODELS set. + const forceNoReasoning = + hasTools && + (model === "gpt-5.6-sol" || + model === "gpt-5.6-terra" || + model === "gpt-5.6-luna"); + + if (options.effort !== undefined && !forceNoReasoning) { + body["reasoning_effort"] = toReasoningEffort( + model, + options.effort, + quirks.maxEffort, + ); + } + if (options.tools !== undefined && options.tools.length > 0) { body["tools"] = options.tools.map((t) => ({ type: "function", @@ -101,17 +146,19 @@ function buildRequest( parameters: t.inputSchema, }, })); - // gpt-5.6 Chat Completions rejects function tools unless - // reasoning_effort is explicitly "none" (reasoned tool use is on - // the Responses API). Keep this list aligned with the discovery - // protocol builder's TOOL_CALL_REASONING_NONE_MODELS set. - if ( - model === "gpt-5.6-sol" || - model === "gpt-5.6-terra" || - model === "gpt-5.6-luna" - ) { - body["reasoning_effort"] = "none"; + } + + // Set after the tools, in the key order of the discovery protocol + // builder's captured bodies. + if (forceNoReasoning) { + if (options.effort !== undefined) { + warnDroppedOption( + model, + `effort "${options.effort}"`, + "gpt-5.6 Chat Completions tool calls require reasoning_effort none", + ); } + body["reasoning_effort"] = "none"; } if (options.systemPrompt) { @@ -1079,6 +1126,7 @@ export function createOpenAIAdapter( reasoningFieldNames: parsedQuirks.reasoningFieldNames ?? DEFAULT_REASONING_FIELDS, maxTokensField: parsedQuirks.maxTokensField ?? "max_tokens", + maxEffort: parsedQuirks.maxEffort ?? null, }; // Per-request indexer state. Adapter instances are created per diff --git a/vendor/intx-inference/src/reactor.test.ts b/vendor/intx-inference/src/reactor.test.ts index 296568a55..e78b41e5e 100644 --- a/vendor/intx-inference/src/reactor.test.ts +++ b/vendor/intx-inference/src/reactor.test.ts @@ -4,12 +4,13 @@ import { validateActions } from "./actions"; import { createAuthzExtension } from "./authz-extension"; import { createGateManager } from "./gates"; import { createCorrelationRegistry } from "./correlation"; -import { createReactor } from "./reactor"; +import { createReactor, mergeInferenceOptions } from "./reactor"; import { createDefaultDependencies } from "./providers"; import { createDefaultDirector } from "./default-director"; import { assertWellFormedToolSequence } from "./turns"; import { createInboundMessage } from "@intx/mime"; import { waitUntil } from "@intx/types/testing"; +import { PerCallInferenceOptions } from "@intx/types/runtime"; import type { ReactorDirector, @@ -6580,23 +6581,27 @@ describe("createReactor — source failover", () => { expect(getEvent(events, "inference.done").data.source.sourceId).toBe("s1"); }); - test("does not fail over on a source-invariant error", async () => { - const { reactor, events, waitFor, attemptedSourceIds } = multiSourceReactor( - { - sourceIds: ["s0", "s1"], - resultFor: () => ({ category: "context_overflow", message: "too big" }), - }, - ); - reactor.start(); - reactor.deliver(makeInboundMessage()); - await waitFor("reactor.done"); - - // context_overflow aborts the cycle; s1 is never tried. - expect(attemptedSourceIds).toEqual(["s0"]); - expect(getEvent(events, "inference.error").data.error.category).toBe( - "context_overflow", - ); - }); + // `fatal` covers a provider's 400 and an adapter's refusal to build the + // request alike. + test.each([["context_overflow"], ["fatal"]] as const)( + "does not fail over on a source-invariant %p error", + async (category) => { + const { reactor, events, waitFor, attemptedSourceIds } = + multiSourceReactor({ + sourceIds: ["s0", "s1"], + resultFor: () => ({ category, message: "refused" }), + }); + reactor.start(); + reactor.deliver(makeInboundMessage()); + await waitFor("reactor.done"); + + // The error aborts the cycle; s1 is never tried. + expect(attemptedSourceIds).toEqual(["s0"]); + expect(getEvent(events, "inference.error").data.error.category).toBe( + category, + ); + }, + ); test("surfaces the last error when every source is exhausted", async () => { const { reactor, events, waitFor, attemptedSourceIds } = multiSourceReactor( @@ -6687,3 +6692,804 @@ describe("createReactor — prompt well-formedness tripwire", () => { expect(inferenceRan).toBe(false); }); }); + +// --------------------------------------------------------------------------- +// Per-send inference options +// --------------------------------------------------------------------------- + +describe("mergeInferenceOptions", () => { + test("a director-named option wins over the per-send value", () => { + expect( + mergeInferenceOptions( + { maxTokens: 500, effort: "low" }, + { effort: "high", systemPrompt: "s" }, + ), + ).toEqual({ maxTokens: 500, effort: "high", systemPrompt: "s" }); + }); + + test("a director thinking setting does not clear a per-send effort", () => { + expect( + mergeInferenceOptions( + { maxTokens: 500, effort: "high" }, + { thinking: { enabled: false } }, + ), + ).toEqual({ + maxTokens: 500, + effort: "high", + thinking: { enabled: false }, + }); + }); + + test("a director effort does not clear a per-send thinking setting", () => { + expect( + mergeInferenceOptions( + { thinking: { enabled: true, budgetTokens: 2048 } }, + { effort: "low" }, + ), + ).toEqual({ + thinking: { enabled: true, budgetTokens: 2048 }, + effort: "low", + }); + }); + + test("a director thinking object replaces the per-send thinking object", () => { + expect( + mergeInferenceOptions( + { thinking: { enabled: true, budgetTokens: 2048 } }, + { thinking: { enabled: true } }, + ), + ).toEqual({ + thinking: { enabled: true }, + }); + }); + + test("absent per-send options pass the director options through", () => { + const director = { systemPrompt: "s" }; + expect(mergeInferenceOptions(undefined, director)).toBe(director); + }); +}); + +describe("createReactor — per-send inference options", () => { + test("apply beneath the director for their own message run only", async () => { + const seen: (InferenceHarnessOptions["inferenceOptions"] | undefined)[] = + []; + const inferenceRunner = async function* (opts: InferenceHarnessOptions) { + seen.push(opts.inferenceOptions); + yield { + type: "inference.done" as const, + seq: opts.nextSeq(), + data: { + turn: { + role: "assistant" as const, + content: [{ type: "text" as const, text: "ok" }], + model: "test-model", + timestamp: 1000, + }, + usage: emptyUsage(), + source: TEST_SOURCE, + }, + }; + }; + const { reactor, events } = createTestReactor({ + inferenceRunner, + director: directorFromTable( + { + "message.received": (_e, _s, caps) => + caps.infer({ temperature: 0.9, systemPrompt: "director" }), + "inference.done": (_e, _s, caps) => caps.wait(), + }, + "wait", + ), + }); + const inferenceDone = (n: number) => + waitUntil( + () => events.filter((e) => e.type === "inference.done").length >= n, + ); + + reactor.start(); + reactor.deliver(makeInboundMessage(), { + inference: { temperature: 0.1, maxTokens: 500, effort: "low" }, + }); + await inferenceDone(1); + reactor.deliver(makeInboundMessage()); + await inferenceDone(2); + + expect(seen).toEqual([ + { + temperature: 0.9, + maxTokens: 500, + effort: "low", + systemPrompt: "director", + }, + { temperature: 0.9, systemPrompt: "director" }, + ]); + reactor.abort("admin_kill"); + }); + + test("a correlated resume of a parked run adopts the resume's options", async () => { + // The run stays open across the park (suspend does not close the message + // run), so this exercises the gate-resume path distinct from the + // uncorrelated-message case above: options must reach the re-infer that + // follows the approval even though currentMessageRunId never went null. + const seen: (InferenceHarnessOptions["inferenceOptions"] | undefined)[] = + []; + const inferenceRunner = async function* (opts: InferenceHarnessOptions) { + seen.push(opts.inferenceOptions); + yield { + type: "inference.done" as const, + seq: opts.nextSeq(), + data: { + turn: + seen.length === 1 + ? suspendToolCallTurn + : { + role: "assistant" as const, + content: [{ type: "text" as const, text: "resumed" }], + model: "test-model", + timestamp: 1000, + }, + usage: emptyUsage(), + source: TEST_SOURCE, + }, + }; + }; + const askExtension = createAuthzExtension({ + authorize: async () => ({ + effect: "ask" as const, + matchingGrants: [], + resolvedBy: null, + }), + approvalTimeoutMs: 60_000, + }); + + const { reactor, events, waitFor } = createTestReactor({ + inferenceRunner, + toolRunner: makeToolRunner(async (call) => ({ + callId: call.id, + content: "charged", + })), + beforeToolExtensions: [askExtension], + director: directorFromTable( + { + "message.received": (_e, _s, caps) => caps.infer(), + "inference.done": (_e, _s, caps) => + caps.executeTools([ + { id: "call-ask", name: "charge_card", arguments: {} }, + ]), + "resume.execute_tools": (e, _s, caps) => + caps.executeTools(e.calls, false, true), + "tool.done": (_e, _s, caps) => caps.infer(), + }, + "wait", + ), + }); + + reactor.start(); + reactor.deliver(makeInboundMessage(), { inference: { temperature: 0.1 } }); + + const blocked = await waitFor("reactor.gate.blocked"); + if (blocked.type !== "reactor.gate.blocked") throw new Error("unreachable"); + const correlationId = blocked.data.correlationId; + if (correlationId === undefined) throw new Error("expected correlationId"); + + reactor.deliver(makeApprovalMessage(correlationId), { + inference: { temperature: 0.42 }, + }); + await waitUntil( + () => events.filter((e) => e.type === "inference.done").length >= 2, + ); + + expect(seen).toEqual([{ temperature: 0.1 }, { temperature: 0.42 }]); + reactor.abort("admin_kill"); + }); + + test("a stale pending-marker correlation does not overwrite the open run's options", async () => { + // S1: infer then a tool returns a pendingMarker for C1; the director + // replies and the run closes, but C1 stays registered with no live gate. + // S2: a new send opens a run, infers, then parks on an ask-rail gate. + // S3: a stale C1 response must not adopt its inference into the still-open + // S2 slot; resuming S2's gate must still see S2's options. + const C1 = "corr-stale-c1"; + const seen: (InferenceHarnessOptions["inferenceOptions"] | undefined)[] = + []; + let capturing = false; + let infers = 0; + const sendMessageTurn: AssistantTurn = { + role: "assistant", + content: [ + { + type: "tool_call", + id: "tc-async", + name: "send_message", + arguments: {}, + }, + ], + model: "test-model", + timestamp: 1000, + }; + const inferenceRunner = async function* (opts: InferenceHarnessOptions) { + if (capturing) seen.push(opts.inferenceOptions); + const turn = + infers === 0 + ? sendMessageTurn + : infers === 1 + ? suspendToolCallTurn + : { + role: "assistant" as const, + content: [{ type: "text" as const, text: "resumed" }], + model: "test-model", + timestamp: 1000, + }; + yield { + type: "inference.done" as const, + seq: opts.nextSeq(), + data: { + turn, + usage: emptyUsage(), + source: TEST_SOURCE, + }, + }; + }; + const askExtension = createAuthzExtension({ + authorize: async (resource) => ({ + effect: + resource === "tool:charge_card" + ? ("ask" as const) + : ("allow" as const), + matchingGrants: [], + resolvedBy: null, + }), + approvalTimeoutMs: 60_000, + }); + + const { reactor, events, waitFor } = createTestReactor({ + inferenceRunner, + toolRunner: { + async run(call) { + if (call.name === "send_message") { + return { + callId: call.id, + content: "sent", + pendingMarker: { + status: "pending" as const, + correlationId: C1, + }, + }; + } + return { callId: call.id, content: "charged" }; + }, + }, + beforeToolExtensions: [askExtension], + director: directorFromTable( + { + "message.received": (_e, _s, caps) => caps.infer(), + "inference.done": (_e, _s, caps) => { + infers += 1; + if (infers === 1) { + return caps.executeTools([ + { id: "tc-async", name: "send_message", arguments: {} }, + ]); + } + if (infers === 2) { + return caps.executeTools([ + { id: "call-ask", name: "charge_card", arguments: {} }, + ]); + } + return caps.wait(); + }, + "resume.execute_tools": (e, _s, caps) => + caps.executeTools(e.calls, false, true), + "tool.done": (e, _s, caps) => { + if (e.result.callId === "tc-async") return caps.reply("acked"); + return caps.infer(); + }, + }, + "wait", + ), + }); + + reactor.start(); + reactor.deliver(makeInboundMessage()); + await waitFor("connector.reply"); + capturing = true; + + reactor.deliver(makeInboundMessage(), { inference: { temperature: 0.2 } }); + await waitUntil( + () => events.filter((e) => e.type === "inference.done").length >= 2, + ); + const blocked = await waitFor("reactor.gate.blocked"); + if (blocked.type !== "reactor.gate.blocked") throw new Error("unreachable"); + const openCorrelationId = blocked.data.correlationId; + if (openCorrelationId === undefined) { + throw new Error("expected correlationId"); + } + + reactor.deliver(makeInboundMessage(C1), { + inference: { temperature: 0.9 }, + }); + await waitFor("message.correlated"); + + reactor.deliver(makeApprovalMessage(openCorrelationId)); + await waitUntil( + () => events.filter((e) => e.type === "inference.done").length >= 3, + ); + + expect(seen).toEqual([{ temperature: 0.2 }, { temperature: 0.2 }]); + reactor.abort("admin_kill"); + }); + + test("a live gate from a closed run does not overwrite the open run's options", async () => { + // S1 parks on G1 (ask-rail gate) and stays open. S2, uncorrelated, closes + // run1 and opens run2, which itself parks on G2. G1 stays registered + // (closeMessageRun never touches gates), so a late correlated response + // for G1 must not adopt its options into the still-open run2 slot. + const askExtension = createAuthzExtension({ + authorize: async () => ({ + effect: "ask" as const, + matchingGrants: [], + resolvedBy: null, + }), + approvalTimeoutMs: 60_000, + }); + const seen: (InferenceHarnessOptions["inferenceOptions"] | undefined)[] = + []; + // Each infer parks on a fresh tool_call id -- the shared history spans + // both runs, so reusing "call-ask" across them would trip the + // duplicate-tool_call-id well-formedness check. + let lastCallId = ""; + const inferenceRunner = async function* (opts: InferenceHarnessOptions) { + seen.push(opts.inferenceOptions); + lastCallId = `call-ask-${seen.length}`; + yield { + type: "inference.done" as const, + seq: opts.nextSeq(), + data: { + turn: { + role: "assistant" as const, + content: [ + { + type: "tool_call" as const, + id: lastCallId, + name: "charge_card", + arguments: {}, + }, + ], + model: "test-model", + timestamp: 1000, + }, + usage: emptyUsage(), + source: TEST_SOURCE, + }, + }; + }; + + const { reactor, events, waitFor } = createTestReactor({ + inferenceRunner, + toolRunner: makeToolRunner(async (call) => ({ + callId: call.id, + content: "charged", + })), + beforeToolExtensions: [askExtension], + director: directorFromTable( + { + "message.received": (_e, _s, caps) => caps.infer(), + "inference.done": (_e, _s, caps) => + caps.executeTools([ + { id: lastCallId, name: "charge_card", arguments: {} }, + ]), + "resume.execute_tools": (e, _s, caps) => + caps.executeTools(e.calls, false, true), + "tool.done": (_e, _s, caps) => caps.infer(), + }, + "wait", + ), + }); + + reactor.start(); + reactor.deliver(makeInboundMessage(), { inference: { temperature: 0.1 } }); + + const blocked1 = await waitFor("reactor.gate.blocked"); + if (blocked1.type !== "reactor.gate.blocked") { + throw new Error("unreachable"); + } + const correlationId1 = blocked1.data.correlationId; + if (correlationId1 === undefined) throw new Error("expected correlationId"); + + reactor.deliver(makeInboundMessage(), { inference: { temperature: 0.2 } }); + await waitUntil( + () => events.filter((e) => e.type === "reactor.gate.blocked").length >= 2, + ); + + reactor.deliver(makeApprovalMessage(correlationId1), { + inference: { temperature: 0.9 }, + }); + await waitUntil(() => seen.length >= 3); + + expect(seen).toEqual([ + { temperature: 0.1 }, + { temperature: 0.2 }, + { temperature: 0.2 }, + ]); + reactor.abort("admin_kill"); + }); + + test("a resume of a director-chosen gate adopts its options", async () => { + // The tool's pending marker registers the operation under + // `pending-`, and the director then suspends under its own + // gate id with the same correlation id. The resume belongs to the open + // run, so its options must be adopted even though the gate id the pending + // operation records is not the gate that is live. + const CORR_ID = "corr-director-gate"; + const seen: (InferenceHarnessOptions["inferenceOptions"] | undefined)[] = + []; + let infers = 0; + const inferenceRunner = async function* (opts: InferenceHarnessOptions) { + seen.push(opts.inferenceOptions); + infers += 1; + yield { + type: "inference.done" as const, + seq: opts.nextSeq(), + data: { + turn: { + role: "assistant" as const, + content: + infers === 1 + ? [ + { + type: "tool_call" as const, + id: "call-send", + name: "send_msg", + arguments: {}, + }, + ] + : [{ type: "text" as const, text: "done" }], + model: "test-model", + timestamp: 1000, + }, + usage: emptyUsage(), + source: TEST_SOURCE, + }, + }; + }; + + const { reactor, waitFor } = createTestReactor({ + inferenceRunner, + toolRunner: makeToolRunner(async (call) => ({ + callId: call.id, + content: "sent", + pendingMarker: { status: "pending" as const, correlationId: CORR_ID }, + })), + director: directorFromTable({ + "message.received": (_e, _s, caps) => caps.infer(), + "inference.done": (_e, _s, caps) => + infers === 1 + ? caps.executeTools([ + { id: "call-send", name: "send_msg", arguments: {} }, + ]) + : caps.done(), + "tool.done": (_e, _s, caps) => + caps.suspend({ + type: "message_response", + gateId: "director-chosen-gate", + timeoutMs: 5000, + correlationId: CORR_ID, + }), + "reactor.gate.cleared": (_e, _s, caps) => caps.infer(), + }), + }); + + reactor.start(); + reactor.deliver(makeInboundMessage(), { inference: { temperature: 0.1 } }); + await waitFor("reactor.gate.blocked"); + + reactor.deliver(makeInboundMessage(CORR_ID), { + inference: { temperature: 0.9 }, + }); + await waitFor("reactor.done"); + + expect(seen).toEqual([{ temperature: 0.1 }, { temperature: 0.9 }]); + }); + + test("a rehydrated gate does not overwrite the open run's options", async () => { + // Same hazard as the previous test, but the stale gate comes from + // rehydration (a restart) rather than from a run this process closed: + // rehydrateGates must not mark it as belonging to any open run either. + const correlationId1 = "rehydrated-corr-1"; + // The pending op's suspended call must have a matching tool_call turn + // already in history, exactly as it would after the original suspend + // (the call landed in history before the park; only the resolution + // carries over the restart) -- otherwise resuming it and re-inferring + // trips the well-formedness check on an orphaned tool_result. + const cell: PersistedCell = { + turns: [ + { + role: "assistant", + content: [ + { + type: "tool_call", + id: "call-ask-rehydrated", + name: "charge_card", + arguments: {}, + }, + ], + model: "test-model", + timestamp: 500, + }, + ], + pendingOperations: [ + { + correlationId: correlationId1, + kind: "approval", + registeredAt: Date.now(), + gateId: `pending-${correlationId1}`, + timeoutAt: Date.now() + 60_000, + suspendedCall: { + id: "call-ask-rehydrated", + name: "charge_card", + arguments: {}, + }, + }, + ], + tokenUsage: emptyUsage(), + }; + const askExtension = createAuthzExtension({ + authorize: async () => ({ + effect: "ask" as const, + matchingGrants: [], + resolvedBy: null, + }), + approvalTimeoutMs: 60_000, + }); + const seen: (InferenceHarnessOptions["inferenceOptions"] | undefined)[] = + []; + let lastCallId = ""; + const inferenceRunner = async function* (opts: InferenceHarnessOptions) { + seen.push(opts.inferenceOptions); + lastCallId = `call-ask-${seen.length}`; + yield { + type: "inference.done" as const, + seq: opts.nextSeq(), + data: { + turn: { + role: "assistant" as const, + content: [ + { + type: "tool_call" as const, + id: lastCallId, + name: "charge_card", + arguments: {}, + }, + ], + model: "test-model", + timestamp: 1000, + }, + usage: emptyUsage(), + source: TEST_SOURCE, + }, + }; + }; + + const { reactor, waitFor } = createTestReactor({ + contextStore: makePersistingContextStore(cell), + inferenceRunner, + toolRunner: makeToolRunner(async (call) => ({ + callId: call.id, + content: "charged", + })), + beforeToolExtensions: [askExtension], + director: directorFromTable( + { + "message.received": (_e, _s, caps) => caps.infer(), + "inference.done": (_e, _s, caps) => + caps.executeTools([ + { id: lastCallId, name: "charge_card", arguments: {} }, + ]), + "resume.execute_tools": (e, _s, caps) => + caps.executeTools(e.calls, false, true), + "tool.done": (_e, _s, caps) => caps.infer(), + }, + "wait", + ), + }); + + reactor.start(); + await waitFor("reactor.start"); + + reactor.deliver(makeInboundMessage(), { inference: { temperature: 0.2 } }); + await waitFor("reactor.gate.blocked"); + + reactor.deliver(makeApprovalMessage(correlationId1), { + inference: { temperature: 0.9 }, + }); + await waitUntil(() => seen.length >= 2); + + expect(seen).toEqual([{ temperature: 0.2 }, { temperature: 0.2 }]); + reactor.abort("admin_kill"); + }); + + test("a correlated resume with no run open does not leak options into the next", async () => { + // Rehydrated gates resume without ever opening a message run (only + // message.received opens one), so the options a resume adopts must + // still be discarded when its cycle closes: a later resume that sets + // none infers with none. + const correlationIds = ["rehydrated-corr-a", "rehydrated-corr-b"]; + const cell: PersistedCell = { + turns: [ + { + role: "assistant", + content: correlationIds.map((id) => ({ + type: "tool_call" as const, + id: `call-${id}`, + name: "charge_card", + arguments: {}, + })), + model: "test-model", + timestamp: 500, + }, + ], + pendingOperations: correlationIds.map((id) => ({ + correlationId: id, + kind: "approval" as const, + registeredAt: Date.now(), + gateId: `pending-${id}`, + timeoutAt: Date.now() + 60_000, + suspendedCall: { id: `call-${id}`, name: "charge_card", arguments: {} }, + })), + tokenUsage: emptyUsage(), + }; + const seen: (InferenceHarnessOptions["inferenceOptions"] | undefined)[] = + []; + const inferenceRunner = async function* (opts: InferenceHarnessOptions) { + seen.push(opts.inferenceOptions); + yield { + type: "inference.done" as const, + seq: opts.nextSeq(), + data: { + turn: { + role: "assistant" as const, + content: [{ type: "text" as const, text: "ok" }], + model: "test-model", + timestamp: 1000, + }, + usage: emptyUsage(), + source: TEST_SOURCE, + }, + }; + }; + + const { reactor, waitFor } = createTestReactor({ + contextStore: makePersistingContextStore(cell), + inferenceRunner, + toolRunner: makeToolRunner(async (call) => ({ + callId: call.id, + content: "charged", + })), + director: directorFromTable( + { + "resume.execute_tools": (e, _s, caps) => + caps.executeTools(e.calls, false, true), + "tool.done": (_e, _s, caps) => caps.infer(), + }, + "wait", + ), + }); + + reactor.start(); + await waitFor("reactor.start"); + + reactor.deliver(makeApprovalMessage("rehydrated-corr-a"), { + inference: { temperature: 0.9 }, + }); + await waitUntil(() => seen.length >= 1); + reactor.deliver(makeApprovalMessage("rehydrated-corr-b")); + await waitUntil(() => seen.length >= 2); + + expect(seen).toEqual([{ temperature: 0.9 }, undefined]); + reactor.abort("admin_kill"); + }); + + test("systemPrompt, tools, and providerOptions on deliver do not reach the harness", async () => { + const seen: (InferenceHarnessOptions["inferenceOptions"] | undefined)[] = + []; + const inferenceRunner = async function* (opts: InferenceHarnessOptions) { + seen.push(opts.inferenceOptions); + yield { + type: "inference.done" as const, + seq: opts.nextSeq(), + data: { + turn: { + role: "assistant" as const, + content: [{ type: "text" as const, text: "ok" }], + model: "test-model", + timestamp: 1000, + }, + usage: emptyUsage(), + source: TEST_SOURCE, + }, + }; + }; + const { reactor } = createTestReactor({ + inferenceRunner, + director: directorFromTable( + { + "message.received": (_e, _s, caps) => + caps.infer({ temperature: 0.9 }), + "inference.done": (_e, _s, caps) => caps.wait(), + }, + "wait", + ), + }); + + reactor.start(); + // A wider bag is assignable to PerCallInferenceOptions; deliver must + // still drop the keys that displace the deployed agent definition. + const inference = { + maxTokens: 100, + temperature: 0.1, + systemPrompt: "sneak", + tools: [], + providerOptions: { foo: 1 }, + }; + reactor.deliver(makeInboundMessage(), { inference }); + await waitUntil(() => seen.length >= 1); + + expect(seen).toEqual([{ temperature: 0.9, maxTokens: 100 }]); + reactor.abort("admin_kill"); + }); + + test("deliver copies every key PerCallInferenceOptions declares", async () => { + const seen: (InferenceHarnessOptions["inferenceOptions"] | undefined)[] = + []; + const inferenceRunner = async function* (opts: InferenceHarnessOptions) { + seen.push(opts.inferenceOptions); + yield { + type: "inference.done" as const, + seq: opts.nextSeq(), + data: { + turn: { + role: "assistant" as const, + content: [{ type: "text" as const, text: "ok" }], + model: "test-model", + timestamp: 1000, + }, + usage: emptyUsage(), + source: TEST_SOURCE, + }, + }; + }; + const { reactor } = createTestReactor({ + inferenceRunner, + director: directorFromTable( + { + "message.received": (_e, _s, caps) => caps.infer(), + "inference.done": (_e, _s, caps) => caps.wait(), + }, + "wait", + ), + }); + + reactor.start(); + // Names every declared key alongside one the type does not declare. + // A key added to the type without an entry here fails the key-set + // comparison below, so the copy can never silently fall behind it. + const inference = { + maxTokens: 100, + temperature: 0.1, + thinking: { enabled: true, budgetTokens: 2048, sneak: true }, + effort: "high" as const, + systemPrompt: "sneak", + }; + reactor.deliver(makeInboundMessage(), { inference }); + await waitUntil(() => seen.length >= 1); + + const declared = PerCallInferenceOptions.props.map((p) => p.key).sort(); + expect(Object.keys(seen[0] ?? {}).sort()).toEqual(declared); + expect(seen[0]).toEqual({ + maxTokens: 100, + temperature: 0.1, + thinking: { enabled: true, budgetTokens: 2048 }, + effort: "high", + }); + reactor.abort("admin_kill"); + }); +}); diff --git a/vendor/intx-inference/src/reactor.ts b/vendor/intx-inference/src/reactor.ts index 50c769e9c..2f9af77c6 100644 --- a/vendor/intx-inference/src/reactor.ts +++ b/vendor/intx-inference/src/reactor.ts @@ -41,7 +41,8 @@ import type { } from "@intx/types/runtime"; import { getLogger } from "@intx/log"; -import { ApprovalDecision, signalKindToGateType } from "@intx/types"; +import { ApprovalDecision, signalKindToGateType } from "@intx/types/signals"; +import { PerCallInferenceOptions } from "@intx/types/runtime-core"; import type { CredentialMaterialResolver } from "@intx/types"; import { canonicalJsonStringify } from "@intx/types/wire-definition-hash"; import { type } from "arktype"; @@ -89,6 +90,42 @@ export type ExtendedInferenceOptions = InferenceOptions & { ephemeralTurns?: ConversationTurn[]; }; +/** + * Layer director-named options over per-send ones; a director key wins over + * the per-send value for that same key. The merge is shallow: a director that + * names `thinking` replaces the whole object, so nested per-send fields such + * as `budgetTokens` are dropped rather than merged in. + */ +export function mergeInferenceOptions( + perSend: InferenceOptions | undefined, + director: InferenceOptions | undefined, +): InferenceOptions | undefined { + if (perSend === undefined) return director; + return { ...perSend, ...director }; +} + +/** + * Copy only the keys a send may set. A structurally wider bag still + * type-checks as `PerCallInferenceOptions`; without this copy, + * `systemPrompt`, `tools`, and `providerOptions` would enter the + * message-run slot and displace the deployed agent definition. The key + * set is the arktype declaration itself, so a key added there is copied + * without a matching edit here. The declaration also checks the values, + * so an out-of-range option is rejected here rather than at the provider. + */ +const StripToPerCallInference = + PerCallInferenceOptions.onDeepUndeclaredKey("delete"); + +function perCallInference( + options: PerCallInferenceOptions, +): PerCallInferenceOptions { + const stripped = StripToPerCallInference(options); + if (stripped instanceof type.errors) { + throw new Error(`Invalid per-call inference options: ${stripped.summary}`); + } + return stripped; +} + function buildHarnessOpts( turns: ConversationTurn[], source: InferenceSource, @@ -173,11 +210,20 @@ export type ReactorConfig = { doomLoopThreshold?: number | false; }; +export type DeliverOptions = { + inference?: PerCallInferenceOptions; +}; + export type Reactor = { /** Begin processing. Emits reactor.start. Must be called exactly once. */ start(): void; - /** Inject an inbound message into the reactor. */ - deliver(message: InboundMessage): void; + /** + * Inject an inbound message into the reactor. `opts.inference` applies to + * every inference call of the message run the delivery opens, or of the + * parked run it resumes, beneath any option the director names; it is + * never retained past that run. + */ + deliver(message: InboundMessage, opts?: DeliverOptions): void; /** Initiate graceful shutdown with a reason. */ abort(reason: AbortReason): void; }; @@ -438,6 +484,26 @@ export function createReactor(config: ReactorConfig): Reactor { // produces unambiguous start/end pairs downstream. let currentMessageRunId: string | null = null; let currentMessageId: string | null = null; + // Per-send inference options, keyed by the delivered message until it is + // dequeued (or correlated), then held for the message run it drives. + const deliveredInference = new WeakMap< + InboundMessage, + PerCallInferenceOptions + >(); + let messageRunInference: PerCallInferenceOptions | undefined; + // Correlation ids the currently open message run suspended on, cleared + // whenever that run closes. This is what tells a late correlated response + // apart from one that correlates to an operation some earlier, already- + // closed run left registered: `gates` itself never expires an entry on run + // close (only on clear/timeout), so a stale gate can outlive the run that + // opened it and still answer `gates.has`. The key is the correlation id, + // not the gate id, because a director may suspend under its own gate id + // while the tool's pending marker registered the operation under + // `pending-`; the correlation id is what both share, and it + // is also how `tryCorrelate` finds the gate to resume. Rehydrated gates + // (`rehydrateGates`) are deliberately not added here -- they predate any + // run this process has opened, so they can never be "this run's own". + const openRunCorrelationIds = new Set(); // Doom-loop detection state, scoped to the current message run. Each executed // tool-call turn is reduced to a batch signature; consecutive identical @@ -450,20 +516,25 @@ export function createReactor(config: ReactorConfig): Reactor { let toolBatchRepeatCount = 0; let lastToolBatchNames: string[] = []; - function openMessageRun(messageId: string): void { + function openMessageRun(messageId: string | undefined): void { currentMessageRunId = crypto.randomUUID(); - currentMessageId = messageId; + currentMessageId = messageId ?? null; lastToolBatchSignature = null; toolBatchRepeatCount = 0; lastToolBatchNames = []; + const data: { + messageId?: string; + messageRunId: string; + receivedAt: number; + } = { + messageRunId: currentMessageRunId, + receivedAt: Date.now(), + }; + if (messageId !== undefined) data.messageId = messageId; emit({ type: "message.run.started", seq: nextSeq(), - data: { - messageId, - messageRunId: currentMessageRunId, - receivedAt: Date.now(), - }, + data, }); } @@ -471,17 +542,21 @@ export function createReactor(config: ReactorConfig): Reactor { status: "completed" | "failed", error?: { message: string; kind?: string }, ): void { - if (currentMessageRunId === null || currentMessageId === null) return; + messageRunInference = undefined; + openRunCorrelationIds.clear(); + // Gated on the run id alone: a message that named no id still has to + // close its bracket, or the run stays open for the rest of the session. + if (currentMessageRunId === null) return; const data: { messageRunId: string; - messageId: string; + messageId?: string; status: "completed" | "failed"; error?: { message: string; kind?: string }; } = { messageRunId: currentMessageRunId, - messageId: currentMessageId, status, }; + if (currentMessageId !== null) data.messageId = currentMessageId; if (error !== undefined) data.error = error; emit({ type: "message.run.ended", seq: nextSeq(), data }); currentMessageRunId = null; @@ -856,7 +931,7 @@ export function createReactor(config: ReactorConfig): Reactor { const harnessOpts = buildHarnessOpts( prompt, config.source, - options, + mergeInferenceOptions(messageRunInference, options), signal, nextSeq, config.readMaterial, @@ -1034,9 +1109,6 @@ export function createReactor(config: ReactorConfig): Reactor { kind: "approval", registeredAt: Date.now(), gateId, - ...(marker.expectedFrom !== undefined - ? { expectedFrom: marker.expectedFrom } - : {}), }; correlations.register(op); stateManager.addPendingOperation(op); @@ -1420,6 +1492,12 @@ export function createReactor(config: ReactorConfig): Reactor { correlationId, onGateCleared, ); + // Mark this correlation as belonging to the currently open run so a late + // correlated response can tell it apart from one an earlier, already- + // closed run left registered (see `openRunCorrelationIds`). + if (currentMessageRunId !== null && correlationId !== undefined) { + openRunCorrelationIds.add(correlationId); + } if (stateManager !== null) { stateManager.setGatesSnapshot(gates.snapshot()); @@ -1477,7 +1555,10 @@ export function createReactor(config: ReactorConfig): Reactor { // effective timeout the director-suspend fallback uses — rather than a // silent zero. This does not run through `suspendOnGate`: rehydration must // not re-emit `reactor.gate.blocked` (the suspension already happened before - // the restart) and must not commit (nothing changed). + // the restart) and must not commit (nothing changed). It also must not add + // to `openRunCorrelationIds`: a rehydrated gate predates any run this + // process has opened, so it can never be mistaken for the currently open + // run's own suspension. function rehydrateGates(ops: PendingOperation[]): void { for (const op of ops) { const timeoutMs = @@ -1530,9 +1611,11 @@ export function createReactor(config: ReactorConfig): Reactor { // Append inbound messages to conversation history so the provider sees them. // Each dequeued message.received opens a fresh per-message run bracket. - // If a prior bracket is still open (defensive — should not occur given - // the dequeue priority that drains cycle events before new messages), - // close it as completed first so the new bracket starts cleanly. + // An uncorrelated message can arrive while an earlier run is still + // parked on a gate (that run left no way to reject or supersede it), so + // finding a prior bracket still open here is a routine path, not a + // defensive fallback: close it as completed first so the new bracket + // starts cleanly. if (event.type === "message.received") { if (stateManager !== null) { const msg = createInboundTurn(event.message); @@ -1544,6 +1627,8 @@ export function createReactor(config: ReactorConfig): Reactor { closeMessageRun("completed"); } openMessageRun(event.message.headers.messageId); + messageRunInference = deliveredInference.get(event.message); + deliveredInference.delete(event.message); } // A parked approval that ended without running its tool (rejected or @@ -1899,6 +1984,28 @@ export function createReactor(config: ReactorConfig): Reactor { function processDelivery(message: InboundMessage): void { void (async () => { + // A correlated resume continues the parked run rather than opening a + // new one, and the loop may re-infer before tryCorrelate settles, so + // the resume's options are adopted up front whenever this message's + // correlation ID is one the currently open run suspended on + // (`openRunCorrelationIds`). `gates.has` alone is not enough: a gate an + // earlier, already-closed run left registered is still live there, and + // a late correlated response for it must not overwrite the run open + // now. An uncorrelated message either finds no run open yet (harmless + // to set now — opening its own run in the message.received handler + // sets the same value) or arrives while an unrelated run is still + // open, in which case it must not touch that run's options and + // instead overwrites them when its own run opens. + const inference = deliveredInference.get(message); + if (inference !== undefined) { + const correlationId = message.headers.interchangeCorrelationId; + const correlatesToOpenRun = + correlationId !== undefined && + openRunCorrelationIds.has(correlationId); + if (correlatesToOpenRun || currentMessageRunId === null) { + messageRunInference = inference; + } + } // Locally patched — see vendor/intx-inference/PATCHES.md#reactor-ts-atomic-approval-acceptance let disposition: CorrelationDisposition; try { @@ -1933,8 +2040,11 @@ export function createReactor(config: ReactorConfig): Reactor { })(); } - function deliver(message: InboundMessage): void { + function deliver(message: InboundMessage, opts?: DeliverOptions): void { if (done) return; + if (opts?.inference !== undefined) { + deliveredInference.set(message, perCallInference(opts.inference)); + } if (startupDeliveries !== null) { startupDeliveries.push(message); return; diff --git a/vendor/intx-inference/src/transform.ts b/vendor/intx-inference/src/transform.ts index 3b820413e..b02d89674 100644 --- a/vendor/intx-inference/src/transform.ts +++ b/vendor/intx-inference/src/transform.ts @@ -9,11 +9,8 @@ // buildRequest paths also apply provider-specific history fixes. The // originating model is tracked per-message, not per-conversation. -import { - formatSafetyRatingText, - type ConversationTurn, - type ContentBlock, -} from "@intx/types/runtime"; +import type { ConversationTurn, ContentBlock } from "@intx/types/runtime-core"; +import { formatSafetyRatingText } from "@intx/types/format-safety-rating"; export type TransformOptions = { targetModel: string; diff --git a/vendor/intx-inference/src/turns.test.ts b/vendor/intx-inference/src/turns.test.ts index a4c730f93..ba97649e1 100644 --- a/vendor/intx-inference/src/turns.test.ts +++ b/vendor/intx-inference/src/turns.test.ts @@ -20,7 +20,7 @@ function msg(opts: { return { ref: { uid: 1, mailbox: "INBOX" }, headers: { - from: opts.from ?? "", + ...(opts.from !== undefined ? { from: opts.from } : {}), to: [], date: "2026-01-01T00:00:00.000Z", messageId: "", @@ -112,6 +112,17 @@ describe("createInboundTurn", () => { }); }); + test("omits the From line when the message carries no originator", () => { + // An absent originator is a real state: a message can arrive with no + // usable `From`. The envelope must then make no claim about the sender + // rather than naming one, so the model is never told an address the + // system chose. + const turn = createInboundTurn(msg({ content: "body", subject: "Hi" })); + expect(turn).toMatchObject({ + content: [{ type: "text", text: "[Subject: Hi]\n\nbody" }], + }); + }); + test("degrades an unmappable attachment type to a text marker, not a throw", () => { // Inbound attachments come from remote senders and are not allowlist // filtered; an unmappable type must surface visibly rather than throw diff --git a/vendor/intx-inference/src/turns.ts b/vendor/intx-inference/src/turns.ts index e0a08a62c..a35779dd7 100644 --- a/vendor/intx-inference/src/turns.ts +++ b/vendor/intx-inference/src/turns.ts @@ -8,7 +8,8 @@ import type { ToolCall, ToolResult, } from "@intx/types/runtime"; -import { attachmentCategory, base64Encode } from "@intx/types"; +import { attachmentCategory } from "@intx/types/attachments"; +import { base64Encode } from "@intx/types/base64"; import { getLogger } from "@intx/log"; const logger = getLogger(["interchange", "inference", "turns"]); @@ -77,7 +78,7 @@ export function createInboundTurn( if (content.length > 0) { const { from, subject } = message.headers; const envelope: string[] = []; - if (from.length > 0) envelope.push(`[From: ${from}]`); + if (from !== undefined && from.length > 0) envelope.push(`[From: ${from}]`); if (subject !== undefined && subject.length > 0) { envelope.push(`[Subject: ${subject}]`); } diff --git a/vendor/intx-mailbox/package.json b/vendor/intx-mailbox/package.json index 794b2eed3..4a29a589e 100644 --- a/vendor/intx-mailbox/package.json +++ b/vendor/intx-mailbox/package.json @@ -11,6 +11,7 @@ }, "dependencies": { "@intx/crypto": "0.3.0", + "@intx/log": "workspace:*", "@intx/mime": "workspace:*", "@intx/types": "workspace:*", "arktype": "catalog:" diff --git a/vendor/intx-mailbox/src/compose.test.ts b/vendor/intx-mailbox/src/compose.test.ts new file mode 100644 index 000000000..b6dccc75d --- /dev/null +++ b/vendor/intx-mailbox/src/compose.test.ts @@ -0,0 +1,72 @@ +import { describe, expect, test } from "bun:test"; +import { createEd25519Crypto, generateKeyPair } from "@intx/crypto"; +import { extractAttachments } from "@intx/mime"; + +import { composeOutbound } from "./compose"; +import { verifyMimeSignature } from "./verify-signature"; + +async function senderCrypto() { + return createEd25519Crypto(await generateKeyPair()); +} + +describe("composeOutbound", () => { + test("a conversation attachment is base64 inside the signed part", async () => { + const crypto = await senderCrypto(); + const data = new Uint8Array([0x89, 0x50, 0x4e, 0x47, 0x0d, 0x0a, 0xff]); + const composed = await composeOutbound( + "alpha@test.interchange", + { + to: "beta@test.interchange", + type: "conversation.message", + content: "with image", + attachments: [{ name: "shot.png", contentType: "image/png", data }], + }, + crypto, + ); + + const wire = new TextDecoder().decode(composed.rawBytes); + const encoded = Buffer.from(data).toString("base64"); + const at = wire.indexOf(encoded); + expect(at).toBeGreaterThan(0); + expect(wire).toContain("Content-Transfer-Encoding: base64"); + + expect(extractAttachments(composed.rawBytes)).toEqual([ + { name: "shot.png", contentType: "image/png", data, part: "1.2" }, + ]); + expect( + await verifyMimeSignature(composed.rawBytes, crypto.getPublicKey()), + ).toBe("valid"); + + // The signature covers the encoded attachment. One flipped octet in that + // base64 is a different signed part. + const tampered = new Uint8Array(composed.rawBytes); + const octet = tampered[at]; + if (octet === undefined) throw new Error("attachment bytes missing"); + tampered[at] = octet ^ 0x01; + expect(await verifyMimeSignature(tampered, crypto.getPublicKey())).toBe( + "invalid", + ); + }); + + test("a structured message cannot carry attachments", async () => { + const crypto = await senderCrypto(); + await expect( + composeOutbound( + "alpha@test.interchange", + { + to: "beta@test.interchange", + type: "offering.request", + payload: { offeringId: "code-review" }, + attachments: [ + { + name: "shot.png", + contentType: "image/png", + data: new Uint8Array([1]), + }, + ], + }, + crypto, + ), + ).rejects.toThrow("Structured messages must not carry attachments"); + }); +}); diff --git a/vendor/intx-mailbox/src/compose.ts b/vendor/intx-mailbox/src/compose.ts new file mode 100644 index 000000000..11ea2a510 --- /dev/null +++ b/vendor/intx-mailbox/src/compose.ts @@ -0,0 +1,193 @@ +// Composition of an `OutboundMessage` into signed RFC 2822 bytes. +// +// Signing and the stored envelope are properties of the message, not of the +// mailbox that delivers it. This module owns that half: the body checks, the +// signed content, the header set, and the References chain. The caller keeps +// delivery. +// +// It sits beside `verifyMimeSignature` deliberately: composing a signed +// message and verifying one are peers over the same byte contract. + +import { + assembleMessage, + assembleSignedContent, + createDetachedSignatureFromProvider, + generateMessageId, + isMessageId, + type ConversationContent, + type MessageHeaders as MimeMessageHeaders, + type StructuredContent, +} from "@intx/mime"; +import type { CryptoProvider, OutboundMessage } from "@intx/types/runtime"; +import { isConversationType } from "@intx/types/runtime"; + +import type { StoredEnvelope } from "./mailbox"; + +/** + * The composed message: the wire bytes, the id stamped into them, the + * pre-parsed envelope a mailbox stores beside them, and the address lists the + * caller routes on. `recipients` is the deduplicated union of `to` and `cc` -- + * the set a transport delivers to -- while `to` and `cc` are kept apart because + * they are separate header fields and separate audit facts. + */ +export type ComposedMessage = { + messageId: string; + rawBytes: Uint8Array; + envelope: StoredEnvelope; + to: string[]; + cc: string[]; + recipients: string[]; +}; + +/** + * Assemble the wire `References` chain for an outbound message. + * + * When the caller supplies a full ancestry (`existingReferences` -- the + * parent's References plus the parent's Message-Id, built by the threaded + * reply path), that chain is used and `inReplyTo` is appended only when it is + * not already the tail. Otherwise the chain is derived from `inReplyTo` alone. + * + * The caller-supplied chain is filtered to RFC 2822 message identifiers: + * inbound mail can carry a headerless-derived (sha256) or otherwise malformed + * Message-Id that is a valid claim-check key but not a valid `` + * identifier, and such a value must not leak into a `References` header. The + * `inReplyTo` value is appended without filtering, matching the behavior for a + * bare reply. + */ +function buildReferences( + inReplyTo: string | undefined, + existingReferences: string[] | undefined, +): string[] | undefined { + const refs = (existingReferences ?? []).filter(isMessageId); + if (inReplyTo === undefined) return refs.length > 0 ? refs : undefined; + if (!refs.includes(inReplyTo)) { + return [...refs, inReplyTo]; + } + return refs; +} + +/** + * Split a `to`/`cc` field that accepts either one address or several into a + * list. Returns a fresh array so a caller cannot alias the message's own. + */ +function addressList(field: string | string[] | undefined): string[] { + if (field === undefined) return []; + return Array.isArray(field) ? [...field] : [field]; +} + +/** + * Compose an `OutboundMessage` into signed RFC 2822 bytes. + * + * Throws when the message contradicts itself -- a conversation type carrying a + * structured payload, a structured type carrying text content or attachments, + * a blank `inReplyTo`, or no recipient at all. These are the conditions that + * make the message unsendable whatever the transport, so they are refused here + * rather than once per transport. + */ +export async function composeOutbound( + senderAddress: string, + message: OutboundMessage, + senderCrypto: CryptoProvider, +): Promise { + const to = addressList(message.to); + if (to.length === 0) { + throw new Error("OutboundMessage must have at least one recipient"); + } + const cc = addressList(message.cc); + const recipients = [...new Set([...to, ...cc])]; + + const isConversation = isConversationType(message.type); + if (isConversation && message.payload !== undefined) { + throw new Error( + "Conversation messages must not carry a structured payload", + ); + } + if (!isConversation && message.content !== undefined) { + throw new Error("Structured messages must not carry a text content field"); + } + if (!isConversation && message.attachments !== undefined) { + throw new Error("Structured messages must not carry attachments"); + } + + // RFC 5322 section 3.6.4 defines `In-Reply-To` as `1*msg-id`, so a blank + // value names nothing rather than naming a shorter parent. + if ( + message.inReplyTo !== undefined && + message.inReplyTo.trim().length === 0 + ) { + throw new Error( + "OutboundMessage inReplyTo, when provided, must name a message identifier", + ); + } + + const messageId = generateMessageId(senderAddress); + const now = new Date(); + + let content: ConversationContent | StructuredContent; + if (isConversation) { + const conversation: ConversationContent = { + kind: "conversation", + text: message.content ?? "", + }; + if (message.attachments !== undefined) { + conversation.attachments = message.attachments; + } + content = conversation; + } else { + const structured: StructuredContent = { + kind: "structured", + json: { type: message.type, version: "1", body: message.payload ?? {} }, + }; + if (message.summary !== undefined) structured.summary = message.summary; + content = structured; + } + + const signedContentBytes = assembleSignedContent(content); + const signatureBytes = await createDetachedSignatureFromProvider( + signedContentBytes, + senderCrypto, + ); + + const references = buildReferences(message.inReplyTo, message.references); + + const mimeHeaders: MimeMessageHeaders = { + from: senderAddress, + to, + cc: cc.length > 0 ? cc : undefined, + date: now, + messageId, + subject: message.subject, + inReplyTo: message.inReplyTo, + references, + mimeVersion: "1.0", + interchangeType: message.type, + interchangeCorrelationId: message.correlationId, + interchangeTenantId: message.tenantId, + interchangeAgentId: undefined, + interchangeSessionId: message.sessionId, + interchangeOfferingId: undefined, + interchangeSchemaVersion: undefined, + traceparent: undefined, + tracestate: undefined, + }; + + const rawBytes = assembleMessage( + mimeHeaders, + signedContentBytes, + signatureBytes, + ); + + const envelope: StoredEnvelope = { + messageId, + from: senderAddress, + to, + subject: message.subject ?? "", + date: now, + inReplyTo: message.inReplyTo, + references: references ?? [], + interchangeType: message.type, + interchangeCorrelationId: message.correlationId, + }; + + return { messageId, rawBytes, envelope, to, cc, recipients }; +} diff --git a/vendor/intx-mailbox/src/fetch.test.ts b/vendor/intx-mailbox/src/fetch.test.ts index 224ca44e3..d9106ed75 100644 --- a/vendor/intx-mailbox/src/fetch.test.ts +++ b/vendor/intx-mailbox/src/fetch.test.ts @@ -1,8 +1,18 @@ -import { describe, test, expect } from "bun:test"; +import { describe, test, expect, beforeEach, afterEach } from "bun:test"; +import { generateKeyPair, createEd25519Crypto } from "@intx/crypto"; +import { + assembleMessage, + assembleSignedContent, + createDetachedSignatureFromProvider, + decodeMail, + parseHeaderSection, + type MessageHeaders, +} from "@intx/mime"; import type { CryptoProvider } from "@intx/types/runtime"; import { createInMemoryMailboxStore, executeSearch, + executeThread, fetchHeaders, fetchStructure, fetchPart, @@ -44,6 +54,61 @@ function rawMessage(subject: string, body: string): Uint8Array { ); } +function signedHeaders(): MessageHeaders { + return { + from: "alice@x", + to: ["bob@y"], + cc: undefined, + date: new Date("2026-01-01T00:00:00Z"), + messageId: "<1@x>", + subject: "Hello", + inReplyTo: undefined, + references: undefined, + mimeVersion: "1.0", + interchangeType: "conversation.message", + interchangeCorrelationId: undefined, + interchangeTenantId: undefined, + interchangeAgentId: undefined, + interchangeSessionId: undefined, + interchangeOfferingId: undefined, + interchangeSchemaVersion: undefined, + traceparent: undefined, + tracestate: undefined, + }; +} + +/** A `multipart/signed` conversation message validly signed by `crypto`. */ +async function signedRawMessage(crypto: CryptoProvider): Promise { + const content = assembleSignedContent({ + kind: "conversation", + text: "signed body", + }); + const signature = await createDetachedSignatureFromProvider(content, crypto); + return assembleMessage(signedHeaders(), content, signature); +} + +/** + * Drop the `From` line from a message's header section, leaving every other + * byte alone. The detached signature covers the signed-content part inside the + * body, not the outer headers, so a message stripped this way still verifies + * -- which is what makes it a fixture for a message that carries no + * originator and would otherwise have a valid signature. + */ +function stripFromHeader(raw: Uint8Array): Uint8Array { + const { headerEnd } = parseHeaderSection(raw); + const kept = new TextDecoder() + .decode(raw.subarray(0, headerEnd)) + .split("\r\n") + .filter((line) => !/^from:/i.test(line)) + .join("\r\n"); + const keptBytes = encoder.encode(kept); + const rest = raw.subarray(headerEnd); + const out = new Uint8Array(keptBytes.length + rest.length); + out.set(keptBytes, 0); + out.set(rest, keptBytes.length); + return out; +} + /** A two-part multipart/mixed message; part 1 is a text/plain body. */ function rawMultipart(body: string): Uint8Array { const boundary = "b0undary"; @@ -66,6 +131,179 @@ function rawMultipart(body: string): Uint8Array { ); } +/** Conversation mail whose text/plain attachment is quoted-printable `caf=E9`. */ +function rawQuotedPrintableAttachment(): Uint8Array { + return encoder.encode( + [ + "From: alice@x", + "To: bob@y", + "Subject: QP", + "Message-ID: ", + "Date: Thu, 01 Jan 2026 00:00:00 +0000", + "Interchange-Type: conversation.message", + `Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg=pgp-sha512; boundary="outer"`, + "", + "--outer", + `Content-Type: multipart/mixed; boundary="inner"`, + "", + "--inner", + "Content-Type: text/plain; charset=utf-8", + "Content-Transfer-Encoding: 7bit", + "", + "see attached", + "--inner", + "Content-Type: text/plain", + "Content-Transfer-Encoding: quoted-printable", + `Content-Disposition: attachment; filename="cafe.txt"`, + "", + "caf=E9", + "--inner--", + "--outer", + "Content-Type: application/pgp-signature", + "", + "FAKE", + "--outer--", + "", + ].join("\r\n"), + ); +} + +/** + * A multipart/mixed message whose single part declares `encoding`. The part + * declares a bare `text/plain` with no parameters, so the comparison is the + * transfer-encoding relabel and not parameter stripping. Both readers report + * type/subtype. + */ +function rawMultipartWithEncoding(encoding: string, body: string): Uint8Array { + const boundary = "b0undary"; + return encoder.encode( + [ + "From: alice@x", + "To: bob@y", + "Subject: Multipart", + "Message-ID: <1@x>", + "Date: Thu, 01 Jan 2026 00:00:00 +0000", + `Content-Type: multipart/mixed; boundary="${boundary}"`, + "", + `--${boundary}`, + "Content-Type: text/plain", + `Content-Transfer-Encoding: ${encoding}`, + "", + body, + `--${boundary}--`, + "", + ].join("\r\n"), + ); +} + +/** A single-part message whose own headers declare `encoding`. */ +function rawMessageWithEncoding(encoding: string, body: string): Uint8Array { + return encoder.encode( + [ + "From: alice@x", + "To: bob@y", + "Subject: Encoded", + "Message-ID: <1@x>", + "Date: Thu, 01 Jan 2026 00:00:00 +0000", + "Content-Type: text/plain", + `Content-Transfer-Encoding: ${encoding}`, + "", + body, + ].join("\r\n"), + ); +} + +/** + * A `multipart/signed` message built by hand, so a test can choose the transfer + * encoding of the signed content. The assembler only ever emits `7bit`, and the + * shape under test is the one a peer's mail client produces. `signedPart` is the + * whole signed content part, headers included. + * + * The signature part is a placeholder: `fetchFull` resolves no key for this + * sender in these tests, so it reports `unknown` without reading it. + */ +function handBuiltSigned( + signedPart: string[], + interchangeType = "conversation.message", +): Uint8Array { + return encoder.encode( + [ + "From: alice@x", + "To: bob@y", + "Subject: Encoded", + "Message-ID: <1@x>", + "Date: Thu, 01 Jan 2026 00:00:00 +0000", + `Interchange-Type: ${interchangeType}`, + 'Content-Type: multipart/signed; protocol="application/pgp-signature"; ' + + 'micalg=pgp-sha512; boundary="outer"', + "", + "--outer", + ...signedPart, + "--outer", + "Content-Type: application/pgp-signature", + "", + "placeholder-signature", + "--outer--", + "", + ].join("\r\n"), + ); +} + +/** The conversation shape: a multipart/mixed whose part 1.1 is the text body. */ +function signedConversationWithEncoding( + encoding: string, + body: string, +): Uint8Array { + return handBuiltSigned([ + 'Content-Type: multipart/mixed; boundary="inner"', + "", + "--inner", + "Content-Type: text/plain; charset=utf-8", + `Content-Transfer-Encoding: ${encoding}`, + "", + body, + "--inner--", + "", + ]); +} + +/** + * A signed content part that is a bare text/plain part with no multipart/mixed + * wrapper -- the shape a plain mail client signs. + */ +function signedBareTextWithEncoding( + encoding: string, + body: string, +): Uint8Array { + return handBuiltSigned([ + "Content-Type: text/plain; charset=utf-8", + `Content-Transfer-Encoding: ${encoding}`, + "", + body, + ]); +} + +/** The structured shape: the JSON payload at part 1.1. */ +function signedStructuredWithEncoding( + encoding: string, + body: string, +): Uint8Array { + return handBuiltSigned( + [ + 'Content-Type: multipart/mixed; boundary="inner"', + "", + "--inner", + "Content-Type: application/vnd.interchange+json; charset=utf-8", + `Content-Transfer-Encoding: ${encoding}`, + "", + body, + "--inner--", + "", + ], + "offering.catalog", + ); +} + /** * Wrap a store so every `readRaw` is counted. Proves the pure functions read * raw only when a projection or predicate needs the bytes. @@ -211,6 +449,177 @@ describe("async fetch projections route through readRaw", () => { expect(new TextDecoder().decode(part.content)).toContain("part body"); }); + // A run of CFWS between two lexical tokens is semantically a single space + // (RFC 2822 section 3.2.3), so a comment separates the tokens either side. + // `ba(c)se64` is therefore the two tokens `ba` and `se64` and names no + // mechanism, where a reading that deletes the comment sees `base64`. Each + // row is a message read two ways -- this projection and the whole-message + // decode -- and the point is that both ways give one answer, because a + // sender that can pick which reading a consumer gets can smuggle content + // past whichever one inspects it. + const transferEncodingCases: { + declared: string; + body: string; + content: string; + contentType: string; + encoding: string | undefined; + }[] = [ + { + declared: "base64", + body: "aGVsbG8=", + content: "hello", + contentType: "text/plain", + encoding: "base64", + }, + { + declared: "ba(c)se64", + body: "aGVsbG8=", + content: "aGVsbG8=", + contentType: "application/octet-stream", + encoding: "ba se64", + }, + { + declared: "base(x)64", + body: "aGVsbG8=", + content: "aGVsbG8=", + contentType: "application/octet-stream", + encoding: "base 64", + }, + { + declared: "b(1)a(2)s(3)e(4)6(5)4", + body: "aGVsbG8=", + content: "aGVsbG8=", + contentType: "application/octet-stream", + encoding: "b a s e 6 4", + }, + { + declared: "7bit (default)", + body: "hello", + content: "hello", + contentType: "text/plain", + encoding: undefined, + }, + { + declared: "(just a comment)", + body: "hello", + content: "hello", + contentType: "text/plain", + encoding: undefined, + }, + ]; + + test("fetchPart reads a transfer encoding as the whole-message decode does", async () => { + for (const c of transferEncodingCases) { + const raw = rawMultipartWithEncoding(c.declared, c.body); + const store = createInMemoryMailboxStore(); + const uid = store.append(raw, envelopeFor(), []); + + const part = await fetchPart({ uid, mailbox: "INBOX" }, "1", store); + expect(new TextDecoder().decode(part.content)).toBe(c.content); + expect(part.contentType).toBe(c.contentType); + expect(part.encoding).toBe(c.encoding); + + // Pinning both sides to the same row proves the expectation, not just + // that two readers are wrong together. + const whole = decodeMail(raw).parts[0]; + expect(new TextDecoder().decode(whole?.content)).toBe(c.content); + expect(whole?.contentType).toBe(c.contentType); + } + }); + + test("fetchPart contentType is type/subtype without parameters", async () => { + const store = createInMemoryMailboxStore(); + const uid = store.append( + encoder.encode( + [ + "From: alice@x", + "To: bob@y", + "Subject: charset", + "Message-ID: <1@x>", + "Date: Thu, 01 Jan 2026 00:00:00 +0000", + `Content-Type: multipart/mixed; boundary="b"`, + "", + "--b", + "Content-Type: text/plain; charset=utf-8", + "", + "hello", + "--b--", + "", + ].join("\r\n"), + ), + envelopeFor({ subject: "charset" }), + [], + ); + const part = await fetchPart({ uid, mailbox: "INBOX" }, "1", store); + expect(part.contentType).toBe("text/plain"); + }); + + test("fetchPart undoes quoted-printable the same way listing does", async () => { + const store = createInMemoryMailboxStore(); + const uid = store.append( + rawQuotedPrintableAttachment(), + envelopeFor({ + messageId: "", + subject: "QP", + interchangeType: "conversation.message", + }), + [], + ); + const ref = { uid, mailbox: "INBOX" }; + const listed = await fetchFull(ref, store, () => undefined); + const attachment = listed.attachments?.[0]; + if (attachment === undefined) + throw new Error("expected a listed attachment"); + expect(Array.from(attachment.data)).toEqual([0x63, 0x61, 0x66, 0xe9]); + + const fetched = await fetchPart(ref, attachment.part ?? "1.2", store); + expect(Array.from(fetched.content)).toEqual(Array.from(attachment.data)); + }); + + test("fetchFull undoes the text body's transfer encoding like fetchPart", async () => { + const store = createInMemoryMailboxStore(); + const uid = store.append( + encoder.encode( + [ + "From: alice@x", + "To: bob@y", + "Subject: QP body", + "Message-ID: ", + "Date: Thu, 01 Jan 2026 00:00:00 +0000", + "Interchange-Type: conversation.message", + `Content-Type: multipart/signed; protocol="application/pgp-signature"; micalg=pgp-sha512; boundary="outer"`, + "", + "--outer", + `Content-Type: multipart/mixed; boundary="inner"`, + "", + "--inner", + "Content-Type: text/plain; charset=utf-8", + "Content-Transfer-Encoding: quoted-printable", + "", + "Bonjour, caf=C3=A9", + "--inner--", + "--outer", + "Content-Type: application/pgp-signature", + "", + "FAKE", + "--outer--", + "", + ].join("\r\n"), + ), + envelopeFor({ + messageId: "", + interchangeType: "conversation.message", + }), + [], + ); + const ref = { uid, mailbox: "INBOX" }; + const full = await fetchFull(ref, store, () => undefined); + expect(full.content).toBe("Bonjour, café"); + + const part = await fetchPart(ref, "1.1", store); + expect(new TextDecoder().decode(part.content)).toBe("Bonjour, café"); + }); + test("fetch projections reject an absent uid", async () => { const store = createInMemoryMailboxStore(); await expect( @@ -218,6 +627,214 @@ describe("async fetch projections route through readRaw", () => { ).rejects.toThrow(/not found/); }); + test("fetchFull verifies a signed message against its sender's key", async () => { + // The positive control for the no-originator case below: this fixture + // does verify, so an `unknown` there is the guard's doing and not a + // fixture that could never have verified. + const crypto = createEd25519Crypto(await generateKeyPair()); + const store = createInMemoryMailboxStore(); + const uid = store.append(await signedRawMessage(crypto), envelopeFor(), []); + + const full = await fetchFull( + { uid, mailbox: "INBOX" }, + store, + () => crypto, + ); + expect(full.signatureStatus).toBe("valid"); + }); + + // `fetchFull` reads the same `Content-Transfer-Encoding` the whole-message + // decoders read. Each row is a body whose declared mechanism has to be undone + // before the bytes are the message's text; reading the part bytes directly + // would hand a caller the source form of the encoding. + const contentEncodingCases: { encoding: string; body: string }[] = [ + { encoding: "base64", body: "aGVsbG8gd29ybGQ=" }, + { encoding: "quoted-printable", body: "hello=20world" }, + { encoding: "7bit", body: "hello world" }, + // RFC 2045 section 1: a comment carries no meaning, so this names base64. + { encoding: "base64 (RFC 2045)", body: "aGVsbG8gd29ybGQ=" }, + ]; + + test("fetchFull undoes the conversation body's transfer encoding", async () => { + for (const c of contentEncodingCases) { + const store = createInMemoryMailboxStore(); + const uid = store.append( + signedConversationWithEncoding(c.encoding, c.body), + envelopeFor(), + [], + ); + const full = await fetchFull( + { uid, mailbox: "INBOX" }, + store, + () => undefined, + ); + expect(full.content).toBe("hello world"); + } + }); + + test("fetchFull undoes a bare signed text part's transfer encoding", async () => { + for (const c of contentEncodingCases) { + const store = createInMemoryMailboxStore(); + const uid = store.append( + signedBareTextWithEncoding(c.encoding, c.body), + envelopeFor(), + [], + ); + const full = await fetchFull( + { uid, mailbox: "INBOX" }, + store, + () => undefined, + ); + expect(full.content).toBe("hello world"); + } + }); + + test("fetchFull undoes a structured payload's transfer encoding", async () => { + const store = createInMemoryMailboxStore(); + const uid = store.append( + signedStructuredWithEncoding( + "base64", + "eyJ0eXBlIjoib2ZmZXJpbmcuY2F0YWxvZyIsInZlcnNpb24iOiIxIiwiYm9keSI6eyJrIjoxfX0=", + ), + envelopeFor({ interchangeType: "offering.catalog" }), + [], + ); + const full = await fetchFull( + { uid, mailbox: "INBOX" }, + store, + () => undefined, + ); + expect(full.payload?.type).toBe("offering.catalog"); + expect(full.payload?.body).toEqual({ k: 1 }); + }); + + test("fetchFull reads a non-multipart message's own body as its content", async () => { + // An ordinary unsigned email is a single part, so it has no part 1 to index + // into and the message is its own leaf part. Returning it with no content + // would drop the body of every sender that writes this shape. + const store = createInMemoryMailboxStore(); + const uid = store.append( + rawMessage("Hello", "the whole body"), + envelopeFor(), + [], + ); + const full = await fetchFull( + { uid, mailbox: "INBOX" }, + store, + () => undefined, + ); + expect(full.content).toBe("the whole body"); + expect(full.signatureStatus).toBe("unknown"); + }); + + test("fetchFull undoes a non-multipart message's transfer encoding", async () => { + // The message's own headers are the part's headers on this shape, so the + // mechanism it declares is the one that applies to its body. + for (const c of contentEncodingCases) { + const store = createInMemoryMailboxStore(); + const uid = store.append( + rawMessageWithEncoding(c.encoding, c.body), + envelopeFor(), + [], + ); + const full = await fetchFull( + { uid, mailbox: "INBOX" }, + store, + () => undefined, + ); + expect(full.content).toBe("hello world"); + } + }); + + // A body that does not decode into text: RFC 2045 section 6.4 makes the + // first opaque octets, and the second is base64 that will not decode. + const undecodableBodyCases: { encoding: string; body: string }[] = [ + { encoding: "x-uuencode", body: "begin 644 x" }, + { encoding: "base64", body: "!!! not base64 !!!" }, + ]; + + test("fetchFull delivers a message whose body it cannot decode", async () => { + // Refusing the message is not a behaviour any mail client has: one that + // cannot render a body still shows the message. `content` is text, so an + // undecodable body is carried as an absent one rather than as invented + // text; the octets remain reachable through `fetchPart`. + for (const c of undecodableBodyCases) { + const store = createInMemoryMailboxStore(); + const uid = store.append( + signedConversationWithEncoding(c.encoding, c.body), + envelopeFor(), + [], + ); + const full = await fetchFull( + { uid, mailbox: "INBOX" }, + store, + () => undefined, + ); + expect(full.content).toBeUndefined(); + expect(full.payload).toBeUndefined(); + expect(full.headers.subject).toBe("Encoded"); + expect(full.headers.from).toBe("alice@x"); + expect(full.flags).toEqual([]); + expect(full.signatureStatus).toBe("unknown"); + expect(full.ref).toEqual({ uid, mailbox: "INBOX" }); + } + }); + + test("fetchFull omits a structured payload it cannot decode", async () => { + const store = createInMemoryMailboxStore(); + const uid = store.append( + signedStructuredWithEncoding("x-uuencode", "begin 644 x"), + envelopeFor({ interchangeType: "offering.catalog" }), + [], + ); + const full = await fetchFull( + { uid, mailbox: "INBOX" }, + store, + () => undefined, + ); + expect(full.payload).toBeUndefined(); + expect(full.content).toBeUndefined(); + expect(full.headers.interchangeType).toBe("offering.catalog"); + }); + + test("fetchPart reports an undecodable body as application/octet-stream", async () => { + // The two paths agree about the same bytes: what `fetchFull` declines to + // call text, `fetchPart` hands back as octets under the section 6.4 + // relabel. A part path resolves against the signed content, so "1.1" is + // the body part `fetchFull` reads. + const store = createInMemoryMailboxStore(); + const uid = store.append( + signedConversationWithEncoding("x-uuencode", "begin 644 x"), + envelopeFor(), + [], + ); + const ref = { uid, mailbox: "INBOX" }; + + expect( + (await fetchFull(ref, store, () => undefined)).content, + ).toBeUndefined(); + + const part = await fetchPart(ref, "1.1", store); + expect(part.contentType).toBe("application/octet-stream"); + expect(part.encoding).toBe("x-uuencode"); + expect(new TextDecoder().decode(part.content)).toBe("begin 644 x"); + }); + + test("fetchFull still refuses a structured payload that is not valid JSON", async () => { + // A decode failure and a payload failure are different conditions: here the + // bytes decoded and the sender's JSON is wrong, which is a fault to surface + // rather than a body this transport cannot read. + const store = createInMemoryMailboxStore(); + const uid = store.append( + signedStructuredWithEncoding("7bit", "{ not json"), + envelopeFor({ interchangeType: "offering.catalog" }), + [], + ); + await expect( + fetchFull({ uid, mailbox: "INBOX" }, store, () => undefined), + ).rejects.toThrow(); + }); + test("fetchFull propagates a sender whose getPublicKey throws", async () => { // A CryptoProvider that cannot produce its own public key is a local // fault, not a bad signature: the error surfaces rather than being @@ -240,3 +857,293 @@ describe("async fetch projections route through readRaw", () => { ).rejects.toThrow(/no public key/); }); }); + +describe("a message that carries no originator", () => { + test("matches no sender query, including the empty one", async () => { + // There is no sender for the substring to be found in, so no `from` + // query can match. The empty query is the assertion that matters: every + // string contains "", so it catches a stand-in substituted for the + // absent sender whatever address that stand-in names. + const store = createInMemoryMailboxStore(); + const orphanUid = store.append( + stripFromHeader(rawMessage("Hello", "body text")), + envelopeFor({ from: undefined, messageId: "" }), + [], + ); + const namedUid = store.append( + rawMessage("Hello", "body text"), + envelopeFor(), + [], + ); + + const byEmpty = await executeSearch("INBOX", store, { from: "" }); + // The named message proves the empty query is a query that matches, so + // the orphan's absence from the result is the guard and not a query that + // matches nothing. + expect(byEmpty.map((r) => r.uid)).toEqual([namedUid]); + + const byAddress = await executeSearch("INBOX", store, { from: "alice@x" }); + expect(byAddress.map((r) => r.uid)).toEqual([namedUid]); + + // Every other predicate still sees the orphan, so it is in the mailbox + // and reachable -- it is the sender predicate alone that excludes it. + const byRecipient = await executeSearch("INBOX", store, { to: "bob@y" }); + expect(byRecipient.map((r) => r.uid)).toEqual([orphanUid, namedUid]); + }); + + test("fetchFull reports its signature unknown without consulting the key lookup", async () => { + // A message with no originator names no key to verify against. Asking + // the lookup for a stand-in address would return whichever key that + // address happens to have and verify the message against it, so the + // lookup must not be asked at all. + const crypto = createEd25519Crypto(await generateKeyPair()); + const store = createInMemoryMailboxStore(); + const raw = stripFromHeader(await signedRawMessage(crypto)); + const uid = store.append(raw, envelopeFor({ from: undefined }), []); + + const asked: string[] = []; + // Answers with the signing key for any address, so a stand-in would + // verify and report `valid`. + const getCrypto = (fromAddress: string): CryptoProvider => { + asked.push(fromAddress); + return crypto; + }; + + const full = await fetchFull({ uid, mailbox: "INBOX" }, store, getCrypto); + + expect(full.headers.from).toBeUndefined(); + expect(full.signatureStatus).toBe("unknown"); + expect(asked).toEqual([]); + }); +}); + +describe("a message that carries no date", () => { + test("matches no date window, in either direction", async () => { + // Every date predicate asks where the sender placed the message in time. A + // message that named no date placed itself nowhere, so each window has to + // exclude it. The dated message in the same mailbox proves each query is + // one that matches, so the undated message's absence is the guard rather + // than a query matching nothing. + const store = createInMemoryMailboxStore(); + const undatedUid = store.append( + rawMessage("Hello", "body text"), + envelopeFor({ date: undefined, messageId: "" }), + [], + ); + const datedUid = store.append( + rawMessage("Hello", "body text"), + envelopeFor(), + [], + ); + + const on = new Date("2026-01-01T00:00:00Z"); + const before = new Date("2026-06-01T00:00:00Z"); + const after = new Date("2025-06-01T00:00:00Z"); + + for (const query of [ + { on }, + { sentOn: on }, + { before }, + { sentBefore: before }, + { after }, + { sentAfter: after }, + ]) { + const hits = await executeSearch("INBOX", store, query); + expect(hits.map((r) => r.uid)).toEqual([datedUid]); + } + + // Every other predicate still sees the undated message, so it is in the + // mailbox and reachable -- the date predicates alone exclude it. + const byRecipient = await executeSearch("INBOX", store, { to: "bob@y" }); + expect(byRecipient.map((r) => r.uid)).toEqual([undatedUid, datedUid]); + }); + + test("sorts behind a dated message in a thread rather than taking a date", async () => { + // Threading orders by date, and an undated message supplies no key. It + // sorts after every dated message, so the dated message keeps the root + // slot instead of hanging off a message that placed itself nowhere in + // time. The undated message is appended first, so append order cannot be + // what puts it last. + const store = createInMemoryMailboxStore(); + const undatedUid = store.append( + rawMessage("Shared", "second"), + envelopeFor({ + messageId: "", + subject: "Shared", + date: undefined, + }), + [], + ); + const datedUid = store.append( + rawMessage("Shared", "first"), + envelopeFor({ messageId: "", subject: "Shared" }), + [], + ); + + const threads = await executeThread("INBOX", store, "orderedsubject"); + + expect(threads).toHaveLength(1); + expect(threads[0]?.ref.uid).toBe(datedUid); + expect(threads[0]?.children.map((c) => c.ref.uid)).toEqual([undatedUid]); + }); +}); + +describe("an envelope whose declared structure is not there", () => { + // A sender chooses its own `Content-Type`, so it chooses whether the part + // `fetchFull` reads can be found at all. These three shapes are the ones + // that leave the projection with no content part, and it refuses the + // message rather than inventing one. Refusing is not deleting: the caller + // that reads the refusal owns what happens to the mail, and the INBOX watch + // in `@intx/harness` keeps it. + + /** A `multipart/*` message whose declared boundary appears nowhere. */ + function boundaryDeclaredButAbsent(): Uint8Array { + return encoder.encode( + [ + "From: alice@x", + "To: bob@y", + "Subject: Encoded", + "Message-ID: <1@x>", + "Date: Thu, 01 Jan 2026 00:00:00 +0000", + "Interchange-Type: conversation.message", + 'Content-Type: multipart/signed; protocol="application/pgp-signature"; ' + + 'micalg=pgp-sha512; boundary="outer"', + "", + "nothing here delimits a part", + "", + ].join("\r\n"), + ); + } + + /** A `multipart/*` message that declares no `boundary` parameter. */ + function noBoundaryDeclared(): Uint8Array { + return encoder.encode( + [ + "From: alice@x", + "To: bob@y", + "Subject: Encoded", + "Message-ID: <1@x>", + "Date: Thu, 01 Jan 2026 00:00:00 +0000", + "Interchange-Type: conversation.message", + "Content-Type: multipart/signed", + "", + "body", + "", + ].join("\r\n"), + ); + } + + /** A signed content part that declares `multipart/mixed` with no boundary. */ + function signedPartWithoutBoundary(): Uint8Array { + return handBuiltSigned(["Content-Type: multipart/mixed", "", "orphan"]); + } + + const malformed: { name: string; raw: () => Uint8Array }[] = [ + { + name: "a declared boundary that appears nowhere", + raw: boundaryDeclaredButAbsent, + }, + { name: "a multipart type declaring no boundary", raw: noBoundaryDeclared }, + { + name: "a signed content part that is multipart with no boundary", + raw: signedPartWithoutBoundary, + }, + ]; + + for (const shape of malformed) { + test(`fetchFull refuses ${shape.name}`, async () => { + const store = createInMemoryMailboxStore(); + const uid = store.append(shape.raw(), envelopeFor(), []); + + await expect( + fetchFull({ uid, mailbox: "INBOX" }, store, () => undefined), + ).rejects.toThrow(); + + // The refusal leaves the mailbox as it found it: the projections are + // pure reads, so the bytes a caller may still want are all there. + expect(store.find(uid)).toBeDefined(); + expect(await store.readRaw(uid)).toEqual(shape.raw()); + }); + } +}); + +describe("a body that will not decode reports itself", () => { + // The default sink routes `warning` and above to `console.warn`, so spying + // on it asserts what an operator actually sees rather than an internal + // logger call. The development formatter colours every interpolated value, + // so the escape sequences come off before anything is matched. + const ansiEscape = new RegExp(`${String.fromCharCode(27)}\\[[0-9;]*m`, "g"); + + /* eslint-disable no-console -- intentional spy on console.warn */ + const warned: string[] = []; + let originalWarn: typeof console.warn; + + beforeEach(() => { + warned.length = 0; + originalWarn = console.warn; + console.warn = (...args: unknown[]) => { + warned.push( + args + .map((a) => String(a)) + .join(" ") + .replace(ansiEscape, ""), + ); + }; + }); + + afterEach(() => { + console.warn = originalWarn; + }); + /* eslint-enable no-console */ + + test("fetchFull records the decode failure it delivers the message without", async () => { + // Absent `content` is the whole of what the caller learns, and "not text" + // and "would not decode" reach it as the same value. Without a record the + // second is indistinguishable from the first and nobody learns the peer is + // sending bodies its own declared encoding does not describe. + const store = createInMemoryMailboxStore(); + const uid = store.append( + signedConversationWithEncoding("base64", "!!! not base64 !!!"), + envelopeFor(), + [], + ); + + const full = await fetchFull( + { uid, mailbox: "INBOX" }, + store, + () => undefined, + ); + expect(full.content).toBeUndefined(); + + // The record has to name the message and the encoding that failed, so an + // operator can read the octets back through `fetchPart` and can tell which + // peer to ask about. + const record = warned.find((line) => line.includes("body did not decode")); + expect(record).toBeDefined(); + expect(record).toContain(`uid=${String(uid)}`); + expect(record).toContain("INBOX"); + expect(record).toContain("base64"); + }); + + test("an unrecognized encoding is not reported as a decode failure", async () => { + // The RFC 2045 section 6.4 relabel is the declared handling of an encoding + // this transport does not implement, not a fault. Reporting it would cry + // wolf on every opaque part a peer legitimately sends. + const store = createInMemoryMailboxStore(); + const uid = store.append( + signedConversationWithEncoding("x-uuencode", "begin 644 x"), + envelopeFor(), + [], + ); + + const full = await fetchFull( + { uid, mailbox: "INBOX" }, + store, + () => undefined, + ); + expect(full.content).toBeUndefined(); + expect( + warned.filter((line) => line.includes("body did not decode")), + ).toEqual([]); + }); +}); diff --git a/vendor/intx-mailbox/src/fetch.ts b/vendor/intx-mailbox/src/fetch.ts index 6e9b08fae..645b30adb 100644 --- a/vendor/intx-mailbox/src/fetch.ts +++ b/vendor/intx-mailbox/src/fetch.ts @@ -1,5 +1,6 @@ -/* eslint-disable @typescript-eslint/no-non-null-assertion -- MIME multipart parsing with bounds checks */ import { type } from "arktype"; +import { getLogger } from "@intx/log"; +import { mimeTypeAndSubtype } from "@intx/types/attachments"; import type { MessageHeaders, BodyStructure, @@ -10,7 +11,6 @@ import type { MessageRef, } from "@intx/types/runtime"; import { InterchangeType } from "@intx/types/runtime"; -import { base64Decode } from "@intx/types"; import type { MailboxStore } from "./mailbox"; import { requireMessage } from "./mailbox"; import { @@ -18,12 +18,20 @@ import { parseMimePart, extractBoundary, parseMultipart, + extractContentTypeMime, extractPartByPath, extractAttachments, + buildMessageHeaders, + decodePartBytes, + isRecognizedTransferEncoding, + reportedContentType, + transferEncodingMechanism, } from "@intx/mime"; -import { buildMessageHeaders } from "./headers"; +import type { ParsedMimePart } from "@intx/mime"; import { verifyMimeSignature } from "./verify-signature"; +const logger = getLogger(["interchange", "mailbox", "fetch"]); + const MessagePayload = type({ type: InterchangeType, version: "string", @@ -62,7 +70,11 @@ export async function fetchStructure( } /** - * Fetch a single MIME part by dot-separated path. + * Fetch a single MIME part by dot-separated path. `content` is the + * CTE-decoded bytes, matching `extractAttachments` / `decodeMail`. + * `contentType` is the type/subtype `decodeMail` reports — parameters such + * as charset are spent once the bytes are decoded — and it carries the + * RFC 2045 §6.4 relabel, since an undecodable part arrives undecoded. */ export async function fetchPart( ref: MessageRef, @@ -74,24 +86,70 @@ export async function fetchPart( const partBytes = extractPartByPath(raw, partPath); const part = parseMimePart(partBytes); - const enc = part.headers.get("content-transfer-encoding") ?? "7bit"; - let content: Uint8Array; - - if (enc.toLowerCase() === "base64") { - const b64 = new TextDecoder().decode(part.body).replace(/\s/g, ""); - content = base64Decode(b64); - } else { - content = part.body; - } - const result: MessagePart = { - contentType: part.contentType, - content, + contentType: mimeTypeAndSubtype( + reportedContentType(part.contentType, part.headers), + ), + content: decodePartBytes(part.body, part.headers), }; - if (enc !== "7bit") result.encoding = enc; + const mechanism = transferEncodingMechanism(part.headers); + if (mechanism !== "7bit") result.encoding = mechanism; return result; } +/** + * Decode a leaf part's body into text, or undefined when the octets are not + * text: an unrecognized mechanism (RFC 2045 §6.4) or encoded data that will not + * decode. The octets stay reachable through `fetchPart`, which reports them + * under the §6.4 relabel. + */ +function decodePartText( + part: ParsedMimePart, + ref: MessageRef, +): string | undefined { + if (!isRecognizedTransferEncoding(transferEncodingMechanism(part.headers))) { + return undefined; + } + let decoded: Uint8Array; + try { + decoded = decodePartBytes(part.body, part.headers); + } catch (cause) { + // Absent text is the answer, but it is not a self-explaining one: a caller + // reading `undefined` cannot tell "not text" from "would not decode", and + // an operator would otherwise never learn that a peer sends bodies its own + // declared encoding does not describe. The record names the message so the + // octets can be read back through `fetchPart`. + logger.warn`Message uid=${ref.uid} in mailbox ${ref.mailbox} carries a ${part.contentType} part whose ${transferEncodingMechanism(part.headers)} body did not decode; delivering it without text: ${cause instanceof Error ? cause.message : String(cause)}`; + return undefined; + } + return new TextDecoder("utf-8", { fatal: false }).decode(decoded); +} + +/** + * The leaf part a message's content lives in: part 1.1 under our assembler's + * multipart/mixed wrapper, part 1 when a peer signed a bare part, and the + * message itself when it is not multipart at the top level. + */ +function resolveContentPart( + raw: Uint8Array, + messageHeaders: Map, + bodyOffset: number, +): ParsedMimePart { + const declared = messageHeaders.get("content-type") ?? "text/plain"; + if (!extractContentTypeMime(declared).startsWith("multipart/")) { + return { + contentType: declared, + headers: messageHeaders, + body: raw.slice(bodyOffset), + }; + } + const part1 = parseMimePart(extractPartByPath(raw, "1")); + if (!extractContentTypeMime(part1.contentType).startsWith("multipart/")) { + return part1; + } + return parseMimePart(extractPartByPath(raw, "1.1")); +} + /** * Fetch a complete message, verify its PGP/MIME signature, and return * a fully parsed InboundMessage. @@ -103,7 +161,7 @@ export async function fetchFull( ): Promise { const msg = requireMessage(store, ref.uid, ref.mailbox); const raw = await store.readRaw(ref.uid); - const { headers } = parseHeaderSection(raw); + const { headers, bodyOffset } = parseHeaderSection(raw); const parsedHeaders = buildMessageHeaders(headers); const rawType = parsedHeaders.interchangeType; @@ -126,48 +184,27 @@ export async function fetchFull( signatureStatus, }; - try { + // A body that did not decode is not text, and both destinations here are + // text; the message is delivered without one rather than refused. + const text = decodePartText( + resolveContentPart(raw, headers, bodyOffset), + ref, + ); + if (text !== undefined) { if (isConversation) { - const part1 = parseMimePart(extractPartByPath(raw, "1")); - const part1Mime = part1.contentType.split(";")[0]!.trim().toLowerCase(); - if (part1Mime.startsWith("multipart/")) { - // Conversation shape: multipart/mixed with the text body at 1.1. - const textPart = parseMimePart(extractPartByPath(raw, "1.1")); - result.content = new TextDecoder("utf-8", { fatal: false }).decode( - textPart.body, - ); - } else { - // A conversation message is "literally a signed email", so a sender - // (e.g. a plain mail client) may sign a bare text/plain part with no - // multipart/mixed wrapper. This branch reads that body directly. Our - // own assembler always emits multipart/mixed; without this branch a - // bare text/plain message would fail the 1.1 lookup and silently lose - // its content to the catch below. - result.content = new TextDecoder("utf-8", { fatal: false }).decode( - part1.body, - ); - } + result.content = text; } else { - // Structured messages carry their JSON payload at 1.1. Attachments on - // structured messages are intentionally not parsed: they have no - // producer today, so parsing them would handle a shape nobody sends. - const part11Bytes = extractPartByPath(raw, "1.1"); - const part11 = parseMimePart(part11Bytes); - const jsonText = new TextDecoder("utf-8", { fatal: false }).decode( - part11.body, - ); - const validated = MessagePayload(JSON.parse(jsonText)); + // Attachments on structured messages are intentionally not parsed: they + // have no producer today. + const validated = MessagePayload(JSON.parse(text)); if (validated instanceof type.errors) { throw new Error(`invalid message payload: ${validated.summary}`); } result.payload = validated; } - } catch { - // If we can't parse the content, return what we have with the signature status. } - // Attachment parsing is deliberately outside the catch above: a malformed - // attachment must surface as a thrown error, not be silently dropped. + // A malformed attachment throws rather than being silently dropped. if (isConversation) { const attachments = extractAttachments(raw); if (attachments.length > 0) { @@ -180,9 +217,14 @@ export async function fetchFull( async function verifyMessageSignature( raw: Uint8Array, - fromAddress: string, + fromAddress: string | undefined, getCrypto: (fromAddress: string) => CryptoProvider | undefined, ): Promise { + // No originator names no key to verify against, which is the same position + // as a sender we hold no key for. + if (fromAddress === undefined) { + return "unknown"; + } const senderCrypto = getCrypto(fromAddress); if (senderCrypto === undefined) { return "unknown"; diff --git a/vendor/intx-mailbox/src/headers.ts b/vendor/intx-mailbox/src/headers.ts deleted file mode 100644 index 0e4021126..000000000 --- a/vendor/intx-mailbox/src/headers.ts +++ /dev/null @@ -1,5 +0,0 @@ -// `buildMessageHeaders` now lives in `@intx/mime` alongside the rest of the -// MIME/header parsing (it is also what the `decodeMail` decoder builds its -// typed header subset with). Re-exported here so mail-memory's callers keep -// their existing import path. -export { buildMessageHeaders } from "@intx/mime"; diff --git a/vendor/intx-mailbox/src/index.ts b/vendor/intx-mailbox/src/index.ts index 15d11b1a7..06bc3c77f 100644 --- a/vendor/intx-mailbox/src/index.ts +++ b/vendor/intx-mailbox/src/index.ts @@ -8,5 +8,6 @@ export type { MailboxStore, StoredMessage, StoredEnvelope } from "./mailbox"; export { executeSearch } from "./search"; export { executeThread } from "./thread"; export { fetchHeaders, fetchStructure, fetchPart, fetchFull } from "./fetch"; -export { buildMessageHeaders } from "./headers"; export { verifyMimeSignature } from "./verify-signature"; +export { composeOutbound } from "./compose"; +export type { ComposedMessage } from "./compose"; diff --git a/vendor/intx-mailbox/src/mailbox.ts b/vendor/intx-mailbox/src/mailbox.ts index 05a3e9455..95b264999 100644 --- a/vendor/intx-mailbox/src/mailbox.ts +++ b/vendor/intx-mailbox/src/mailbox.ts @@ -4,10 +4,11 @@ */ export type StoredEnvelope = { messageId: string; - from: string; + from: string | undefined; to: string[]; subject: string; - date: Date; + /** The date the sender put on the message, not the instant it arrived. */ + date: Date | undefined; inReplyTo: string | undefined; references: string[]; interchangeType: string | undefined; diff --git a/vendor/intx-mailbox/src/search.ts b/vendor/intx-mailbox/src/search.ts index 4efc54c2e..f9a36ac9b 100644 --- a/vendor/intx-mailbox/src/search.ts +++ b/vendor/intx-mailbox/src/search.ts @@ -55,7 +55,11 @@ async function matchMessage( readRaw: () => Promise, ): Promise { if (query.from !== undefined) { - if (!msg.envelope.from.toLowerCase().includes(query.from.toLowerCase())) { + const sender = msg.envelope.from; + if ( + sender === undefined || + !sender.toLowerCase().includes(query.from.toLowerCase()) + ) { return false; } } @@ -93,19 +97,23 @@ async function matchMessage( } } + // A message that named no date placed itself nowhere in time, so it falls + // outside every date window below rather than inside one. + const sent = msg.envelope.date; + if (query.before !== undefined) { - if (msg.envelope.date >= query.before) return false; + if (sent === undefined || sent >= query.before) return false; } if (query.after !== undefined) { - if (msg.envelope.date <= query.after) return false; + if (sent === undefined || sent <= query.after) return false; } if (query.on !== undefined) { - const d = msg.envelope.date; + if (sent === undefined) return false; const q = query.on; if ( - d.getUTCFullYear() !== q.getUTCFullYear() || - d.getUTCMonth() !== q.getUTCMonth() || - d.getUTCDate() !== q.getUTCDate() + sent.getUTCFullYear() !== q.getUTCFullYear() || + sent.getUTCMonth() !== q.getUTCMonth() || + sent.getUTCDate() !== q.getUTCDate() ) { return false; } @@ -113,18 +121,18 @@ async function matchMessage( // Sent date filters use the Date header (same as envelope date here). if (query.sentBefore !== undefined) { - if (msg.envelope.date >= query.sentBefore) return false; + if (sent === undefined || sent >= query.sentBefore) return false; } if (query.sentAfter !== undefined) { - if (msg.envelope.date <= query.sentAfter) return false; + if (sent === undefined || sent <= query.sentAfter) return false; } if (query.sentOn !== undefined) { - const d = msg.envelope.date; + if (sent === undefined) return false; const q = query.sentOn; if ( - d.getUTCFullYear() !== q.getUTCFullYear() || - d.getUTCMonth() !== q.getUTCMonth() || - d.getUTCDate() !== q.getUTCDate() + sent.getUTCFullYear() !== q.getUTCFullYear() || + sent.getUTCMonth() !== q.getUTCMonth() || + sent.getUTCDate() !== q.getUTCDate() ) { return false; } diff --git a/vendor/intx-mailbox/src/thread.test.ts b/vendor/intx-mailbox/src/thread.test.ts new file mode 100644 index 000000000..32c0b03a0 --- /dev/null +++ b/vendor/intx-mailbox/src/thread.test.ts @@ -0,0 +1,214 @@ +import { describe, test, expect } from "bun:test"; +import { + createInMemoryMailboxStore, + executeThread, + type MailboxStore, + type StoredEnvelope, +} from "./index"; + +const encoder = new TextEncoder(); + +/** + * Threading reads only the stored envelope, never the raw bytes, so every + * message in this file shares the same placeholder body. + */ +const rawBody = encoder.encode("body"); + +function envelopeFor( + overrides: Partial & { messageId: string }, +): StoredEnvelope { + return { + from: "alice@x", + to: ["bob@y"], + subject: "Shared", + date: undefined, + inReplyTo: undefined, + references: [], + interchangeType: undefined, + interchangeCorrelationId: undefined, + ...overrides, + }; +} + +function append( + store: MailboxStore, + overrides: Partial & { messageId: string }, +): number { + return store.append(rawBody, envelopeFor(overrides), []); +} + +const jan1 = new Date("2026-01-01T00:00:00Z"); +const jan2 = new Date("2026-01-02T00:00:00Z"); +const jan3 = new Date("2026-01-03T00:00:00Z"); + +describe("orderedsubject threading", () => { + test("roots the thread at the earliest dated message, not at an undated one", async () => { + // A message that names no date says nothing about where it belongs in the + // conversation, so it cannot claim the root slot from a message that does + // name one. The undated message is appended between the two dated ones, so + // neither append order nor a date decides its position by accident. + const store = createInMemoryMailboxStore(); + const originalUid = append(store, { + messageId: "", + subject: "Invoice", + date: jan1, + }); + const undatedUid = append(store, { + messageId: "", + subject: "Re: Invoice", + }); + const replyUid = append(store, { + messageId: "", + subject: "Re: Invoice", + date: jan2, + }); + + const threads = await executeThread("INBOX", store, "orderedsubject"); + + expect(threads).toHaveLength(1); + expect(threads[0]?.ref.uid).toBe(originalUid); + expect(threads[0]?.children.map((c) => c.ref.uid)).toEqual([ + replyUid, + undatedUid, + ]); + }); + + test("orders a fully dated thread by date", async () => { + const store = createInMemoryMailboxStore(); + const lastUid = append(store, { + messageId: "", + subject: "Re: Invoice", + date: jan3, + }); + const firstUid = append(store, { + messageId: "", + subject: "Invoice", + date: jan1, + }); + const middleUid = append(store, { + messageId: "", + subject: "Re: Invoice", + date: jan2, + }); + + const threads = await executeThread("INBOX", store, "orderedsubject"); + + expect(threads).toHaveLength(1); + expect(threads[0]?.ref.uid).toBe(firstUid); + expect(threads[0]?.children.map((c) => c.ref.uid)).toEqual([ + middleUid, + lastUid, + ]); + }); + + test("orders a dated thread ahead of a thread that is entirely undated", async () => { + const store = createInMemoryMailboxStore(); + const undatedFirstUid = append(store, { + messageId: "", + subject: "Nowhere", + }); + const undatedSecondUid = append(store, { + messageId: "", + subject: "Re: Nowhere", + }); + const datedUid = append(store, { + messageId: "", + subject: "Invoice", + date: jan1, + }); + + const threads = await executeThread("INBOX", store, "orderedsubject"); + + expect(threads.map((t) => t.ref.uid)).toEqual([datedUid, undatedFirstUid]); + // Undated messages share one sort key, so the stable sort leaves them in + // the order the mailbox holds them. + expect(threads[1]?.children.map((c) => c.ref.uid)).toEqual([ + undatedSecondUid, + ]); + }); +}); + +describe("references threading", () => { + test("sorts an undated top-level message after every dated one", async () => { + const store = createInMemoryMailboxStore(); + const laterUid = append(store, { messageId: "", date: jan2 }); + const undatedUid = append(store, { messageId: "" }); + const earlierUid = append(store, { + messageId: "", + date: jan1, + }); + + const threads = await executeThread("INBOX", store, "references"); + + expect(threads.map((t) => t.ref.uid)).toEqual([ + earlierUid, + laterUid, + undatedUid, + ]); + }); + + test("sorts an undated reply after every dated sibling", async () => { + const store = createInMemoryMailboxStore(); + const rootUid = append(store, { messageId: "", date: jan1 }); + const undatedReplyUid = append(store, { + messageId: "", + inReplyTo: "", + }); + const datedReplyUid = append(store, { + messageId: "", + date: jan2, + inReplyTo: "", + }); + + const threads = await executeThread("INBOX", store, "references"); + + expect(threads).toHaveLength(1); + expect(threads[0]?.ref.uid).toBe(rootUid); + expect(threads[0]?.children.map((c) => c.ref.uid)).toEqual([ + datedReplyUid, + undatedReplyUid, + ]); + }); + + test("orders fully dated threads by date", async () => { + const store = createInMemoryMailboxStore(); + const laterUid = append(store, { messageId: "", date: jan3 }); + const earlierUid = append(store, { + messageId: "", + date: jan1, + }); + const replyUid = append(store, { + messageId: "", + date: jan2, + inReplyTo: "", + }); + + const threads = await executeThread("INBOX", store, "references"); + + expect(threads.map((t) => t.ref.uid)).toEqual([earlierUid, laterUid]); + expect(threads[0]?.children.map((c) => c.ref.uid)).toEqual([replyUid]); + }); + + test("promotes children of an absent parent and still sorts the undated one last", async () => { + // Nothing in the set carries ``, so the container standing in + // for it is dropped and its children rise to the top level, where the same + // ordering rule applies to them. + const store = createInMemoryMailboxStore(); + const undatedOrphanUid = append(store, { + messageId: "", + references: [""], + }); + const datedOrphanUid = append(store, { + messageId: "", + date: jan1, + references: [""], + }); + + const threads = await executeThread("INBOX", store, "references"); + + expect(threads.map((t) => t.ref.uid)).toEqual([ + datedOrphanUid, + undatedOrphanUid, + ]); + }); +}); diff --git a/vendor/intx-mailbox/src/thread.ts b/vendor/intx-mailbox/src/thread.ts index 024557901..d8c2628b8 100644 --- a/vendor/intx-mailbox/src/thread.ts +++ b/vendor/intx-mailbox/src/thread.ts @@ -76,9 +76,7 @@ function orderedSubjectThread( const threads: Thread[] = []; for (const [, msgs] of bySubject) { - const sorted = msgs.sort( - (a, b) => a.envelope.date.getTime() - b.envelope.date.getTime(), - ); + const sorted = msgs.sort((a, b) => messageDate(a) - messageDate(b)); const root = sorted[0]!; const rootThread: Thread = { ref: { uid: root.uid, mailbox: mailboxName }, @@ -93,7 +91,7 @@ function orderedSubjectThread( return threads.sort((a, b) => { const aMsg = messages.find((m) => m.uid === a.ref.uid)!; const bMsg = messages.find((m) => m.uid === b.ref.uid)!; - return aMsg.envelope.date.getTime() - bMsg.envelope.date.getTime(); + return messageDate(aMsg) - messageDate(bMsg); }); } @@ -195,7 +193,7 @@ function buildRefList(references: string[], inReplyTo?: string): string[] { } } - if (inReplyTo !== undefined && inReplyTo !== "" && !seen.has(inReplyTo)) { + if (inReplyTo !== undefined && !seen.has(inReplyTo)) { result.push(inReplyTo); } @@ -231,17 +229,35 @@ function pruneContainers(containers: Container[]): Container[] { return result; } +/** + * The sort key for a message that named no date. `Number.MAX_SAFE_INTEGER` + * exceeds the largest time value a `Date` can hold (8.64e15), so a message + * carrying this key sorts after every dated message. The epoch would do the + * opposite: it is the earliest representable instant, which would hand the + * root of an ascending-sorted thread to a message that placed itself nowhere + * in time. + */ +const UNDATED_SORT_KEY = Number.MAX_SAFE_INTEGER; + +/** The sort key for a message's position in a thread (RFC 5256 orders by date). */ +function messageDate(msg: StoredMessage): number { + const date = msg.envelope.date; + return date === undefined ? UNDATED_SORT_KEY : date.getTime(); +} + function containerDate(c: Container): number { if (c.message !== null) { - return c.message.envelope.date.getTime(); + return messageDate(c.message); } - // For dummy containers, use the earliest child date. - let earliest = Infinity; + // A dummy container carries no date of its own, so it borrows the earliest + // date among its descendants. With no dated descendant it keeps + // `UNDATED_SORT_KEY` and sorts last, as an undated message does. + let earliest = UNDATED_SORT_KEY; for (const child of c.children) { const d = containerDate(child); if (d < earliest) earliest = d; } - return earliest === Infinity ? 0 : earliest; + return earliest; } function containersToThreads( diff --git a/vendor/intx-mailbox/src/verify-signature.ts b/vendor/intx-mailbox/src/verify-signature.ts index 565387075..05fbccae6 100644 --- a/vendor/intx-mailbox/src/verify-signature.ts +++ b/vendor/intx-mailbox/src/verify-signature.ts @@ -5,7 +5,7 @@ import { extractBoundary, parseMultipart, } from "@intx/mime"; -import { verifyDetachedSignature } from "@intx/crypto"; +import { importPublicKeyBytes, verifyDetachedSignature } from "@intx/crypto"; /** * Verify a PGP/MIME `multipart/signed` message against a public key. @@ -17,8 +17,11 @@ import { verifyDetachedSignature } from "@intx/crypto"; * - `valid` — the detached signature verified against `publicKey` * - `invalid` — the signature check failed, or the message could not be * parsed as a signed message - * - `missing` — the message is not `multipart/signed`, or carries no - * `application/pgp-signature` part + * - `missing` — the message is not `multipart/signed`, declares no `boundary=` + * parameter, or carries no `application/pgp-signature` part + * + * Throws when `publicKey` cannot verify anything: a fault in the caller's + * input, not a verdict about the message. * * `raw` must be the original, unmodified message bytes: the signature is * recomputed over the exact canonical bytes of the signed part, so a @@ -29,6 +32,7 @@ export async function verifyMimeSignature( raw: Uint8Array, publicKey: Uint8Array, ): Promise<"valid" | "invalid" | "missing"> { + await requireUsableKey(publicKey); try { const { headers, bodyOffset } = parseHeaderSection(raw); const body = raw.slice(bodyOffset); @@ -65,3 +69,16 @@ export async function verifyMimeSignature( return "invalid"; } } + +/** + * Refuse a public key that cannot verify a signature, ahead of the catch that + * turns every other failure into `invalid` -- inside it, an unusable key would + * read as a check that ran and failed. + */ +async function requireUsableKey(publicKey: Uint8Array): Promise { + try { + await importPublicKeyBytes(publicKey); + } catch (cause) { + throw new Error("public key cannot verify a signature", { cause }); + } +} diff --git a/vendor/intx-mime/src/header-folding.test.ts b/vendor/intx-mime/src/header-folding.test.ts new file mode 100644 index 000000000..d966f7ff7 --- /dev/null +++ b/vendor/intx-mime/src/header-folding.test.ts @@ -0,0 +1,364 @@ +// Header line length, non-ASCII header text, and quoted-printable bodies. +// +// Two RFC 5322 requirements this assembler did not meet. A line MUST NOT exceed +// 998 characters (section 2.1.1), and a long `References` chain reaches that on +// its own -- 20 message ids of 50 characters is 1000. And a header body is +// ASCII: a non-ASCII subject needs the RFC 2047 encoded-word form, because +// 8-bit bytes in a header are illegal without an extension no relay here +// negotiates. +// +// Both were latent rather than theoretical. Nothing in the tree writes a +// subject with an accent in it today, and threads stay short in the tests -- +// which is why neither showed up until a message was put on a real wire. +// +// A text part that is not 7-bit is quoted-printable. RFC 2045 section 6.7 +// forbids a literal space or tab at the end of an encoded line, including the +// line a soft break closes, because a relay may strip it. + +import { describe, expect, test } from "bun:test"; + +import { + assembleMessage, + assembleSignedContent, + buildMessageHeaders, + decodeHeaderText, + decodeMail, + parseMailToEmail, + decodePartBytes, + extractBoundary, + parseHeaderSection, + parseMimePart, + parseMultipart, +} from "./mime"; +import type { MessageHeaders } from "./mime"; + +/** Every optional field spelled out: `MessageHeaders` requires all of them. */ +const BASE: MessageHeaders = { + from: "alpha@test.interchange", + to: ["beta@test.interchange"], + cc: undefined, + date: new Date("2026-01-01T00:00:00Z"), + messageId: "", + subject: undefined, + inReplyTo: undefined, + references: undefined, + mimeVersion: "1.0", + interchangeType: undefined, + interchangeCorrelationId: undefined, + interchangeTenantId: undefined, + interchangeAgentId: undefined, + interchangeSessionId: undefined, + interchangeOfferingId: undefined, + interchangeSchemaVersion: undefined, + traceparent: undefined, + tracestate: undefined, +}; + +const SIGNED_CONTENT = new TextEncoder().encode( + "Content-Type: text/plain\r\n\r\nbody\r\n", +); +const SIGNATURE = new TextEncoder().encode("-----BEGIN PGP SIGNATURE-----\r\n"); + +function headerLines(raw: Uint8Array): string[] { + const text = new TextDecoder().decode(raw); + const end = text.indexOf("\r\n\r\n"); + return text.slice(0, end === -1 ? text.length : end).split("\r\n"); +} + +function assembled(headers: MessageHeaders): Uint8Array { + return assembleMessage(headers, SIGNED_CONTENT, SIGNATURE); +} + +/** The value a receiver reconstructs: unfold, then decode encoded-words. */ +function receivedHeader(raw: Uint8Array, name: string): string | undefined { + const { headers } = parseHeaderSection(raw); + const value = headers.get(name.toLowerCase()); + return value === undefined ? undefined : decodeHeaderText(value); +} + +describe("no header line exceeds the hard limit", () => { + test("a long References chain is folded", () => { + // 20 ids at 50 characters each: 1000 characters of value, past the 998 a + // line may not exceed, before the header name is even counted. + const references = Array.from( + { length: 20 }, + (_, i) => `<${String(i).padStart(44, "0")}@x.test>`, + ); + const raw = assembled({ ...BASE, references }); + + for (const line of headerLines(raw)) { + expect(line.length).toBeLessThanOrEqual(998); + } + }); + + test("folding preserves the value a receiver reconstructs", () => { + // The point of folding at an existing space: unfolding deletes the CRLF and + // keeps the space, so the receiver's value is identical to the one given. + const references = Array.from( + { length: 20 }, + (_, i) => `<${String(i).padStart(44, "0")}@x.test>`, + ); + const raw = assembled({ ...BASE, references }); + + const parsed = buildMessageHeaders(parseHeaderSection(raw).headers); + expect(parsed.references).toEqual(references); + }); + + test("a long address list is folded and reconstructs", () => { + const to = Array.from( + { length: 12 }, + (_, i) => `recipient-with-a-long-name-${String(i)}@test.interchange`, + ); + const raw = assembled({ ...BASE, to }); + + for (const line of headerLines(raw)) { + expect(line.length).toBeLessThanOrEqual(998); + } + const parsed = buildMessageHeaders(parseHeaderSection(raw).headers); + expect(parsed.to).toEqual(to); + }); + + test("a single unfoldable token is not broken apart", () => { + // There is no legal fold point inside a msg-id. Breaking one to meet the + // recommended 78 would corrupt it, so an over-long one stays whole. + const messageId = `<${"a".repeat(200)}@test.interchange>`; + const raw = assembled({ ...BASE, messageId }); + + const parsed = buildMessageHeaders(parseHeaderSection(raw).headers); + expect(parsed.messageId).toBe(messageId); + }); + + test("repeated spaces in a header value survive folding", () => { + // Long enough that the fold falls inside a run of spaces. Unfolding + // replaces the break with one space, so the other space has to remain. + const subject = `${"x".repeat(60)} ${"y".repeat(30)}`; + const raw = assembled({ ...BASE, subject }); + + expect(headerLines(raw).some((line) => line.startsWith(" "))).toBe(true); + expect(receivedHeader(raw, "Subject")).toBe(subject); + }); + + test("a space run that crosses the fold stays intact", () => { + // The continuation may open with only one space: unfolding deletes the + // break and every whitespace character after it, then inserts one space. + // Spaces from the run that do not fit before the break have to stay on + // the line being folded. + const subjects = [ + `${"a".repeat(60)}${" ".repeat(10)}b`, + `${"x".repeat(68)} y`, + `${"A".repeat(70)} hello`, + ]; + for (const subject of subjects) { + const raw = assembled({ ...BASE, subject }); + for (const line of headerLines(raw)) { + expect(line.startsWith(" ")).toBe(false); + } + expect(receivedHeader(raw, "Subject")).toBe(subject); + } + }); + + test("a tab-ending run does not hide an earlier fold", () => { + // The last space before the target ends a run in a tab, so it cannot + // be the fold. An earlier space still can. Keeping the tab run on this + // line instead makes one line of 1008 characters. + const subject = `hi ${"a".repeat(64)} \t${"b".repeat(930)}`; + const raw = assembled({ ...BASE, subject }); + + for (const line of headerLines(raw)) { + expect(line.length).toBeLessThanOrEqual(998); + expect(line.startsWith(" ")).toBe(false); + } + expect(headerLines(raw).some((line) => line.startsWith(" "))).toBe(true); + expect(receivedHeader(raw, "Subject")).toBe(subject); + expect(decodeMail(raw).rawHeaders["subject"]).toEqual([subject]); + }); + + test("a tab after a folded space run stays in the value", () => { + // A tab that follows the fold space is leading whitespace on the + // continuation, and unfolding deletes that whole run. + const subjects = [ + `${"A".repeat(70)} \tX`, + `${"a".repeat(60)}${" ".repeat(10)}\tb`, + ]; + for (const subject of subjects) { + const raw = assembled({ ...BASE, subject }); + expect(receivedHeader(raw, "Subject")).toBe(subject); + expect(decodeMail(raw).rawHeaders["subject"]).toEqual([subject]); + } + }); + + test("spaces parked at the end of a continuation survive the raw header", () => { + const subject = `${"a".repeat(80)} ${"b".repeat(80)} c`; + const raw = assembled({ ...BASE, subject }); + + expect(receivedHeader(raw, "Subject")).toBe(subject); + expect(decodeMail(raw).rawHeaders["subject"]).toEqual([subject]); + }); + + test("a short header is not folded at all", () => { + // Folding only where it is needed: a continuation line on a short header + // is legal but makes every message harder to read. + const raw = assembled({ ...BASE, subject: "hello" }); + const text = new TextDecoder().decode(raw); + + expect(text).toContain("Subject: hello\r\n"); + }); +}); + +describe("a non-ASCII subject is encoded for the wire", () => { + test("the wire form is ASCII and the received value is the original", () => { + const subject = "Réunion du café — 10h"; + const raw = assembled({ ...BASE, subject }); + const text = new TextDecoder().decode(raw); + const end = text.indexOf("\r\n\r\n"); + const headerSection = text.slice(0, end); + + // eslint-disable-next-line no-control-regex + expect(/[^\x00-\x7F]/.test(headerSection)).toBe(false); + expect(headerSection).toContain("=?UTF-8?B?"); + expect(receivedHeader(raw, "Subject")).toBe(subject); + // The agent-facing parse is the one the hub delivers. The wire form + // stays in the raw header. + expect(parseMailToEmail(raw, "fold-subject").subject).toBe(subject); + expect(decodeMail(raw).rawHeaders["subject"]?.[0]).toContain("=?UTF-8?B?"); + }); + + test("a long non-ASCII subject round-trips across several encoded-words", () => { + // One encoded-word caps at 75 characters, so a long subject becomes a + // sequence of them. Each must decode independently and the whitespace + // joining them must not survive into the value. + const subject = "Поздравляем с завершением развертывания ".repeat(4).trim(); + const raw = assembled({ ...BASE, subject }); + + for (const line of headerLines(raw)) { + expect(line.length).toBeLessThanOrEqual(998); + } + expect(receivedHeader(raw, "Subject")).toBe(subject); + }); + + test("a character is never split across two encoded-words", () => { + // A multi-byte sequence straddling two words decodes to replacement + // characters in both, so the cut has to land on a character boundary. An + // emoji is 4 bytes, which makes a 45-byte payload boundary fall inside one + // unless the encoder walks back off it. + const subject = "🎉".repeat(40); + const raw = assembled({ ...BASE, subject }); + + expect(receivedHeader(raw, "Subject")).toBe(subject); + expect(receivedHeader(raw, "Subject")).not.toContain("�"); + }); + + test("an ASCII subject is left exactly as it was", () => { + // Encoding an ASCII subject would be legal and awful: every subject in + // every log and every mail client would read as base64. + const raw = assembled({ ...BASE, subject: "Deployment complete" }); + + expect(new TextDecoder().decode(raw)).toContain( + "Subject: Deployment complete\r\n", + ); + }); +}); + +describe("decoding what other senders produce", () => { + test("a Q-encoded word is decoded", () => { + // Only B is produced here, but Q is widespread, so a receiver has to read + // it. `_` is a space and `=XX` is a hex octet. + expect(decodeHeaderText("=?UTF-8?Q?caf=C3=A9_time?=")).toBe("café time"); + }); + + test("whitespace between two encoded-words is dropped", () => { + // RFC 2047 section 6.2: the separator is not content. Keeping it inserts a + // space into the middle of a word that was split for length. + expect(decodeHeaderText("=?UTF-8?B?w6A=?= =?UTF-8?B?w6E=?=")).toBe("àá"); + }); + + test("whitespace around a word is kept", () => { + expect(decodeHeaderText("Re: =?UTF-8?B?w6A=?= today")).toBe("Re: à today"); + }); + + test("an unencoded value passes through untouched", () => { + expect(decodeHeaderText("Plain subject")).toBe("Plain subject"); + }); + + test("a charset this cannot decode keeps its encoded form", () => { + // Showing the encoded text is a visible fault a reader can report. + // Substituting replacement characters would read as the sender's own text. + const value = "=?Shift_JIS?B?gqCCooKk?="; + expect(decodeHeaderText(value)).toBe(value); + }); + + test("a base64 word with characters outside the alphabet keeps its encoded form", () => { + expect(decodeHeaderText("=?UTF-8?B?***?=")).toBe("=?UTF-8?B?***?="); + expect(decodeHeaderText("=?UTF-8?B?YQ!!?=")).toBe("=?UTF-8?B?YQ!!?="); + }); + + test("a word whose bytes contradict its charset keeps its encoded form", () => { + // `/w==` is the single byte 0xFF, which is not valid UTF-8. + const value = "=?UTF-8?B?/w==?="; + expect(decodeHeaderText(value)).toBe(value); + }); + + test("an ISO-8859-1 word is decoded by byte value", () => { + expect(decodeHeaderText("=?ISO-8859-1?Q?caf=E9?=")).toBe("café"); + }); + + test("a language suffix on the charset is tolerated", () => { + // RFC 2231 section 5 permits `charset*language`. + expect(decodeHeaderText("=?UTF-8*fr?B?w6A=?=")).toBe("à"); + }); +}); + +describe("quoted-printable text survives a whitespace-stripping relay", () => { + test("a soft break does not close a line on a literal space or tab", () => { + // Long enough to wrap, with a space and a tab in the run, and one + // non-ASCII character so the part cannot stay 7bit. + const text = `${"alpha ".repeat(20)}beta\tgamma café`; + const signed = assembleSignedContent({ kind: "conversation", text }); + const { headers, bodyOffset } = parseHeaderSection(signed); + const contentType = headers.get("content-type"); + if (contentType === undefined) throw new Error("missing content-type"); + const boundary = extractBoundary(contentType); + if (boundary === undefined) throw new Error("missing boundary"); + const first = parseMultipart(signed.slice(bodyOffset), boundary)[0]; + if (first === undefined) throw new Error("missing text part"); + const textPart = parseMimePart(first); + expect(textPart.headers.get("content-transfer-encoding")).toBe( + "quoted-printable", + ); + + const wire = new TextDecoder().decode(textPart.body); + for (const line of wire.split("\r\n")) { + expect(line.endsWith(" ") || line.endsWith("\t")).toBe(false); + expect(/[ \t]=$/.test(line)).toBe(false); + } + + const decoded = new TextDecoder().decode( + decodePartBytes(textPart.body, textPart.headers), + ); + expect(decoded).toBe(`${"alpha ".repeat(20)}beta\tgamma café`); + }); + + test("escaped whitespace does not push a line past 76 octets", () => { + const text = `${"a".repeat(50)}${" ".repeat(25)}é${"\t".repeat(75)}ø`; + const signed = assembleSignedContent({ kind: "conversation", text }); + const { headers, bodyOffset } = parseHeaderSection(signed); + const contentType = headers.get("content-type"); + if (contentType === undefined) throw new Error("missing content-type"); + const boundary = extractBoundary(contentType); + if (boundary === undefined) throw new Error("missing boundary"); + const first = parseMultipart(signed.slice(bodyOffset), boundary)[0]; + if (first === undefined) throw new Error("missing text part"); + const textPart = parseMimePart(first); + + const wire = new TextDecoder().decode(textPart.body); + for (const line of wire.split("\r\n")) { + expect(line.length).toBeLessThanOrEqual(76); + expect(line.endsWith(" ") || line.endsWith("\t")).toBe(false); + } + expect( + new TextDecoder().decode( + decodePartBytes(textPart.body, textPart.headers), + ), + ).toBe(text); + }); +}); diff --git a/vendor/intx-mime/src/index.test.ts b/vendor/intx-mime/src/index.test.ts index 29fffe6de..ab49f3650 100644 --- a/vendor/intx-mime/src/index.test.ts +++ b/vendor/intx-mime/src/index.test.ts @@ -150,6 +150,119 @@ describe("extractAddrSpec", () => { test("throws on multiple '@' in an unquoted form", () => { expect(() => extractAddrSpec("a@b@example.com")).toThrow(); }); + + test("keeps a comma inside a quoted display name", () => { + expect(extractAddrSpec('"Doe, John" ')).toBe( + "john@example.com", + ); + }); + + test("throws on a comma-separated bare list", () => { + expect(() => extractAddrSpec("a@b.example, c@d.example")).toThrow( + /address lists are not supported/, + ); + }); + + test("throws on an angle-bracketed list rather than taking one member", () => { + // The reduction this refuses picked the last member, so a `From` naming a + // victim first and the sender second read as the sender alone while every + // parser of the full header still saw both. + expect(() => + extractAddrSpec( + '"CEO" , "x" ', + ), + ).toThrow(/address lists are not supported/); + }); + + test("throws on adjacent angle-bracketed addresses separated by a space", () => { + // A space is not the only spelling of a list, so a rule about commas + // alone left this reducing to the last member. + expect(() => + extractAddrSpec(" "), + ).toThrow(/address lists are not supported/); + }); + + test("throws on adjacent angle-bracketed addresses separated by a tab", () => { + expect(() => + extractAddrSpec("\t"), + ).toThrow(/address lists are not supported/); + }); + + test("throws on angle-bracketed addresses with nothing between them", () => { + expect(() => + extractAddrSpec(""), + ).toThrow(/address lists are not supported/); + }); + + test("throws on a bare address ahead of an angle-bracketed one", () => { + // A display name holds no unquoted '@', so the leading addr-spec is a + // second address and not a name for the one in brackets. + expect(() => + extractAddrSpec("ceo@victim.example "), + ).toThrow(/address lists are not supported/); + }); + + test("throws on a display name followed by two angle-bracketed addresses", () => { + expect(() => extractAddrSpec("Name ")).toThrow( + /address lists are not supported/, + ); + }); + + test("throws on quoted display names between adjacent addresses", () => { + // Quoting each member hides the separator but not the second '<'. + expect(() => + extractAddrSpec('"CEO" "x"'), + ).toThrow(/address lists are not supported/); + }); + + test("throws on an address carried in a leading comment", () => { + // A comment may hold '@' as text, and extractAddrSpec refuses a comment + // wherever it appears, so reading it as a second address costs nothing. + expect(() => + extractAddrSpec("(ceo@victim.example) "), + ).toThrow(/address lists are not supported/); + }); + + test("throws on a group rather than taking its one member", () => { + expect(() => + extractAddrSpec("Team: , ;"), + ).toThrow(/address lists are not supported/); + }); + + test("throws on a '>' that closes the address early", () => { + expect(() => extractAddrSpec(">")).toThrow(/stray '>'/); + }); + + test("keeps an empty quoted display name", () => { + expect(extractAddrSpec('"" ')).toBe("a@b.example"); + }); + + test("keeps an escaped quote inside a quoted display name", () => { + expect(extractAddrSpec('"a\\"b, c" ')).toBe("x@y.example"); + }); + + test("keeps a dot in an unquoted display name", () => { + expect(extractAddrSpec("John Q. Public ")).toBe( + "john@example.com", + ); + }); + + test("keeps a domain literal", () => { + expect(extractAddrSpec("")).toBe("user@[192.168.0.1]"); + }); + + test("throws on an unterminated quoted string", () => { + // Leaving the quote open would otherwise hide the separator from the scan. + expect(() => + extractAddrSpec('"CEO , "x" '), + ).toThrow(/unterminated quoted string/); + }); + + test("reads a fully quoted prefix as one display name", () => { + expect( + extractAddrSpec('"CEO , x" '), + ).toBe("attacker@evil.example"); + }); }); describe("formatRFC2822Date", () => { @@ -317,6 +430,64 @@ describe("assembleMessage", () => { const sigPart = parseMimePart(defined(parts[1])); expect(sigPart.contentType).toBe("application/pgp-signature"); }); + + test("rejects a subject containing CRLF (header injection)", () => { + // A reply copies the subject from the inbound peer message, so this is + // the remotely reachable path: a doubled line ending ends the header + // block and turns the signed envelope into inert body text. + const content = assembleSignedContent({ + kind: "conversation", + text: "test", + }); + const headers = makeHeaders({ + subject: "hi\r\nInterchange-Type: session.accept\r\n\r\nplanted body", + }); + expect(() => assembleMessage(headers, content, enc.encode("SIG"))).toThrow( + /CR or LF/, + ); + }); + + test("rejects a bare LF and a bare CR in any emitted header", () => { + const content = assembleSignedContent({ + kind: "conversation", + text: "test", + }); + expect(() => + assembleMessage( + makeHeaders({ from: "alice@test.interchange\nCc: mallory@evil" }), + content, + enc.encode("SIG"), + ), + ).toThrow(/CR or LF/); + expect(() => + assembleMessage( + makeHeaders({ + interchangeAgentId: "agt-1\rInterchange-Tenant-ID: t-2", + }), + content, + enc.encode("SIG"), + ), + ).toThrow(/CR or LF/); + }); + + test("emits a double quote in a header value verbatim", () => { + // Unstructured text may carry a quote, and the serializer emits its own + // quoted boundary parameter on every message, so the emission guard must + // reject line breaks only. + const content = assembleSignedContent({ + kind: "conversation", + text: "test", + }); + const headers = makeHeaders({ + subject: 'Re: the "urgent" request', + from: '"Doe, Jane" ', + }); + const text = dec.decode( + assembleMessage(headers, content, enc.encode("SIG")), + ); + expect(text).toContain('Subject: Re: the "urgent" request\r\n'); + expect(text).toContain('From: "Doe, Jane" \r\n'); + }); }); // --------------------------------------------------------------------------- @@ -332,11 +503,69 @@ describe("parseHeaderSection", () => { expect(dec.decode(raw.slice(bodyOffset))).toBe("Body here"); }); - test("parses LF-terminated headers", () => { + test("refuses an LF-terminated message", () => { + // RFC 5321 section 2.3.8 forbids recognizing any character or sequence but + // CRLF as a line terminator, and section 4.1.1.4 refuses the lone-LF ending + // by name. Folding the breaks instead would leave one field whose value + // swallows every later field and the body. const raw = enc.encode("From: alice@test\nTo: bob@test\n\nBody"); + expect(() => parseHeaderSection(raw)).toThrow( + /must break its lines with CRLF/, + ); + }); + + test("refuses a CR-terminated message", () => { + const raw = enc.encode("From: alice@test\rTo: bob@test\r\rBody"); + expect(() => parseHeaderSection(raw)).toThrow( + /must break its lines with CRLF/, + ); + }); + + test("refuses an LF-only header section that a CRLF CRLF terminates", () => { + // The section's own line breaks decide whether it conforms, not the flavour + // of the blank line that ends it. Folding these instead leaves one field + // whose value swallows `Interchange-Type` and `Subject`, so a sender + // suppresses both fields by writing the section with bare LFs. + const raw = enc.encode( + "From: alice@x\nInterchange-Type: conversation.message\n" + + "Subject: Hi\r\n\r\nBody", + ); + expect(() => parseHeaderSection(raw)).toThrow( + /must break its lines with CRLF/, + ); + }); + + test("refuses an LF-only header section when the body carries a CRLF CRLF", () => { + // Searching the whole message for the separator finds the one in the body + // and reads the header section from there, which both suppresses every + // field after the first and truncates the body to what followed it. + const raw = enc.encode( + "From: alice@x\nInterchange-Type: conversation.message\n" + + "Subject: Hi\n\nBody\r\n\r\ntail\n", + ); + expect(() => parseHeaderSection(raw)).toThrow( + /must break its lines with CRLF/, + ); + }); + + test("refuses an LF LF separator after a CRLF-only section", () => { + // The blank line is a pair of line breaks of its own, so honouring a bare + // pair lets a sender who controls one field body end the section early and + // strip the fields after it. + const raw = enc.encode( + "From: alice@x\r\nSubject: Hi\n\n" + + "Interchange-Type: conversation.message\r\n\r\nBody", + ); + expect(() => parseHeaderSection(raw)).toThrow( + /must break its lines with CRLF/, + ); + }); + + test("takes the first blank line when the body carries another", () => { + const raw = enc.encode("Subject: Hi\r\n\r\nBody\r\n\r\nmore"); const { headers, bodyOffset } = parseHeaderSection(raw); - expect(headers.get("from")).toBe("alice@test"); - expect(dec.decode(raw.slice(bodyOffset))).toBe("Body"); + expect(headers.get("subject")).toBe("Hi"); + expect(dec.decode(raw.slice(bodyOffset))).toBe("Body\r\n\r\nmore"); }); test("unfolds continuation lines", () => { @@ -397,6 +626,46 @@ describe("parseHeaderSection", () => { expect(bodyOffset).toBe(4); expect(headers.size).toBe(0); }); + + // A bare CR and a bare LF are external input that no field body may carry: + // RFC 5322 section 2.2 admits neither inside a field body, and section 2.3 + // requires the two to occur only together as CRLF. A sender controls every one + // of these fields, and each row hides the same second field behind one bare + // character. Neither reading of that character is safe -- as a terminator it + // resolves a field from a value the sender smuggled inside one it controls, + // and folded to a space it suppresses whatever field followed it -- so the + // section is refused for every row. + const bareBreaks = ["\r", "\n"]; + const bareBreakHeaders: string[] = bareBreaks.flatMap((brk) => [ + `Subject: hello${brk}Bcc: attacker@evil.test`, + `From: alice@example.com${brk}Bcc: attacker@evil.test`, + `Message-ID: ${brk}Bcc: attacker@evil.test`, + `Interchange-Correlation-ID: corr-1${brk}Bcc: attacker@evil.test`, + `References: ${brk}Bcc: attacker@evil.test`, + `Subject: benign${brk}Interchange-Agent-Id: victim`, + ]); + + test("refuses a bare carriage return or line feed in a field body", () => { + for (const header of bareBreakHeaders) { + expect(() => + parseHeaderSection( + enc.encode(`${header}\r\nTo: bob@example.com\r\n\r\nbody`), + ), + ).toThrow(/must break its lines with CRLF/); + } + }); + + test("leaves CRLF folding and line termination alone", () => { + // The fold targets a CR that is not part of a CRLF, so a real terminator + // and a real continuation line still mean what they meant. + const { headers } = parseHeaderSection( + enc.encode( + "References: \r\n \r\nSubject: kept\r\n\r\nBody", + ), + ); + expect(headers.get("references")).toBe(" "); + expect(headers.get("subject")).toBe("kept"); + }); }); // --------------------------------------------------------------------------- @@ -444,9 +713,16 @@ describe("parseMultipart", () => { expect(dec.decode(defined(parts[1]))).toContain("

Part two

"); }); - test("handles LF-only line endings", () => { + test("splits parts whose delimiter lines are LF-terminated", () => { + // The boundary scan tolerates an LF-terminated delimiter line. Each part's + // own fields are CRLF-terminated, because CRLF is the only field terminator + // the header section parse recognizes (RFC 5322 section 2.3); this fixture + // keeps the two questions apart rather than asserting both tolerances from + // one input. const body = enc.encode( - "--boundary\nContent-Type: text/plain\n\nPart one\n--boundary\nContent-Type: text/html\n\n

Part two

\n--boundary--\n", + "--boundary\nContent-Type: text/plain\r\n\r\nPart one\n" + + "--boundary\nContent-Type: text/html\r\n\r\n

Part two

\n" + + "--boundary--\n", ); const parts = parseMultipart(body, "boundary"); expect(parts).toHaveLength(2); @@ -1052,6 +1328,7 @@ describe("conversation attachments round-trip", () => { const got = defined(extracted[i]); expect(got.name).toBe(orig.name); expect(got.contentType).toBe(orig.contentType); + expect(got.part).toBe(`1.${i + 2}`); expect(Array.from(got.data)).toEqual(Array.from(orig.data)); } }); @@ -1137,6 +1414,56 @@ describe("conversation attachments round-trip", () => { expect(valid).toBe(true); }); + test("an inline html sibling does not steal the PDF's IMAP path", () => { + // A mail client that also sends text/html leaves an extra sibling + // between BODY[1.1] and the attachment. extractAttachments skips that + // inline part; the PDF's path must still be the sibling number + // extractPartByPath uses (1.3), not the attachment-array index (1.2). + const pdf = new Uint8Array([0x25, 0x50, 0x44, 0x46, 0x2d]); + const signed = [ + `Content-Type: multipart/mixed; boundary="inner"`, + "", + `--inner`, + "Content-Type: text/plain; charset=utf-8", + "Content-Transfer-Encoding: 7bit", + "", + "hello", + `--inner`, + "Content-Type: text/html; charset=utf-8", + "Content-Disposition: inline", + "", + "

hello

", + `--inner`, + "Content-Type: application/pdf", + "Content-Transfer-Encoding: base64", + `Content-Disposition: attachment; filename="report.pdf"`, + "", + "JVBERi0=", + `--inner--`, + "", + ].join("\r\n"); + const msg = assembleMessage( + makeHeaders({ interchangeType: "conversation.message" }), + enc.encode(signed), + enc.encode("FAKE-SIGNATURE"), + ); + + const htmlPart = parseMimePart(extractPartByPath(msg, "1.2")); + expect( + defined(htmlPart.contentType.split(";")[0]).trim().toLowerCase(), + ).toBe("text/html"); + const pdfPart = parseMimePart(extractPartByPath(msg, "1.3")); + expect( + defined(pdfPart.contentType.split(";")[0]).trim().toLowerCase(), + ).toBe("application/pdf"); + + const extracted = extractAttachments(msg); + expect(extracted).toHaveLength(1); + expect(defined(extracted[0]).name).toBe("report.pdf"); + expect(defined(extracted[0]).part).toBe("1.3"); + expect(Array.from(defined(extracted[0]).data)).toEqual(Array.from(pdf)); + }); + test("rejects an attachment name containing CRLF (header injection)", () => { expect(() => assembleSignedContent({ @@ -1152,4 +1479,45 @@ describe("conversation attachments round-trip", () => { }), ).toThrow(); }); + + test("rejects an attachment header that is not 7-bit or over 998 octets", () => { + // These headers sit inside the signed part and have no transfer + // encoding. An 8-bit name, or a line past 998, is one a relay can rewrite. + const blob = new Uint8Array([1]); + expect(() => + assembleSignedContent({ + kind: "conversation", + text: "x", + attachments: [ + { name: "résumé.pdf", contentType: "application/pdf", data: blob }, + ], + }), + ).toThrow(/US-ASCII/); + expect(() => + assembleSignedContent({ + kind: "conversation", + text: "x", + attachments: [ + { + name: "a".repeat(1100), + contentType: "application/pdf", + data: blob, + }, + ], + }), + ).toThrow(/998/); + expect(() => + assembleSignedContent({ + kind: "conversation", + text: "x", + attachments: [ + { + name: "notes.txt", + contentType: `application/${"é"}`, + data: blob, + }, + ], + }), + ).toThrow(/US-ASCII/); + }); }); diff --git a/vendor/intx-mime/src/index.ts b/vendor/intx-mime/src/index.ts index a2029561d..64540e770 100644 --- a/vendor/intx-mime/src/index.ts +++ b/vendor/intx-mime/src/index.ts @@ -8,11 +8,17 @@ export { parseMimePart, parseMultipart, extractBoundary, + extractContentTypeMime, extractPartByPath, parseMailToEmail, extractAttachments, buildMessageHeaders, + decodeHeaderText, decodeMail, + decodePartBytes, + isRecognizedTransferEncoding, + reportedContentType, + transferEncodingMechanism, } from "./mime"; export type { diff --git a/vendor/intx-mime/src/mail-builder.test.ts b/vendor/intx-mime/src/mail-builder.test.ts index 95ca4e37d..27383a8c2 100644 --- a/vendor/intx-mime/src/mail-builder.test.ts +++ b/vendor/intx-mime/src/mail-builder.test.ts @@ -50,9 +50,13 @@ describe("createInboundMessage", () => { test("auto-generated date is a parseable ISO string", () => { const msg = createInboundMessage({ from: FROM, to: TO, content: "hi" }); - const parsed = new Date(msg.headers.date); + const { date } = msg.headers; + // The builder mints a date for a caller that supplies none, so an absent + // one here is the failure this test exists to catch. + if (date === undefined) throw new Error("expected a generated Date header"); + const parsed = new Date(date); expect(Number.isNaN(parsed.getTime())).toBe(false); - expect(msg.headers.date).toBe(parsed.toISOString()); + expect(date).toBe(parsed.toISOString()); }); test("structured payload sets interchangeType header automatically", () => { @@ -233,10 +237,39 @@ describe("createInboundMessage", () => { ).toThrow(/`from` must be a non-empty string/); }); - test("throws when to is an empty array", () => { + test("builds a message with no From when from is omitted", () => { + // Omitting the originator is how a caller says it has none. An empty + // string is still rejected above: that is a caller supplying a + // placeholder, which is the fabrication this distinction exists to stop. + const msg = createInboundMessage({ to: TO, content: "x" }); + expect(msg.headers.from).toBeUndefined(); + expect(msg.headers.messageId).toMatch(/^<[^<>\s@]+@local>$/); + }); + + test("builds a message with no recipient when to is an empty array", () => { + // An empty list is how a caller says the message names no recipient it + // can vouch for. Inbound mail reaches this builder with an absent or + // unreadable `To` routinely -- the hub routes on an out-of-band address, + // not on the header -- and the alternative to recording none is + // recording one the caller chose, which reads back as the message's own. + const msg = createInboundMessage({ from: FROM, to: [], content: "x" }); + expect(msg.headers.to).toEqual([]); + }); + + test("throws when to is an empty string", () => { + // The empty list above is the caller having no recipient. An empty + // string is a caller supplying a placeholder, which stays refused. expect(() => - createInboundMessage({ from: FROM, to: [], content: "x" }), - ).toThrow(/`to` must contain at least one recipient address/); + createInboundMessage({ from: FROM, to: "", content: "x" }), + ).toThrow(/`to` must be a non-empty string/); + }); + + test("throws when cc is an empty array", () => { + // Only `to` carries the no-recipient state; the decoder omits an empty + // `cc` rather than emptying it, so an empty list here is a caller error. + expect(() => + createInboundMessage({ from: FROM, to: TO, cc: [], content: "x" }), + ).toThrow(/`cc` must contain at least one recipient address/); }); test("throws when to contains an empty string", () => { diff --git a/vendor/intx-mime/src/mail-builder.ts b/vendor/intx-mime/src/mail-builder.ts index 65c3ff388..63f7402f2 100644 --- a/vendor/intx-mime/src/mail-builder.ts +++ b/vendor/intx-mime/src/mail-builder.ts @@ -22,8 +22,12 @@ import type { MessageHeaders, MessageRef, OutboundMessage, -} from "@intx/types/runtime"; -import { InterchangeType, SignatureStatus } from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; +import { + InterchangeType, + SignatureStatus, + isConversationType, +} from "@intx/types/runtime-core"; import { generateMessageId } from "./mime"; /** @@ -37,8 +41,6 @@ const DEFAULT_PAYLOAD_VERSION = "1"; const MESSAGE_ID_RE = /^<[^<>\s@]+@[^<>\s@]+>$/; const ADDRESS_RE = /^[^@\s]+@[^@\s]+$/; -const CONVERSATION_TYPE_PREFIX = "conversation."; - // --------------------------------------------------------------------------- // InboundMessage builder // --------------------------------------------------------------------------- @@ -54,7 +56,10 @@ export type InboundPayloadInput = { }; export type CreateInboundMessageOpts = { - from: string; + /** Originator; omit it when the message has none. */ + from?: string; + + /** Recipients; pass an empty array when the message names none. */ to: string | string[]; /** Plain-text body. Mutually exclusive with `payload`. */ @@ -111,8 +116,8 @@ export function createInboundMessage( ): InboundMessage { const fn = "createInboundMessage"; - requireAddress(opts.from, "from", fn); - const to = normalizeAndValidateAddressArray(opts.to, "to", fn); + if (opts.from !== undefined) requireAddress(opts.from, "from", fn); + const to = normalizeAndValidateAddressArray(opts.to, "to", fn, true); validateBodyExclusivity(opts.content, opts.payload, fn); @@ -168,7 +173,7 @@ export function createInboundMessage( const cc = opts.cc === undefined ? undefined - : normalizeAndValidateAddressArray(opts.cc, "cc", fn); + : normalizeAndValidateAddressArray(opts.cc, "cc", fn, false); rejectEmptyStringIfPresent(opts.content, "content", fn); rejectEmptyStringIfPresent(opts.subject, "subject", fn); @@ -202,7 +207,8 @@ export function createInboundMessage( const messageId = opts.messageId ?? generateMessageId(opts.from); const derivedInterchangeType = opts.interchangeType ?? opts.payload?.type; - const headers: MessageHeaders = { from: opts.from, to, date, messageId }; + const headers: MessageHeaders = { to, date, messageId }; + if (opts.from !== undefined) headers.from = opts.from; if (cc !== undefined) headers.cc = cc; if (opts.subject !== undefined) headers.subject = opts.subject; if (opts.inReplyTo !== undefined) headers.inReplyTo = opts.inReplyTo; @@ -310,9 +316,9 @@ export function createOutboundMessage( validateInterchangeType(opts.type, "type", fn); // Validate addresses without mutating the source shape; the OutboundMessage // type preserves `string | string[]` and downstream consumers handle both. - normalizeAndValidateAddressArray(opts.to, "to", fn); + normalizeAndValidateAddressArray(opts.to, "to", fn, false); if (opts.cc !== undefined) { - normalizeAndValidateAddressArray(opts.cc, "cc", fn); + normalizeAndValidateAddressArray(opts.cc, "cc", fn, false); } validateBodyExclusivity(opts.content, opts.payload, fn); @@ -410,16 +416,21 @@ function requireAddress(value: unknown, field: string, fn: string): void { } } +/** + * Validate an address list and normalize a bare string into a one-entry array. + * `allowNone` permits an empty list, which only an inbound `to` does. + */ function normalizeAndValidateAddressArray( input: string | string[], field: string, fn: string, + allowNone: boolean, ): string[] { if (typeof input === "string") { requireAddress(input, field, fn); return [input]; } - if (!Array.isArray(input) || input.length === 0) { + if (!Array.isArray(input) || (input.length === 0 && !allowNone)) { throw new Error( `${fn}: \`${field}\` must contain at least one recipient address`, ); @@ -440,10 +451,6 @@ function validatePayloadBody(value: unknown, field: string, fn: string): void { } } -function isConversationType(t: InterchangeType): boolean { - return t.startsWith(CONVERSATION_TYPE_PREFIX); -} - function validateInterchangeType( value: unknown, field: string, diff --git a/vendor/intx-mime/src/mail-decode.test.ts b/vendor/intx-mime/src/mail-decode.test.ts index c297c2914..95e7e24e7 100644 --- a/vendor/intx-mime/src/mail-decode.test.ts +++ b/vendor/intx-mime/src/mail-decode.test.ts @@ -1,14 +1,150 @@ import { describe, test, expect } from "bun:test"; import type { MailPart, MessageAttachment } from "@intx/types/runtime"; import { isMail } from "@intx/types/runtime"; +import { deriveMessageId, parseMessageIdHeader } from "@intx/types"; -import { assembleSignedContent, assembleMessage, decodeMail } from "./index"; +import { + assembleSignedContent, + assembleMessage, + decodeMail, + extractAttachments, + parseMailToEmail, +} from "./index"; import type { MessageHeaders } from "./index"; function rawBytes(text: string): Uint8Array { return new TextEncoder().encode(text); } +/** + * Build message bytes whose body is exactly the given byte values, so a test + * can carry a body that is not valid UTF-8. + */ +function rawWithBody(headerText: string, bodyBytes: number[]): Uint8Array { + const head = new TextEncoder().encode(headerText); + const out = new Uint8Array(head.length + bodyBytes.length); + out.set(head, 0); + out.set(Uint8Array.from(bodyBytes), head.length); + return out; +} + +function codeUnits(value: string): number[] { + const out: number[] = []; + for (let i = 0; i < value.length; i++) out.push(value.charCodeAt(i)); + return out; +} + +/** A multipart/mixed message with one text/plain part under `encoding`. */ +function multipartWithEncoding(encoding: string, body: string): Uint8Array { + const boundary = "cte_boundary"; + return rawBytes( + [ + "From: alice@example.com", + "To: bob@example.com", + `Content-Type: multipart/mixed; boundary="${boundary}"`, + "", + `--${boundary}`, + "Content-Type: text/plain", + `Content-Transfer-Encoding: ${encoding}`, + "", + body, + `--${boundary}--`, + "", + ].join("\r\n"), + ); +} + +/** + * A single-part text/plain message under `encoding`. The message's own headers + * are the part's headers here, so this is the shape that tells whether an + * entry point reads the declared mechanism or a reconstruction that dropped + * it. + */ +function singlePartWithEncoding(encoding: string, body: string): Uint8Array { + return rawBytes( + [ + "From: alice@example.com", + "To: bob@example.com", + "Content-Type: text/plain", + `Content-Transfer-Encoding: ${encoding}`, + "", + body, + ].join("\r\n"), + ); +} + +/** + * A multipart/mixed message whose part 1 is the text body and whose part 2 is + * an application/pdf attachment under `encoding`. + */ +function multipartWithAttachmentEncoding( + encoding: string, + body: string, +): Uint8Array { + const boundary = "cte_att_boundary"; + return rawBytes( + [ + "From: alice@example.com", + "To: bob@example.com", + `Content-Type: multipart/mixed; boundary="${boundary}"`, + "", + `--${boundary}`, + "Content-Type: text/plain", + "", + "see attached", + `--${boundary}`, + "Content-Type: application/pdf", + `Content-Transfer-Encoding: ${encoding}`, + 'Content-Disposition: attachment; filename="report.pdf"', + "", + body, + `--${boundary}--`, + "", + ].join("\r\n"), + ); +} + +/** + * A multipart/signed message in the shape `extractAttachments` reads: an outer + * signed envelope whose content part is a multipart/mixed carrying the text + * body and one application/pdf attachment under `encoding`. + */ +function signedWithAttachmentEncoding( + encoding: string, + body: string, +): Uint8Array { + return rawBytes( + [ + "From: alice@example.com", + "To: bob@example.com", + 'Content-Type: multipart/signed; protocol="application/pgp-signature"; ' + + 'micalg="pgp-sha256"; boundary="cte_outer"', + "", + "--cte_outer", + 'Content-Type: multipart/mixed; boundary="cte_inner"', + "", + "--cte_inner", + "Content-Type: text/plain", + "", + "see attached", + "--cte_inner", + "Content-Type: application/pdf", + `Content-Transfer-Encoding: ${encoding}`, + 'Content-Disposition: attachment; filename="report.pdf"', + "", + body, + "--cte_inner--", + "", + "--cte_outer", + "Content-Type: application/pgp-signature", + "", + "placeholder-signature", + "--cte_outer--", + "", + ].join("\r\n"), + ); +} + function headers(overrides: Partial = {}): MessageHeaders { return { from: '"Alice" ', @@ -78,6 +214,75 @@ describe("decodeMail", () => { ); }); + test("records an absent Date and Message-ID as absent, not as empty", () => { + // RFC 5322 defines `Date` as `date-time` and `Message-ID` as `msg-id`. + // Neither admits an empty body, so a message carrying neither has to come + // back with both fields absent: a defaulted empty string reads as a value + // downstream and gets written back out as a malformed header line. + const mail = decodeMail( + rawBytes( + [ + "From: alice@example.com", + "To: bob@example.com", + "Subject: neither header", + "Content-Type: text/plain", + "", + "body", + ].join("\r\n"), + ), + ); + + expect(mail.headers.date).toBeUndefined(); + expect(mail.headers.messageId).toBeUndefined(); + // Absent, not present-and-undefined: the key must not survive into a + // serialized envelope as an explicit null-ish value. + expect("date" in mail.headers).toBe(false); + expect("messageId" in mail.headers).toBe(false); + }); + + test("treats a blank Date and Message-ID as absent", () => { + // A sender that writes the header and leaves it empty has named nothing, + // which is the same treatment a blank `From` already gets. + const mail = decodeMail( + rawBytes( + [ + "From: alice@example.com", + "To: bob@example.com", + "Date: ", + "Message-ID: ", + "Content-Type: text/plain", + "", + "body", + ].join("\r\n"), + ), + ); + + expect("date" in mail.headers).toBe(false); + expect("messageId" in mail.headers).toBe(false); + }); + + test("treats a blank In-Reply-To as absent", () => { + // RFC 5322 defines `In-Reply-To` as `1*msg-id` under the same clause as + // `Message-ID`, so a header left blank names no parent. An empty value is + // an id no message can carry, so every message whose header was blank + // would otherwise read as a reply to one shared nonexistent parent. + const mail = decodeMail( + rawBytes( + [ + "From: alice@example.com", + "To: bob@example.com", + "Message-ID: ", + "In-Reply-To: ", + "Content-Type: text/plain", + "", + "body", + ].join("\r\n"), + ), + ); + + expect("inReplyTo" in mail.headers).toBe(false); + }); + test("decodes every part of a message with attachments, no data loss", () => { const mail = decodeMail( signedConversation("see attached", [ @@ -127,6 +332,62 @@ describe("decodeMail", () => { expect(mail.rawHeaders["subject"]).toEqual(["folded subject line"]); }); + test("keeps a space parked at the end of a folded line", () => { + // The leading whitespace of a continuation is the fold. A space before + // the next fold is part of the value, so trimming the continuation + // would drop it. + const mail = decodeMail( + rawBytes( + "From: a@b\r\n" + + "Subject: hello\r\n" + + " world \r\n" + + " more\r\n" + + "Content-Type: text/plain\r\n\r\nbody", + ), + ); + expect(mail.rawHeaders["subject"]).toEqual(["hello world more"]); + }); + + test("keeps a header named after an Object.prototype member", () => { + // `__proto__` and `constructor` are well-formed field names (RFC 5322 + // section 3.6.8), so a peer can send them. Accumulating them into a plain + // object reaches inherited members instead of the accumulator's own + // properties, which either throws or discards the value. + const mail = decodeMail( + rawBytes( + "From: a@b\r\n" + + "__proto__: injected\r\n" + + "Constructor: hostile\r\n" + + "Content-Type: text/plain\r\n\r\nbody", + ), + ); + + // Lossless: each value is an own data property, so it survives rather + // than being swallowed by the inherited `__proto__` setter. + expect(Object.hasOwn(mail.rawHeaders, "__proto__")).toBe(true); + expect( + Object.getOwnPropertyDescriptor(mail.rawHeaders, "__proto__")?.value, + ).toEqual(["injected"]); + expect(Object.hasOwn(mail.rawHeaders, "constructor")).toBe(true); + expect(mail.rawHeaders["constructor"]).toEqual(["hostile"]); + + // Prototype-free: the map carries no inherited member, so a field name the + // message does not carry resolves to nothing rather than to whatever + // `Object.prototype` holds under that name -- the declared type promises + // `string[] | undefined` for every name, and `toString` would otherwise be + // a function. The shared accessor itself is untouched. + expect(Object.getPrototypeOf(mail.rawHeaders)).toBeNull(); + expect(mail.rawHeaders["toString"]).toBeUndefined(); + expect(mail.rawHeaders["hasOwnProperty"]).toBeUndefined(); + expect( + typeof Object.getOwnPropertyDescriptor(Object.prototype, "__proto__") + ?.get, + ).toBe("function"); + + // Ordinary headers in the same message still parse. + expect(mail.rawHeaders["from"]).toEqual(["a@b"]); + }); + test("undoes base64 transfer-encoding on a leaf part", () => { const mail = decodeMail( rawBytes( @@ -159,6 +420,182 @@ describe("decodeMail", () => { expect(mail.headers.from).toBe("alice@example.com"); }); + test("refuses a bare carriage return or line feed before either reading", () => { + // RFC 5322 section 2.3 requires CR and LF to occur only together as CRLF, + // and RFC 5321 section 4.1.1.4 refuses the lone-LF line ending by name, so a + // bare break terminates no field. `decodeMail` reads one header section + // twice -- once into the typed subset, once into the raw map a workflow + // walks -- and refusing the section ahead of both is what keeps the two from + // disagreeing about which fields the message carries. + for (const brk of ["\r", "\n"]) { + expect(() => + decodeMail( + rawBytes( + "From: alice@example.com\r\n" + + "To: bob@example.com\r\n" + + `Subject: hello${brk}Bcc: attacker@evil.test\r\n` + + `Interchange-Correlation-ID: corr-1${brk}Bcc: attacker@evil.test\r\n` + + "\r\nbody", + ), + ), + ).toThrow(/must break its lines with CRLF/); + } + }); + + test("records a blank or absent From as no originator, not an empty one", () => { + // A defaulted "" made these two states indistinguishable and handed every + // consumer an originator the message never carried. + expect( + decodeMail(rawBytes("To: bob@example.com\r\n\r\nbody")).headers.from, + ).toBeUndefined(); + expect( + decodeMail(rawBytes("From: \r\nTo: bob@example.com\r\n\r\nbody")) + .headers.from, + ).toBeUndefined(); + }); + + test("carries an unparseable From through verbatim", () => { + // This projection is lossless. An originator that is present but not a + // parseable address is a different judgement from one that was never + // there, and the admission gate keys on exactly that difference. + expect( + decodeMail(rawBytes("From: not an address\r\n\r\nbody")).headers.from, + ).toBe("not an address"); + }); + + test("keeps every header when the header section opens with a blank line", () => { + // The raw parse used to break on the leading empty line and return no + // headers at all, where the typed parse skipped it and kept both fields. + const mail = decodeMail( + rawBytes( + "\r\nFrom: alice@example.com\r\nTo: bob@example.com\r\n\r\nbody", + ), + ); + expect(mail.rawHeaders["from"]).toEqual(["alice@example.com"]); + expect(mail.rawHeaders["to"]).toEqual(["bob@example.com"]); + expect(mail.headers.from).toBe("alice@example.com"); + expect(mail.headers.to).toEqual(["bob@example.com"]); + }); + + test("names no id in either parser for a lone-LF message", async () => { + // CRLF is the sole line terminator (RFC 5321 section 2.3.8, section + // 4.1.1.4). `decodeMail` refuses these bytes, so the claim-check id + // derived for them must not be one read out of the text it refused. + const raw = rawBytes("Message-ID: \n\nbody"); + expect(() => decodeMail(raw)).toThrow(/must break its lines with CRLF/); + expect(parseMessageIdHeader(raw)).toBeNull(); + expect(await deriveMessageId(raw)).toMatch(/^[0-9a-f]{64}$/); + }); + + test("names no id in either parser for an LF-only header section", async () => { + // A whole-message search for the `CRLF CRLF` separator finds the one in the + // body and reads the header section from there: every field after the first + // is folded into it, so `Subject` goes missing, the body is truncated to + // what followed the separator, and the id absorbs sender-controlled body + // bytes. That id is the claim-check dedup key, a stored filename and a + // database join value, so nothing from the body may reach it. + const bodyTerminated = rawBytes( + "Message-ID: \nSubject: Hi\n\nbody\r\n\r\ntail\n", + ); + expect(() => decodeMail(bodyTerminated)).toThrow( + /must break its lines with CRLF/, + ); + expect(parseMessageIdHeader(bodyTerminated)).toBeNull(); + expect(await deriveMessageId(bodyTerminated)).toMatch(/^[0-9a-f]{64}$/); + + // The same LF-only section, ended by a `CRLF CRLF` of its own rather than + // by one the sender left in the body. + const crlfTerminated = rawBytes( + "Message-ID: \nSubject: Hi\r\n\r\nbody", + ); + expect(() => decodeMail(crlfTerminated)).toThrow( + /must break its lines with CRLF/, + ); + expect(parseMessageIdHeader(crlfTerminated)).toBeNull(); + expect(await deriveMessageId(crlfTerminated)).toMatch(/^[0-9a-f]{64}$/); + }); + + test("refuses an LF LF separator in either parser", async () => { + // The blank line is a pair of line breaks of its own. Honouring a bare pair + // lets a sender who controls one field body end the header section early and + // strip the fields after it, `Message-ID` included. + const raw = rawBytes( + "From: alice@example.com\r\nSubject: Hi\n\n" + + "Message-ID: \r\n\r\nbody", + ); + expect(() => decodeMail(raw)).toThrow(/must break its lines with CRLF/); + expect(parseMessageIdHeader(raw)).toBeNull(); + expect(await deriveMessageId(raw)).toMatch(/^[0-9a-f]{64}$/); + }); + + test("takes the first blank line when the body carries another", () => { + const raw = rawBytes( + "From: alice@example.com\r\nSubject: Hi\r\n\r\nbody\r\n\r\ntail\r\n", + ); + const mail = decodeMail(raw); + expect(mail.headers.from).toBe("alice@example.com"); + expect(mail.headers.subject).toBe("Hi"); + expect(mail.parts).toHaveLength(1); + expect(new TextDecoder().decode(mail.parts[0]?.content)).toBe( + "body\r\n\r\ntail\r\n", + ); + }); + + test("treats a Date that does not parse the same as an absent one", () => { + // A consumer builds a `Date` from this value and compares it to a search + // window's bounds. Every comparison against an Invalid Date is false, so a + // kept value places the message inside `before: 1990` and `after: 2999` at + // once. The absence reported instead falls outside every window, which is + // already how an absent header reads. + const absent = decodeMail(rawBytes("From: a@example.com\r\n\r\nbody")); + const garbage = decodeMail( + rawBytes("From: a@example.com\r\nDate: not a date\r\n\r\nbody"), + ); + expect("date" in absent.headers).toBe(false); + expect("date" in garbage.headers).toBe(false); + }); + + test("never reports a Date a consumer cannot parse", () => { + for (const value of ["not a date", "Mon, 32 Foo 2026", " ", ""]) { + const mail = decodeMail( + rawBytes(`From: a@example.com\r\nDate: ${value}\r\n\r\nbody`), + ); + expect(mail.headers.date).toBeUndefined(); + } + + const good = decodeMail( + rawBytes( + "From: a@example.com\r\nDate: Mon, 21 Apr 2026 12:00:00 +0000\r\n\r\nbody", + ), + ); + const { date } = good.headers; + if (date === undefined) throw new Error("expected a Date header"); + expect(date).toBe("Mon, 21 Apr 2026 12:00:00 +0000"); + expect(Number.isNaN(new Date(date).getTime())).toBe(false); + }); + + test("resolves a leading-WSP line to no field in either parser", async () => { + // A field begins with a printable name character (RFC 2822 section 2.2), + // so the smuggled line continues nothing and names no field. Reading it as + // one made the stored envelope id disagree with the dedup key. + const raw = rawBytes( + " Message-ID: \r\n" + + "Message-ID: \r\n" + + "\r\nbody", + ); + const mail = decodeMail(raw); + expect(mail.headers.messageId).toBe(""); + expect(mail.rawHeaders["message-id"]).toEqual([""]); + expect(await deriveMessageId(raw)).toBe(""); + + const solo = rawBytes(" Message-ID: \r\n\r\nbody"); + const soloMail = decodeMail(solo); + expect(soloMail.headers.messageId).toBeUndefined(); + expect(soloMail.rawHeaders["message-id"]).toBeUndefined(); + expect(parseMessageIdHeader(solo)).toBeNull(); + expect(await deriveMessageId(solo)).toMatch(/^[0-9a-f]{64}$/); + }); + test("throws on a multipart part with no boundary rather than dropping it", () => { expect(() => decodeMail( @@ -168,6 +605,452 @@ describe("decodeMail", () => { }); }); +describe("Content-Transfer-Encoding normalization", () => { + test("accepts a mechanism that carries a trailing RFC 822 comment", () => { + // RFC 2045 section 1: comments in a MIME header field have no semantic + // content and are ignored during processing, so this names 7bit. + const mail = decodeMail( + rawBytes( + "Content-Type: text/plain\r\n" + + "Content-Transfer-Encoding: 7bit (default)\r\n\r\nhello world", + ), + ); + expect(new TextDecoder().decode(mail.parts[0]?.content)).toBe( + "hello world", + ); + }); + + test("accepts a leading comment and a nested comment", () => { + const mail = decodeMail( + rawBytes( + "Content-Type: text/plain\r\n" + + "Content-Transfer-Encoding: (per RFC 2045 (section 6.8)) base64" + + "\r\n\r\naGVsbG8gd29ybGQ=", + ), + ); + expect(new TextDecoder().decode(mail.parts[0]?.content)).toBe( + "hello world", + ); + }); + + test("does not let a quoted close-paren end a comment early", () => { + // RFC 822 section 3.4.5: a backslash quotes the next character, so the + // escaped ')' is comment text rather than the comment terminator. + const mail = decodeMail( + rawBytes( + "Content-Type: text/plain\r\n" + + "Content-Transfer-Encoding: base64 (a \\) b)\r\n\r\naGk=", + ), + ); + expect(new TextDecoder().decode(mail.parts[0]?.content)).toBe("hi"); + }); + + test("treats a field naming no mechanism as the 7bit default", () => { + // RFC 2045 section 6.1 defaults an ABSENT field to 7bit. A field that is + // present but names nothing -- empty, or nothing but a comment -- has + // declared no mechanism either, so it takes the same default. Routing it + // to the unrecognised arm instead treats a body the sender never claimed + // to have encoded as opaque bytes, which surfaces to a reader as mojibake + // rather than the characters that were sent. + for (const encoding of ["", " ", "(just a comment)"]) { + const email = parseMailToEmail( + multipartWithEncoding(encoding, "caf\u00e9"), + "sml_cte", + ); + expect(email.bodyValues["1"]?.value).toBe("caf\u00e9"); + expect(email.bodyValues["1"]?.isEncodingProblem).toBe(false); + } + }); + + test("ignores a parameter tail on the mechanism", () => { + const mail = decodeMail( + rawBytes( + "Content-Type: text/plain\r\n" + + "Content-Transfer-Encoding: base64; x-note=stray\r\n\r\naGk=", + ), + ); + expect(new TextDecoder().decode(mail.parts[0]?.content)).toBe("hi"); + }); + + test("returns an unrecognised mechanism's body as opaque bytes", () => { + // RFC 2045 section 6.4: an entity with an unrecognised mechanism is + // treated as application/octet-stream. Discarding it, or running it + // through a text decode, both destroy the bytes the treatment preserves. + const body = [0xff, 0xfe, 0x41, 0x00, 0x80]; + const mail = decodeMail( + rawWithBody( + "Content-Type: text/plain\r\n" + + "Content-Transfer-Encoding: x-uuencode\r\n\r\n", + body, + ), + ); + expect(mail.parts).toHaveLength(1); + expect(Array.from(mail.parts[0]?.content ?? new Uint8Array())).toEqual( + body, + ); + }); + + test("still throws on a malformed body under a recognised mechanism", () => { + // A malformed base64 body is a different condition from an unrecognised + // mechanism, and attachment integrity depends on it surfacing. + expect(() => + decodeMail( + rawBytes( + "Content-Type: text/plain\r\n" + + "Content-Transfer-Encoding: base64\r\n\r\n!!! not base64 !!!", + ), + ), + ).toThrow(); + }); + + test("parseMailToEmail decodes base64 through an RFC 822 comment", () => { + const email = parseMailToEmail( + multipartWithEncoding("base64 (RFC 2045)", "aGVsbG8="), + "sml_cte_comment", + ); + expect(email.bodyValues["1"]?.value).toBe("hello"); + expect(email.bodyValues["1"]?.isEncodingProblem).toBe(false); + }); + + test("parseMailToEmail carries an unrecognised mechanism's bytes intact", () => { + // The body is valid UTF-8 for a non-ASCII character, so a UTF-8 decode + // would fold its two bytes into one code point and lose the octets that + // octet-stream treatment exists to preserve. + const email = parseMailToEmail( + multipartWithEncoding("x-weird", "café"), + "sml_cte_unknown", + ); + expect(codeUnits(email.bodyValues["1"]?.value ?? "")).toEqual([ + 0x63, 0x61, 0x66, 0xc3, 0xa9, + ]); + expect(email.bodyValues["1"]?.isEncodingProblem).toBe(true); + }); + + // RFC 2045 section 6.4 is one instruction with two halves: an entity whose + // mechanism is unrecognised is not interpreted, AND it is treated as + // application/octet-stream. The reported content type is the second half. + // Each row pairs a mechanism with the type a part reports under it and the + // body that mechanism yields, so a recognised mechanism is the control that + // keeps its declared type and is decoded. + const reportedTypeCases: { + mechanism: string; + textPartBody: string; + decodedText: string; + attachmentBody: string; + reportedTextType: string; + reportedAttachmentType: string; + }[] = [ + { + mechanism: "7bit", + textPartBody: "hello", + decodedText: "hello", + attachmentBody: "pdf-bytes", + reportedTextType: "text/plain", + reportedAttachmentType: "application/pdf", + }, + { + // No comment to replace, so the value reaches the match untouched. + mechanism: "base64", + textPartBody: "aGVsbG8=", + decodedText: "hello", + attachmentBody: "cGRmLWJ5dGVz", + reportedTextType: "text/plain", + reportedAttachmentType: "application/pdf", + }, + { + mechanism: "quoted-printable", + textPartBody: "hello", + decodedText: "hello", + attachmentBody: "pdf-bytes", + reportedTextType: "text/plain", + reportedAttachmentType: "application/pdf", + }, + { + mechanism: "x-uuencode", + textPartBody: "hello", + decodedText: "hello", + attachmentBody: "begin 644 report.pdf", + reportedTextType: "application/octet-stream", + reportedAttachmentType: "application/octet-stream", + }, + { + // A trailing comment leaves the token before it alone, so this row + // names x-weird and relabels for the same reason the row above does. + mechanism: "x-weird (not a mechanism)", + textPartBody: "hello", + decodedText: "hello", + attachmentBody: "weird-bytes", + reportedTextType: "application/octet-stream", + reportedAttachmentType: "application/octet-stream", + }, + { + // A run of CFWS between two lexical tokens is semantically a single + // space (RFC 2822 section 3.2.3), so a comment separates the tokens it + // sits between rather than joining them. This value is the two + // tokens `ba` and `se64`; neither names a mechanism, and a single + // mechanism is all RFC 2045 section 6.1 admits. Replacing the comment + // with nothing instead reads it as `base64` and hands an attacker a + // body a strict peer leaves opaque and we decode. + mechanism: "ba(c)se64", + textPartBody: "aGVsbG8=", + decodedText: "aGVsbG8=", + attachmentBody: "cGRmLWJ5dGVz", + reportedTextType: "application/octet-stream", + reportedAttachmentType: "application/octet-stream", + }, + { + // The same smuggle with the comment at a different interior offset. + mechanism: "base(x)64", + textPartBody: "aGVsbG8=", + decodedText: "aGVsbG8=", + attachmentBody: "cGRmLWJ5dGVz", + reportedTextType: "application/octet-stream", + reportedAttachmentType: "application/octet-stream", + }, + { + // One comment between every pair of characters, so no two characters + // may be joined at all. + mechanism: "b(1)a(2)s(3)e(4)6(5)4", + textPartBody: "aGVsbG8=", + decodedText: "aGVsbG8=", + attachmentBody: "cGRmLWJ5dGVz", + reportedTextType: "application/octet-stream", + reportedAttachmentType: "application/octet-stream", + }, + { + // Separating the tokens must not cost a legal value its mechanism: + // this one still names 7bit once the trailing comment becomes a space + // and the value is trimmed. + mechanism: "7bit (default)", + textPartBody: "hello", + decodedText: "hello", + attachmentBody: "pdf-bytes", + reportedTextType: "text/plain", + reportedAttachmentType: "application/pdf", + }, + { + // Nothing but a comment still names no mechanism, which RFC 2045 + // section 6.1 defaults to 7bit rather than treating as unrecognised. + mechanism: "(just a comment)", + textPartBody: "hello", + decodedText: "hello", + attachmentBody: "pdf-bytes", + reportedTextType: "text/plain", + reportedAttachmentType: "application/pdf", + }, + ]; + + test("decodes a body part by the mechanism its value resolves to", () => { + // The companion to the relabel below: the type says how the bytes may be + // treated, this says which bytes the reader gets. A value that resolves + // to no mechanism is not decoded, so its body arrives as it was sent. + for (const c of reportedTypeCases) { + const mail = decodeMail( + multipartWithEncoding(c.mechanism, c.textPartBody), + ); + expect(new TextDecoder().decode(mail.parts[0]?.content)).toBe( + c.decodedText, + ); + + // The JMAP projection is a second decoder over the same field, so it + // reads the same value the same way. + const email = parseMailToEmail( + multipartWithEncoding(c.mechanism, c.textPartBody), + "sml_cte_decoded_body", + ); + expect(email.bodyValues["1"]?.value).toBe(c.decodedText); + } + }); + + test("reads a single-part message's own transfer encoding on both paths", () => { + // A single-part message carries its transfer encoding in the message + // headers, and `parseMailToEmail` reaches the decoder through a part it + // reconstructs from them. A reconstruction that carries only the content + // type leaves the decoder with the RFC 2045 section 6.1 default, so a + // base64 body arrives still encoded and an unrecognised mechanism's + // octets are interpreted as text -- and the two entry points disagree on + // the same bytes. Each row asserts the pair agrees. + for (const c of reportedTypeCases) { + const raw = singlePartWithEncoding(c.mechanism, c.textPartBody); + + const mail = decodeMail(raw); + expect(mail.parts).toHaveLength(1); + expect(mail.parts[0]?.contentType).toBe(c.reportedTextType); + expect(new TextDecoder().decode(mail.parts[0]?.content)).toBe( + c.decodedText, + ); + + const email = parseMailToEmail(raw, "sml_cte_single"); + expect(email.textBody).toEqual([ + { partId: "1", type: c.reportedTextType }, + ]); + expect(email.bodyValues["1"]?.value).toBe(c.decodedText); + expect(email.bodyValues["1"]?.isEncodingProblem).toBe( + c.reportedTextType === "application/octet-stream", + ); + } + }); + + test("keeps a single-part unrecognised mechanism's octets intact", () => { + // The octets, not just the mechanism: a UTF-8 decode folds the two bytes + // of a non-ASCII character into one code point, which is the loss that + // octet-stream treatment exists to prevent. `decodeMail` already returns + // the bytes; the JMAP projection must widen the same ones. + const raw = singlePartWithEncoding("x-weird", "café"); + const octets = [0x63, 0x61, 0x66, 0xc3, 0xa9]; + + expect(Array.from(decodeMail(raw).parts[0]?.content ?? [])).toEqual(octets); + + const email = parseMailToEmail(raw, "sml_cte_single_octets"); + expect(codeUnits(email.bodyValues["1"]?.value ?? "")).toEqual(octets); + expect(email.bodyValues["1"]?.isEncodingProblem).toBe(true); + }); + + test("widens every one of the 256 byte values", () => { + // RFC 2045 section 6.4 hands a body under an unrecognised transfer + // encoding back as octets, not text. The JMAP string value for such a + // body must therefore carry one code unit per byte, with every byte + // preserved exactly -- the case above covers one character's worth of the + // range, this one covers all of it. + // + // This is a regression guard against the obvious optimisation. + // `new TextDecoder("latin1")` looks like a one-pass replacement for the + // widening in `bytesToBinaryString`. It is not: the WHATWG Encoding + // Standard makes "latin1" and "iso-8859-1" labels for windows-1252, which + // maps 27 of the 256 byte values to a different code point. This test + // fails if anyone makes that swap. + const body: number[] = []; + for (let i = 0; i < 256; i++) body.push(i); + const raw = rawWithBody( + [ + "From: alice@example.com", + "To: bob@example.com", + "Content-Type: text/plain", + "Content-Transfer-Encoding: x-uuencode", + "", + "", + ].join("\r\n"), + body, + ); + + const email = parseMailToEmail(raw, "sml_cte_all_octets"); + expect(email.bodyValues["1"]).toBeDefined(); + expect(email.bodyValues["1"]?.isEncodingProblem).toBe(true); + expect(email.bodyValues["1"]?.value.length).toBe(256); + expect(codeUnits(email.bodyValues["1"]?.value ?? "")).toEqual(body); + }); + + test("names the windows-1252 range a latin1 decode would rewrite", () => { + // Documents the trap rather than the product: these are the byte values the + // swap would rewrite, asserted against windows-1252 because that is the + // encoding the "latin1" label resolves to. Kept beside the test above so + // the reason that test exists is visible where it is asserted. + const all = new Uint8Array(256); + for (let i = 0; i < 256; i++) all[i] = i; + const decoded = new TextDecoder("windows-1252").decode(all); + const diverging = [...all].filter((b) => decoded.charCodeAt(b) !== b); + expect(diverging.length).toBe(27); + expect(diverging.every((b) => b >= 0x80 && b <= 0x9f)).toBe(true); + }); + + test("reports a body part under an unrecognised mechanism as octet-stream", () => { + // Without the relabel a consumer is told the part holds text in the + // charset it declared while it holds bytes that are not that text, and + // every consumer that keys on the type acts on that label. + for (const c of reportedTypeCases) { + const mail = decodeMail( + multipartWithEncoding(c.mechanism, c.textPartBody), + ); + expect(mail.parts).toHaveLength(1); + expect(mail.parts[0]?.contentType).toBe(c.reportedTextType); + + // The JMAP projection reports the same type. The part stays listed in + // textBody under it rather than dropping out of the list, so a consumer + // that walks textBody to find the body still learns the part exists. + const email = parseMailToEmail( + multipartWithEncoding(c.mechanism, c.textPartBody), + "sml_cte_reported_body", + ); + expect(email.textBody).toEqual([ + { partId: "1", type: c.reportedTextType }, + ]); + expect(email.attachments).toEqual([]); + expect(email.bodyValues["1"]).toBeDefined(); + } + }); + + test("reports an attachment under an unrecognised mechanism as octet-stream", () => { + for (const c of reportedTypeCases) { + const mail = decodeMail( + multipartWithAttachmentEncoding(c.mechanism, c.attachmentBody), + ); + expect(mail.parts).toHaveLength(2); + const att = mail.parts[1]; + expect(att?.contentType).toBe(c.reportedAttachmentType); + // Relabelling the type does not cost the reader the part's identity. + expect(att?.filename).toBe("report.pdf"); + expect(att?.disposition).toBe("attachment"); + // The body part beside it is untouched: the relabel is per part. + expect(mail.parts[0]?.contentType).toBe("text/plain"); + + const email = parseMailToEmail( + multipartWithAttachmentEncoding(c.mechanism, c.attachmentBody), + "sml_cte_reported_att", + ); + expect(email.attachments[0]?.type).toBe(c.reportedAttachmentType); + expect(email.attachments[0]?.name).toBe("report.pdf"); + + // The MessageAttachment projection a conversation turn is built from. + const extracted = extractAttachments( + signedWithAttachmentEncoding(c.mechanism, c.attachmentBody), + ); + expect(extracted).toHaveLength(1); + expect(extracted[0]?.contentType).toBe(c.reportedAttachmentType); + expect(extracted[0]?.name).toBe("report.pdf"); + } + }); + + test("keeps the encoding-problem flag beside the relabelled type", () => { + // The flag and the relabel are complementary, not alternatives: the flag + // says the decode did not happen, the type says what the undecoded bytes + // may be treated as. Routing on the relabelled type would reclassify this + // part as an attachment and the flag would never be written. + const email = parseMailToEmail( + multipartWithEncoding("x-weird", "café"), + "sml_cte_flag_and_type", + ); + expect(email.textBody[0]?.type).toBe("application/octet-stream"); + expect(email.bodyValues["1"]?.isEncodingProblem).toBe(true); + }); + + test("still walks into a multipart wrapper declaring an unrecognised mechanism", () => { + // A wrapper's children are located from its declared type, so relabelling + // the wrapper would make them unreachable and drop their content -- the + // opposite of what section 6.4 protects. The relabel is leaf-only. + const mail = decodeMail( + rawBytes( + [ + "From: alice@example.com", + 'Content-Type: multipart/mixed; boundary="wrap"', + "Content-Transfer-Encoding: x-weird", + "", + "--wrap", + "Content-Type: text/plain", + "", + "inner content", + "--wrap--", + "", + ].join("\r\n"), + ), + ); + expect(mail.parts).toHaveLength(1); + expect(mail.parts[0]?.contentType).toBe("text/plain"); + expect(new TextDecoder().decode(mail.parts[0]?.content)).toBe( + "inner content", + ); + }); +}); + describe("isMail", () => { function validMail(): Record { const parts: MailPart[] = [ @@ -180,6 +1063,20 @@ describe("isMail", () => { expect(isMail(validMail())).toBe(true); }); + test("rejects rawHeaders that is not a record of string arrays", () => { + expect(isMail({ ...validMail(), rawHeaders: { subject: ["hi"] } })).toBe( + true, + ); + expect(isMail({ ...validMail(), rawHeaders: { subject: "hi" } })).toBe( + false, + ); + expect(isMail({ ...validMail(), rawHeaders: { subject: [1] } })).toBe( + false, + ); + expect(isMail({ ...validMail(), rawHeaders: "subject" })).toBe(false); + expect(isMail({ ...validMail(), rawHeaders: null })).toBe(false); + }); + test("rejects a MessagePart-shaped part (bytes, no ref)", () => { // decodeMail returns MessagePart[] (content bytes); only the committed // MailPart[] (ref) is a Mail, so the in-memory decode is NOT a Mail. @@ -211,8 +1108,94 @@ describe("isMail", () => { ).toBe(false); }); - test("rejects headers missing the from/to a consumer dereferences", () => { + test("rejects headers missing the to a consumer dereferences", () => { expect(isMail({ ...validMail(), headers: {} })).toBe(false); expect(isMail({ ...validMail(), headers: { from: "a@b" } })).toBe(false); }); + + test("accepts a Mail carrying no originator", () => { + // Mail with no usable From is still mail. Rejecting it here routed such a + // message down the arbitrary-step-value path, where it was stringified + // into a text turn with no error and no log. + expect(isMail({ ...validMail(), headers: { to: ["c@d"] } })).toBe(true); + }); + + /** Narrow a value reached out of JSON output, without a type assertion. */ + function asObject(value: unknown): object { + if (typeof value !== "object" || value === null) { + throw new Error(`expected an object, got ${typeof value}`); + } + return value; + } + + /** + * The map the decoder builds for a message carrying `__proto__` and + * `Constructor` as field names. Every value is an own enumerable data + * property and the map has no prototype, which is the shape the sibling + * "keeps a header named after an Object.prototype member" case pins. + */ + function hostileRawHeaders(): Record { + return decodeMail( + rawBytes( + "From: a@b\r\n" + + "To: c@d\r\n" + + "__proto__: injected\r\n" + + "Constructor: hostile\r\n" + + "Content-Type: text/plain\r\n\r\nbody", + ), + ).rawHeaders; + } + + test("accepts the rawHeaders map a hostile field name produces", () => { + // The `rawHeaders` index signature must still ACCEPT the map the decoder + // actually builds. A rejection here is silent: `isMail` returning false + // sends the step input down `synthesizeInputContent` instead of the mail + // projection, so an inbound mail carrying a `__proto__` header would reach + // the agent as stringified text with its parts and threading headers + // dropped, and nothing would raise. + const rawHeaders = hostileRawHeaders(); + expect(Object.getPrototypeOf(rawHeaders)).toBeNull(); + expect(Object.keys(rawHeaders)).toContain("__proto__"); + expect(Object.keys(rawHeaders)).toContain("constructor"); + + expect(isMail({ ...validMail(), rawHeaders })).toBe(true); + + // The acceptance above is the values passing, not the hostile keys going + // uninspected: the same map with a bare string under `__proto__` is + // refused, so the index signature does reach that key. + const tampered = { ...rawHeaders }; + Object.defineProperty(tampered, "__proto__", { + value: "injected", + enumerable: true, + writable: true, + configurable: true, + }); + expect(isMail({ ...validMail(), rawHeaders: tampered })).toBe(false); + }); + + test("accepts that map after the JSON round trip the control channel imposes", () => { + // Decoded mail crosses the workflow host's control channel as JSON, so the + // value `isMail` narrows is `JSON.parse` output rather than the decoder's + // own object. + const transported: unknown = JSON.parse( + JSON.stringify({ ...validMail(), rawHeaders: hostileRawHeaders() }), + ); + const map = asObject( + Object.getOwnPropertyDescriptor(asObject(transported), "rawHeaders") + ?.value, + ); + // `JSON.parse` writes `__proto__` as an own data property rather than + // through the inherited setter, so the entry survives the crossing and does + // not become the map's prototype. Asserted before the acceptance below, + // which a map that LOST the key would satisfy too. + expect(Object.getOwnPropertyDescriptor(map, "__proto__")?.value).toEqual([ + "injected", + ]); + expect(Object.getOwnPropertyDescriptor(map, "constructor")?.value).toEqual([ + "hostile", + ]); + expect(Object.getPrototypeOf(map)).toBe(Object.prototype); + + expect(isMail(transported)).toBe(true); + }); }); diff --git a/vendor/intx-mime/src/mime.ts b/vendor/intx-mime/src/mime.ts index f842b2594..043845a47 100644 --- a/vendor/intx-mime/src/mime.ts +++ b/vendor/intx-mime/src/mime.ts @@ -20,13 +20,13 @@ */ import { type } from "arktype"; -import { base64Decode, base64Encode } from "@intx/types"; +import { base64Decode, base64Encode } from "@intx/types/base64"; import type { MessageAttachment, MessageHeaders as ParsedMessageHeaders, MessagePart, -} from "@intx/types/runtime"; -import { InterchangeType } from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; +import { InterchangeType } from "@intx/types/runtime-core"; // --------------------------------------------------------------------------- // Types @@ -123,8 +123,11 @@ export type JMAPEmail = { // Message-ID generation // --------------------------------------------------------------------------- -export function generateMessageId(address: string): string { - const domain = address.includes("@") ? address.split("@")[1]! : "local"; +export function generateMessageId(address?: string): string { + const domain = + address !== undefined && address.includes("@") + ? address.split("@")[1]! + : "local"; const uuid = crypto.randomUUID(); return `<${uuid}@${domain}>`; } @@ -133,13 +136,59 @@ export function generateMessageId(address: string): string { // Address normalization // --------------------------------------------------------------------------- +function addressListError(addressLine: string): Error { + return new Error( + `extractAddrSpec: address lists are not supported: ${JSON.stringify(addressLine)}`, + ); +} + +/** + * Refuse a value naming more than one address; report where its `<` opens, or + * -1 for a bare `addr-spec`. A `quoted-string` carries `@`, `<` and `,` as + * text, so the scan exempts what it spans and refuses unpaired quotes. + */ +function scanSingleAddress(trimmed: string, addressLine: string): number { + let quoted = false; + let angleOpen = -1; + let sawUnbracketedAt = false; + let i = 0; + while (i < trimmed.length) { + const ch = trimmed[i]; + if (quoted && ch === "\\") { + i += 2; + continue; + } + if (ch === '"') { + quoted = !quoted; + } else if (!quoted) { + if (ch === ",") { + throw addressListError(addressLine); + } else if (ch === "@" && angleOpen === -1) { + sawUnbracketedAt = true; + } else if (ch === "<") { + if (angleOpen !== -1 || sawUnbracketedAt) { + throw addressListError(addressLine); + } + angleOpen = i; + } + } + i += 1; + } + if (quoted) { + throw new Error( + `extractAddrSpec: unterminated quoted string in ${JSON.stringify(addressLine)}`, + ); + } + return angleOpen; +} + /** * Extract the bare addr-spec (local-part@domain) from a single RFC 5322 * address value. Strips any display name and surrounding angle brackets, * then lowercases the result so case-insensitive comparison falls out * naturally. * - * Accepted inputs (single-address only — do not pass comma-separated lists): + * Accepted inputs (exactly one address): * `"Display Name" ` → `user@host` * `Display Name ` → `user@host` * `` → `user@host` @@ -148,6 +197,8 @@ export function generateMessageId(address: string): string { * * Rejected (throws) inputs: * - empty or whitespace-only + * - a value naming more than one address, however the members are separated + * - an input whose double quotes do not pair up * - input with no `@` * - input that produces an empty local-part or domain * - quoted local-parts (e.g. `"a@b"@host`) — technically valid per RFC @@ -170,8 +221,9 @@ export function extractAddrSpec(addressLine: string): string { throw new Error("extractAddrSpec: address is empty"); } + const angleOpen = scanSingleAddress(trimmed, addressLine); + let candidate: string; - const angleOpen = trimmed.lastIndexOf("<"); if (angleOpen !== -1) { // Angle-bracketed form. Require the `>` to be the trailing // non-whitespace character so that input like `Name (comment)` @@ -182,6 +234,11 @@ export function extractAddrSpec(addressLine: string): string { ); } candidate = trimmed.slice(angleOpen + 1, -1).trim(); + if (candidate.includes(">")) { + throw new Error( + `extractAddrSpec: stray '>' inside angle brackets in ${JSON.stringify(addressLine)}`, + ); + } } else { // Bare form. An unquoted addr-spec carries no internal whitespace, so // treat any as trailing content (e.g. `a@b (comment)`) and refuse rather @@ -283,8 +340,311 @@ function generateBoundary(): string { const CRLF = "\r\n"; +/** + * RFC 5322 section 2.1.1 recommends no more than 78 characters per line and + * REQUIRES no more than 998. Folding targets the recommendation: a generator + * that aims at 998 produces lines no human and few mail clients display well, + * and leaves nothing in hand for a relay that prepends to a header. + */ +const FOLD_TARGET_OCTETS = 78; + +/** + * Insert folding whitespace so a line stays near `FOLD_TARGET_OCTETS`. + * + * Folds at a space that is already in the value, which is the only place RFC + * 5322 section 2.2.3 permits. The continuation carries exactly one leading + * space, and that space is one space from the value. `parseHeaders` replaces + * a fold plus every whitespace character after it with a single space, so a + * second leading space would not survive. + * + * Any earlier space in the same run stays on the line being folded, even when + * that carries the line past the target. A token with no space in it is + * emitted whole. There is no legal fold point inside a `msg-id` or an `atom`, + * so breaking one would corrupt it. + * + * A run that ends in a tab is not a fold point: the continuation can carry + * only one leading space, so the tab would be deleted or turned into a + * space. An earlier space that ends its own run is still a fold. Skipping + * it to keep the tab on this line can push the line past 998. + */ +function foldHeaderLine(name: string, value: string): string { + if (value.length === 0) return `${name}:${CRLF}`; + + const head = `${name}: `; + let out = head; + let column = head.length; + let index = 0; + + // Last index of the whitespace run that contains `wsIndex`. + const runLast = (wsIndex: number): number => { + let last = wsIndex; + while ( + last + 1 < value.length && + (value[last + 1] === " " || value[last + 1] === "\t") + ) { + last += 1; + } + return last; + }; + + // `wsIndex` is whitespace in `value`. The continuation can carry only one + // leading space, so the fold has to be a space that ends the run. A tab + // after that space would be swallowed with the leading whitespace, and a + // tab used as the fold itself would come back as a space. When the run + // ends in a tab and no earlier fold remains, the whole run stays here. + const foldOnRun = (wsIndex: number): void => { + const last = runLast(wsIndex); + if (value[last] !== " ") { + out += value.slice(index, last + 1); + column += last + 1 - index; + index = last + 1; + return; + } + out += value.slice(index, last); + column += last - index; + index = last + 1; + out += `${CRLF} `; + column = 1; + }; + + while (index < value.length) { + const room = FOLD_TARGET_OCTETS - column; + if (room > 0 && value.length - index <= room) { + out += value.slice(index); + break; + } + + // The last space in reach whose run ends in a space. A run that ends + // in a tab is skipped so an earlier space can still be the fold. + let breakAt = -1; + const limit = Math.max(0, Math.min(room, value.length - index)); + for (let offset = 0; offset < limit; offset += 1) { + const at = index + offset; + if (value[at] === " " && value[runLast(at)] === " ") breakAt = at; + } + + if (breakAt !== -1 || value[index] === " ") { + foldOnRun(breakAt === -1 ? index : breakAt); + continue; + } + + // No space between here and the target, and the next character is not a + // space. Emit the token whole rather than inventing a fold point. + const nextSpace = value.indexOf(" ", index); + const end = nextSpace === -1 ? value.length : nextSpace; + out += value.slice(index, end); + column += end - index; + index = end; + } + + return out + CRLF; +} + +/** + * Serialize a STRUCTURED header: an address list, a message id, a date, a + * content type. Folded, never encoded -- RFC 2047 encoded-words are not + * permitted in a structured field's tokens, so a non-ASCII address here needs + * SMTPUTF8 and IDNA rather than an encoding this function could apply. + */ function hdr(name: string, value: string): string { - return `${name}: ${value}${CRLF}`; + assertNoLineBreaks(value, `${name} header`); + return foldHeaderLine(name, value); +} + +/** Bytes of UTF-8 per encoded-word. + * + * RFC 2047 section 2 caps an encoded-word at 75 characters. `=?UTF-8?B?` and + * `?=` spend 12 of them, leaving 63 for base64 -- so 45 input bytes, the + * largest multiple of 3 that fits, encode to 60 characters and keep every word + * inside the cap. A multiple of 3 also means no word carries base64 padding + * except the last. + */ +const ENCODED_WORD_PAYLOAD_BYTES = 45; + +function isAscii(value: string): boolean { + // eslint-disable-next-line no-control-regex + return !/[^\x00-\x7F]/.test(value); +} + +/** + * Encode `value` as RFC 2047 encoded-words when it is not already ASCII. + * + * Base64 (`B`) rather than quoted-printable (`Q`) for every value that needs + * encoding at all. `Q` produces a more readable wire form for a value that is + * mostly ASCII, but it needs a per-character escape table whose omissions are + * silent, and it offers nothing a receiver can do that `B` cannot. One code + * path that is always correct beats two where one is prettier. + * + * Words are split on a CHARACTER boundary, never inside a multi-byte sequence: + * RFC 2047 section 5 requires each encoded-word to decode independently, and a + * sequence straddling two of them decodes to a replacement character in both. + */ +function encodeHeaderText(value: string): string { + if (isAscii(value)) return value; + + const bytes = new TextEncoder().encode(value); + const words: string[] = []; + let start = 0; + + while (start < bytes.length) { + let end = Math.min(start + ENCODED_WORD_PAYLOAD_BYTES, bytes.length); + // Walk back off a continuation byte (10xxxxxx) so the cut lands between + // characters rather than inside one. + while (end > start && end < bytes.length) { + const byte = bytes[end]; + if (byte === undefined || (byte & 0xc0) !== 0x80) break; + end -= 1; + } + const chunk = bytes.subarray(start, end); + words.push(`=?UTF-8?B?${Buffer.from(chunk).toString("base64")}?=`); + start = end; + } + + // Adjacent encoded-words are joined by whitespace, which a receiver deletes + // (RFC 2047 section 6.2). `foldHeaderLine` then folds at those same spaces, + // which is why each word has to fit a line on its own. + return words.join(" "); +} + +/** + * Serialize an UNSTRUCTURED header: text a person reads, such as `Subject`. + * + * Encoded before folding, because encoding is what creates the fold points: a + * long non-ASCII subject has no spaces to fold at until it becomes a sequence + * of encoded-words. + */ +function hdrText(name: string, value: string): string { + assertNoLineBreaks(value, `${name} header`); + return foldHeaderLine(name, encodeHeaderText(value)); +} + +/** + * Decode RFC 2047 encoded-words in a header value that was received. + * + * Needed for interoperability rather than for our own mail: every sender that + * writes a non-ASCII subject encodes it, and a receiver that does not decode + * shows an agent `=?UTF-8?B?...?=` as the subject it is supposed to read. Both + * `B` and `Q` are decoded even though only `B` is produced here, because the + * choice belongs to whoever sent the message. + * + * Whitespace BETWEEN two encoded-words is deleted and whitespace elsewhere is + * kept, per RFC 2047 section 6.2. A word whose charset or encoding this cannot + * handle is left exactly as it arrived: showing the encoded form is a visible + * fault a reader can report, whereas substituting replacement characters looks + * like the sender's own text. + */ +export function decodeHeaderText(value: string): string { + const word = /=\?([^?]+)\?([BbQq])\?([^?]*)\?=/g; + if (!word.test(value)) return value; + word.lastIndex = 0; + + let out = ""; + let cursor = 0; + let previousWasWord = false; + + for (let match = word.exec(value); match !== null; match = word.exec(value)) { + const whole = match[0]; + const charset = match[1]; + const encoding = match[2]; + const payload = match[3]; + if ( + charset === undefined || + encoding === undefined || + payload === undefined + ) { + // Unreachable: all three groups are mandatory in the pattern. Narrowed + // rather than asserted so a future pattern edit cannot make it a crash. + continue; + } + const gap = value.slice(cursor, match.index); + + // Whitespace separating two encoded-words is a separator, not content. + if (!(previousWasWord && gap.trim() === "")) out += gap; + + const decoded = decodeEncodedWord(charset, encoding, payload); + out += decoded ?? whole; + previousWasWord = decoded !== undefined; + cursor = match.index + whole.length; + } + + return out + value.slice(cursor); +} + +/** + * Decode a base64 payload only when every character is in the alphabet and + * the bytes re-encode to the same text. `Buffer` accepts illegal characters + * by ignoring them, which would turn a corrupt encoded-word into different + * text instead of leaving the word as it arrived. + */ +function decodeBase64Strict(payload: string): Uint8Array | undefined { + if (!/^[A-Za-z0-9+/]*={0,2}$/.test(payload)) return undefined; + if (payload.length % 4 === 1) return undefined; + const bytes = Buffer.from(payload, "base64"); + const canonical = Buffer.from(bytes).toString("base64").replace(/=+$/, ""); + const given = payload.replace(/=+$/, ""); + if (canonical !== given) return undefined; + return new Uint8Array(bytes); +} + +function decodeEncodedWord( + charset: string, + encoding: string, + payload: string, +): string | undefined { + // RFC 2231 permits a language suffix (`utf-8*en`); the charset is the part + // before it. + const normalized = charset.toLowerCase().split("*")[0] ?? ""; + // Only the charsets decoded here. An unsupported one returns undefined so the + // caller keeps the encoded form rather than guessing at the bytes. + const utf8 = + normalized === "utf-8" || + normalized === "utf8" || + normalized === "us-ascii"; + const latin1 = normalized === "iso-8859-1" || normalized === "latin1"; + if (!utf8 && !latin1) return undefined; + + let bytes: Uint8Array; + if (encoding.toLowerCase() === "b") { + const decoded = decodeBase64Strict(payload); + if (decoded === undefined) return undefined; + bytes = decoded; + } else { + // Q: `_` is a space, `=XX` is a hex octet. Anything else is literal. + const octets: number[] = []; + for (let i = 0; i < payload.length; i += 1) { + const char = payload[i]; + if (char === undefined) break; + if (char === "_") { + octets.push(0x20); + continue; + } + if (char === "=" && i + 2 < payload.length) { + const hex = payload.slice(i + 1, i + 3); + if (/^[0-9A-Fa-f]{2}$/.test(hex)) { + octets.push(parseInt(hex, 16)); + i += 2; + continue; + } + } + octets.push(char.charCodeAt(0) & 0xff); + } + bytes = new Uint8Array(octets); + } + + // Latin-1 needs no decoder: its code points ARE its byte values, which is the + // one case where that identity holds. + if (latin1) { + return Array.from(bytes, (byte) => String.fromCharCode(byte)).join(""); + } + + try { + return new TextDecoder("utf-8", { fatal: true }).decode(bytes); + } catch { + // Bytes that are not valid UTF-8 despite the charset saying so. Keeping the + // encoded form reports the sender's fault; replacement characters would + // read as the sender's own text. + return undefined; + } } function serializeMessageHeaders( @@ -293,14 +653,16 @@ function serializeMessageHeaders( ): string { let out = ""; out += hdr("From", h.from); - out += hdr("To", Array.isArray(h.to) ? h.to.join(", ") : (h.to as string)); + out += hdr("To", h.to.join(", ")); if (h.cc && h.cc.length > 0) { out += hdr("Cc", h.cc.join(", ")); } out += hdr("Date", formatRFC2822Date(h.date)); out += hdr("Message-ID", h.messageId); if (h.subject !== undefined) { - out += hdr("Subject", h.subject); + // The one unstructured header this assembler writes, and so the only one + // whose value is text a person chose rather than a token or an address. + out += hdrText("Subject", h.subject); } if (h.inReplyTo !== undefined) { out += hdr("In-Reply-To", h.inReplyTo); @@ -347,24 +709,186 @@ function serializeMessageHeaders( // MIME part assembly // --------------------------------------------------------------------------- -/** - * Reject values that would break out of a MIME header. CR/LF in a header - * value is a header-injection vector; a double quote breaks the quoted - * `filename="..."` / `name="..."` forms the parser relies on. The MIME - * layer owns header well-formedness, so it fails loudly here rather than - * emitting a corrupt envelope. - */ -function assertHeaderSafe(value: string, field: string): void { +/** Reject a value that would break out of its header: a CR or LF ends the field. */ +function assertNoLineBreaks(value: string, field: string): void { if (/[\r\n]/.test(value)) { throw new Error( `${field} must not contain CR or LF: ${JSON.stringify(value)}`, ); } +} + +/** + * A double quote would close the `filename="..."` form, and this serializer + * emits no escape for an inner quote. A non-ASCII octet is the same class of + * defect: these part headers sit inside the signed bytes and carry no + * transfer encoding, so an 8-bit header is one a relay may rewrite. + */ +function assertAttachmentHeaderSafe(value: string, field: string): void { + assertNoLineBreaks(value, field); if (value.includes('"')) { throw new Error( `${field} must not contain a double quote: ${JSON.stringify(value)}`, ); } + if (!isAscii(value)) { + throw new Error(`${field} must be US-ASCII: ${JSON.stringify(value)}`); + } +} + +// --------------------------------------------------------------------------- +// Text part transfer encoding (RFC 2045 §6.7) +// --------------------------------------------------------------------------- +// +// A text part is labelled `7bit` only when it IS 7-bit: every octet US-ASCII, +// and every line inside the RFC 5322 §2.1.1 limit. Anything else is +// quoted-printable. The distinction is load-bearing once the bytes reach a real +// relay: a part labelled `7bit` that carries 8-bit octets is rewritten by any +// hop that does not advertise 8BITMIME, and that rewrite changes the exact +// bytes the PGP/MIME signature covers, so the recipient reports `invalid`. +// Encoding here keeps the part 7-bit clean, so no hop has a reason to touch it. + +/** The longest line quoted-printable permits, including the soft-break `=`. */ +const QP_MAX_LINE = 76; + +/** RFC 5322 §2.1.1: a line MUST NOT exceed 998 characters, excluding CRLF. */ +const MAX_LINE_OCTETS = 998; + +function hexEscape(byte: number): string { + return `=${byte.toString(16).toUpperCase().padStart(2, "0")}`; +} + +/** + * Octets RFC 2045 §6.7 rule 2 permits literally: 33-60 and 62-126. The gap at + * 61 is `=`, which always escapes because it introduces an escape. + */ +function isQuotedPrintableLiteral(byte: number): boolean { + return (byte >= 33 && byte <= 60) || (byte >= 62 && byte <= 126); +} + +/** + * True when the bytes are already a legal `7bit` body: US-ASCII only, no bare + * CR or LF, no NUL, and no line over the RFC 5322 limit. The caller has already + * canonicalized line endings to CRLF, so a CR or LF outside a CRLF pair is a + * defect rather than a line break to measure. + */ +function isSevenBitClean(bytes: Uint8Array): boolean { + let lineOctets = 0; + for (let i = 0; i < bytes.length; i++) { + const byte = bytes[i]!; + if (byte === 0x0d && bytes[i + 1] === 0x0a) { + i += 1; + lineOctets = 0; + continue; + } + if (byte === 0x00 || byte === 0x0d || byte === 0x0a || byte > 0x7f) { + return false; + } + lineOctets += 1; + if (lineOctets > MAX_LINE_OCTETS) return false; + } + return true; +} + +/** + * Quoted-printable encode one hard line. Space and tab ride literally except + * where they would end an encoded line. RFC 2045 §6.7 rule 3 requires an + * escape there: a relay that strips trailing whitespace would otherwise change + * the signed bytes. That includes the line a soft break closes, not only the + * last line of the input. + * + * Soft breaks land between whole tokens, never inside an `=XX` escape, so the + * escape cannot be split across a line. Escaping a run of spaces writes `=20` + * onto the line the break opened, which can fill that line; the token that + * caused the break is written only after it still fits. + */ +function encodeQuotedPrintableLine(bytes: Uint8Array): string { + let out = ""; + let column = 0; + + // Leave a column for the soft-break `=`. A token is one octet or a + // three-octet hex escape, so it always fits once `column` is 0. + const fits = (length: number): boolean => + column === 0 || column + length <= QP_MAX_LINE - 1; + + const softBreak = (): void => { + out += `=${CRLF}`; + column = 0; + }; + + const writeToken = (token: string): void => { + if (!fits(token.length)) { + const trailing: string[] = []; + while (out.endsWith(" ") || out.endsWith("\t")) { + const last = out.at(-1); + if (last === undefined) break; + trailing.push(last); + out = out.slice(0, -1); + column -= 1; + } + softBreak(); + for (const ws of trailing.reverse()) { + const escaped = ws === " " ? "=20" : "=09"; + if (!fits(escaped.length)) softBreak(); + out += escaped; + column += escaped.length; + } + } + // The escaped whitespace may have filled the line it was moved onto. + // That line ends in `0` or `9`, so one more break opens a fresh one. + if (!fits(token.length)) softBreak(); + out += token; + column += token.length; + }; + + for (let i = 0; i < bytes.length; i++) { + const byte = bytes[i]!; + const atEnd = i === bytes.length - 1; + const token = + byte === 0x20 || byte === 0x09 + ? atEnd + ? hexEscape(byte) + : String.fromCharCode(byte) + : isQuotedPrintableLiteral(byte) + ? String.fromCharCode(byte) + : hexEscape(byte); + writeToken(token); + } + return out; +} + +function encodeQuotedPrintable(text: string): string { + const encoder = new TextEncoder(); + return text + .split(CRLF) + .map((line) => encodeQuotedPrintableLine(encoder.encode(line))) + .join(CRLF); +} + +/** + * Resolve a canonicalized text body to the transfer encoding that honestly + * describes it, and the body in that encoding. Pass `canonical` already + * CRLF-normalized with trailing whitespace stripped per line. + */ +function encodeTextBody(canonical: string): { + encoding: string; + body: string; +} { + if (isSevenBitClean(new TextEncoder().encode(canonical))) { + return { encoding: "7bit", body: canonical }; + } + return { + encoding: "quoted-printable", + body: encodeQuotedPrintable(canonical), + }; +} + +/** CRLF-normalize and strip trailing whitespace per line (MIME canonical form). */ +function canonicalizeText(text: string): string { + return text + .split(/\r\n|\r|\n/) + .map((line) => line.replace(/[ \t]+$/, "")) + .join(CRLF); } /** @@ -396,29 +920,39 @@ function assembleConversationSignedPart( ): Uint8Array { const boundary = generateBoundary(); - // Canonicalize the text part: CRLF line endings, strip trailing - // whitespace per line. - const lines = text.split(/\r\n|\r|\n/); - const canonLines = lines.map((l) => l.replace(/[ \t]+$/, "")); - const canonical = canonLines.join(CRLF); + const text7bit = encodeTextBody(canonicalizeText(text)); let body = `Content-Type: multipart/mixed; boundary="${boundary}"${CRLF}${CRLF}`; // Text part (BODY[1.1]) body += `--${boundary}${CRLF}`; body += `Content-Type: text/plain; charset=utf-8${CRLF}`; - body += `Content-Transfer-Encoding: 7bit${CRLF}`; + body += `Content-Transfer-Encoding: ${text7bit.encoding}${CRLF}`; body += `${CRLF}`; - body += `${canonical}${CRLF}`; + body += `${text7bit.body}${CRLF}`; // Attachment parts (BODY[1.2..N]) for (const att of attachments) { - assertHeaderSafe(att.contentType, "attachment contentType"); - assertHeaderSafe(att.name, "attachment name"); + assertAttachmentHeaderSafe(att.contentType, "attachment contentType"); + assertAttachmentHeaderSafe(att.name, "attachment name"); + // A filename is one token, so folding cannot bring it back under the + // limit. Refuse it here rather than emit a line a relay must break. + const typeLine = `Content-Type: ${att.contentType}`; + const dispositionLine = `Content-Disposition: attachment; filename="${att.name}"`; + if (typeLine.length > MAX_LINE_OCTETS) { + throw new Error( + `attachment contentType header exceeds ${MAX_LINE_OCTETS} octets`, + ); + } + if (dispositionLine.length > MAX_LINE_OCTETS) { + throw new Error( + `attachment name header exceeds ${MAX_LINE_OCTETS} octets`, + ); + } body += `--${boundary}${CRLF}`; - body += `Content-Type: ${att.contentType}${CRLF}`; + body += `${typeLine}${CRLF}`; body += `Content-Transfer-Encoding: base64${CRLF}`; - body += `Content-Disposition: attachment; filename="${att.name}"${CRLF}`; + body += `${dispositionLine}${CRLF}`; body += `${CRLF}`; body += `${base64Lines(att.data)}${CRLF}`; } @@ -437,26 +971,28 @@ function assembleStructuredSignedPart( summary?: string, ): Uint8Array { const boundary = generateBoundary(); - const jsonStr = JSON.stringify(json); + // The payload is machine-read, so it is encoded rather than canonicalized: + // stripping trailing whitespace inside a JSON string literal would change the + // value, and JSON.stringify emits no bare CR or LF to normalize. + const jsonPart = encodeTextBody(JSON.stringify(json)); let body = `Content-Type: multipart/mixed; boundary="${boundary}"${CRLF}${CRLF}`; // JSON payload part body += `--${boundary}${CRLF}`; body += `Content-Type: application/vnd.interchange+json; charset=utf-8${CRLF}`; - body += `Content-Transfer-Encoding: 7bit${CRLF}`; + body += `Content-Transfer-Encoding: ${jsonPart.encoding}${CRLF}`; body += `${CRLF}`; - body += `${jsonStr}${CRLF}`; + body += `${jsonPart.body}${CRLF}`; // Optional human-readable summary if (summary !== undefined) { + const summaryPart = encodeTextBody(canonicalizeText(summary)); body += `--${boundary}${CRLF}`; body += `Content-Type: text/plain; charset=utf-8${CRLF}`; - body += `Content-Transfer-Encoding: 7bit${CRLF}`; + body += `Content-Transfer-Encoding: ${summaryPart.encoding}${CRLF}`; body += `${CRLF}`; - const lines = summary.split(/\r\n|\r|\n/); - const canonLines = lines.map((l) => l.replace(/[ \t]+$/, "")); - body += `${canonLines.join(CRLF)}${CRLF}`; + body += `${summaryPart.body}${CRLF}`; } body += `--${boundary}--${CRLF}`; @@ -554,25 +1090,90 @@ export function assembleSignedContent( // MIME parsing (for fetchHeaders, fetchStructure, fetchPart, fetchFull) // --------------------------------------------------------------------------- -const CRLF_CRLF = new Uint8Array([0x0d, 0x0a, 0x0d, 0x0a]); -const LF_LF = new Uint8Array([0x0a, 0x0a]); +const CR = 0x0d; +const LF = 0x0a; + +/** + * Refuse a header section that breaks a line with anything but CRLF (RFC 5321 + * §2.3.8, §4.1.1.4). The caller passes the section together with the blank line + * that ends it, so neither an LF-only section nor an `LF LF` separator survives. + * + * Neither reading of a bare break is safe. Splitting on it resolves a field + * from a value the sender smuggled inside one it controls; folding it collapses + * every field after it into the value of the field that carries it, suppressing + * each one. A non-conforming peer is owed an error rather than either parse. + */ +function assertNoBareLineBreaks(section: Uint8Array): void { + for (let i = 0; i < section.length; i++) { + const byte = section[i]!; + if (byte === CR && section[i + 1] === LF) { + i += 1; + continue; + } + if (byte === CR || byte === LF) { + throw new Error( + "parseHeaderSection: a header section must break its lines with CRLF", + ); + } + } +} + +/** The length of the line break starting at `at`, or 0 when none starts there. */ +function lineBreakLength(raw: Uint8Array, at: number): number { + const byte = raw[at]; + if (byte === CR) return raw[at + 1] === LF ? 2 : 1; + if (byte === LF) return 1; + return 0; +} -function findByteSequence(haystack: Uint8Array, needle: Uint8Array): number { - if (needle.length === 0) return 0; - const limit = haystack.length - needle.length; - outer: for (let i = 0; i <= limit; i++) { - for (let j = 0; j < needle.length; j++) { - if (haystack[i + j] !== needle[j]) continue outer; +/** + * Locate the blank line that ends the header section: the first line break + * immediately followed by another, whichever flavour either uses. Returns null + * for a message carrying no blank line. + * + * Searching for `CRLF CRLF` alone walks past a blank line written with bare + * breaks and takes the section from a later offset, so every field before that + * offset reads as one field body. Admitting the bare blank line as a separator + * is the mirror of that: a sender who controls one field body ends the section + * early and strips the fields after it. This search locates the boundary and + * `assertNoBareLineBreaks` then refuses the section unless CRLF wrote it, + * including the blank line itself. + * + * The pair is the whole test, so a body part that carries no header fields -- + * legal under RFC 2046 §5.1.1, which makes every field of a part optional -- + * offers one leading break, and its content reads as the header section. A + * leading break is not admitted as a separator here because the message path + * relies on this scan to step over a spurious blank line and keep the fields + * behind it; a strict reading of a leading break would take those fields as + * body and erase the message's originator. The two paths want opposite + * readings of the same leading break, and this scan serves the message. + */ +function findHeaderBoundary( + raw: Uint8Array, +): { headerEnd: number; bodyOffset: number } | null { + let i = 0; + while (i < raw.length) { + const first = lineBreakLength(raw, i); + if (first === 0) { + i += 1; + continue; + } + const second = lineBreakLength(raw, i + first); + if (second !== 0) { + return { headerEnd: i, bodyOffset: i + first + second }; } - return i; + i += first; } - return -1; + return null; } /** * Parse the header section of a raw RFC 2822 message. * Returns a map of lowercase header names to their values, and the * byte offset where the body starts. + * + * Throws for a header section that CRLF did not write, the blank line ending it + * included. A message with no blank line at all is all header section. */ export function parseHeaderSection(raw: Uint8Array): { headers: Map; @@ -581,22 +1182,13 @@ export function parseHeaderSection(raw: Uint8Array): { } { const headers = new Map(); - // Search for the blank line separator in byte space so the returned - // offset is valid for Uint8Array.slice() even when headers contain - // multi-byte UTF-8 characters. - const crlfIdx = findByteSequence(raw, CRLF_CRLF); - const lfIdx = findByteSequence(raw, LF_LF); + // The boundary is located in byte space so the returned offset is valid for + // Uint8Array.slice() even when headers contain multi-byte UTF-8 characters. + const boundary = findHeaderBoundary(raw); + const headerEnd = boundary === null ? raw.length : boundary.headerEnd; + const bodyOffset = boundary === null ? raw.length : boundary.bodyOffset; - let bodyOffset = raw.length; - let headerEnd = raw.length; - - if (crlfIdx !== -1 && (lfIdx === -1 || crlfIdx <= lfIdx)) { - headerEnd = crlfIdx; - bodyOffset = crlfIdx + 4; - } else if (lfIdx !== -1) { - headerEnd = lfIdx; - bodyOffset = lfIdx + 2; - } + assertNoBareLineBreaks(raw.subarray(0, bodyOffset)); const headerText = new TextDecoder("utf-8", { fatal: false }).decode( raw.subarray(0, headerEnd), @@ -606,14 +1198,30 @@ export function parseHeaderSection(raw: Uint8Array): { return { headers, bodyOffset, headerEnd }; } +/** + * Fold a bare CR or LF -- one not part of a CRLF -- to a space. Splitting on it + * would let a sender who controls a field body append a field of its own. + * + * Both callers read a region `parseHeaderSection` has already refused a bare + * break in, so nothing reaches this today. It stays because folding is the safe + * outcome for a region that refusal ever stops covering, and splitting is not. + */ +function foldBareLineBreaks(text: string): string { + return text.replace(/\r(?!\n)|(?): void { // Unfold continuation lines (lines starting with whitespace per RFC 2822). - const unfolded = headerSection - .replace(/\r\n[ \t]+/g, " ") - .replace(/\n[ \t]+/g, " "); - const lines = unfolded.split(/\r\n|\n/); + const unfolded = foldBareLineBreaks(headerSection).replace( + /\r\n[ \t]+/g, + " ", + ); + const lines = unfolded.split(CRLF); for (const line of lines) { if (line.trim() === "") continue; + // A field begins with a printable name character (RFC 2822 2.2), so a + // leading-WSP line continues nothing and names no field. + if (line.startsWith(" ") || line.startsWith("\t")) continue; const colon = line.indexOf(":"); if (colon === -1) continue; const name = line.slice(0, colon).trim().toLowerCase(); @@ -859,6 +1467,112 @@ function parseDateHeader(value: string | undefined): string | null { return date.toISOString(); } +/** + * Replace every RFC 822 comment in a header value with a single space -- not + * nothing, or `ba(c)se64` composes a token the sender never wrote. Comments + * nest (RFC 822 §3.4.3), and a backslash quotes the next character (§3.4.5). + */ +function replaceCommentsWithSpace(value: string): string { + const kept: string[] = []; + let depth = 0; + for (let i = 0; i < value.length; i++) { + const ch = value.charAt(i); + if (depth > 0 && ch === "\\") { + i++; + continue; + } + if (ch === "(") { + if (depth === 0) kept.push(" "); + depth++; + continue; + } + if (ch === ")" && depth > 0) { + depth--; + continue; + } + if (depth === 0) kept.push(ch); + } + return kept.join(""); +} + +/** + * Resolve the Content-Transfer-Encoding mechanism a part declares. RFC 2045 + * §6.1 makes it a single case-insensitive token admitting no parameters. + */ +export function transferEncodingMechanism( + headers: Map, +): string { + const withoutComments = replaceCommentsWithSpace( + headers.get("content-transfer-encoding") ?? "", + ); + const semicolon = withoutComments.indexOf(";"); + const mechanism = + semicolon === -1 ? withoutComments : withoutComments.slice(0, semicolon); + const named = mechanism.trim().toLowerCase(); + // A field naming no mechanism takes the same default as an absent one. + return named === "" ? "7bit" : named; +} + +const RECOGNIZED_TRANSFER_ENCODINGS = new Set([ + "base64", + "quoted-printable", + "7bit", + "8bit", + "binary", +]); + +export function isRecognizedTransferEncoding(mechanism: string): boolean { + return RECOGNIZED_TRANSFER_ENCODINGS.has(mechanism); +} + +/** + * The content type a leaf part reports, after the RFC 2045 §6.4 rule that an + * entity under an unrecognized transfer encoding is treated as + * application/octet-stream. A `multipart/*` wrapper is exempt: its children + * are located from its declared type. The relabel governs what a part + * reports, not how it is routed. + */ +export function reportedContentType( + declaredContentType: string, + headers: Map, +): string { + if (extractContentTypeMime(declaredContentType).startsWith("multipart/")) { + return declaredContentType; + } + const mechanism = transferEncodingMechanism(headers); + if (isRecognizedTransferEncoding(mechanism)) return declaredContentType; + return "application/octet-stream"; +} + +/** + * The number of bytes widened per `String.fromCharCode` call. Large enough + * that the per-call cost is amortised over the buffer, small enough to stay + * well clear of the engine's limit on argument count. + */ +const BINARY_WIDEN_CHUNK_SIZE = 8192; + +/** + * Widen bytes into a string of one code unit per byte. A UTF-8 decode would + * fold multi-byte sequences and replace any byte that is not valid UTF-8, and + * no `TextDecoder` label widens bytes either: the WHATWG Encoding Standard + * makes "latin1" and "iso-8859-1" labels for windows-1252, which maps 27 of + * the byte values in 0x80-0x9f to other code points. + * + * Widening runs a chunk at a time. One `String.fromCharCode` per byte + * allocates a single-character string per byte plus an array to hold them, + * which on a body of tens of megabytes costs several times the body's own + * size in resident memory, on a caller that runs synchronously. + */ +function bytesToBinaryString(bytes: Uint8Array): string { + let out = ""; + for (let i = 0; i < bytes.length; i += BINARY_WIDEN_CHUNK_SIZE) { + out += String.fromCharCode( + ...bytes.subarray(i, i + BINARY_WIDEN_CHUNK_SIZE), + ); + } + return out; +} + /** * Decode a MIME body part, handling Content-Transfer-Encoding. */ @@ -866,9 +1580,7 @@ function decodeBodyBytes( body: Uint8Array, headers: Map, ): { value: string; isEncodingProblem: boolean } { - const cte = (headers.get("content-transfer-encoding") ?? "7bit") - .trim() - .toLowerCase(); + const cte = transferEncodingMechanism(headers); if (cte === "base64") { try { @@ -886,16 +1598,34 @@ function decodeBodyBytes( if (cte === "quoted-printable") { const raw = new TextDecoder("utf-8", { fatal: false }).decode(body); - return { value: decodeQuotedPrintable(raw), isEncodingProblem: false }; + // Decode to octets first, then read those octets as UTF-8. A + // quoted-printable escape names one OCTET, so a multi-byte character + // arrives as several escapes; reading each escape as a character instead + // yields one code point per octet, which renders as mojibake. + return { + value: new TextDecoder("utf-8", { fatal: false }).decode( + decodeQuotedPrintableBytes(raw), + ), + isEncodingProblem: false, + }; } - // 7bit, 8bit, binary — decode as UTF-8 - return { - value: new TextDecoder("utf-8", { fatal: false }).decode(body), - isEncodingProblem: false, - }; + if (cte === "7bit" || cte === "8bit" || cte === "binary") { + return { + value: new TextDecoder("utf-8", { fatal: false }).decode(body), + isEncodingProblem: false, + }; + } + + // RFC 2045 §6.4: an unrecognized mechanism is a byte stream, not text. + return { value: bytesToBinaryString(body), isEncodingProblem: true }; } +/** + * Undo quoted-printable into a string of one code unit per decoded OCTET. The + * result is an octet sequence widened into a string, not text: a caller that + * wants text decodes those octets under the part's charset. + */ function decodeQuotedPrintable(text: string): string { return text .replace(/=\r\n/g, "") @@ -905,6 +1635,16 @@ function decodeQuotedPrintable(text: string): string { ); } +/** The octets a quoted-printable body carries, narrowed from the widened form. */ +function decodeQuotedPrintableBytes(text: string): Uint8Array { + const widened = decodeQuotedPrintable(text); + const out = new Uint8Array(widened.length); + for (let i = 0; i < widened.length; i++) { + out[i] = widened.charCodeAt(i); + } + return out; +} + /** * Determine whether a MIME part is an attachment based on Content-Disposition * and content type. @@ -925,7 +1665,7 @@ function isAttachmentPart( return true; } -function extractContentTypeMime(contentType: string): string { +export function extractContentTypeMime(contentType: string): string { return contentType.split(";")[0]!.trim().toLowerCase(); } @@ -963,9 +1703,9 @@ function walkMimePart( ctx: WalkContext, ): void { const part = parseMimePart(partBytes); - const mime = extractContentTypeMime(part.contentType); + const declaredMime = extractContentTypeMime(part.contentType); - if (mime.startsWith("multipart/")) { + if (declaredMime.startsWith("multipart/")) { const boundary = extractBoundary(part.contentType); if (boundary === undefined) return; const subParts = parseMultipart(part.body, boundary); @@ -975,12 +1715,14 @@ function walkMimePart( return; } + const reportedMime = reportedContentType(declaredMime, part.headers); + if (isAttachmentPart(part.contentType, part.headers)) { const blobId = `blob_${ctx.mailId}_${partPath}`; ctx.attachments.push({ blobId, name: extractFilename(part.headers), - type: mime, + type: reportedMime, size: part.body.length, }); return; @@ -989,10 +1731,12 @@ function walkMimePart( const decoded = decodeBodyBytes(part.body, part.headers); ctx.bodyValues[partPath] = decoded; - if (mime === "text/plain") { - ctx.textBody.push({ partId: partPath, type: mime }); - } else if (mime === "text/html") { - ctx.htmlBody.push({ partId: partPath, type: mime }); + // Listed on the declared type but reported under the relabeled one: a + // consumer walking `textBody` would otherwise never learn the part exists. + if (declaredMime === "text/plain") { + ctx.textBody.push({ partId: partPath, type: reportedMime }); + } else if (declaredMime === "text/html") { + ctx.htmlBody.push({ partId: partPath, type: reportedMime }); } } @@ -1042,13 +1786,18 @@ export function parseMailToEmail(raw: Uint8Array, mailId: string): JMAPEmail { }); } } else { - // Single-part message (e.g. text/plain). - // Reconstruct minimal part bytes with content-type header so parseMimePart works. + // Single-part message. Reconstruct minimal part bytes so parseMimePart + // works; the message's own Content-Transfer-Encoding travels with the type. const enc = new TextEncoder(); - const ctHeader = `Content-Type: ${contentType}\r\n\r\n`; - const partBytes = new Uint8Array(enc.encode(ctHeader).length + body.length); - partBytes.set(enc.encode(ctHeader), 0); - partBytes.set(body, enc.encode(ctHeader).length); + const cte = msgHeaders.get("content-transfer-encoding"); + const partHeaderText = + `Content-Type: ${contentType}\r\n` + + (cte === undefined ? "" : `Content-Transfer-Encoding: ${cte}\r\n`) + + "\r\n"; + const partHeaderBytes = enc.encode(partHeaderText); + const partBytes = new Uint8Array(partHeaderBytes.length + body.length); + partBytes.set(partHeaderBytes, 0); + partBytes.set(body, partHeaderBytes.length); walkMimePart(partBytes, "1", ctx); } @@ -1060,10 +1809,13 @@ export function parseMailToEmail(raw: Uint8Array, mailId: string): JMAPEmail { } } + // The agent reads this field as text. `decodeMail` keeps the wire form. + const rawSubject = msgHeaders.get("subject"); + return { from: parseAddressList(msgHeaders.get("from") ?? ""), to: parseAddressList(msgHeaders.get("to") ?? ""), - subject: msgHeaders.get("subject") ?? null, + subject: rawSubject === undefined ? null : decodeHeaderText(rawSubject), sentAt: parseDateHeader(msgHeaders.get("date")), bodyValues: ctx.bodyValues, textBody: ctx.textBody, @@ -1077,17 +1829,20 @@ export function parseMailToEmail(raw: Uint8Array, mailId: string): JMAPEmail { * Decode a MIME part body into raw bytes, honoring Content-Transfer-Encoding. * * Unlike `decodeBodyBytes` (which produces a JMAP string value), this returns - * the actual bytes for reconstructing a `MessageAttachment`. A malformed - * base64 body surfaces as a thrown error rather than a silent best-effort - * decode — attachment integrity is load-bearing. + * the actual bytes. A malformed base64 body throws; an unrecognized mechanism + * does not — RFC 2045 §6.4 hands back the bytes as they came. + * + * Limitation: the quoted-printable branch reads the wire text as UTF-8 + * before undoing the escapes, so a raw 8-bit byte in a quoted-printable + * body (which RFC 2045 forbids) comes out as a replacement character + * truncated to one byte rather than the original octet. Escaped bytes + * (`=E9`, `=C3=A9`) round-trip exactly. */ -function decodeAttachmentBytes( +export function decodePartBytes( body: Uint8Array, headers: Map, ): Uint8Array { - const cte = (headers.get("content-transfer-encoding") ?? "7bit") - .trim() - .toLowerCase(); + const cte = transferEncodingMechanism(headers); if (cte === "base64") { const raw = new TextDecoder("utf-8", { fatal: false }).decode(body); @@ -1096,26 +1851,17 @@ function decodeAttachmentBytes( if (cte === "quoted-printable") { const raw = new TextDecoder("utf-8", { fatal: false }).decode(body); - const decoded = decodeQuotedPrintable(raw); - const out = new Uint8Array(decoded.length); - for (let i = 0; i < decoded.length; i++) { - out[i] = decoded.charCodeAt(i); - } - return out; + return decodeQuotedPrintableBytes(raw); } - if (cte === "7bit" || cte === "8bit" || cte === "binary") { - return body; - } - - throw new Error( - `decodeAttachmentBytes: unsupported content-transfer-encoding "${cte}"`, - ); + // Identity encoding, including anything unrecognized (RFC 2045 §6.4). + return body; } /** * Extract conversation attachments from raw message bytes as - * `MessageAttachment[]` with decoded payloads. + * `MessageAttachment[]` with decoded payloads and the IMAP part path + * of each sibling (`part`), matching `extractPartByPath` numbering. * * The conversation signed content is a multipart/mixed whose first part is * the text body and whose remaining attachment parts (Content-Disposition: @@ -1147,13 +1893,20 @@ export function extractAttachments(raw: Uint8Array): MessageAttachment[] { if (innerBoundary === undefined) return []; const attachments: MessageAttachment[] = []; - for (const subPartBytes of parseMultipart(signed.body, innerBoundary)) { + for (const [index, subPartBytes] of parseMultipart( + signed.body, + innerBoundary, + ).entries()) { const subPart = parseMimePart(subPartBytes); if (!isAttachmentPart(subPart.contentType, subPart.headers)) continue; attachments.push({ name: extractFilename(subPart.headers) ?? "attachment", - contentType: extractContentTypeMime(subPart.contentType), - data: decodeAttachmentBytes(subPart.body, subPart.headers), + contentType: reportedContentType( + extractContentTypeMime(subPart.contentType), + subPart.headers, + ), + data: decodePartBytes(subPart.body, subPart.headers), + part: `1.${String(index + 1)}`, }); } return attachments; @@ -1175,7 +1928,6 @@ function isInterchangeType(s: string): s is InterchangeType { export function buildMessageHeaders( headers: Map, ): ParsedMessageHeaders { - const from = headers.get("from") ?? ""; const toRaw = headers.get("to") ?? ""; const to = toRaw ? toRaw @@ -1184,10 +1936,30 @@ export function buildMessageHeaders( .filter(Boolean) : []; - const date = headers.get("date") ?? ""; - const messageId = headers.get("message-id") ?? ""; + const result: ParsedMessageHeaders = { to }; + + // A blank header below is recorded as an absence, not an empty value: RFC + // 5322 admits no empty `Date`, `Message-ID`, `In-Reply-To` or `From` body. + // + // A `Date` that does not parse is an absence too, which the parse check below + // covers along with the blank one. A consumer builds a `Date` from this string + // and compares it to a window's bounds; every comparison against an Invalid + // Date is false, so a value kept here places the message inside every date + // window, including two that exclude each other. An absence falls outside all + // of them. + const date = headers.get("date"); + if (date !== undefined && !Number.isNaN(new Date(date).getTime())) { + result.date = date; + } + + const messageId = headers.get("message-id"); + if (messageId !== undefined && messageId.trim().length > 0) { + result.messageId = messageId; + } - const result: ParsedMessageHeaders = { from, to, date, messageId }; + // A non-blank value rides through verbatim even when it is not parseable. + const from = headers.get("from"); + if (from !== undefined && from.trim().length > 0) result.from = from; const ccRaw = headers.get("cc"); if (ccRaw !== undefined) { @@ -1205,10 +1977,15 @@ export function buildMessageHeaders( } const inReplyTo = headers.get("in-reply-to"); - if (inReplyTo !== undefined) result.inReplyTo = inReplyTo; + if (inReplyTo !== undefined && inReplyTo.trim().length > 0) { + result.inReplyTo = inReplyTo; + } + // Decoded, because a subject is text to read: a sender that wrote a + // non-ASCII one encoded it, and the agent reading this field wants the + // characters rather than their RFC 2047 transport form. const subject = headers.get("subject"); - if (subject !== undefined) result.subject = subject; + if (subject !== undefined) result.subject = decodeHeaderText(subject); const listId = headers.get("list-id"); if (listId !== undefined) result.listId = listId; @@ -1257,21 +2034,36 @@ function parseRawHeaders( raw: Uint8Array, headerEnd: number, ): Record { - const text = new TextDecoder("utf-8", { fatal: false }).decode( - raw.subarray(0, headerEnd), + const text = foldBareLineBreaks( + new TextDecoder("utf-8", { fatal: false }).decode( + raw.subarray(0, headerEnd), + ), ); - const out: Record = {}; + // `__proto__` and `constructor` are legal RFC 5322 field names that collide + // with Object.prototype members, so the accumulator is a Map. + const out = new Map(); let current: { name: string; value: string } | null = null; const flush = (): void => { if (current === null) return; const key = current.name.trim().toLowerCase(); - (out[key] ??= []).push(current.value.trim()); + const values = out.get(key) ?? []; + values.push(current.value.trim()); + out.set(key, values); current = null; }; - for (const line of text.split(/\r\n|\n/)) { - if (line === "") break; - if ((line.startsWith(" ") || line.startsWith("\t")) && current !== null) { - current.value += ` ${line.trim()}`; + for (const line of text.split(CRLF)) { + // `headerEnd` bounds the section already, so a blank line here is a + // leading one, not the terminator; breaking would drop every field. + if (line === "") continue; + // Nothing to continue means no field (RFC 2822 2.2), not a field of its own. + if (line.startsWith(" ") || line.startsWith("\t")) { + if (current !== null) { + // The leading whitespace is the fold, and unfolding replaces that + // run with one space. Trailing whitespace is content: trimming it + // would drop a space the fold left at the end of the line. + const body = line.replace(/^[ \t]+/, ""); + current.value += ` ${body}`; + } continue; } const idx = line.indexOf(":"); @@ -1280,7 +2072,10 @@ function parseRawHeaders( current = { name: line.slice(0, idx), value: line.slice(idx + 1) }; } flush(); - return out; + // Null prototype for the same reason: an absent name must resolve to nothing. + const result: Record = Object.create(null); + for (const [name, values] of out) result[name] = values; + return result; } function parseDisposition( @@ -1316,8 +2111,8 @@ function collectLeafParts(partBytes: Uint8Array): MessagePart[] { return parseMultipart(part.body, boundary).flatMap(collectLeafParts); } const result: MessagePart = { - contentType: mime, - content: decodeAttachmentBytes(part.body, part.headers), + contentType: reportedContentType(mime, part.headers), + content: decodePartBytes(part.body, part.headers), }; const filename = extractFilename(part.headers); if (filename !== null) result.filename = filename; diff --git a/vendor/intx-storage-isogit/PATCHES.md b/vendor/intx-storage-isogit/PATCHES.md index a81a0bf89..9125a584e 100644 --- a/vendor/intx-storage-isogit/PATCHES.md +++ b/vendor/intx-storage-isogit/PATCHES.md @@ -19,6 +19,20 @@ Re-carried as-is: upstream did not touch `src/store.ts` or `src/store.test.ts` in this range (`loadErrors` still absent upstream), so both files and the entry are byte-identical to the prior pin. +### 2026-10-14 re-sync to `4cc6972` (CL-10656, HEAD drift 0.4.0 → base) + +Re-carried: upstream touched `store.ts` in this range for the first time +since vendoring (nine commits in `packages/storage-isogit`, net +15/−16); +the `store-ts-load-errors` entry three-way merged cleanly (`git +merge-file`) with zero adaptation, and `loadErrors` is still absent +upstream at `4cc6972`. Upstream `e75eee2` ("Remove sync.request and the +undeploy's agent-state push") dropped `createStatePack`'s only remaining +hub-side consumer and the two comment blocks in `pack-send.ts` / +`repo-lock.ts` that described the state-pack flow (`createDeployPack` +itself keeps its export); the vendored-side safety of that removal is +recorded in docs/VENDORING.md. The tree now types off the same +`4cc6972` pin as the coupled `@intx/types`/`@intx/inference` trio. + ## store-ts-load-errors `store.ts` — Implements `AuditStore.loadErrors` by reading diff --git a/vendor/intx-storage-isogit/src/gc.ts b/vendor/intx-storage-isogit/src/gc.ts index b8ce7f67e..353c59281 100644 --- a/vendor/intx-storage-isogit/src/gc.ts +++ b/vendor/intx-storage-isogit/src/gc.ts @@ -1,6 +1,6 @@ import git from "isomorphic-git"; import { getLogger } from "@intx/log"; -import { hasCode } from "@intx/types"; +import { hasCode } from "@intx/types/has-code"; import { collectReachableObjects } from "./object-walk"; import { publishPackAtomically } from "./pack-receive"; import { diff --git a/vendor/intx-storage-isogit/src/mail-store.ts b/vendor/intx-storage-isogit/src/mail-store.ts index 168d1e08d..ea5b71592 100644 --- a/vendor/intx-storage-isogit/src/mail-store.ts +++ b/vendor/intx-storage-isogit/src/mail-store.ts @@ -1,6 +1,7 @@ import git from "isomorphic-git"; import { parseHeaderSection } from "@intx/mime"; -import { hasCode, hexEncode } from "@intx/types"; +import { hasCode } from "@intx/types/has-code"; +import { hexEncode } from "@intx/types/hex"; import { AUTHOR } from "./init"; import type { CommitSigner } from "./signer"; import { diff --git a/vendor/intx-storage-isogit/src/node-metrics.ts b/vendor/intx-storage-isogit/src/node-metrics.ts index 189b5b75d..9281c13a4 100644 --- a/vendor/intx-storage-isogit/src/node-metrics.ts +++ b/vendor/intx-storage-isogit/src/node-metrics.ts @@ -1,7 +1,7 @@ import fs from "node:fs"; import path from "node:path"; -import { hasCode } from "@intx/types"; +import { hasCode } from "@intx/types/has-code"; import type { RepoDiskUsage, RepoObjectCounts } from "./repo-disk"; // Synchronous Node counterparts to the runtime-backed helpers in repo-disk.ts. diff --git a/vendor/intx-storage-isogit/src/pack-receive.ts b/vendor/intx-storage-isogit/src/pack-receive.ts index dadd7ccdf..0abdcdb69 100644 --- a/vendor/intx-storage-isogit/src/pack-receive.ts +++ b/vendor/intx-storage-isogit/src/pack-receive.ts @@ -2,7 +2,7 @@ import git from "isomorphic-git"; import { readRawObject } from "./isogit-helpers"; import { withRepoDirLock } from "./repo-lock"; import { decodeUTF8, flushRuntime, type StorageRuntime } from "./runtime"; -import { hasCode } from "@intx/types"; +import { hasCode } from "@intx/types/has-code"; /** * Verifies the signature embedded in a git commit object. diff --git a/vendor/intx-storage-isogit/src/pack-send.ts b/vendor/intx-storage-isogit/src/pack-send.ts index 20a75e3b3..f18456d82 100644 --- a/vendor/intx-storage-isogit/src/pack-send.ts +++ b/vendor/intx-storage-isogit/src/pack-send.ts @@ -6,9 +6,8 @@ import type { StorageRuntime } from "./runtime"; /** * Create a git packfile containing all objects reachable from a ref. * - * Used by the hub to produce deploy packs for transfer to sidecars, and by - * the sidecar to produce state packs. The caller sends the resulting bytes - * as chunked repo.pack.push frames. + * Used by the hub to produce deploy packs for transfer to sidecars. The + * caller sends the resulting bytes as chunked repo.pack.push frames. */ export async function createDeployPack( runtime: StorageRuntime, diff --git a/vendor/intx-storage-isogit/src/repo-disk.ts b/vendor/intx-storage-isogit/src/repo-disk.ts index 61f5d7b4d..73a5fc55a 100644 --- a/vendor/intx-storage-isogit/src/repo-disk.ts +++ b/vendor/intx-storage-isogit/src/repo-disk.ts @@ -1,5 +1,5 @@ import git from "isomorphic-git"; -import { hasCode } from "@intx/types"; +import { hasCode } from "@intx/types/has-code"; import type { StorageRuntime } from "./runtime"; /** diff --git a/vendor/intx-storage-isogit/src/repo-lock.ts b/vendor/intx-storage-isogit/src/repo-lock.ts index ae0e71b4b..916f59281 100644 --- a/vendor/intx-storage-isogit/src/repo-lock.ts +++ b/vendor/intx-storage-isogit/src/repo-lock.ts @@ -7,8 +7,8 @@ import type { StorageRuntime } from "./runtime"; * * On the sidecar a single agent repo is written by several independent * drivers — the reactor's context commits, the mail-audit commits, deploy - * pack applies — and read by the state-pack producer, none of which share a - * higher-level lock. Garbage collection prunes loose objects and packs, so + * pack applies — none of which share a higher-level lock. Garbage + * collection prunes loose objects and packs, so * it cannot run concurrently with any of them without risking the deletion * of an object a writer just produced. This lock makes the storage layer * the single owner of that constraint: every mutator and the collector diff --git a/vendor/intx-storage-isogit/src/runtime.ts b/vendor/intx-storage-isogit/src/runtime.ts index 814194670..8668fb89d 100644 --- a/vendor/intx-storage-isogit/src/runtime.ts +++ b/vendor/intx-storage-isogit/src/runtime.ts @@ -1,5 +1,5 @@ import type { FsClient } from "isomorphic-git"; -import { hasCode } from "@intx/types"; +import { hasCode } from "@intx/types/has-code"; const textDecoder = new TextDecoder(); diff --git a/vendor/intx-storage-isogit/src/store.ts b/vendor/intx-storage-isogit/src/store.ts index 968edf019..639a2f9fd 100644 --- a/vendor/intx-storage-isogit/src/store.ts +++ b/vendor/intx-storage-isogit/src/store.ts @@ -13,7 +13,7 @@ import { type ConversationTurn, type ConnectorThreadState, type PendingOperation, -} from "@intx/types/runtime"; +} from "@intx/types/runtime-core"; import { type } from "arktype"; import { AuditRecord, @@ -31,7 +31,7 @@ import { import { withRepoDirLock } from "./repo-lock"; import { maybeGCUnderLock, type GCPolicy } from "./gc"; import { decodeUTF8, flushRuntime, type StorageRuntime } from "./runtime"; -import { hasCode } from "@intx/types"; +import { hasCode } from "@intx/types/has-code"; const TURNS_FILE = "turns.jsonl"; const PROMPT_FILE = "prompt.jsonl"; @@ -52,8 +52,8 @@ function blobExtensionFor(contentType: string | undefined): string { } const ConnectorThreadStateSchema = type({ - threadRoot: "string", - lastMessageId: "string", + "threadRoot?": "string", + "lastMessageId?": "string", replyTo: "string", cc: "string[]", "subject?": "string", @@ -69,7 +69,6 @@ const ConversationTurnSchema = type({ const PendingOperationSchema = type({ correlationId: "string", kind: "'approval'", - "expectedFrom?": "string", registeredAt: "number", gateId: "string", "timeoutAt?": "number", diff --git a/vendor/intx-tools-posix/PATCHES.md b/vendor/intx-tools-posix/PATCHES.md index 1414a3399..fd6b0aa97 100644 --- a/vendor/intx-tools-posix/PATCHES.md +++ b/vendor/intx-tools-posix/PATCHES.md @@ -26,7 +26,9 @@ describe. Companion to `size-cap-ts-paged-read-file` in `tools-posix.test.ts` — carries a small registry-contract test asserting the description names `Use offset=` and paging, so the wording cannot -drift from the behavior it documents. +drift from the behavior it documents. Upstream did not touch this package +in the `5453d0b0` → `4cc6972` range (CL-10656 re-sync verified: the diff +is empty), so the entry carried byte-identically. **Disposition:** Companion to `size-cap-ts-paged-read-file` (vendor/ intx-inference). **Removal path:** the same upstream PR that adds the diff --git a/vendor/intx-types/PATCHES.md b/vendor/intx-types/PATCHES.md index 16bb1a18b..014601483 100644 --- a/vendor/intx-types/PATCHES.md +++ b/vendor/intx-types/PATCHES.md @@ -29,6 +29,65 @@ byte-identically with zero adaptation (`stopReason` chain verified at `src/testing/` subpackage (`waitUntil`), which the vendored `package.json` now exposes via the upstream `./testing` subpath export. +### 2026-10-14 re-sync to `4cc6972` (CL-10656) + +All three entries re-carried; none dropped as upstream-absorbed (grep of the +pristine `4cc6972` `packages/types` tree: `loadErrors`, `stopReason`, +`normalizeError`, `credentialFailureOrdinal` all absent upstream). Upstream +split `runtime.ts` into `runtime-core.ts` (runtime contracts) + the new +`inference-events.ts` (event union). `runtime.ts` is those split +re-exports plus the compat re-export of `formatSafetyRatingText` +(about lines 1–11; see the entry below). The patch sites follow the split: + +- `inference.usage` stopReason → both declarations in + `inference-events.ts` (arktype mapping and the TS union) carry the + `types-ts-usage-stop-reason` marker. +- `AuditStore.loadErrors` → `runtime-core.ts`, marker intact. +- Retry-context (`RetryDecision.error?`, `RetrySituation.source` / + `credentialFailureOrdinal` / `credentialFailureHistory`, + `RetryPolicy.normalizeError`) → `runtime-core.ts`; the sites moved + unchanged from the old `runtime.ts`, and each now carries the + `runtime-ts-auth-recovery-context` marker (previously only the + `mediated-credential.ts` site carried one). +- `mediated-credential.ts` was untouched upstream; its entry carried + byte-identically. + +Upstream also added `adapter-manifest.ts` (adapter manifest schema), +`attachment-validation.ts`, `inbound-mail-policy.ts`, +`credential-delivery.ts`, `format-safety-rating.ts`, `repo.ts`, +`workflow-definition.ts`, `workflow-lifecycle.ts` — all carried verbatim, +with the subpath exports the adopted vendored consumers import +(`runtime-core`, `attachments`, `credential-delivery`, +`format-safety-rating`, `inference-events`) wired into the vendored +`package.json`. Resume audit (same day): the first pass wired only the +five consumer-used new subpaths and dropped fifteen upstream exports +(`base64`, `signals`, `hex`, `has-code`, `agent-address`, +`workflow-run-id`, `message-id`, `concat`, `grants`, `grant-snapshot`, +`credential-cipher`, `repo`, `inbound-mail-policy`, +`workflow-definition`, `adapter-manifest`) that upstream's export map +carries; since the vendored tree is wholesale, the vendored export map +now mirrors upstream's 33 entries verbatim (src paths instead of dist). +The vendored `@intx/inference` `package.json` likewise gained the +`./manifest` subpath it was missing. +The `sync.request` frame and the `agent.undeploy.ack` +`statePushed` field left `sidecar.ts` with upstream `e75eee2`; first-party +`src/` has zero references to either (grep evidence recorded in +docs/VENDORING.md). + +## vendor-types-ts-runtime-format-safety-rating-compat + +`runtime.ts` — after the upstream moved `formatSafetyRatingText` to +`format-safety-rating.ts`, the barrel keeps one compat re-export because +the published `@corbits/openai-responses@0.2.1` dist imports it from the +`./runtime` subpath (`dist/protocol/body.js`), and npm has no newer +release that compiles against the new subpath. Without the re-export, +`bun run build` fails to bundle the installed dist. + +**Disposition:** Compat shim, not a divergence of behavior. **Removal +path:** the `@corbits/openai-responses` release whose dist imports +`@intx/types/format-safety-rating`; drop this entry and the re-export +lines in the same change. + ## runtime-ts-auth-recovery-context `mediated-credential.ts` and `runtime.ts` — credential material can atomically diff --git a/vendor/intx-types/README.md b/vendor/intx-types/README.md index 1ffe2e90c..4acdee19e 100644 --- a/vendor/intx-types/README.md +++ b/vendor/intx-types/README.md @@ -6,9 +6,9 @@ wire frames. Nearly every other package imports from here, which makes this the canonical home for any shape that crosses a package boundary. -Each entry point pairs an ArkType validator with its inferred -TypeScript type so consumers can validate at the boundary and -trust the resulting value internally. +Runtime validators expose their inferred TypeScript types so consumers can +validate at the boundary and trust the resulting value internally. Dedicated +helper entry points avoid loading unrelated schema collections. ## Surface @@ -23,6 +23,15 @@ pull in the shapes they need: discovery view), observability, sidecar status enums (distinct from the wire frames under `@intx/types/sidecar` below), run addresses, hex and base64 helpers, and the `hasCode` error guard. +- `@intx/types/hex` — hexadecimal byte encoding and decoding. +- `@intx/types/message-id` — canonical Message-ID parsing and derivation. +- `@intx/types/credential-cipher` — credential cipher contracts and the + authenticated-data helpers for sealed credentials and signing keys. +- `@intx/types/grants` — grant creation, update, and evaluation schemas, + grant requirements, approval items, and effect/origin enums. +- `@intx/types/adapter-manifest` — the operator-declared custom inference + adapter manifest schema and `parseAdapterManifestEnv`, the reader for + `SIDECAR_ADAPTER_MANIFEST`. - `@intx/types/authz` — grant rules, condition contexts, and authorization result shapes shared between `@intx/authz` and the hub. @@ -30,14 +39,33 @@ pull in the shapes they need: `ErrorRecord` shapes for tool-authorization and error records. - `@intx/types/content-type` — `detectResponseKind`, which classifies a response's `Headers` as an SSE stream or a JSON body. -- `@intx/types/runtime` — inference and harness contracts: +- `@intx/types/runtime` — aggregate entry for runtime contracts and + inference-event wire schemas. +- `@intx/types/runtime-core` — inference and harness contracts: `ContextStore`, `ToolRunner`, `ToolDefinition`, `AuditStore`, `InferenceSource` (the resolved provider/model/credential a call - executes against), retry policy, director and reactor types. + executes against), retry policy, director and reactor types. Use this + lighter entry in child processes that need runtime contracts without + constructing the inference-event union. +- `@intx/types/inference-events` — the `InferenceEvent` wire validator, + its discriminated TypeScript type, and `parseInferenceEvent`. +- `@intx/types/inbound-mail-policy` — `InboundMailOutcome`, + `AuthorControllableOutcome`, and the authored `InboundMailPolicy` schema. +- `@intx/types/format-safety-rating` — `formatSafetyRatingText`, the plain-text + rendering helper, without runtime schema initialization. - `@intx/types/runtime-capabilities` — the capability-registry contract harness extensions resolve against (e.g. mail transport, blob reader). - `@intx/types/sidecar` — hub-sidecar WebSocket wire frames. +- `@intx/types/credential-delivery` — credential material and binding + descriptors delivered to running workflow children. +- `@intx/types/repo` — `RepoKind`, `RepoAction`, and `RepoId` without the + sidecar frame schemas. +- `@intx/types/workflow-definition` — `workflowDefinitionEnvelopeSchema`, + the structural workflow envelope validator used by definition loaders + and the repository substrate. +- `@intx/types/grant-snapshot` — `GrantWalkSnapshot`, the per-step grant + declarations and workflow grant requirements serialized for deployment. - `@intx/types/grant-wire` — grant-update wire frames pushed from the hub to the sidecar. - `@intx/types/tool-packages` — schemas for the tool-package @@ -59,4 +87,16 @@ pull in the shapes they need: - `@intx/types/workflow-sources` — schemas for where a code-sourced workflow definition's bytes come from: `WorkflowDefinitionRegistrySource`, `WorkflowDefinitionAssetSource` (with its `tarball` and `source` - `package` arms), unioned as `WorkflowDefinitionSource`. + `package` arms), unioned as `WorkflowDefinitionSource`, and the shared + `workflowSourceAssetMountPath` layout helper. +- `@intx/types/attachments` — MIME classification, attachment categories, + and attachment size/count limits. +- `@intx/types/signals` — signal kinds, approval decisions, and correlation + signal-name helpers. +- `@intx/types/agent-address` — formatting and parsing run mail addresses. +- `@intx/types/workflow-run-id` — `deriveWorkflowRunId`, which extracts a + deployment's top-level run ID from its mail address and throws on an + invalid address. +- `@intx/types/base64` — byte encoding and decoding. +- `@intx/types/concat` — `concatBytes` for combining byte arrays. +- `@intx/types/has-code` — the `hasCode` error guard. diff --git a/vendor/intx-types/package.json b/vendor/intx-types/package.json index bdfa481af..5b41811fb 100644 --- a/vendor/intx-types/package.json +++ b/vendor/intx-types/package.json @@ -24,6 +24,10 @@ "types": "./src/runtime.ts", "default": "./src/runtime.ts" }, + "./runtime-core": { + "types": "./src/runtime-core.ts", + "default": "./src/runtime-core.ts" + }, "./runtime-capabilities": { "types": "./src/runtime-capabilities.ts", "default": "./src/runtime-capabilities.ts" @@ -55,6 +59,82 @@ "./testing": { "types": "./src/testing/index.ts", "default": "./src/testing/index.ts" + }, + "./attachments": { + "types": "./src/attachments.ts", + "default": "./src/attachments.ts" + }, + "./credential-delivery": { + "types": "./src/credential-delivery.ts", + "default": "./src/credential-delivery.ts" + }, + "./format-safety-rating": { + "types": "./src/format-safety-rating.ts", + "default": "./src/format-safety-rating.ts" + }, + "./inference-events": { + "types": "./src/inference-events.ts", + "default": "./src/inference-events.ts" + }, + "./inbound-mail-policy": { + "types": "./src/inbound-mail-policy.ts", + "default": "./src/inbound-mail-policy.ts" + }, + "./workflow-definition": { + "types": "./src/workflow-definition.ts", + "default": "./src/workflow-definition.ts" + }, + "./hex": { + "types": "./src/hex.ts", + "default": "./src/hex.ts" + }, + "./grant-snapshot": { + "types": "./src/grant-snapshot.ts", + "default": "./src/grant-snapshot.ts" + }, + "./repo": { + "types": "./src/repo.ts", + "default": "./src/repo.ts" + }, + "./base64": { + "types": "./src/base64.ts", + "default": "./src/base64.ts" + }, + "./signals": { + "types": "./src/signals.ts", + "default": "./src/signals.ts" + }, + "./agent-address": { + "types": "./src/agent-address.ts", + "default": "./src/agent-address.ts" + }, + "./has-code": { + "types": "./src/has-code.ts", + "default": "./src/has-code.ts" + }, + "./concat": { + "types": "./src/concat.ts", + "default": "./src/concat.ts" + }, + "./workflow-run-id": { + "types": "./src/workflow-run-id.ts", + "default": "./src/workflow-run-id.ts" + }, + "./message-id": { + "types": "./src/message-id.ts", + "default": "./src/message-id.ts" + }, + "./credential-cipher": { + "types": "./src/credential-cipher.ts", + "default": "./src/credential-cipher.ts" + }, + "./grants": { + "types": "./src/grants.ts", + "default": "./src/grants.ts" + }, + "./adapter-manifest": { + "types": "./src/adapter-manifest.ts", + "default": "./src/adapter-manifest.ts" } }, "dependencies": { diff --git a/vendor/intx-types/src/adapter-manifest.test.ts b/vendor/intx-types/src/adapter-manifest.test.ts new file mode 100644 index 000000000..70b1af87d --- /dev/null +++ b/vendor/intx-types/src/adapter-manifest.test.ts @@ -0,0 +1,40 @@ +import { describe, expect, test } from "bun:test"; + +import { parseAdapterManifestEnv } from "./adapter-manifest"; + +describe("parseAdapterManifestEnv", () => { + test("treats an absent or blank value as no declaration", () => { + expect(parseAdapterManifestEnv(undefined)).toBeUndefined(); + expect(parseAdapterManifestEnv("")).toBeUndefined(); + expect(parseAdapterManifestEnv(" ")).toBeUndefined(); + }); + + test("returns an empty manifest for an empty array", () => { + expect(parseAdapterManifestEnv("[]")).toEqual([]); + }); + + test("returns the entries of a valid manifest", () => { + const entries = [ + { provider: "openai-responses", specifier: "./adapter", export: "make" }, + ]; + expect(parseAdapterManifestEnv(JSON.stringify(entries))).toEqual(entries); + }); + + test("rejects malformed JSON", () => { + expect(() => parseAdapterManifestEnv("{not json")).toThrow( + /SIDECAR_ADAPTER_MANIFEST is not valid JSON/, + ); + }); + + test("rejects a value of the wrong shape", () => { + for (const raw of [ + '{"provider":"x"}', + '[{"provider":1}]', + '[{"provider":"a:b","specifier":"x","export":"y"}]', + ]) { + expect(() => parseAdapterManifestEnv(raw)).toThrow( + /SIDECAR_ADAPTER_MANIFEST failed validation/, + ); + } + }); +}); diff --git a/vendor/intx-types/src/adapter-manifest.ts b/vendor/intx-types/src/adapter-manifest.ts new file mode 100644 index 000000000..cd2ff1229 --- /dev/null +++ b/vendor/intx-types/src/adapter-manifest.ts @@ -0,0 +1,43 @@ +import { type } from "arktype"; + +import { ModelProviderPlugin } from "./catalog"; + +// Describes one custom adapter: the provider identifier it serves, the module +// specifier to import, and the named export within that module to use as its +// factory. Specifiers are operator-config-only and resolve to arbitrary code +// via `import()`, so they must originate solely from trusted operator +// configuration, never from tenant or deploy data. The shape is validated at +// every deserialization boundary; the value is trusted operator input. +export const AdapterManifestEntry = type({ + provider: ModelProviderPlugin, + specifier: "string", + export: "string", +}); +export type AdapterManifestEntry = typeof AdapterManifestEntry.infer; + +export const AdapterManifest = AdapterManifestEntry.array(); +export type AdapterManifest = typeof AdapterManifest.infer; + +// Parses the raw `SIDECAR_ADAPTER_MANIFEST` environment value. Absent or +// whitespace-only yields `undefined`; the caller decides what an omitted +// declaration means at its own boundary. A present value must be a JSON +// array of manifest entries, and anything else throws naming the variable +// so a typo fails the boot rather than silently dropping adapters. +export function parseAdapterManifestEnv( + raw: string | undefined, +): AdapterManifest | undefined { + if (raw === undefined || raw.trim() === "") return undefined; + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (cause) { + throw new Error("SIDECAR_ADAPTER_MANIFEST is not valid JSON", { cause }); + } + const validated = AdapterManifest(parsed); + if (validated instanceof type.errors) { + throw new Error( + `SIDECAR_ADAPTER_MANIFEST failed validation: ${validated.summary}`, + ); + } + return validated; +} diff --git a/vendor/intx-types/src/agent-address.ts b/vendor/intx-types/src/agent-address.ts index 56610d931..b08b00a69 100644 --- a/vendor/intx-types/src/agent-address.ts +++ b/vendor/intx-types/src/agent-address.ts @@ -5,11 +5,6 @@ // The shape of the right-hand side of the "@" is not validated beyond the // requirement that it be non-empty: tightening the contract (DNS-ish // validation, normalisation, etc.) is a separate follow-up. -// -// `@intx/hub-sessions`'s `parseAgentId` is the canonical throwing wrapper -// over `parseRunAddress` — call it when a `null` return would -// propagate as a silent bug, and keep this parser's `null` return -// reserved for callers that already have a structured fallback. const RUN_PREFIX = "run_"; diff --git a/vendor/intx-types/src/attachment-validation.test.ts b/vendor/intx-types/src/attachment-validation.test.ts new file mode 100644 index 000000000..86a86e614 --- /dev/null +++ b/vendor/intx-types/src/attachment-validation.test.ts @@ -0,0 +1,334 @@ +import { describe, test, expect } from "bun:test"; + +import { + DEFAULT_ATTACHMENT_POLICY, + validateAttachments, + type AttachmentInput, + type AttachmentPolicy, +} from "./attachment-validation"; +import { PER_ATTACHMENT_LIMIT_BYTES } from "./attachments"; +import { base64Encode } from "./base64"; + +function b64(bytes: number[]): string { + return base64Encode(new Uint8Array(bytes)); +} + +function bytesOfLength(n: number): string { + return base64Encode(new Uint8Array(n).fill(0x61)); +} + +// Small limits so oversize cases need only tiny buffers. +const policy: AttachmentPolicy = { + isAllowed: (m) => m === "image/png" || m === "application/pdf", + perAttachmentLimitBytes: 100, + perMessageTotalLimitBytes: 150, + maxAttachments: 3, +}; + +describe("validateAttachments", () => { + test("rejects too many attachments before decoding any", () => { + const inputs = Array.from({ length: 4 }, () => ({ + mimeType: "image/png", + data: "not base64 and never decoded", + })); + expect(validateAttachments(inputs, policy)).toEqual({ + ok: false, + error: { + code: "too_many_attachments", + message: "4 attachments, over the limit of 3", + count: 4, + limit: 3, + }, + }); + }); + + test("accepts valid attachments and defaults names by index", () => { + const inputs: AttachmentInput[] = [ + { mimeType: "image/png", data: b64([1, 2, 3]), name: "shot.png" }, + { mimeType: "application/pdf", data: b64([4, 5]) }, + ]; + const result = validateAttachments(inputs, policy); + expect(result).toEqual({ + ok: true, + attachments: [ + { + name: "shot.png", + contentType: "image/png", + data: new Uint8Array([1, 2, 3]), + }, + { + name: "attachment-1", + contentType: "application/pdf", + data: new Uint8Array([4, 5]), + }, + ], + }); + }); + + test("already-decoded bytes pass through the same checks", () => { + const bytes = new Uint8Array([1, 2, 3]); + expect( + validateAttachments([{ mimeType: "image/png", data: bytes }], policy), + ).toEqual({ + ok: true, + attachments: [ + { name: "attachment-0", contentType: "image/png", data: bytes }, + ], + }); + + const result = validateAttachments( + [{ mimeType: "image/png", data: new Uint8Array(101) }], + policy, + ); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.error.code).toBe("oversize_attachment"); + }); + + test("empty input is valid", () => { + expect(validateAttachments([], policy)).toEqual({ + ok: true, + attachments: [], + }); + }); + + test("an allowlisted type with parameters is stored as type/subtype", () => { + const result = validateAttachments([ + { + mimeType: "text/plain; charset=utf-8", + data: new TextEncoder().encode("hi"), + name: "notes.txt", + }, + ]); + expect(result).toEqual({ + ok: true, + attachments: [ + { + name: "notes.txt", + contentType: "text/plain", + data: new TextEncoder().encode("hi"), + }, + ], + }); + }); + + test("rejects a disallowed mimeType with the offending index", () => { + const result = validateAttachments( + [ + { mimeType: "image/png", data: b64([1]) }, + { mimeType: "image/tiff", data: b64([2]) }, + ], + policy, + ); + expect(result).toMatchObject({ + ok: false, + error: { + code: "disallowed_mime_type", + attachmentIndex: 1, + mimeType: "image/tiff", + }, + }); + }); + + test("rejects a name with header-unsafe characters", () => { + const result = validateAttachments( + [ + { mimeType: "image/png", data: b64([1]) }, + { mimeType: "image/png", data: b64([2]), name: 'a"b.png' }, + ], + policy, + ); + expect(result).toMatchObject({ + ok: false, + error: { code: "invalid_attachment_name", attachmentIndex: 1 }, + }); + if (result.ok) return; + expect(result.error.message).toContain("invalid characters"); + }); + + test("rejects an empty or whitespace-only name", () => { + for (const name of ["", " "]) { + const result = validateAttachments( + [{ mimeType: "image/png", data: b64([1]), name }], + policy, + ); + expect(result).toMatchObject({ + ok: false, + error: { code: "invalid_attachment_name", attachmentIndex: 0 }, + }); + if (result.ok) return; + expect(result.error.message).toContain("empty name"); + } + }); + + test("rejects malformed base64 with the offending index", () => { + const result = validateAttachments( + [{ mimeType: "image/png", data: "@@not-base64@@" }], + policy, + ); + expect(result).toMatchObject({ + ok: false, + error: { code: "malformed_base64", attachmentIndex: 0 }, + }); + }); + + test("rejects an oversize attachment with byte length and limit", () => { + const result = validateAttachments( + [{ mimeType: "image/png", data: new Uint8Array(101) }], + policy, + ); + expect(result).toMatchObject({ + ok: false, + error: { + code: "oversize_attachment", + attachmentIndex: 0, + byteLength: 101, + limitBytes: 100, + }, + }); + }); + + test("accepts compact base64 of exactly the per-attachment limit", () => { + const bytes = new Uint8Array(policy.perAttachmentLimitBytes); + expect( + validateAttachments([{ mimeType: "image/png", data: bytes }], policy).ok, + ).toBe(true); + + const encoded = base64Encode(bytes); + const result = validateAttachments( + [{ mimeType: "image/png", data: encoded }], + policy, + ); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.attachments[0]?.data.length).toBe( + policy.perAttachmentLimitBytes, + ); + }); + + test("rejects compact base64 one byte over the per-attachment limit", () => { + const encoded = base64Encode( + new Uint8Array(policy.perAttachmentLimitBytes + 1), + ); + const result = validateAttachments( + [{ mimeType: "image/png", data: encoded }], + policy, + ); + expect(result).toMatchObject({ + ok: false, + error: { + code: "oversize_attachment", + attachmentIndex: 0, + byteLength: policy.perAttachmentLimitBytes + 1, + limitBytes: policy.perAttachmentLimitBytes, + }, + }); + }); + + test("accepts compact base64 of exactly 10MiB under the default policy", () => { + const bytes = new Uint8Array(PER_ATTACHMENT_LIMIT_BYTES); + const encoded = base64Encode(bytes); + const result = validateAttachments([ + { mimeType: "image/png", data: encoded }, + ]); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.attachments[0]?.data.length).toBe( + DEFAULT_ATTACHMENT_POLICY.perAttachmentLimitBytes, + ); + }); + + test("accepts whitespace-padded compact base64 of an in-policy payload", () => { + const encoded = ` ${base64Encode(new Uint8Array(policy.perAttachmentLimitBytes))} \n`; + const result = validateAttachments( + [{ mimeType: "image/png", data: encoded }], + policy, + ); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect(result.attachments[0]?.data.length).toBe( + policy.perAttachmentLimitBytes, + ); + }); + + test("rejects encoded base64 whose length already exceeds the decoded limit", () => { + // 200 alphabet chars → 150 decoded bytes before atob (floor(200/4)*3). + const encoded = "x".repeat(200); + const result = validateAttachments( + [{ mimeType: "image/png", data: encoded }], + policy, + ); + expect(result).toMatchObject({ + ok: false, + error: { + code: "oversize_attachment", + attachmentIndex: 0, + byteLength: 150, + limitBytes: 100, + }, + }); + }); + + test("rejects an oversize total only after each attachment passes", () => { + const result = validateAttachments( + [ + { mimeType: "image/png", data: bytesOfLength(80) }, + { mimeType: "application/pdf", data: bytesOfLength(80) }, + ], + policy, + ); + expect(result).toMatchObject({ + ok: false, + error: { code: "oversize_total", totalBytes: 160, limitBytes: 150 }, + }); + }); + + test("per-attachment oversize wins over total (ordering)", () => { + // One oversize attachment plus a small one. Per-attachment size is the + // most specific error and must win over oversize_total. + const result = validateAttachments( + [ + { mimeType: "image/png", data: new Uint8Array(200) }, + { mimeType: "application/pdf", data: new Uint8Array(10) }, + ], + policy, + ); + expect(result).toMatchObject({ + ok: false, + error: { + code: "oversize_attachment", + attachmentIndex: 0, + byteLength: 200, + limitBytes: 100, + }, + }); + }); + + test("returns the first error without decoding later entries", () => { + const result = validateAttachments( + [ + { mimeType: "image/tiff", data: b64([1]) }, + { mimeType: "image/png", data: "x".repeat(200) }, + ], + policy, + ); + expect(result).toMatchObject({ + ok: false, + error: { + code: "disallowed_mime_type", + attachmentIndex: 0, + mimeType: "image/tiff", + }, + }); + }); + + test("structured errors carry a human-readable message", () => { + const result = validateAttachments( + [{ mimeType: "image/tiff", data: b64([1]) }], + policy, + ); + if (result.ok) throw new Error("expected a validation error"); + expect(typeof result.error.message).toBe("string"); + expect(result.error.message.length).toBeGreaterThan(0); + }); +}); diff --git a/vendor/intx-types/src/attachment-validation.ts b/vendor/intx-types/src/attachment-validation.ts new file mode 100644 index 000000000..d8db5a9bb --- /dev/null +++ b/vendor/intx-types/src/attachment-validation.ts @@ -0,0 +1,201 @@ +import { + isAllowedMimeType, + MAX_ATTACHMENTS_PER_MESSAGE, + mimeTypeAndSubtype, + PER_ATTACHMENT_LIMIT_BYTES, + PER_MESSAGE_TOTAL_LIMIT_BYTES, +} from "./attachments"; +import { base64Decode } from "./base64"; +import type { MessageAttachment } from "./runtime"; +import type { AttachmentError } from "./sessions"; + +/** + * A single attachment as it arrives at a boundary: a MIME type, the bytes, + * and an optional filename. A string `data` is base64 (the request-body + * form); a caller that already holds the bytes passes them as-is. + */ +export type AttachmentInput = { + mimeType: string; + data: string | Uint8Array; + name?: string; +}; + +/** + * The attachment policy the route validates against. Defaults to the + * system-level allowlist and limits; a future per-agent or per-workflow + * lookup substitutes a narrowed policy here (the injection seam — no + * validation logic changes). + */ +export type AttachmentPolicy = { + isAllowed: (mimeType: string) => boolean; + perAttachmentLimitBytes: number; + perMessageTotalLimitBytes: number; + maxAttachments: number; +}; + +export const DEFAULT_ATTACHMENT_POLICY: AttachmentPolicy = { + isAllowed: isAllowedMimeType, + perAttachmentLimitBytes: PER_ATTACHMENT_LIMIT_BYTES, + perMessageTotalLimitBytes: PER_MESSAGE_TOTAL_LIMIT_BYTES, + maxAttachments: MAX_ATTACHMENTS_PER_MESSAGE, +}; + +// The error shape is the wire contract `AttachmentError` from @intx/types: +// a machine-actionable `code`, the fields needed to locate the rejection, +// and a human-readable `message`. Defining it once in @intx/types lets the +// route document the structured 400 in its OpenAPI surface. +export type AttachmentValidationError = AttachmentError; + +export type AttachmentValidationResult = + | { ok: true; attachments: MessageAttachment[] } + | { ok: false; error: AttachmentError }; + +function compactDecodedSize(compact: string): number { + const pad = compact.endsWith("==") ? 2 : compact.endsWith("=") ? 1 : 0; + return Math.floor(compact.length / 4) * 3 - pad; +} + +function compactBase64(data: string): string { + return data.replace(/\s+/g, ""); +} + +function decode(data: string | Uint8Array): Uint8Array | null { + if (typeof data !== "string") return data; + try { + return base64Decode(data); + } catch { + return null; + } +} + +/** + * Validate and decode attachments against a policy at either boundary + * (mail tools or the request body). + * + * The attachment count is checked first, before anything is decoded. + * Encoded base64 is size-checked from its compact length (whitespace + * stripped, padding counted) before decode, and the first error returns + * without decoding later entries. Remaining checks, in encounter order: + * encoded oversize, malformed base64, decoded per-attachment oversize, + * disallowed MIME type, invalid name. After every attachment passes, the + * per-message total is checked. On success the decoded + * `MessageAttachment[]` is returned with names defaulted to + * `attachment-{index}` by input position. + */ +export function validateAttachments( + inputs: readonly AttachmentInput[], + policy: AttachmentPolicy = DEFAULT_ATTACHMENT_POLICY, +): AttachmentValidationResult { + if (inputs.length > policy.maxAttachments) { + return { + ok: false, + error: { + code: "too_many_attachments", + message: `${inputs.length} attachments, over the limit of ${policy.maxAttachments}`, + count: inputs.length, + limit: policy.maxAttachments, + }, + }; + } + + const decoded: MessageAttachment[] = []; + + for (const [index, input] of inputs.entries()) { + let payload: string | Uint8Array; + if (typeof input.data === "string") { + payload = compactBase64(input.data); + const encodedSize = compactDecodedSize(payload); + if (encodedSize > policy.perAttachmentLimitBytes) { + return { + ok: false, + error: { + code: "oversize_attachment", + message: `attachment ${index} is ${encodedSize} bytes, over the ${policy.perAttachmentLimitBytes}-byte limit`, + attachmentIndex: index, + byteLength: encodedSize, + limitBytes: policy.perAttachmentLimitBytes, + }, + }; + } + } else { + payload = input.data; + } + + const bytes = decode(payload); + if (bytes === null) { + return { + ok: false, + error: { + code: "malformed_base64", + message: `attachment ${index} is not valid base64`, + attachmentIndex: index, + }, + }; + } + if (bytes.length > policy.perAttachmentLimitBytes) { + return { + ok: false, + error: { + code: "oversize_attachment", + message: `attachment ${index} is ${bytes.length} bytes, over the ${policy.perAttachmentLimitBytes}-byte limit`, + attachmentIndex: index, + byteLength: bytes.length, + limitBytes: policy.perAttachmentLimitBytes, + }, + }; + } + const mimeType = mimeTypeAndSubtype(input.mimeType); + if (!policy.isAllowed(mimeType)) { + return { + ok: false, + error: { + code: "disallowed_mime_type", + message: `attachment ${index} has unsupported content type "${input.mimeType}"`, + attachmentIndex: index, + mimeType: input.mimeType, + }, + }; + } + // A user-supplied name becomes the MIME part's quoted filename, so it + // must not contain characters that would break out of the header + // (line breaks or a double quote). Empty and whitespace-only names + // cannot round-trip through `filename=""`. The default name is always + // safe. + if ( + input.name !== undefined && + (input.name.trim() === "" || /[\r\n"]/.test(input.name)) + ) { + const empty = input.name.trim() === ""; + return { + ok: false, + error: { + code: "invalid_attachment_name", + message: empty + ? `attachment ${index} has an empty name` + : `attachment ${index} has a name with invalid characters (no quotes or line breaks)`, + attachmentIndex: index, + }, + }; + } + decoded.push({ + name: input.name ?? `attachment-${index}`, + contentType: mimeType, + data: bytes, + }); + } + + const totalBytes = decoded.reduce((sum, a) => sum + a.data.length, 0); + if (totalBytes > policy.perMessageTotalLimitBytes) { + return { + ok: false, + error: { + code: "oversize_total", + message: `attachments total ${totalBytes} bytes, over the ${policy.perMessageTotalLimitBytes}-byte limit`, + totalBytes, + limitBytes: policy.perMessageTotalLimitBytes, + }, + }; + } + + return { ok: true, attachments: decoded }; +} diff --git a/vendor/intx-types/src/attachments.test.ts b/vendor/intx-types/src/attachments.test.ts index d33329a21..6289db275 100644 --- a/vendor/intx-types/src/attachments.test.ts +++ b/vendor/intx-types/src/attachments.test.ts @@ -20,6 +20,14 @@ describe("attachment allowlist", () => { expect(isAllowedMimeType("")).toBe(false); }); + test("Content-Type parameters are not part of allowlist identity", () => { + expect(isAllowedMimeType("text/plain; charset=utf-8")).toBe(true); + expect(isAllowedMimeType("TEXT/PLAIN; charset=UTF-8")).toBe(true); + expect(isAllowedMimeType("image/png; name=shot.png")).toBe(true); + expect(isAllowedMimeType("image/tiff; charset=utf-8")).toBe(false); + expect(attachmentCategory("text/plain; charset=utf-8")).toBe("document"); + }); + test("category dispatch maps each major type and the document category", () => { expect(attachmentCategory("image/png")).toBe("image"); expect(attachmentCategory("video/mp4")).toBe("video"); diff --git a/vendor/intx-types/src/attachments.ts b/vendor/intx-types/src/attachments.ts index ffae81706..97f05952f 100644 --- a/vendor/intx-types/src/attachments.ts +++ b/vendor/intx-types/src/attachments.ts @@ -38,10 +38,20 @@ export const ATTACHMENT_ALLOWLIST = { export type AllowedMimeType = keyof typeof ATTACHMENT_ALLOWLIST; -export function isAllowedMimeType( - mimeType: string, -): mimeType is AllowedMimeType { - return mimeType in ATTACHMENT_ALLOWLIST; +const ALLOWLIST_BY_TYPE: Record = + ATTACHMENT_ALLOWLIST; + +/** + * The type/subtype of a Content-Type, ignoring parameters and case. + * Allowlist identity is this token, not the raw header value. + */ +export function mimeTypeAndSubtype(contentType: string): string { + const [mimeType = ""] = contentType.split(";"); + return mimeType.trim().toLowerCase(); +} + +export function isAllowedMimeType(mimeType: string): boolean { + return Object.hasOwn(ALLOWLIST_BY_TYPE, mimeTypeAndSubtype(mimeType)); } /** @@ -53,10 +63,22 @@ export function isAllowedMimeType( export function attachmentCategory( mimeType: string, ): AttachmentCategory | undefined { - if (isAllowedMimeType(mimeType)) { - return ATTACHMENT_ALLOWLIST[mimeType]; - } - return undefined; + return ALLOWLIST_BY_TYPE[mimeTypeAndSubtype(mimeType)]; +} + +/** + * True when a Content-Type names readable text: `text/*`, JSON, and the + * `+json` structured suffix. Callers use this to decide whether an + * attachment's content is carried as text or as opaque bytes. Any + * Content-Type parameters (e.g. `; charset=utf-8`) are ignored. + */ +export function isTextLikeMimeType(contentType: string): boolean { + const mime = mimeTypeAndSubtype(contentType); + return ( + mime.startsWith("text/") || + mime === "application/json" || + mime.endsWith("+json") + ); } // Default size limits, on decoded bytes. These are the system-level @@ -64,3 +86,7 @@ export function attachmentCategory( // limit that defaults to these. export const PER_ATTACHMENT_LIMIT_BYTES = 10 * 1024 * 1024; export const PER_MESSAGE_TOTAL_LIMIT_BYTES = 30 * 1024 * 1024; +// Cap on attachment count. The byte limits measure decoded payload, so +// without this a request could carry an unbounded number of empty +// attachments, each of which becomes a MIME part. +export const MAX_ATTACHMENTS_PER_MESSAGE = 20; diff --git a/vendor/intx-types/src/catalog.test.ts b/vendor/intx-types/src/catalog.test.ts index 57295fc3c..82912db27 100644 --- a/vendor/intx-types/src/catalog.test.ts +++ b/vendor/intx-types/src/catalog.test.ts @@ -11,15 +11,17 @@ import { PricingRowResponse, } from "./catalog"; -describe("ModelProviderPlugin enum", () => { - test("accepts each adapter key", () => { - for (const plugin of modelProviderPlugins) { +describe("ModelProviderPlugin", () => { + test("accepts every built-in key and an operator-registered key", () => { + for (const plugin of [...modelProviderPlugins, "openai-responses"]) { expect(ModelProviderPlugin(plugin)).toBe(plugin); } }); - test("rejects an unknown plugin", () => { - expect(ModelProviderPlugin("cohere") instanceof type.errors).toBe(true); + test("rejects a key the grant identifier cannot delimit", () => { + for (const plugin of ["", " ", "a:b", "Anthropic", "-x", "a/b"]) { + expect(ModelProviderPlugin(plugin) instanceof type.errors).toBe(true); + } }); }); diff --git a/vendor/intx-types/src/catalog.ts b/vendor/intx-types/src/catalog.ts index 47d1c3717..fadfbc60a 100644 --- a/vendor/intx-types/src/catalog.ts +++ b/vendor/intx-types/src/catalog.ts @@ -8,12 +8,16 @@ export const modelProviderPlugins = [ "openai-compatible", "google-genai", ] as const; -export type ModelProviderPlugin = (typeof modelProviderPlugins)[number]; -export const ModelProviderPlugin = type - .enumerated(...modelProviderPlugins) - .describe( - "The inference adapter that serves this provider's models, dispatched by the runtime provider registry.", - ); +// Open-ended so an operator-registered adapter (SIDECAR_ADAPTER_MANIFEST) is +// selectable; a key no sidecar registers fails at deploy admission. The +// grammar excludes `:` because the key is embedded in the +// `inference.source::` grant identifier, which is matched as +// one string: a colon in the plugin would let two provider/model pairs share +// an approval. +export const ModelProviderPlugin = type(/^[a-z0-9][a-z0-9._-]*$/).describe( + "The inference adapter that serves this provider's models, dispatched by the runtime provider registry: a built-in key or one registered in the sidecar's adapter manifest. Lowercase alphanumerics, `.`, `_` and `-`, starting with an alphanumeric.", +); +export type ModelProviderPlugin = typeof ModelProviderPlugin.infer; export const providerPreferenceModes = ["pin", "prefer"] as const; export type ProviderPreferenceMode = (typeof providerPreferenceModes)[number]; diff --git a/vendor/intx-types/src/credential-delivery.ts b/vendor/intx-types/src/credential-delivery.ts new file mode 100644 index 000000000..398824a85 --- /dev/null +++ b/vendor/intx-types/src/credential-delivery.ts @@ -0,0 +1,37 @@ +import { type } from "arktype"; + +/** + * The decrypted credential material and per-handle binding descriptors + * delivered to a running agent so its tools can use provider-backed + * credentials. Secrets are decrypted hub-side and ride this payload on the + * live channel ONLY -- the deploy frame at launch, a `credentials.update` + * frame on rotation, and the child's in-memory cell. They are NEVER written to + * disk (they do not ride the git-committed grants file) and NEVER copied into + * any snapshot, event, or state -- redaction is by construction, mirroring how + * an `InferenceSource`'s `apiKey` stays off every egress type. + * + * `materials` is keyed by `credentialId` (a credential can back several handles, + * so its secret is stored once); `bindings` maps each declared tool handle to + * the credential that backs it and the consumer identity allowed to use it. + */ +export const CredentialMaterialEntry = type({ + credentialId: "string", + providerKey: "string", + origin: "string", + secret: "string", +}); +export type CredentialMaterialEntry = typeof CredentialMaterialEntry.infer; + +export const CredentialBindingDescriptor = type({ + handle: "string", + credentialId: "string", + consumer: "string", +}); +export type CredentialBindingDescriptor = + typeof CredentialBindingDescriptor.infer; + +export const CredentialDelivery = type({ + bindings: CredentialBindingDescriptor.array(), + materials: CredentialMaterialEntry.array(), +}); +export type CredentialDelivery = typeof CredentialDelivery.infer; diff --git a/vendor/intx-types/src/credentials.ts b/vendor/intx-types/src/credentials.ts index aad64b732..7a537f38f 100644 --- a/vendor/intx-types/src/credentials.ts +++ b/vendor/intx-types/src/credentials.ts @@ -45,7 +45,7 @@ const BindingLocator = type.enumerated(...credentialBindingLocators); export const CredentialBinding = type({ package: type("string").describe( - "The tool package the declared handle belongs to; matches the resolved manifest's top-level package name.", + "The tool package the declared handle belongs to. On a source-ref workflow this field is the tool factory's bundle id. On a pinned package it is that package's name in the tool-package manifest's top level.", ), handle: ToolCredentialHandle.describe( "The credential handle the tool package declared; unique within its package.", diff --git a/vendor/intx-types/src/format-safety-rating.ts b/vendor/intx-types/src/format-safety-rating.ts new file mode 100644 index 000000000..5369f8200 --- /dev/null +++ b/vendor/intx-types/src/format-safety-rating.ts @@ -0,0 +1,11 @@ +import type { SafetyRatingBlock } from "./runtime-core"; + +/** + * Human-readable rendering of a SafetyRatingBlock for reply text, + * timeline summaries, and request-history rewrites when a provider + * has no input wire shape for safety_rating. Single owner of the + * display string so reply / history / transform stay in lockstep. + */ +export function formatSafetyRatingText(block: SafetyRatingBlock): string { + return `Request blocked: ${block.blockReason}`; +} diff --git a/vendor/intx-types/src/inbound-mail-policy.ts b/vendor/intx-types/src/inbound-mail-policy.ts new file mode 100644 index 000000000..f65467aef --- /dev/null +++ b/vendor/intx-types/src/inbound-mail-policy.ts @@ -0,0 +1,91 @@ +import { type } from "arktype"; + +/** + * The admission outcome of an inbound message. This is the single vocabulary a + * delivery decision keys on, distinct from the two-axis signature verdict that + * produces it. + * + * - `clean` — nothing suspect; always admitted + * - `untrustedFrom` — the visible `From` is present but cannot be reduced to a + * single addr-spec + * - `mismatchedFrom` — the visible `From` names a different identity from the + * sender the hub stamped + * - `absentFrom` — the message carries no usable `From` + * - `invalid` — the signature check failed (tampering or the wrong key) + * - `missing` — the message carried no signature + * - `unknown` — no key was available to verify against + * - `error` — a fault stopped the check from running at all; always rejected + */ +export const InboundMailOutcome = type.enumerated( + "clean", + "untrustedFrom", + "mismatchedFrom", + "absentFrom", + "invalid", + "missing", + "unknown", + "error", +); +export type InboundMailOutcome = typeof InboundMailOutcome.infer; + +/** + * The subset of {@link InboundMailOutcome} a workflow author may relax to admit + * a message that would otherwise be rejected. It omits `clean` (which always + * admits, so there is nothing to relax) and `error` (pinned to reject, since a + * fault we could not check through is never something an author should be able + * to wave past). A per-workflow policy keys on exactly these outcomes. + */ +export const AuthorControllableOutcome = type.enumerated( + "untrustedFrom", + "mismatchedFrom", + "absentFrom", + "invalid", + "missing", + "unknown", +); +export type AuthorControllableOutcome = typeof AuthorControllableOutcome.infer; + +/** + * A per-workflow inbound-mail admission policy: for each admission outcome the + * author may control, whether a message that raises that outcome as a finding is + * `reject`ed or `admit`ted. The key set is exactly the + * {@link AuthorControllableOutcome} values -- `clean` (always admitted) and + * `error` (pinned to reject) are deliberately not keys. + * + * The object is SPARSE: every key is optional, and an omitted key is NOT a + * default of any kind here. It is left for a later resolution step to interpret + * an absent outcome. Keeping it sparse means the content hash covers only what + * the author actually declared, so a definition that omits the policy hashes + * identically to one authored before the field existed. Undeclared keys are + * rejected so a typo such as `clean` or `errror` fails at the wire boundary + * rather than riding through as an inert unknown key. + */ +export const InboundMailPolicy = type({ + "untrustedFrom?": "'reject' | 'admit'", + "mismatchedFrom?": "'reject' | 'admit'", + "absentFrom?": "'reject' | 'admit'", + "invalid?": "'reject' | 'admit'", + "missing?": "'reject' | 'admit'", + "unknown?": "'reject' | 'admit'", +}).onUndeclaredKey("reject"); +export type InboundMailPolicy = typeof InboundMailPolicy.infer; + +type AssertEqual = [A] extends [B] + ? [B] extends [A] + ? true + : false + : false; + +/** + * The three outcome vocabularies must move together, but nothing else connects + * them: each spells the same names out again in its own notation. + */ +const _policyKeysMatchAuthorControllable: AssertEqual< + keyof InboundMailPolicy, + AuthorControllableOutcome +> = true; + +const _authorControllableAreOutcomes: AssertEqual< + Exclude, + never +> = true; diff --git a/vendor/intx-types/src/index.ts b/vendor/intx-types/src/index.ts index 755229df8..5ae3df72f 100644 --- a/vendor/intx-types/src/index.ts +++ b/vendor/intx-types/src/index.ts @@ -9,6 +9,7 @@ export * from "./signals"; export * from "./instances"; export * from "./workflows"; export * from "./attachments"; +export * from "./attachment-validation"; export * from "./sessions"; export * from "./approvals"; export * from "./wallets"; @@ -23,6 +24,7 @@ export * from "./offerings"; export * from "./models"; export * from "./capabilities"; export * from "./catalog"; +export * from "./adapter-manifest"; export * from "./observability"; export * from "./agent-address"; export * from "./agent-data"; @@ -36,3 +38,4 @@ export * from "./has-code"; export * from "./audit"; export * from "./sidecar-allocation"; export * from "./sidecar-capabilities"; +export * from "./workflow-lifecycle"; diff --git a/vendor/intx-types/src/inference-events.ts b/vendor/intx-types/src/inference-events.ts new file mode 100644 index 000000000..6bcc006c8 --- /dev/null +++ b/vendor/intx-types/src/inference-events.ts @@ -0,0 +1,619 @@ +import { type } from "arktype"; +import { + PartialMessage, + RedactedThinkingBlock, + TokenUsage, + LastCycleSource, + AssistantTurn, + InferenceError, + CitationBlock, + SafetyRatingBlock, + CodeExecutionRequestBlock, + CodeExecutionResultBlock, + ImageBlock, + ToolCall, + ToolResult, + GateType, + ApprovalSnapshot, + ForkMode, + type InboundMessage, +} from "./runtime-core"; + +// --------------------------------------------------------------------------- +// Inference Event Protocol (INFERENCE.md § Event Protocol) +// --------------------------------------------------------------------------- + +/** + * Wire-safe representation of InboundMessage for use in InferenceEvent + * variants. The runtime InboundMessage type contains Uint8Array fields + * (MessageAttachment.data) that cannot survive JSON serialization, so the + * wire validator uses `unknown` for attachment data and accepts whatever + * JSON.parse produces. + */ +const WireInboundMessage = type({ + ref: { uid: "number", mailbox: "string" }, + headers: "Record", + flags: "string[]", + "content?": "string", + "payload?": "object", + "attachments?": "unknown[]", + signatureStatus: type.enumerated("valid", "invalid", "unknown", "missing"), +}); + +/** + * A single event in the inference event protocol. Every event carries a + * monotonic session-scoped sequence number. + * + * Event types are namespaced: `inference.*`, `tool.*`, `reactor.*`, + * `fork.*`, `message.*`, `custom.*`. + * + * (INFERENCE.md § Event Protocol) + */ +export const InferenceEvent = type.or( + { + type: "'inference.start'", + seq: "number", + data: { model: "string" }, + }, + { + type: "'inference.thinking.delta'", + seq: "number", + data: { + token: "string", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.block.signature'", + seq: "number", + data: { signature: "string", "index?": "number" }, + }, + { + type: "'inference.thinking.redacted'", + seq: "number", + data: { redactedThinking: RedactedThinkingBlock, "index?": "number" }, + }, + { + type: "'inference.text.delta'", + seq: "number", + data: { + token: "string", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.refusal.delta'", + seq: "number", + data: { + token: "string", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.tool_call.start'", + seq: "number", + data: { + callId: "string", + name: "string", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.tool_call.delta'", + seq: "number", + data: { + callId: "string", + argumentFragment: "string", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.tool_call.end'", + seq: "number", + data: { + callId: "string", + name: "string", + arguments: "Record", + partial: PartialMessage, + "index?": "number", + }, + }, + { + type: "'inference.usage'", + seq: "number", + // Locally patched — see vendor/intx-types/PATCHES.md#types-ts-usage-stop-reason + data: { usage: TokenUsage, source: LastCycleSource, "stopReason?": "string" }, + }, + { + type: "'inference.done'", + seq: "number", + data: { + turn: AssistantTurn, + usage: TokenUsage, + source: LastCycleSource, + "pacingDelayMs?": "number", + }, + }, + { + type: "'inference.error'", + seq: "number", + data: { error: InferenceError, partial: PartialMessage }, + }, + { + type: "'inference.retry'", + seq: "number", + data: { + attempt: "number", + delayMs: "number", + previousError: InferenceError, + }, + }, + { + type: "'inference.citation'", + seq: "number", + // `index`, when present, names the source content block (typically + // a TextBlock) the citation annotates. The harness uses it to + // interleave the citation into the finalized turn's `content[]` + // immediately after the matching block. Adapters whose wire + // protocol does not carry per-citation block indices omit the + // field; the harness then appends those citations at the end of + // `content[]` and consumers attribute them to the nearest + // preceding TextBlock per the CitationBlock docstring. + data: { citation: CitationBlock, "index?": "number" }, + }, + { + type: "'inference.safety_rating'", + seq: "number", + // Prompt-level structured safety signal (observed Gemini + // `promptFeedback.blockReason`). No candidate index: the first + // capture has zero candidates. Harness appends the block to the + // finalized turn's `content[]`. + data: { safetyRating: SafetyRatingBlock }, + }, + { + type: "'inference.code_execution.start'", + seq: "number", + data: { request: CodeExecutionRequestBlock, "index?": "number" }, + }, + { + type: "'inference.code_execution.delta'", + seq: "number", + // requestId correlates fragments back to the originating + // CodeExecutionRequestBlock; index is the positional hint into + // the response's content-block stream. They are independent: a + // single response may stream code execution for multiple + // requests interleaved, distinguished by requestId; index lets + // the harness's per-block accumulator route the fragment to + // the correct block when the array isn't yet finalized. + data: { + requestId: "string", + codeFragment: "string", + "index?": "number", + }, + }, + { + type: "'inference.code_execution.result'", + seq: "number", + data: { result: CodeExecutionResultBlock, "index?": "number" }, + }, + { + type: "'inference.image_output'", + seq: "number", + // Fires mid-stream when an adapter finalizes an image-output + // block, signaling that the image is ready for downstream + // handoff before the full inference.done lands. The wrapped + // ImageBlock typically carries a base64 MediaSource — the + // payload can be large (Gemini's image-output captures show + // ~1MB inline blobs); consumers that subscribe to this event + // should treat it as a non-trivial transport size. + data: { image: ImageBlock, "index?": "number" }, + }, + { + type: "'tool.start'", + seq: "number", + data: { call: ToolCall }, + }, + { + type: "'tool.update'", + seq: "number", + data: { callId: "string", partial: "string" }, + }, + { + type: "'tool.done'", + seq: "number", + data: { result: ToolResult }, + }, + { + type: "'message.queued'", + seq: "number", + data: { message: WireInboundMessage }, + }, + { + type: "'message.run.started'", + seq: "number", + data: { + "messageId?": "string", + messageRunId: "string", + receivedAt: "number", + }, + }, + { + type: "'message.run.ended'", + seq: "number", + data: { + messageRunId: "string", + "messageId?": "string", + status: type.enumerated("completed", "failed"), + "error?": { + message: "string", + "kind?": "string", + }, + }, + }, + { + type: "'message.correlated'", + seq: "number", + data: { message: WireInboundMessage, correlationId: "string" }, + }, + { + type: "'connector.reply'", + seq: "number", + data: { content: "string", "checkpointHash?": "string" }, + }, + { + type: "'reactor.start'", + seq: "number", + data: "object", + }, + { + type: "'reactor.gate.blocked'", + seq: "number", + data: { + reason: GateType, + gateId: "string", + "correlationId?": "string", + "approvalSnapshot?": ApprovalSnapshot, + }, + }, + { + type: "'reactor.gate.cleared'", + seq: "number", + data: { + gateId: "string", + reason: type.enumerated("resolved", "timeout", "shutdown"), + }, + }, + { + type: "'reactor.done'", + seq: "number", + data: "object", + }, + { + type: "'reactor.error'", + seq: "number", + data: { error: "string", fatal: "boolean" }, + }, + { + type: "'fork.created'", + seq: "number", + data: { forkId: "string", parentId: "string", mode: ForkMode }, + }, + { + type: "'fork.done'", + seq: "number", + data: { forkId: "string", "result?": "unknown" }, + }, + { + type: "'fork.error'", + seq: "number", + data: { forkId: "string", error: "string" }, + }, + { + type: "'fork.aborted'", + seq: "number", + data: { forkId: "string" }, + }, + { + type: /^custom\./, + seq: "number", + data: "Record", + }, +); +// The TypeScript type is defined manually rather than inferred from the +// validator because the `custom.*` variant uses a regex pattern which +// arktype infers as `string`. A bare `string` in the discriminant position +// prevents TypeScript from narrowing the union in switch statements. +// The manually defined type uses a `custom.${string}` template literal +// for that variant, preserving the narrowing behavior downstream code +// relies on. +export type InferenceEvent = + | { type: "inference.start"; seq: number; data: { model: string } } + | { + type: "inference.thinking.delta"; + seq: number; + data: { token: string; partial: PartialMessage; index?: number }; + } + | { + type: "inference.block.signature"; + seq: number; + data: { signature: string; index?: number }; + } + | { + type: "inference.thinking.redacted"; + seq: number; + data: { redactedThinking: RedactedThinkingBlock; index?: number }; + } + | { + type: "inference.text.delta"; + seq: number; + data: { token: string; partial: PartialMessage; index?: number }; + } + | { + type: "inference.refusal.delta"; + seq: number; + data: { token: string; partial: PartialMessage; index?: number }; + } + | { + type: "inference.tool_call.start"; + seq: number; + data: { + callId: string; + name: string; + partial: PartialMessage; + index?: number; + }; + } + | { + type: "inference.tool_call.delta"; + seq: number; + data: { + callId: string; + argumentFragment: string; + partial: PartialMessage; + index?: number; + }; + } + | { + type: "inference.tool_call.end"; + seq: number; + data: { + callId: string; + name: string; + arguments: Record; + partial: PartialMessage; + index?: number; + }; + } + | { + type: "inference.usage"; + seq: number; + // Locally patched — see vendor/intx-types/PATCHES.md#types-ts-usage-stop-reason + data: { usage: TokenUsage; source: LastCycleSource; stopReason?: string }; + } + | { + type: "inference.done"; + seq: number; + data: { + turn: AssistantTurn; + usage: TokenUsage; + source: LastCycleSource; + pacingDelayMs?: number; + }; + } + | { + type: "inference.error"; + seq: number; + data: { error: InferenceError; partial: PartialMessage }; + } + | { + /** + * Emitted between attempts when the per-call retry policy decides + * to retry after an error. `attempt` is the 1-indexed number of + * the attempt that just **failed** — the same value the policy + * saw on its `RetrySituation.attempt` reading. `delayMs` is the + * delay the wrapper will apply before the next attempt starts; + * `previousError` carries the classified error that triggered + * the retry. The event is not emitted when the policy aborts. + */ + type: "inference.retry"; + seq: number; + data: { + attempt: number; + delayMs: number; + previousError: InferenceError; + }; + } + | { + type: "inference.citation"; + seq: number; + data: { citation: CitationBlock; index?: number }; + } + | { + type: "inference.safety_rating"; + seq: number; + data: { safetyRating: SafetyRatingBlock }; + } + | { + type: "inference.code_execution.start"; + seq: number; + data: { request: CodeExecutionRequestBlock; index?: number }; + } + | { + type: "inference.code_execution.delta"; + seq: number; + data: { requestId: string; codeFragment: string; index?: number }; + } + | { + type: "inference.code_execution.result"; + seq: number; + data: { result: CodeExecutionResultBlock; index?: number }; + } + | { + type: "inference.image_output"; + seq: number; + data: { image: ImageBlock; index?: number }; + } + | { type: "tool.start"; seq: number; data: { call: ToolCall } } + | { + type: "tool.update"; + seq: number; + data: { callId: string; partial: string }; + } + | { type: "tool.done"; seq: number; data: { result: ToolResult } } + | { + type: "message.queued"; + seq: number; + data: { message: InboundMessage }; + } + | { + /** + * Per-message run-bracket open. Emitted by the reactor when it + * dequeues an inbound mail message and begins per-message work. + * + * `messageRunId` is reactor-minted, unique per dequeue. It is + * non-negotiable for crash-replay correlation: the reactor can + * legitimately dequeue the same `messageId` more than once across + * a crash + replay cycle, so two bracket-open events with the + * same `messageId` and no run-id cannot be unambiguously paired + * with their `message.run.ended` counterparts. + */ + type: "message.run.started"; + seq: number; + data: { + messageId?: string; + messageRunId: string; + receivedAt: number; + }; + } + | { + /** + * Per-message run-bracket close. Pairs with `message.run.started` + * by `messageRunId`. `messageId` is carried redundantly so log + * readers can correlate without a join against the open event; it is + * absent for a message that named no id of its own. + * + * The `status` enum is `"completed" | "failed"` only. + * Cancellation lives in the workflow-runtime's + * `CancelRequested` -> `RunFailed` vocabulary, not on the + * reactor's bracket: the reactor does not run a state machine + * and what it observes when cancellation arrives is a harness + * abort, which is structurally `"failed"` with a specific + * `error.kind`. + * + * `error.kind` is documented as one of + * `"inference_error" | "tool_error" | "reactor_fatal" | + * "harness_aborted" | "doom_loop"` initially, extensible as new + * failure categories surface. `"doom_loop"` marks a protective + * break the reactor took on the agent's behalf when the agent + * repeated an identical tool batch past the configured threshold; + * unlike `"reactor_fatal"` it is not an internal fault. + */ + type: "message.run.ended"; + seq: number; + data: { + messageRunId: string; + messageId?: string; + status: "completed" | "failed"; + error?: { + message: string; + kind?: string; + }; + }; + } + | { + type: "message.correlated"; + seq: number; + data: { message: InboundMessage; correlationId: string }; + } + | { + type: "connector.reply"; + seq: number; + data: { content: string; checkpointHash?: string }; + } + | { type: "reactor.start"; seq: number; data: Record } + | { + type: "reactor.gate.blocked"; + seq: number; + data: { + reason: GateType; + gateId: string; + correlationId?: string; + approvalSnapshot?: ApprovalSnapshot; + }; + } + | { + type: "reactor.gate.cleared"; + seq: number; + data: { + gateId: string; + reason: "resolved" | "timeout" | "shutdown"; + }; + } + | { type: "reactor.done"; seq: number; data: Record } + | { + type: "reactor.error"; + seq: number; + data: { error: string; fatal: boolean }; + } + | { + type: "fork.created"; + seq: number; + data: { forkId: string; parentId: string; mode: ForkMode }; + } + | { + type: "fork.done"; + seq: number; + data: { forkId: string; result?: unknown }; + } + | { + type: "fork.error"; + seq: number; + data: { forkId: string; error: string }; + } + | { type: "fork.aborted"; seq: number; data: { forkId: string } } + | { + type: `custom.${string}`; + seq: number; + data: Record; + }; + +// Load-bearing drift guards for the dual-maintained `reactor.gate.blocked` +// event. The arktype `InferenceEvent` validator and the hand-written +// `InferenceEvent` type are kept in lockstep by hand (the `custom.*` regex +// variant forces the manual mirror). arktype passes undeclared keys through at +// runtime, so a schema that dropped `approvalSnapshot` would not fail at +// runtime. Projecting the field off each inferred shape makes it load-bearing: +// `tsc` errors if either mirror stops carrying it, mirroring the +// `_persistedSuspendedCall` guard in storage-isogit. +const _arkGateBlockedApprovalSnapshot = ( + data: Extract< + typeof InferenceEvent.infer, + { type: "reactor.gate.blocked" } + >["data"], +): ApprovalSnapshot | undefined => data.approvalSnapshot; +void _arkGateBlockedApprovalSnapshot; + +const _tsGateBlockedApprovalSnapshot = ( + data: Extract["data"], +): ApprovalSnapshot | undefined => data.approvalSnapshot; +void _tsGateBlockedApprovalSnapshot; + +/** + * Validate unknown data as an InferenceEvent. ArkType's regex-based validator + * infers `custom.*` event types as `string`, but the manual InferenceEvent type + * uses a `custom.${string}` template literal for switch narrowing. This function + * centralizes that single unavoidable cast. + */ +export function parseInferenceEvent( + data: unknown, +): InferenceEvent | type.errors { + const result = InferenceEvent(data); + if (result instanceof type.errors) return result; + // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- arktype regex infers as string; manual type uses template literal + return result as InferenceEvent; +} diff --git a/vendor/intx-types/src/instances.ts b/vendor/intx-types/src/instances.ts index 9f2786ceb..2b33e5059 100644 --- a/vendor/intx-types/src/instances.ts +++ b/vendor/intx-types/src/instances.ts @@ -48,6 +48,12 @@ export const WorkflowRunResponse = type({ createdAt: "string", updatedAt: "string", "endedAt?": "string | null", + "failureCode?": type("string | null").describe( + "Why the Hub failed this run's deployment, such as `sidecar_connect_failed`; null for a run the Hub did not fail.", + ), + "failureMessage?": type("string | null").describe( + "The detail of `failureCode`, such as the error the sidecar reported.", + ), }); export const WorkflowRunHealth = type({ diff --git a/vendor/intx-types/src/message-id.test.ts b/vendor/intx-types/src/message-id.test.ts index 10a574eb6..8111748bb 100644 --- a/vendor/intx-types/src/message-id.test.ts +++ b/vendor/intx-types/src/message-id.test.ts @@ -20,16 +20,67 @@ describe("deriveMessageId", () => { test("is case-insensitive on the header name", async () => { const raw = encoder.encode( - ["message-id: ", "", "body"].join("\n"), + ["message-id: ", "", "body"].join("\r\n"), ); expect(await deriveMessageId(raw)).toBe(""); }); - test("tolerates a lone-LF header boundary", async () => { + test("names no id in a lone-LF message, taking the digest instead", async () => { + // CRLF is the sole line terminator (RFC 5321 section 2.3.8, section + // 4.1.1.4), so a lone LF ends no field and `@intx/mime` refuses the + // message outright. Reading an id out of that text would give the two + // parsers different answers about the same bytes. const raw = encoder.encode( ["Message-ID: ", "", "body"].join("\n"), ); - expect(await deriveMessageId(raw)).toBe(""); + expect(parseMessageIdHeader(raw)).toBeNull(); + expect(await deriveMessageId(raw)).toMatch(/^[0-9a-f]{64}$/); + }); + + test("resolves no id from a bare break inside a field body", () => { + // The smuggled field would otherwise name the id for a message whose real + // `Message-ID` is the one the sender wrote above it. + const raw = encoder.encode( + "From: a@example.com\nMessage-ID: \r\n\r\nbody", + ); + expect(parseMessageIdHeader(raw)).toBeNull(); + }); + + test("names no id for an LF-only header section, whatever ends it", async () => { + // A whole-message search for the `CRLF CRLF` separator finds the one a + // sender left in the body and reads the header section from there, so the id + // absorbs the fields after the first and the head of the body. This id is + // the claim-check dedup key, a stored filename and a database join value, so + // no body bytes may reach it. + const bodyTerminated = encoder.encode( + "Message-ID: \nSubject: Hi\n\nbody\r\n\r\ntail\n", + ); + expect(parseMessageIdHeader(bodyTerminated)).toBeNull(); + expect(await deriveMessageId(bodyTerminated)).toMatch(/^[0-9a-f]{64}$/); + + const crlfTerminated = encoder.encode( + "Message-ID: \nSubject: Hi\r\n\r\nbody", + ); + expect(parseMessageIdHeader(crlfTerminated)).toBeNull(); + expect(await deriveMessageId(crlfTerminated)).toMatch(/^[0-9a-f]{64}$/); + }); + + test("names no id when an LF LF ends the header section", () => { + // The blank line is a pair of line breaks of its own. Honouring a bare pair + // lets a sender who controls one field body end the section early and strip + // the `Message-ID` after it. + const raw = encoder.encode( + "From: a@example.com\r\nSubject: Hi\n\n" + + "Message-ID: \r\n\r\nbody", + ); + expect(parseMessageIdHeader(raw)).toBeNull(); + }); + + test("reads the id at the first blank line when the body carries another", () => { + const raw = encoder.encode( + "Message-ID: \r\n\r\nbody\r\n\r\ntail\r\n", + ); + expect(parseMessageIdHeader(raw)).toBe(""); }); test("falls back to a sha256 hex digest with no Message-ID header", async () => { @@ -45,4 +96,33 @@ describe("deriveMessageId", () => { const raw = encoder.encode("From: a@example.com\r\n\r\nbody"); expect(parseMessageIdHeader(raw)).toBeNull(); }); + + test("treats a blank Message-ID header as naming no id", () => { + // RFC 2822 defines `Message-ID` as `msg-id`, which admits no empty value, + // so a header a sender left blank names nothing. + const raw = encoder.encode( + ["From: a@example.com", "Message-ID: ", "", "body"].join("\r\n"), + ); + expect(parseMessageIdHeader(raw)).toBeNull(); + }); + + test("two different messages with blank Message-ID headers derive different ids", async () => { + // The derived id is the dedup key that makes the same bytes consume once. + // Answering `""` for a blank header would make it one key two different + // messages share, so the second would be discarded as a redelivery of the + // first. Each falls back to its own digest instead. + const first = encoder.encode( + ["From: a@example.com", "Message-ID:", "", "first body"].join("\r\n"), + ); + const second = encoder.encode( + ["From: b@example.com", "Message-ID:", "", "second body"].join("\r\n"), + ); + + const firstId = await deriveMessageId(first); + const secondId = await deriveMessageId(second); + + expect(firstId).toMatch(/^[0-9a-f]{64}$/); + expect(secondId).toMatch(/^[0-9a-f]{64}$/); + expect(firstId).not.toBe(secondId); + }); }); diff --git a/vendor/intx-types/src/message-id.ts b/vendor/intx-types/src/message-id.ts index 523afffe8..cd96d94cf 100644 --- a/vendor/intx-types/src/message-id.ts +++ b/vendor/intx-types/src/message-id.ts @@ -18,10 +18,15 @@ import { hexEncode } from "./hex"; +const BARE_LINE_BREAK = /\r(?!\n)|(? { const messageIdFromHeader = parseMessageIdHeader(rawMessage); @@ -38,30 +43,32 @@ export async function deriveMessageId(rawMessage: Uint8Array): Promise { /** * Parse the `Message-ID` header value from a raw message, or `null` when - * the message carries no such header. + * the message carries no such header or carries a blank one. * - * The parser walks the message until the headers/body separator - * (`CRLF CRLF` per RFC 2822 §2.1, with the lone-`LF` variant tolerated to - * match common in-memory senders). Header-field unfolding follows RFC - * 2822 §2.2.3: a continuation line begins with whitespace and appends to - * the prior line. Header-name comparison is case-insensitive per RFC 2822 - * §1.2.2. + * The header section ends at the first blank line, whichever line-break flavour + * wrote it, and the whole message is the section when it carries no blank line. + * CRLF is the sole line terminator (RFC 5321 §2.3.8, §4.1.1.4), matching + * `@intx/mime`'s decoder, which refuses a section any other break wrote -- an + * `LF LF` separator included. Such a message names no id here and takes the + * digest in `deriveMessageId` instead, because reading one out of text that + * parser refuses would fold the fields after the break, and the head of the + * body, into this identifier. Header-field unfolding follows RFC 2822 §2.2.3: a + * continuation line begins with whitespace and appends to the prior line. + * Header-name comparison is case-insensitive per RFC 2822 §1.2.2. */ export function parseMessageIdHeader(rawMessage: Uint8Array): string | null { - const text = new TextDecoder("utf-8", { fatal: false }).decode(rawMessage); - // Headers end at the first blank line. RFC 2822 mandates `CRLF CRLF` - // but tolerate `LF LF` for callers that normalize line endings. - let headerSection = text; - const crlfBoundary = text.indexOf("\r\n\r\n"); - const lfBoundary = text.indexOf("\n\n"); - if (crlfBoundary >= 0 && (lfBoundary < 0 || crlfBoundary < lfBoundary)) { - headerSection = text.slice(0, crlfBoundary); - } else if (lfBoundary >= 0) { - headerSection = text.slice(0, lfBoundary); - } + const raw = new TextDecoder("utf-8", { fatal: false }).decode(rawMessage); + // Locating the blank line before checking the breaks is what keeps a `CRLF + // CRLF` the sender left in the body from becoming the separator: the section + // would then be read from that later offset, absorbing every field before it. + const boundary = BLANK_LINE.exec(raw); + const throughBoundary = + boundary === null ? raw.length : boundary.index + boundary[0].length; + if (BARE_LINE_BREAK.test(raw.slice(0, throughBoundary))) return null; + const headerSection = boundary === null ? raw : raw.slice(0, boundary.index); // Unfold continuation lines (a line starting with WSP belongs to // the prior header field). - const lines = headerSection.split(/\r?\n/); + const lines = headerSection.split("\r\n"); const unfolded: string[] = []; for (const line of lines) { if (line.length > 0 && (line[0] === " " || line[0] === "\t")) { @@ -76,7 +83,12 @@ export function parseMessageIdHeader(rawMessage: Uint8Array): string | null { if (colon < 0) continue; const name = line.slice(0, colon).trim().toLowerCase(); if (name !== "message-id") continue; - return line.slice(colon + 1).trim(); + const value = line.slice(colon + 1).trim(); + // A blank header names no id (RFC 2822 §3.6.4 admits no empty `msg-id`). + // Returning `""` would give every such message the same id, and the dedup + // key built on it would discard the second as a redelivery of the first. + if (value.length === 0) return null; + return value; } return null; } diff --git a/vendor/intx-types/src/repo.ts b/vendor/intx-types/src/repo.ts new file mode 100644 index 000000000..cffd8e4b3 --- /dev/null +++ b/vendor/intx-types/src/repo.ts @@ -0,0 +1,41 @@ +import { type } from "arktype"; + +/** + * Tag identifying a kind of repository in the hub's kind-keyed RepoStore. + * Lives in `@intx/types` because the wire-level pack frames reference it; + * the substrate package re-exports it for handler authors. + */ +export const RepoKind = type.enumerated( + "agent-state", + "skill", + "package-registry", + "workflow", + "workflow-run", +); +export type RepoKind = typeof RepoKind.infer; + +/** + * Operations a principal may invoke against a repo in the RepoStore. + * Lives in `@intx/types` so storage layers (e.g. `@intx/db`) can validate + * persisted action vocabularies without depending on the substrate + * package. The substrate re-exports it for handler authors. + */ +export const RepoAction = type.enumerated( + "init", + "writeTree", + "receivePack", + "createPack", + "resolveRef", +); +export type RepoAction = typeof RepoAction.infer; + +/** + * Hub-side identity of a repository in the RepoStore. Pack frames carry + * this alongside `agentAddress` so the hub can map a pack back to the + * originating repo independently of which sidecar/agent it is destined for. + */ +export const RepoId = type({ + kind: RepoKind, + id: "string", +}); +export type RepoId = typeof RepoId.infer; diff --git a/vendor/intx-types/src/runtime-core.ts b/vendor/intx-types/src/runtime-core.ts new file mode 100644 index 000000000..6806b4234 --- /dev/null +++ b/vendor/intx-types/src/runtime-core.ts @@ -0,0 +1,2405 @@ +// Runtime definitions for the Interchange agent harness. +// +// Wire-facing data types (AbortReason, InferenceSource, ToolDefinition, +// HarnessConfig) are arktype validators so they can be composed into +// WebSocket frame validators and used for runtime validation at parse +// boundaries. Behavioral interfaces (ContextStore, MessageTransport, +// ToolRunner, etc.) remain plain TypeScript. + +import { type } from "arktype"; +import type { AuditRecord, ErrorRecord } from "./audit"; +import { WireGrantRule } from "./grant-wire"; +import type { SignalKind } from "./signals"; + +// --------------------------------------------------------------------------- +// Cryptographic Identity (ARCHITECTURE.md § Cryptographic Identity, +// IMPLEMENTATION.md § Cryptographic Identity: Key Formats) +// --------------------------------------------------------------------------- + +/** + * An Ed25519 key pair as raw bytes. The private key is 32 bytes; the public + * key is the corresponding 32-byte compressed point. + * + * Key material is represented as Uint8Array throughout so it stays + * runtime-agnostic (Bun, Node, browser) and never accidentally leaks through + * JSON serialization. + */ +export type KeyPair = { + privateKey: Uint8Array; + publicKey: Uint8Array; +}; + +/** + * A key-bound cryptographic provider. Each instance is constructed with a + * specific agent's Ed25519 key pair and holds the private key internally. + * + * `sign` uses the instance's own private key — no key parameter is accepted. + * `verify` accepts a public key parameter so the holder can verify messages + * from arbitrary senders without constructing a new provider instance. + * + * The in-memory transport stores one CryptoProvider per registered agent and + * calls `crypto.sign(content)` during `send()` without passing keys around. + * + * Key formats (IMPLEMENTATION.md): + * - Ed25519 in SSH format — control plane interactions + * - Ed25519 in PGP format — message-level signatures over SMTP/IMAP + * - Ed25519 in X.509 format — TLS mutual auth certificates + * + * `getPublicKey` returns the raw public key bytes so callers can publish + * them to the control plane or embed them in discovery metadata. + */ +export interface CryptoProvider { + /** + * Sign `content` with the instance's private key. Returns the Ed25519 + * detached signature as raw bytes. + */ + sign(content: Uint8Array): Promise; + + /** + * Sign `payload` with the instance's private key using the SSH signature + * envelope (sshsig). Returns an ASCII-armored SSH SIGNATURE block suitable + * for the `gpgsig` header of a git commit or any other site that consumes + * `git verify-commit`-compatible signatures. The framing differs from + * `sign`'s raw output; callers that need either format should pick the + * matching method rather than reframing the result themselves. + */ + signSSH(payload: string): Promise; + + /** + * Verify that `signature` over `content` was produced by `publicKey`. + * Returns true if the signature is valid; false otherwise. + */ + verify( + content: Uint8Array, + signature: Uint8Array, + publicKey: Uint8Array, + ): Promise; + + /** The public key for this instance, as raw bytes. */ + getPublicKey(): Uint8Array; +} + +/** + * Generate a fresh Ed25519 key pair. The returned pair is used to construct + * a CryptoProvider instance. + */ +export type GenerateKeyPair = () => Promise; + +// --------------------------------------------------------------------------- +// Message Transport (MESSAGE.md § Transport Interface) +// --------------------------------------------------------------------------- + +/** + * Opaque reference to a message in a specific mailbox. Carries the IMAP UID + * and the mailbox name. Passed to fetch, flag, and move operations without + * requiring re-search. + */ +export type MessageRef = { + uid: number; + mailbox: string; +}; + +/** + * Interchange payload types as defined in MESSAGE.md § Payload Types. + * The type field in structured messages matches the Interchange-Type header. + * + * Exposed as both an arktype validator (for runtime validation at parse + * boundaries and tool-argument schemas) and a derived TypeScript union. + */ +export const InterchangeType = type.enumerated( + "conversation.message", + "conversation.join", + "conversation.leave", + "offering.request", + "offering.response", + "offering.error", + "offering.discover", + "offering.catalog", + "payment.required", + "payment.receipt", + "payment.verified", + "approval.request", + "approval.granted", + "approval.denied", + "system.health", + "system.register", + "system.deregister", + "system.credential.refresh", +); +export type InterchangeType = typeof InterchangeType.infer; + +const CONVERSATION_TYPES: ReadonlySet = new Set([ + "conversation.message", + "conversation.join", + "conversation.leave", +]); + +/** + * Membership is enumerated rather than derived from the `conversation.` name + * prefix, so a member added to InterchangeType is not a conversation type until + * it is listed in CONVERSATION_TYPES. + */ +export function isConversationType(value: InterchangeType): boolean { + return CONVERSATION_TYPES.has(value); +} + +/** Attachment for an outbound message. Content is raw bytes. */ +export type MessageAttachment = { + name: string; + contentType: string; + data: Uint8Array; + /** + * IMAP BODY section of this part, stamped by `extractAttachments` from + * the parsed sibling numbering. Absent on outbound attachments. + */ + part?: string; +}; + +/** + * A message the harness submits for delivery via SMTP. The transport + * assembles the PGP/MIME multipart structure, signs it with the agent's + * CryptoProvider, and submits it. + * + * Conversation types (conversation.*) carry `content` as text/plain. + * Structured types carry `payload` as application/vnd.interchange+json. + * Providing both is an error. + * + * (MESSAGE.md § Transport Interface › Outbound) + */ +export type OutboundMessage = { + to: string | string[]; + cc?: string | string[]; + subject?: string; + + type: InterchangeType; + + /** Plain text body — used when type is a conversation.* type. */ + content?: string; + + /** Structured JSON body — used when type is a non-conversation type. */ + payload?: Record; + + /** Human-readable summary for structured messages (the text/plain part). */ + summary?: string; + + attachments?: MessageAttachment[]; + + /** Message-ID of the message being replied to. */ + inReplyTo?: string; + + /** + * The RFC 5322 References chain for a threaded reply: the parent's own + * References plus the parent's Message-ID, in order. When present the + * transport ships it verbatim (after appending `inReplyTo` if it is not + * already the tail) rather than deriving a single-element `[inReplyTo]` + * chain, so a reply carries the full conversational ancestry. Absent for a + * non-reply or a reply whose parent could not be located. + */ + references?: string[]; + + /** Correlation ID linking this message to a pending async request. */ + correlationId?: string; + + /** Reactor session ID from the Interchange-Session-ID header. */ + sessionId?: string; + + /** Tenant ID for the Interchange-Tenant-ID header. */ + tenantId?: string; +}; + +/** + * Receipt returned by `send()`. Contains the assigned Message-ID and + * delivery status. + * + * (MESSAGE.md § Transport Interface › Outbound) + */ +export type SendReceipt = { + messageId: string; + status: "delivered" | "queued"; +}; + +/** + * Parsed headers from an inbound message. Field names follow RFC 5322 and + * the Interchange-specific header conventions from MESSAGE.md § Headers. + * + * `from` is optional because a message can arrive carrying no originator at + * all, and there is no string that honestly stands for one. + */ +export type MessageHeaders = { + from?: string; + to: string[]; + cc?: string[]; + date?: string; + messageId?: string; + inReplyTo?: string; + references?: string[]; + subject?: string; + listId?: string; + + interchangeType?: InterchangeType; + interchangeCorrelationId?: string; + interchangeTenantId?: string; + interchangeAgentId?: string; + interchangeSessionId?: string; + interchangeOfferingId?: string; + interchangeSchemaVersion?: string; + + traceparent?: string; + tracestate?: string; +}; + +/** + * Signature verification status of an inbound message. + * + * - `valid` — signature verified against the sender's public key + * - `invalid` — signature check failed (tampering or wrong key) + * - `unknown` — public key not available for verification + * - `missing` — message was not signed + * + * (MESSAGE.md § Transport Interface › fetchFull) + */ +export const SignatureStatus = type.enumerated( + "valid", + "invalid", + "unknown", + "missing", +); +export type SignatureStatus = typeof SignatureStatus.infer; + +/** + * A parsed MIME part. `content` is the DECODED bytes in memory (the + * transfer-encoding has already been undone). `filename` and `disposition` are + * surfaced from the part's `Content-Disposition` / `Content-Type` so a consumer + * can distinguish an inline part from a named attachment without re-parsing + * headers. + */ +export type MessagePart = { + contentType: string; + content: Uint8Array; + filename?: string; + disposition?: "inline" | "attachment"; + /** + * The Content-Transfer-Encoding the part declared. Not set for a decoded + * mail part -- `content` is already decoded, so the wire encoding is spent + * transport metadata. + */ + encoding?: string; +}; + +/** + * A single part of a persisted `Mail`. The bytes live in the durable store; + * this descriptor carries the part's metadata plus an opaque, relocation-stable + * `ref` a `MailPartReader` resolves to the part's bytes. Small UTF-8 text parts + * also carry their decoded `text` inline so a selector can read them without + * resolving. + */ +export type MailPart = { + contentType: string; + filename?: string; + disposition?: "inline" | "attachment"; + ref: string; + text?: string; +}; + +/** + * The single, environment-agnostic interface for reading a persisted mail + * part's bytes. Modeled on `BlobReader`: a consumer -- a workflow step, an + * agent tool, the agent's content-block projection -- resolves a + * `MailPart.ref` to its bytes without knowing or caring where the bytes live + * (a committed file on the sidecar, a blob in a browser runtime). The `ref` + * is opaque; the reader owns its scheme. Threaded to consumers through the + * runtime so a browser runtime can supply its own implementation. + */ +export interface MailPartReader { + /** Resolve a `MailPart.ref` to the part's decoded bytes. Throws if the ref + * is unrecognized or its bytes are missing. */ + read(ref: string): Promise; +} + +/** + * A fully decoded mail message: every header (both the typed, ergonomic subset + * and a raw catch-all with nothing dropped) plus the flat list of decoded leaf + * parts. This is the lossless representation a deployed workflow receives as + * its trigger input; a workflow programs against it directly (select headers, + * walk parts, route on content type), and an agent step projects the parts into + * model content blocks. It is JSON-safe: each part carries a `ref` (not raw + * bytes), so binary content never enters the event log; the runtime resolves a + * `ref` into a loadable `MessagePart` on demand. + */ +export type Mail = { + headers: MessageHeaders; + /** Every header, lowercased name to its ordered values; nothing dropped. */ + rawHeaders: Record; + parts: MailPart[]; +}; + +const MailShape = type({ + // Require the recipient list a consumer dereferences unconditionally; other + // header fields stay optional and are carried losslessly in `rawHeaders`. + // `from` is optional here deliberately: a mail with no usable originator is + // still mail, and requiring it makes `isMail` reject one. + headers: { + "from?": "string", + to: "string[]", + }, + rawHeaders: { + "[string]": "string[]", + }, + parts: type({ + contentType: "string", + ref: "string", + "filename?": "string", + "disposition?": "'inline' | 'attachment'", + "text?": "string", + }) + .onUndeclaredKey("reject") + .array(), +}).onUndeclaredKey("reject"); + +/** + * Narrow an opaque value (a workflow step input) to a `Mail`. Used at the + * `agent.send` boundary to decide whether the input is a mail-derived message + * whose parts must be projected into content blocks, or an arbitrary value + * delivered as synthesized text. The strict undeclared-key rejection keeps an + * arbitrary step value that merely carries a `parts` field from matching. + */ +export function isMail(value: unknown): value is Mail { + return !(MailShape(value) instanceof type.errors); +} + +/** + * MIME tree metadata returned by `fetchStructure()`. Describes content types, + * sizes, and dispositions without transferring content. + * + * (MESSAGE.md § Partial Fetch) + */ +export type BodyStructure = { + contentType: string; + size?: number; + disposition?: string; + parts?: BodyStructure[]; +}; + +/** + * A fully parsed inbound message including structured payload, headers, + * attachments, and signature verification status. + * + * (MESSAGE.md § Transport Interface › fetchFull) + */ +export type InboundMessage = { + ref: MessageRef; + headers: MessageHeaders; + flags: string[]; + + /** Plain text body for conversation.* types. */ + content?: string; + + /** Parsed JSON payload for structured types. */ + payload?: { + type: InterchangeType; + version: string; + body: Record; + }; + + attachments?: MessageAttachment[]; + signatureStatus: SignatureStatus; +}; + +/** + * IMAP mailbox descriptor. + * + * (MESSAGE.md § Inbox Management) + */ +export type Mailbox = { + name: string; + role?: string; + delimiter?: string; +}; + +/** + * Current status of an IMAP mailbox, including QRESYNC identifiers. + * + * (MESSAGE.md § Inbox Management) + */ +export type MailboxStatus = { + total: number; + unseen: number; + recent: number; + uidNext: number; + uidValidity: number; + highestModSeq: number; +}; + +/** + * Structured IMAP search query. Maps the IMAP SEARCH grammar to a typed + * object. Supports recursive boolean composition via `and`, `or`, `not`. + * + * (MESSAGE.md § Search) + */ +export type SearchQuery = { + from?: string; + to?: string; + cc?: string; + bcc?: string; + header?: { field: string; contains: string }; + before?: Date; + after?: Date; + on?: Date; + sentBefore?: Date; + sentAfter?: Date; + sentOn?: Date; + hasFlags?: string[]; + missingFlags?: string[]; + body?: string; + text?: string; + largerThan?: number; + smallerThan?: number; + and?: SearchQuery[]; + or?: SearchQuery[]; + not?: SearchQuery; +}; + +/** + * A thread node returned by `thread()`. Carries a message reference and + * child threads representing replies. Implements the RFC 5256 REFERENCES + * threading algorithm. + * + * (MESSAGE.md § Thread Retrieval) + */ +export type Thread = { + ref: MessageRef; + children: Thread[]; +}; + +/** + * QRESYNC state the harness provides when reconnecting to the transport. + * + * (MESSAGE.md § Synchronization) + */ +export type SyncState = { + uidValidity: number; + uidNext: number; + highestModSeq: number; + knownUids?: number[]; +}; + +/** + * Result of a QRESYNC-style sync operation. + * + * (MESSAGE.md § Synchronization) + */ +export type SyncResult = { + vanished: number[]; + changed: { uid: number; flags: string[] }[]; + newMessages: MessageRef[]; + fullResyncRequired: boolean; +}; + +/** + * Distribution list metadata returned by `createList()`. + * + * (MESSAGE.md § Message Topologies) + */ +export type ListInfo = { + address: string; + name: string; + memberCount: number; + createdAt: string; +}; + +/** + * Event emitted by the mailbox watcher callback. Corresponds to IMAP IDLE + * notifications. + * + * (MESSAGE.md § Real-Time Notification) + */ +export type MailboxEvent = + | { type: "exists"; uid: number; headers: MessageHeaders } + | { type: "flagsChanged"; uid: number; flags: string[] } + | { type: "expunged"; uid: number }; + +/** Unsubscribe function returned by `watch()`. */ +export type Unsubscribe = () => void; + +/** + * The message transport interface. Abstracts SMTP and IMAP behind a + * TypeScript API. Implementations range from real SMTP/IMAP servers to + * in-process stubs that route messages through memory. + * + * All long-running operations accept an AbortSignal for cooperative + * cancellation. + * + * (MESSAGE.md § Transport Interface) + */ +export interface MessageTransport { + // --- Outbound --- + + /** Compose, sign, and deliver a message via SMTP. */ + send(message: OutboundMessage, signal?: AbortSignal): Promise; + + /** Append a raw message to a mailbox (IMAP APPEND). */ + append( + mailbox: string, + message: InboundMessage, + flags?: string[], + signal?: AbortSignal, + ): Promise; + + // --- Mailbox management --- + + listMailboxes(signal?: AbortSignal): Promise; + createMailbox(name: string, signal?: AbortSignal): Promise; + deleteMailbox(name: string, signal?: AbortSignal): Promise; + getMailboxStatus(name: string, signal?: AbortSignal): Promise; + + // --- Message search and retrieval --- + + search( + mailbox: string, + query: SearchQuery, + signal?: AbortSignal, + ): Promise; + + thread( + mailbox: string, + algorithm: "references" | "orderedsubject", + query?: SearchQuery, + signal?: AbortSignal, + ): Promise; + + fetchHeaders(ref: MessageRef, signal?: AbortSignal): Promise; + fetchStructure(ref: MessageRef, signal?: AbortSignal): Promise; + fetchPart( + ref: MessageRef, + partPath: string, + signal?: AbortSignal, + ): Promise; + fetchFull(ref: MessageRef, signal?: AbortSignal): Promise; + + // --- Flag management --- + + setFlags( + ref: MessageRef, + flags: string[], + signal?: AbortSignal, + ): Promise; + + clearFlags( + ref: MessageRef, + flags: string[], + signal?: AbortSignal, + ): Promise; + + // --- Message organization --- + + move(ref: MessageRef, toMailbox: string, signal?: AbortSignal): Promise; + + copy(ref: MessageRef, toMailbox: string, signal?: AbortSignal): Promise; + + /** + * Permanently remove every `\Deleted` message from the mailbox. Returns the + * uids that were expunged, so a caller can report how many messages it + * consumed and which ones. + */ + expunge( + mailbox: string, + signal?: AbortSignal, + ): Promise<{ expungedUids: number[] }>; + + // --- Real-time notification --- + + /** + * Monitor a mailbox for new messages and flag changes (IMAP IDLE). + * The promise resolves once the watch is accepted. Callers await it + * before relying on the callback, and a refusal rejects with the + * transport's condition. The unsubscribe function stays synchronous. + */ + watch( + mailbox: string, + callback: (event: MailboxEvent) => void, + ): Promise; + + // --- Synchronization --- + + /** Efficient reconnection using QRESYNC semantics. */ + sync( + mailbox: string, + knownState: SyncState, + signal?: AbortSignal, + ): Promise; + + // --- Distribution lists --- + + createList( + address: string, + name: string, + signal?: AbortSignal, + ): Promise; + + listMembers(address: string, signal?: AbortSignal): Promise; + + subscribe( + listAddress: string, + subscriberAddress: string, + signal?: AbortSignal, + ): Promise; + + unsubscribe( + listAddress: string, + subscriberAddress: string, + signal?: AbortSignal, + ): Promise; +} + +/** + * The condition a `MessageTransport` operation failed under. A transport that + * grows a condition this union does not carry takes its name from RFC 5530 § 3 + * rather than coining one. + * + * https://www.rfc-editor.org/rfc/rfc5530.html + */ +export const MessageTransportCondition = type( + "'NONEXISTENT' | 'CANNOT' | 'SERVERBUG'", +); +export type MessageTransportCondition = typeof MessageTransportCondition.infer; + +export class MessageTransportError extends Error { + readonly condition: MessageTransportCondition; + + constructor(condition: MessageTransportCondition, message: string) { + super(message); + this.name = "MessageTransportError"; + this.condition = condition; + } +} + +/** + * `instanceof` is not usable for this: a tool package is published as a bundle + * whose workspace imports are inlined (`bin/build-builtins.ts`), so a consumer + * loaded from a bundle holds its own copy of the class above and answers false + * against an error the host's copy constructed. + */ +export function isMessageTransportError( + value: unknown, +): value is Error & { readonly condition: MessageTransportCondition } { + if (!(value instanceof Error)) return false; + if (!("condition" in value)) return false; + return MessageTransportCondition.allows(value.condition); +} + +// --------------------------------------------------------------------------- +// Tool Execution (ARCHITECTURE.md § Tools, INFERENCE.md § Tool Execution) +// --------------------------------------------------------------------------- + +/** + * A tool call as requested by the model. Carries the provider-assigned call + * ID, the tool name, and the parsed arguments. + * + * (INFERENCE.md § Message Format › Content Types) + */ +export const ToolCall = type({ + id: "string", + name: "string", + arguments: "Record", +}); +export type ToolCall = typeof ToolCall.infer; + +/** + * Approver-facing snapshot of the tool call awaiting approval. Built at the + * authz `ask` branch from the tool's definition and the live call, then + * threaded unchanged from the reactor's pending operation through every + * suspend hop to the hub co-write that records it on the approval row. + * + * `name`, `description`, and `inputSchema` mirror the {@link ToolDefinition}; + * `arguments` is the live call's arguments. Carried as a sibling of the pending + * operation's `suspendedCall`, never folded into {@link ToolCall}, so the + * re-dispatch artifact and the approval snapshot stay separate concerns. + */ +export const ApprovalSnapshot = type({ + name: "string", + description: "string", + inputSchema: "Record", + arguments: "Record", +}); +export type ApprovalSnapshot = typeof ApprovalSnapshot.infer; + +/** + * The kind of a control-plane park: a step suspended awaiting an external + * event. `"approval"` and `"input"` park on a reserved + * `signalName(correlationId)` channel; `"signal-relay"` parks on an + * author-chosen name. + * + * - `"approval"` -- the step parked on a tool/authz gate and REQUIRES an + * {@link ApprovalSnapshot}; the runtime notifies the host (`env.onPark`) so + * the sidecar co-writes the approval/correlation rows the hub registers. + * - `"input"` -- the step parked awaiting its next input (e.g. a long-lived + * agent run awaiting the next mail so it can take another turn). It carries + * NO snapshot and does NOT notify the host: the run's owner delivers the + * input on the same channel and the step re-arms. It is a runtime-local + * concept -- deliberately NOT a {@link SignalKind}, so it never touches the + * approval-routing machinery (IPC register frames, the hub co-write, the + * approval columns). + * - `"signal-relay"` -- an onTrigger section container parked on an + * author-named signal so a body child's `awaitSignal` on that name is + * serviced through the deployment run: the external signal is delivered to + * the parent run and the runtime relays it down into the live body child. + * The channel name is the author's free-form signal name, NOT a reserved + * `signalName(correlationId)`, so recovery must branch on this kind BEFORE + * assuming the awaited name is a reserved control-plane channel. Carries no + * snapshot and is not hub-registered. + * + * The kinds are distinguished by an EXPLICIT discriminant everywhere the kind + * flows -- never inferred from the presence or absence of a snapshot, which + * would silently reclassify a malformed snapshot-less approval as another + * park kind rather than failing loud. + */ +export const ControlParkKind = type.enumerated( + "approval", + "input", + "signal-relay", +); +export type ControlParkKind = typeof ControlParkKind.infer; + +/** + * Maximum serialized size, in UTF-8 bytes, of an {@link ApprovalSnapshot} that + * crosses a trust boundary. A tool `inputSchema` is normally single-digit KB; + * a snapshot approaching this bound is malformed or hostile and is rejected at + * the parse boundary rather than co-written onto an approval row. + */ +export const APPROVAL_SNAPSHOT_MAX_BYTES = 131072; + +/** + * {@link ApprovalSnapshot} bounded to {@link APPROVAL_SNAPSHOT_MAX_BYTES}. + * Parse the snapshot through this validator where it crosses a trust boundary + * (the `park.notify` IPC frame, the `parked-correlations.response` IPC frame, + * and the sidecar→hub register frame); internal hops use the unbounded + * {@link ApprovalSnapshot}. `.narrow` bounds the runtime check only — its + * inferred type is identical to {@link ApprovalSnapshot} — so the cap holds only + * where a frame is actually parsed, not merely typed. + */ +export const BoundedApprovalSnapshot = ApprovalSnapshot.narrow( + (snapshot, ctx) => { + const bytes = Buffer.byteLength(JSON.stringify(snapshot), "utf8"); + return ( + bytes <= APPROVAL_SNAPSHOT_MAX_BYTES || + ctx.mustBe(`at most ${APPROVAL_SNAPSHOT_MAX_BYTES} bytes when serialized`) + ); + }, +); +export type BoundedApprovalSnapshot = typeof BoundedApprovalSnapshot.infer; + +/** + * Result of a tool execution. `content` is text or structured data the model + * sees as the tool result. `detail` is additional data that the harness may + * use (e.g., for validation or audit) but that is not shown to the model. + * + * When `isError` is true the model sees the result as an error. When + * `pendingMarker` is present the tool is async — the reactor registers the + * correlation ID and waits for a matching inbound message. + * + * (INFERENCE.md § Tool Execution Semantics) + */ +export const ToolResult = type({ + callId: "string", + content: "string | Record", + "detail?": "unknown", + "isError?": "boolean", + "pendingMarker?": { + status: "'pending'", + correlationId: "string", + }, +}); +export type ToolResult = typeof ToolResult.infer; + +/** + * The tool runner interface. The harness implements this; the reactor calls + * it when the director requests tool execution. + * + * Parallel execution is modeled by calling `run` concurrently for each call + * in a batch — the interface is per-call, not per-batch. + * + * (ARCHITECTURE.md § Agent Harness › Tools) + */ +export interface ToolRunner { + /** + * Execute a single tool call. Resolves with the result. Must not throw — + * errors are returned as `ToolResult` with `isError: true`. + */ + run(call: ToolCall, signal: AbortSignal): Promise; +} + +// --------------------------------------------------------------------------- +// Inference Event Building Blocks (INFERENCE.md § Event Protocol) +// --------------------------------------------------------------------------- + +/** + * Partial assistant message accumulated during streaming. Carries all + * content blocks seen so far so late-joining subscribers receive current + * state without replaying deltas. + * + * `text` and `thinking` are cumulative across every emitted delta of + * that kind in the current turn — intentionally flat, even when the + * harness's per-index block tracking has split the stream into + * multiple ThinkingBlocks or TextBlocks. Consumers that need per-block + * structure walk the finalized inference.done turn's content[]; this + * snapshot is the live "what bytes has the assistant streamed so + * far" view. + * + * (INFERENCE.md § Event Protocol › Partial State) + */ +export const PartialMessage = type({ + text: "string", + "thinking?": "string", + "toolCalls?": type({ + id: "string", + name: "string", + partialArguments: "string", + }).array(), +}); +export type PartialMessage = typeof PartialMessage.infer; + +/** + * Token usage for a single inference call. Cache read/write counts are + * provider-specific and may be zero when the provider does not report them. + * + * (INFERENCE.md § Token Accounting) + */ +export const TokenUsage = type({ + input: "number", + output: "number", + cacheRead: "number", + cacheWrite: "number", + thinking: "number", +}); +export type TokenUsage = typeof TokenUsage.infer; + +/** + * Slim source descriptor stamped onto `inference.usage` / `inference.done` + * events and onto `ReactorState.lastCycleSource`. + * + * Carries enough identity for state-aware policies (cost gating, budget + * caps, governance triggers, audit) to attribute usage to a specific + * inference source without re-reading the live, mutable `InferenceSource` + * the harness owns. + * + * Deliberately a strict subset of `InferenceSource` — `apiKey` and + * `baseURL` are intentionally excluded. Credentials and endpoints must + * not leak to director-side policy code or to external event consumers. + * Any code path that needs the full source obtains it through the + * harness's source registry, not through this descriptor. + * + * `sourceId` aliases `InferenceSource.id` to disambiguate from message + * ids, turn ids, and session ids in director-side code where `id` alone + * would be ambiguous. + */ +export const LastCycleSource = type({ + sourceId: "string", + provider: "string", + model: "string", +}); +export type LastCycleSource = typeof LastCycleSource.infer; + +// --------------------------------------------------------------------------- +// Internal Turn Format (INFERENCE.md § Message Format) +// --------------------------------------------------------------------------- + +/** + * A single content block within a conversation turn. Provider-agnostic. + * + * (INFERENCE.md § Message Format › Content Types) + */ +const TextBlock = type({ + type: "'text'", + text: "string", + // Opaque provider signature authenticating this block, echoed back + // verbatim on follow-up turns. Gemini attaches a `thoughtSignature` to + // output parts (including plain text); absent for providers that do not + // sign this block kind. + "signature?": "string", +}); + +/** + * How a media payload is carried by a content block. One of three + * variants: inline as a base64-encoded string, by reference to an + * opaque provider-native handle (e.g. a Gemini fileUri, an Anthropic + * file_id), or by public URL the provider fetches itself. The wire + * shape each provider expects is built by the provider adapter; + * MediaSource is the internal, provider-agnostic representation. + * + * (INFERENCE.md § Generalized Multimodal Taxonomy) + */ +const MediaSourceBase64 = type({ + kind: "'base64'", + mimeType: "string", + data: "string", +}); + +const MediaSourceFileReference = type({ + kind: "'file-reference'", + mimeType: "string", + reference: "string", +}); + +const MediaSourceUrl = type({ + kind: "'url'", + mimeType: "string", + url: "string", +}); + +export const MediaSource = type.or( + MediaSourceBase64, + MediaSourceFileReference, + MediaSourceUrl, +); +export type MediaSource = typeof MediaSource.infer; + +// Exported because `inference.image_output` events reference it by +// name, following the same pattern as `CitationBlock`, +// `CodeExecutionRequestBlock`, and `RedactedThinkingBlock`. +export const ImageBlock = type({ + type: "'image'", + source: MediaSource, + // Opaque provider signature authenticating this block, echoed back + // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the + // inlineData part; absent otherwise. + "signature?": "string", +}); +export type ImageBlock = typeof ImageBlock.infer; + +const AudioBlock = type({ + type: "'audio'", + source: MediaSource, +}); + +const VideoBlock = type({ + type: "'video'", + source: MediaSource, +}); + +const DocumentBlock = type({ + type: "'document'", + source: MediaSource, + "title?": "string", + "context?": "string", +}); + +const ThinkingBlock = type({ + type: "'thinking'", + thinking: "string", + "signature?": "string", +}); + +/** + * A thinking block whose content the provider has filtered. The + * opaque `data` blob must echo back verbatim on every follow-up turn + * — Anthropic 400s the request if it changes or goes missing. Treat + * the bytes as opaque: do not log them and do not render them to + * users. + * + * Exported because `inference.thinking.redacted` events reference it + * by name. + */ +export const RedactedThinkingBlock = type({ + type: "'redacted_thinking'", + data: "string", +}); +export type RedactedThinkingBlock = typeof RedactedThinkingBlock.infer; + +/** + * A model-emitted refusal. Produced when a provider's strict-mode + * structured-outputs path declines to satisfy the requested schema — + * OpenAI's `delta.refusal` / `message.refusal` field is the canonical + * wire shape. The `reason` is the accumulated human-readable text the + * model emitted in lieu of conformant output. + * + * Refusal is semantically distinct from `inference.error`: the HTTP + * call succeeded and the model produced a coherent response, but that + * response is "I will not satisfy this schema" rather than schema- + * conformant content. Callers that distinguish policy declines from + * transport/protocol failures should branch on the block type rather + * than treat the assistant turn as an error. + * + * Exported because `inference.refusal.delta` events reference it by + * name and adapters construct RefusalBlocks in the finalized + * AssistantTurn from accumulated delta fragments. + */ +export const RefusalBlock = type({ + type: "'refusal'", + // Refusals must carry text — a zero-length reason corrupts the + // "human-readable text the model emitted in lieu of conformant + // output" contract and would round-trip indistinguishably from a + // refusal block whose payload was lost. The arktype constraint is + // belt-and-braces alongside the adapter's wire-boundary filter on + // empty `delta.refusal` chunks: synthetic fixtures or future + // adapters without that filter still cannot construct a vacuous + // refusal. + reason: "string > 0", +}); +export type RefusalBlock = typeof RefusalBlock.infer; +const ToolCallBlock = type({ + type: "'tool_call'", + id: "string", + name: "string", + arguments: "Record", + // Opaque provider signature authenticating this block, echoed back + // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the + // functionCall part; absent otherwise. + "signature?": "string", +}); +/** + * Location of a citation's cited span within its source document. + * The unit of `start` and `end` varies by `kind`: + * - "page": 1-indexed page numbers (Anthropic `page_location`). + * - "char": UTF-16 character offsets, matching JS string semantics + * (Anthropic `char_location`; Gemini `groundingSupports[].segment`). + * - "content-block": index into a structured source's content blocks + * (Anthropic `content_block_location`). + */ +const CitationLocation = type({ + kind: "'page' | 'char' | 'content-block'", + start: "number", + end: "number", +}); + +const CitationSource = type({ + "title?": "string", + // Self-contained dereferenceable URL — populated by providers whose + // citations carry URLs directly (Gemini `groundingChunks[].web.uri`). + "uri?": "string", + // Back-pointer into the request's `documents` array, populated by + // providers that cite uploaded documents by position (Anthropic + // `document_index`). + "documentRef?": type({ index: "number" }), +}); + +/** + * A citation that supports a span of assistant text. Consumers + * receiving a CitationBlock without a paired source-block index MUST + * attribute it by adjacency to the nearest preceding TextBlock in the + * same turn. + * + * Citations are deliberately excluded from ToolResultBlock.content + * — they annotate model output, not tool output. + * + * Exported because `inference.citation` events reference it by name, + * following the same pattern as `AssistantTurn`, `ToolCall`, and + * `ToolResult`. See the `inference.citation` event docstring for how + * a paired source-block index is carried on the wire and consumed by + * the harness. + */ +export const CitationBlock = type({ + type: "'citation'", + // The exact substring of the preceding TextBlock this citation + // supports. Both providers emit it; required for inspection and + // for fallback offset reconstruction. + citedText: "string", + source: CitationSource, + "location?": CitationLocation, + // UTF-16 character offsets into the preceding TextBlock's text. + // Providers that emit offsets natively populate these directly; + // adapters that derive offsets from a cited substring populate + // them only when the substring appears unambiguously in the + // preceding text. Omitted when the offset cannot be determined. + "textOffset?": type({ start: "number", end: "number" }), +}); +export type CitationBlock = typeof CitationBlock.infer; + +/** + * A structured safety signal on model output or request filtering. + * + * The name `SafetyRatingBlock` follows the issue vocabulary; the + * payload is derived from the first real Gemini capture that engaged + * the structured classifier (2026-07-28). That wire shape is + * prompt-level only: + * + * `promptFeedback: { blockReason: "PROHIBITED_CONTENT" }` + * + * with no candidates and no per-category `safetyRatings` arrays. So + * this block carries `blockReason` and does **not** invent category / + * probability / blocked fields. When a future capture surfaces + * candidate-level ratings, extend the type from those bytes rather + * than from the API reference. + * + * Deliberately excluded from ToolResultBlock.content — safety + * signals annotate model/request filtering, not tool output. + * + * Exported because `inference.safety_rating` events reference it by + * name. + */ +export const SafetyRatingBlock = type({ + type: "'safety_rating'", + // Provider-native block reason string (observed: "PROHIBITED_CONTENT"). + // Open string so a new reason token does not force a type bump. + blockReason: "string > 0", +}); +export type SafetyRatingBlock = typeof SafetyRatingBlock.infer; + +/** + * The model's request to execute code via a server-side execution tool. + * Paired with a CodeExecutionResultBlock carrying the same `id` as the + * result's `requestId`. Streaming order within a single execution is + * `inference.code_execution.start` → zero or more + * `inference.code_execution.delta` → `inference.code_execution.result`, + * uninterrupted by other events that share the same `requestId`; events + * with different `requestId`s or for other block kinds at distinct + * `index`es may interleave. + * + * Exported because `inference.code_execution.start` references it by + * name. + */ +export const CodeExecutionRequestBlock = type({ + type: "'code_execution_request'", + // Identifier for the execution request. Populated from the + // provider's call id where one exists (Anthropic + // `srvtoolu_...`); synthesized by the adapter for providers that + // don't emit one (Gemini), using a deterministic per-response + // position-based scheme so replays match. + id: "string", + // Source code the model is asking to execute. + code: "string", + // Language hint. Absent when the provider does not emit one; + // adapters MUST NOT default this — callers narrow on its + // presence rather than fall through to a guessed language. + "language?": "string", + // Opaque provider signature authenticating this block, echoed back + // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the + // executableCode part; absent otherwise. + "signature?": "string", +}); +export type CodeExecutionRequestBlock = typeof CodeExecutionRequestBlock.infer; + +/** + * The result of executing a CodeExecutionRequestBlock. The `requestId` + * back-points to the request block's `id`. Status is normalized across + * providers; raw provider signals (return code, native outcome string, + * abort reason) are preserved on optional fields for callers that need + * them. + * + * File outputs from code execution (e.g. generated plots that + * Anthropic returns in `code_execution_tool_result.content`) are NOT + * modeled by this block today. The block carries no field for them; + * surfacing file outputs is a separate concern. + * + * Exported because `inference.code_execution.result` references it by + * name. + */ +export const CodeExecutionResultBlock = type({ + type: "'code_execution_result'", + // Back-pointer to the originating CodeExecutionRequestBlock.id. + requestId: "string", + // Normalized outcome. Translated from provider-specific signals: + // - Anthropic: derived from `return_code` (0 → "ok", non-zero → + // "error") and `abort_reason` (non-null → "aborted" or + // "timeout" per the reason). + // - Gemini: derived from the `outcome` enum + // (OUTCOME_OK → "ok", OUTCOME_FAILED → "error", + // OUTCOME_DEADLINE_EXCEEDED → "timeout", etc.). + status: "'ok' | 'error' | 'aborted' | 'timeout'", + // Standard output. Providers that don't split stdout from stderr + // (Gemini) map their combined `output` here and leave `stderr` empty. + "stdout?": "string", + // Standard error. Empty for providers that don't split. + "stderr?": "string", + // Provider-native numeric return code when available + // (Anthropic `return_code`). Absent for providers whose outcome + // is enum-only (Gemini). + "returnCode?": "number", + // Provider-native outcome string preserved verbatim for callers + // that need the raw signal (Gemini `OUTCOME_OK` / + // `OUTCOME_FAILED` / `OUTCOME_DEADLINE_EXCEEDED` / ...). Absent + // when the provider does not emit one (Anthropic). + "providerOutcome?": "string", + // Human-readable reason populated when status is "aborted" + // (Anthropic `abort_reason`). Absent otherwise. + "abortReason?": "string", +}); +export type CodeExecutionResultBlock = typeof CodeExecutionResultBlock.infer; + +const ToolResultBlock = type({ + type: "'tool_result'", + callId: "string", + // Deliberately narrow: tool results carry user-facing media, not + // CitationBlocks (citations annotate the model's text output), not + // SafetyRatingBlocks (safety signals annotate model/request + // filtering), and not CodeExecution blocks (server-side code + // execution is a distinct lifecycle from the user-tool round-trip). + content: type + .or(TextBlock, ImageBlock, AudioBlock, VideoBlock, DocumentBlock) + .array(), + "detail?": "unknown", + "isError?": "boolean", +}); + +export const ContentBlock = type.or( + TextBlock, + ThinkingBlock, + RedactedThinkingBlock, + RefusalBlock, + ImageBlock, + AudioBlock, + VideoBlock, + DocumentBlock, + CitationBlock, + SafetyRatingBlock, + CodeExecutionRequestBlock, + CodeExecutionResultBlock, + ToolCallBlock, + ToolResultBlock, +); +export type ContentBlock = typeof ContentBlock.infer; + +/** + * A turn in the internal conversation history. The `model` field records + * which provider model produced this turn (present only on assistant + * turns). Used by cross-provider transformation to strip or preserve + * thinking blocks. + * + * (INFERENCE.md § Message Format) + */ +export type ConversationTurn = { + role: "user" | "assistant" | "system"; + content: ContentBlock[]; + model?: string; + timestamp: number; +}; + +/** + * A completed assistant turn returned in `inference.done`. Narrower type + * than ConversationTurn to make the inference boundary explicit. + */ +export const AssistantTurn = type({ + role: "'assistant'", + content: ContentBlock.array(), + model: "string", + timestamp: "number", +}); +export type AssistantTurn = typeof AssistantTurn.infer; + +// --------------------------------------------------------------------------- +// Error Classification (INFERENCE.md § Error Classification) +// --------------------------------------------------------------------------- + +/** + * Classified inference error. The category determines the reactor's default + * response; the director can override per its policy. + * + * (INFERENCE.md § Error Classification) + */ +export const InferenceError = type({ + category: type.enumerated( + "retryable", + "context_overflow", + "credential_failure", + "quota_exhausted", + "fatal", + "aborted", + "timeout", + "protocol_mismatch", + ), + message: "string", + "statusCode?": "number", + "retryAfterMs?": "number", + "raw?": "unknown", +}); +export type InferenceError = typeof InferenceError.infer; + +// --------------------------------------------------------------------------- +// Agent Reactor (INFERENCE.md § Agent Reactor) +// --------------------------------------------------------------------------- + +/** + * Gate types that can block the reactor. + * + * (INFERENCE.md § Gates) + */ +export const GateType = type.enumerated( + "approval", + "payment", + "credential", + "budget", + "child_completion", + "message_response", +); +export type GateType = typeof GateType.infer; + +/** + * Fork mode. `independent` creates a divergent reactor with its own context. + * `child` creates a reactor that reports results back to the parent. + * + * (INFERENCE.md § Forking) + */ +export const ForkMode = type.enumerated("independent", "child"); +export type ForkMode = typeof ForkMode.infer; + +/** + * A pending async operation registered in the reactor's async state. + * Correlates an outbound message (or payment/approval request) to the + * expected inbound response. + * + * (INFERENCE.md § Correlation) + */ +export type PendingOperation = { + correlationId: string; + kind: SignalKind; + registeredAt: number; + gateId: string; + /** + * Absolute deadline (epoch ms) for the gate that parks this operation. + * Persisted so that rehydration after a restart re-arms the gate with the + * remaining time against the original deadline rather than restarting the + * countdown. Absent for operations parked with no deadline. + */ + timeoutAt?: number; + /** + * The tool call that was suspended when this operation parked. Captured for + * `kind: "approval"` operations minted from the ask flow so the approved + * call can be re-run on resume. Absent for operations parked by the + * director path (async-tool pending markers), which carry no tool call. + */ + suspendedCall?: ToolCall; + /** + * Approver-facing snapshot of `suspendedCall`, built at the authz `ask` + * branch from the tool definition and the live arguments. A sibling of + * `suspendedCall`, not a widening of it: `suspendedCall` is the re-dispatch + * artifact, this is what the approver decides on. Present only for ask-rail + * operations that carry a `suspendedCall`; absent for async-tool pending + * markers. Threaded through the suspend hops to the hub co-write. + */ + approvalSnapshot?: ApprovalSnapshot; +}; + +/** + * Complete reactor state visible to the director decision function. + * + * `tokenUsage` is the cumulative usage across the session. + * + * `lastCycleUsage` and `lastCycleSource` describe the most recent + * *successful* inference call. They move together: both null before the + * first completion; every `inference.done` sets both atomically. + * `inference.error` does not clear either — the pair always reflects the + * last cycle that produced a well-defined turn and usage. The director's + * `afterInferenceDone` hook fires only on `inference.done`, so policy + * code never observes a torn or stale-vs-fresh window. + * + * The per-cycle values support compaction triggers that key off recent + * input cost rather than session totals, and state-aware policies (cost + * gating, budget caps, governance triggers) that need to attribute + * usage to the source that produced it. + * + * (INFERENCE.md § Agent Reactor › Director Decision Function) + */ +export type ReactorState = { + turns: ConversationTurn[]; + activeForks: { forkId: string; mode: ForkMode }[]; + pendingOperations: PendingOperation[]; + activeGates: { gateId: string; type: GateType; timeoutAt: number }[]; + tokenUsage: TokenUsage; + lastCycleUsage: TokenUsage | null; + lastCycleSource: LastCycleSource | null; + sessionId: string; +}; + +/** + * Actions the director can direct the reactor to take. + * + * (INFERENCE.md § Agent Reactor › Actions) + */ +export type ReactorAction = + | { + type: "infer"; + options?: InferenceOptions; + } + | { + type: "execute_tools"; + calls: ToolCall[]; + parallel?: boolean; + addToHistory?: boolean; + } + | { + type: "suspend"; + gate: { + type: GateType; + gateId: string; + timeoutMs: number; + correlationId?: string; + }; + } + | { + type: "fork"; + mode: ForkMode; + forkId: string; + } + | { + type: "emit"; + eventType: `custom.${string}`; + data: Record; + } + | { + type: "reply"; + content: string; + } + | { type: "checkpoint"; message: string } + | { type: "compact"; compactor: string; reason: string } + | { type: "wait" } + | { type: "done" }; + +/** + * The capabilities object passed to the director. Mirrors the `ReactorAction` + * union — provides a type-safe way for the director to construct actions. + * + * (INFERENCE.md § Agent Reactor › Director Decision Function) + */ +export type ReactorCapabilities = { + infer(options?: InferenceOptions): ReactorAction; + executeTools( + calls: ToolCall[], + parallel?: boolean, + addToHistory?: boolean, + ): ReactorAction; + suspend(gate: { + type: GateType; + gateId: string; + timeoutMs: number; + correlationId?: string; + }): ReactorAction; + fork(mode: ForkMode, forkId: string): ReactorAction; + emit( + eventType: `custom.${string}`, + data: Record, + ): ReactorAction; + reply(content: string): ReactorAction; + checkpoint(message?: string): ReactorAction; + compact(compactor: string, reason: string): ReactorAction; + wait(): ReactorAction; + done(): ReactorAction; +}; + +/** + * The inbound events delivered to the director decision function. + * + * `resume.execute_tools` is raised by the reactor when an approval resolves + * and a parked tool call must be re-run on resume. It carries the calls the + * reactor is about to dispatch so the director can seed its outstanding + * tool-result count before those calls' `tool.done` events arrive — the + * reactor drives the execution, the director counts the results. Without this + * seed the count would sit at zero and the first `tool.done` would drive an + * accidental re-inference off a negative count. + * + * `resume.tool_result` is raised by the reactor when a parked approval ends + * without running its tool — a rejected decision or a gate timeout. It carries + * a synthetic error tool result that answers the parked call so history stays + * well-formed; the director appends it and re-infers exactly once. No tool + * runs, so it seeds no outstanding-result count. + * + * (INFERENCE.md § Agent Reactor › Reactor Structure) + */ +export type ReactorInboundEvent = + | { type: "message.received"; message: InboundMessage } + | { + type: "inference.done"; + turn: AssistantTurn; + usage: TokenUsage; + source: LastCycleSource; + } + | { type: "inference.error"; error: InferenceError; partial: PartialMessage } + | { type: "tool.done"; result: ToolResult } + | { + type: "reactor.gate.cleared"; + gateId: string; + reason: "resolved" | "timeout" | "shutdown"; + } + | { type: "resume.execute_tools"; calls: ToolCall[] } + | { type: "resume.tool_result"; result: ToolResult } + | { type: "abort"; reason: AbortReason }; + +/** + * The core director is a single decision function: given an event and the + * current reactor state, return one or more actions. + * + * If the director throws, the reactor catches the exception, emits + * `reactor.error`, and initiates graceful shutdown. + * + * (INFERENCE.md § Reactor Director › Core Director) + */ +export interface ReactorDirector { + decide( + event: ReactorInboundEvent, + state: ReactorState, + capabilities: ReactorCapabilities, + ): Promise; +} + +// --------------------------------------------------------------------------- +// Director Extension Hooks (INFERENCE.md § Reactor Director › Extension Hooks) +// --------------------------------------------------------------------------- + +/** + * Decision returned by a `BeforeToolExtension`. + * + * - `allow` — the tool proceeds. + * - `block` — the tool is answered with an error result carrying `reason`; + * the call is done. + * - `suspend` — the call is parked awaiting an external decision. The reactor + * registers `gate`, persists `pendingOp`, and does not answer the call: it + * is neither run nor error-completed. `gate.timeoutAt` is the absolute + * deadline (epoch ms) so the reactor can compute the remaining time; the + * `correlationId` on both `gate` and `pendingOp` ties an inbound resolution + * back to the suspension. + */ +export type BeforeToolDecision = + | { type: "allow" } + | { type: "block"; reason: string } + | { + type: "suspend"; + gate: { + type: GateType; + gateId: string; + correlationId: string; + timeoutAt: number; + }; + pendingOp: PendingOperation; + }; + +/** + * Extension that runs before a tool call is executed. Returns a + * `BeforeToolDecision`: `allow` lets the call run, `block` answers it with an + * error result, `suspend` parks it awaiting an external decision. + * + * `grantOneShot` registers a within-cycle bypass token keyed on a + * `ToolCall.id`: the next `beforeTool` for that id skips a suspension it would + * otherwise raise, consuming the token as it does so. It is optional because + * only extensions that can suspend a call have anything to bypass; extensions + * that never suspend omit it. + */ +export interface BeforeToolExtension { + beforeTool( + call: ToolCall, + state: ReactorState, + signal: AbortSignal, + ): Promise; + grantOneShot?(id: string): void; +} + +/** + * Extension that runs after a tool result is produced. Can modify the result + * (redaction, enrichment, audit logging). Extensions run in order. + */ +export interface AfterToolExtension { + afterTool( + result: ToolResult, + call: ToolCall, + state: ReactorState, + signal: AbortSignal, + ): Promise; +} + +// --------------------------------------------------------------------------- +// Context Strategies: Transforms and Compactors +// (INFERENCE.md § Context Management, § Tool Result Lifecycle) +// --------------------------------------------------------------------------- + +/** + * Durable description of a single strategy invocation. Written to the + * per-cycle manifest in the context store so that future operators can + * reconstruct exactly which strategy made which change, with what + * parameters, and why. + * + * - `strategy` is the implementation name (e.g. `"size-cap"`). + * - `version` is the implementation version. Changes to the strategy's + * behavior bump the version so old manifest entries remain unambiguous. + * - `parameters` records the configuration the strategy ran with. + * - `reason` is a short machine-readable cause label + * (e.g. `"exceeded-cap"`, `"overflow-recovery"`). + * - `decisions` records strategy-specific details about what was actually + * done (e.g. the keep count, the spill key, the original byte size). + */ +export const TransformRecord = type({ + strategy: "string", + version: "string", + parameters: "Record", + reason: "string", + decisions: "Record", +}); +export type TransformRecord = typeof TransformRecord.infer; + +/** + * Per-invocation context passed to every `ContextStrategy.apply` call. + * `state` is the reactor's snapshot at the moment the strategy runs; + * `trigger` is a short label describing why the strategy was invoked + * (e.g. `"tool-result-ingest"`, `"pre-inference"`, `"director-request"`). + */ +export interface StrategyContext { + readonly state: ReactorState; + readonly trigger: string; +} + +/** + * Optional blob attachment emitted by a strategy. The reactor writes each + * blob to the context store's working tree via `ContextStore.writeBlob` + * (Phase 2) so the data is durable and migrates with the conversation. + */ +export type StrategyBlob = { + key: string; + bytes: Uint8Array; + contentType?: string; +}; + +/** + * Result returned by `ContextStrategy.apply`. Carries the transformed + * output, a `TransformRecord` describing what happened, and any blobs + * that should be persisted in the context store. + */ +export interface StrategyResult { + output: O; + record: TransformRecord; + blobs?: StrategyBlob[]; +} + +/** + * Generic base interface for content-mutating strategies. The role-specific + * aliases below specialize `I` and `O` for tool-result ingestion, pre- + * inference context shaping, and explicit compaction. + * + * Strategies are pure with respect to the context store: they describe what + * should change via their return value. The reactor decides where to write + * the result (history, prompt, manifest) and which blobs to persist. + */ +export interface ContextStrategy { + readonly name: string; + readonly version: string; + apply(input: I, ctx: StrategyContext): Promise>; +} + +/** + * Runs on each tool result entering history. Output is appended to the + * conversation; any emitted blobs are written to the context store's + * `tool-output/` directory. + */ +export type ToolResultTransform = ContextStrategy< + { call: ToolCall; result: ToolResult }, + ToolResult +>; + +/** + * Runs in order before every inference call, producing the materialized + * prompt. Output is written to `prompt.jsonl` for that cycle; the durable + * history in `turns.jsonl` is left untouched. + * + * (INFERENCE.md § Async State Awareness › Pending Status Injection) + */ +export type ContextTransform = ContextStrategy< + ConversationTurn[], + ConversationTurn[] +>; + +/** + * Named compaction strategy. Registered in a registry on the reactor and + * invoked explicitly via the director's `compact` action. Output overwrites + * `turns.jsonl`; a `TransformRecord` is appended to the manifest. + */ +export type Compactor = ContextStrategy; + +// --------------------------------------------------------------------------- +// Blob Reader (INFERENCE.md § Tool Result Lifecycle) +// --------------------------------------------------------------------------- + +/** + * Read-only capability for resolving `tool-output:///{callId}` URIs to the + * underlying blob bytes. A `ToolResultTransform` that spills oversized tool + * output writes a blob via `ContextStore.writeBlob` and returns a pointer of + * the form `tool-output:///{callId}`; the agent's read tool reaches the spill + * by calling `BlobReader.read(uri)`. + * + * The URI scheme is deliberately rigid: + * + * - Scheme: `tool-output` + * - Authority: empty (the `///` makes pathname carry the callId) + * - Path: `/{callId}` — preserves case so provider-assigned callIds with + * uppercase letters survive parsing + * - Query and fragment: rejected + * + * Any deviation (different scheme, missing or non-empty hostname, extra path + * segments, search string, or fragment) throws. Missing blobs throw. + * `BlobReader` never accepts a filesystem path; the agent has no direct view + * of the context store's working tree. + */ +export interface BlobReader { + /** + * Resolve `uri` to the underlying blob bytes. Throws if the URI is not a + * well-formed `tool-output:///{callId}` reference or if no blob exists for + * the extracted callId. + */ + read(uri: string): Promise; +} + +/** Source for blob bytes used by `createBlobReader`. */ +export interface BlobSource { + readBlob(key: string, signal?: AbortSignal): Promise; +} + +/** + * Parse a `tool-output:///{callId}` URI and return the callId. Throws on any + * deviation from the documented shape: wrong scheme, non-empty authority, + * missing or extra path components, search string, or fragment. + * + * The two-slash form `tool-output://abc` is rejected because the URL parser + * lowercases the hostname, which silently corrupts provider-assigned callIds + * that contain uppercase letters. The three-slash form puts the callId in + * `pathname`, where case is preserved. + */ +export function parseToolOutputURI(uri: string): string { + let parsed: URL; + try { + parsed = new URL(uri); + } catch (cause) { + throw new Error(`invalid tool-output URI: ${uri}`, { cause }); + } + if (parsed.protocol !== "tool-output:") { + throw new Error( + `invalid tool-output URI scheme: expected "tool-output:", got "${parsed.protocol}"`, + ); + } + if (parsed.hostname !== "") { + throw new Error( + `invalid tool-output URI: authority must be empty (use the form tool-output:///{callId}), got "${parsed.hostname}"`, + ); + } + if (parsed.search !== "") { + throw new Error( + `invalid tool-output URI: query string is not allowed, got "${parsed.search}"`, + ); + } + if (parsed.hash !== "") { + throw new Error( + `invalid tool-output URI: fragment is not allowed, got "${parsed.hash}"`, + ); + } + const path = parsed.pathname; + if (!path.startsWith("/")) { + throw new Error(`invalid tool-output URI: empty path: ${uri}`); + } + const callId = path.slice(1); + if (callId === "") { + throw new Error(`invalid tool-output URI: missing callId: ${uri}`); + } + if (callId.includes("/")) { + throw new Error( + `invalid tool-output URI: path must contain a single callId segment, got "${callId}"`, + ); + } + return callId; +} + +/** + * Construct a `BlobReader` that resolves `tool-output:///{callId}` URIs by + * delegating to `source.readBlob(callId)`. The most common source is a + * `ContextStore` (Phase 2 added `readBlob` to that interface), but any object + * implementing `BlobSource` works — this keeps tests trivial. + * + * URI parsing is performed in this layer; the source only ever sees the + * extracted callId. Missing blobs surface as whatever error the source + * raises (`ContextStore.readBlob` already throws for unknown keys). + */ +export function createBlobReader(source: BlobSource): BlobReader { + return { + async read(uri: string): Promise { + const callId = parseToolOutputURI(uri); + return source.readBlob(callId); + }, + }; +} + +// --------------------------------------------------------------------------- +// Abort Reasons (INFERENCE.md § Abort Handling) +// --------------------------------------------------------------------------- + +/** + * Reason codes for the `abort` reactor event. The reason determines the + * appropriate cleanup action. + * + * (INFERENCE.md § Abort Handling › Abort Reasons) + */ +export const AbortReason = type.enumerated( + "user_disconnect", + "wallet_exhaustion", + "admin_kill", + "session_timeout", + "credential_revocation", +); +export type AbortReason = typeof AbortReason.infer; + +// --------------------------------------------------------------------------- +// Inference Source (INFERENCE.md § Providers) +// --------------------------------------------------------------------------- + +/** + * Model-bound default knobs for an inference source. Per-call + * `InferenceOptions.X` overrides `defaults.X`; the merge happens once at + * the top of `runInference` before the adapter sees anything. New fields + * land here as separately-scoped issues. + */ +export const InferenceSourceDefaults = type({ + "maxTokens?": "number", + // A bag of provider-native knobs the caller wants merged into the + // outbound request body (Anthropic's `metadata.user_id`, + // OpenAI's `user`, Gemini's `safetySettings`, etc.). Adapters that + // recognize keys translate; unrecognized keys are passed through or + // dropped per the adapter's documented behavior. The merge into + // per-call `InferenceOptions.providerOptions` is shallow — a per- + // call providerOptions object wholesale replaces the source-bound + // one, it does not deep-merge per key. + "providerOptions?": "Record", +}); +export type InferenceSourceDefaults = typeof InferenceSourceDefaults.infer; + +/** + * A specific (provider, model) bundle the agent runtime can route to. + * Carries wire reachability, credentials, the model identity at the + * provider, and the model-bound default knobs. + * + * `id` is the catalog offering's primary key, set by the resolver from the + * matched offering. It is the routing key used by `AgentConfig.defaultSource` + * and `Agent.setSource`. + * + * Multi-model providers become multiple sources — `model` is part of the + * identity, not an optional override. + * + * `capabilities` is carried for the selection-policy layer (the model + * selector consumes it). The runtime ignores it; populating the field + * later is not a wire-format change. + * + * `quirks` is the opaque per-deployment bag of provider-specific adapter + * accommodations. The harness reads it once, at adapter instantiation, and + * passes it to `AdapterRegistry.resolve` as a sibling of the slim + * `LastCycleSource` — quirks are deliberately kept off `LastCycleSource`, + * which rides on every usage event. The field is present-and-populated or + * absent; it is never `null`. A source row with no quirks stores SQL `NULL`, + * and the catalog resolver translates that absence into an omitted key here, + * so downstream code sees `undefined`, never `null`. + * + * (INFERENCE.md § Providers) + */ +export const InferenceSource = type({ + id: "string", + provider: "string", + baseURL: "string", + // Reference into the run's credential-material cell. The provider's secret + // (formerly an inline `apiKey`) is resolved from that cell by `credentialId` + // at call time, so the source config carries no secret and the child never + // holds the key inline. The same cell backs tool credentials. + credentialId: "string", + model: "string", + "defaults?": InferenceSourceDefaults, + "capabilities?": "string[]", + "quirks?": "Record", +}); +export type InferenceSource = typeof InferenceSource.infer; + +/** + * Replace every field on `active` with the corresponding field from + * `next`, in place. Optional fields (`defaults`, `capabilities`, + * `quirks`) are `delete`d from `active` when absent on `next` so the + * swap is exact — no stale value from a previous rotation can survive. + * + * Used by both the agent's source registry and the harness's source + * hot-swap path to mutate the single shared `InferenceSource` object the + * reactor reads lazily at the start of each inference call. Putting the + * field list in one place means the next field added to + * `InferenceSource` only has to be remembered here. + */ +export function applyInferenceSourceFields( + active: InferenceSource, + next: InferenceSource, +): void { + active.id = next.id; + active.provider = next.provider; + active.baseURL = next.baseURL; + active.credentialId = next.credentialId; + active.model = next.model; + if (next.defaults !== undefined) { + active.defaults = next.defaults; + } else { + delete active.defaults; + } + if (next.capabilities !== undefined) { + active.capabilities = next.capabilities; + } else { + delete active.capabilities; + } + if (next.quirks !== undefined) { + active.quirks = next.quirks; + } else { + delete active.quirks; + } + + // Compile-time exhaustiveness check. `Required<>` forces optional + // keys to also be required in the guard — so a future optional field + // (e.g. `region?: string`) added to `InferenceSource` without being + // handled above is flagged by TypeScript, not silently dropped. + const _handled: { readonly [K in keyof Required]: true } = { + id: true, + provider: true, + baseURL: true, + credentialId: true, + model: true, + defaults: true, + capabilities: true, + quirks: true, + }; + void _handled; +} + +/** + * Outcome of a `RetryPolicy` consultation. Either abort the call + * (surface the most recent `inference.error` to the caller), or retry + * after `delayMs` milliseconds, measured against the harness Scheduler. + * + * (INFERENCE.md § Providers › Streaming Harness) + */ +export type RetryDecision = + // Locally patched — see vendor/intx-types/PATCHES.md#runtime-ts-auth-recovery-context + | { kind: "abort"; error?: InferenceError } + | { kind: "retry"; delayMs: number }; + +/** + * Context supplied to a `RetryPolicy` each time an attempt produces an + * `inference.error`. + * + * (INFERENCE.md § Providers › Streaming Harness) + */ +export type RetrySituation = { + /** The classified error the most recent attempt produced. */ + readonly error: InferenceError; + /** + * 1-indexed attempt counter. The first failure has `attempt: 1`; + * the second failure (after one retry) has `attempt: 2`; and so on. + */ + readonly attempt: number; + /** + * Milliseconds since the *first* attempt of this call started, + * measured via the harness `Scheduler.now()`. The default Scheduler + * uses `performance.now()` (sub-millisecond resolution), so the + * value may be fractional; virtual-clock test schedulers report + * integer virtual time. Both are valid; policies that compare + * against integer thresholds should `Math.floor` if they need that. + */ + readonly elapsedMs: number; + // Locally patched — see vendor/intx-types/PATCHES.md#runtime-ts-auth-recovery-context + /** Immutable shallow snapshot of the source selected when this call began. */ + readonly source?: Readonly; + /** 1-indexed within credential failures only; zero for other categories. */ + readonly credentialFailureOrdinal?: number; + /** Credential failures observed earlier in this call. */ + readonly credentialFailureHistory?: readonly InferenceError[]; +}; + +/** + * Per-call retry policy. The harness invokes the policy once per + * `inference.error` an attempt produces, in 1-indexed attempt order. + * Returning `{ kind: "abort" }` ends the call by surfacing the most + * recent error to the caller; returning `{ kind: "retry", delayMs }` + * causes the harness to discard the failed attempt's events, sleep + * `delayMs` milliseconds against the Scheduler, and re-issue the + * underlying HTTP request with the identical body. The policy may be + * async; the harness awaits the returned `Promise` if + * it is a thenable. + * + * (INFERENCE.md § Providers › Streaming Harness) + */ +export type RetryPolicy = (( + situation: RetrySituation, +) => RetryDecision | Promise) & { + // Locally patched — see vendor/intx-types/PATCHES.md#runtime-ts-auth-recovery-context + readonly normalizeError?: ( + error: InferenceError, + source?: Readonly, + ) => InferenceError; +}; + +/** + * Provider-neutral reasoning effort. No provider accepts every name as a + * wire value, so each adapter maps the name onto what its model accepts. + * There is no "off": `thinking.enabled: false` already says that. + * + * (INFERENCE.md § Thinking and Reasoning) + */ +export const InferenceEffort = type.enumerated("low", "medium", "high", "max"); +export type InferenceEffort = typeof InferenceEffort.infer; + +/** + * The subset of `InferenceOptions` a caller may set for a single send (a + * workflow step's resolved `inference` selector). Excludes anything that + * displaces the deployed agent definition (`systemPrompt`, `tools`, + * `providerOptions`). + */ +export const PerCallInferenceOptions = type({ + "+": "reject", + "maxTokens?": "number.integer > 0", + "temperature?": "number >= 0", + "thinking?": { + "+": "reject", + enabled: "boolean", + "budgetTokens?": "number.integer > 0", + }, + // Reasoning intent. Effort and `thinking.budgetTokens` are two spellings + // of one intent; each adapter resolves them to the field its model + // accepts. When omitted the provider default applies. + "effort?": InferenceEffort, +}); +export type PerCallInferenceOptions = typeof PerCallInferenceOptions.infer; + +/** + * Options for a single inference call. Override the defaults from the agent + * configuration on a per-call basis. + * + * (INFERENCE.md § Providers › Streaming Harness) + */ +export type InferenceOptions = PerCallInferenceOptions & { + systemPrompt?: string; + tools?: ToolDefinition[]; + /** + * Modalities the caller wants the model to emit. Adapters translate + * to the provider-native shape (Gemini's + * `generationConfig.responseModalities` accepts `"TEXT"` / `"IMAGE"` + * uppercase; see `packages/inference-discovery-google-genai/sessions/ + * google-genai/gemini-2.5-flash-image/image-output/exchanges/0/request.json` + * for the captured shape). Providers that do not expose a modality + * switch ignore the + * field. When omitted the provider's default modalities apply. + */ + responseModalities?: ("text" | "image" | "audio")[]; + /** + * Structured-output constraint. Asks the model to produce text, free- + * form JSON, or JSON conforming to a specific schema. Adapters + * translate to the provider-native wire shape: + * + * - **OpenAI** (`response_format`): + * - `text` → `{ type: "text" }` + * - `json` → `{ type: "json_object" }` + * - `json-schema` → `{ type: "json_schema", json_schema: { name, schema, strict } }` + * When the model declines in strict mode, the wire emits + * `delta.refusal` chunks; the adapter surfaces them as + * `inference.refusal.delta` events and a final `RefusalBlock` in + * the assistant turn's `content[]`. + * - **Google GenAI** (`generationConfig`): + * - `text` → no constraint (default). + * - `json` → `{ responseMimeType: "application/json" }`. + * - `json-schema` → `{ responseMimeType: "application/json", responseSchema: }`. + * `name` and `strict` are OpenAI-specific and have no Gemini + * counterpart; adapters ignore them. Gemini enforces a subset of + * JSON Schema (no `oneOf`, limited `pattern`, no `$ref`, etc.) — + * the adapter forwards the schema verbatim and surfaces Gemini's + * HTTP error if the subset is violated. + * - **Anthropic**: no native structured-output API. + * - `text` is a no-op (the default). + * - `json` and `json-schema` throw at the adapter boundary; there + * is no shim that synthesizes a tool to extract structured + * output. + * + * When omitted the provider's default applies (typically free-form + * text). + */ + responseFormat?: + | { kind: "text" } + | { kind: "json" } + | { + kind: "json-schema"; + name: string; + schema: unknown; + strict?: boolean; + }; + /** + * A bag of provider-native knobs the adapter merges into the outbound + * request body. Primary home is `InferenceSourceDefaults.providerOptions` + * (model-bound); this field exists for per-call overrides through the + * standard merge precedence at the top of `runInference`. The merge is + * shallow: a per-call providerOptions object wholesale replaces the + * source-bound one, it does not deep-merge per key. + */ + providerOptions?: Record; + /** + * Per-call inactivity timeout in milliseconds. If the harness yields no + * event (other than `inference.start`) for this many ms, the underlying + * fetch is aborted and the call ends with `inference.error` of category + * `"timeout"`. Default 120_000 (2 min). Tune higher for reasoning models + * that exhibit long silent-thinking stretches between token bursts; tune + * lower to fail fast. `0` arms the timer to fire on the next tick (a + * "fail-fast even if the fetch is instant" mode useful in tests). + */ + inactivityTimeoutMs?: number; + /** + * Per-call total wall-clock cap in milliseconds. Starts at fetch. + * Default 600_000 (10 min). Backstop for streams that keep emitting + * forever without terminating. Same error category as `inactivityTimeoutMs`. + * `0` arms the timer to fire on the next tick. + */ + totalTimeoutMs?: number; + /** + * Per-call mechanical retry policy. Consulted once per attempt that + * ends in `inference.error`; see `RetryPolicy` for the contract. If + * omitted, a built-in default policy is applied. + */ + retryPolicy?: RetryPolicy; +}; + +// --------------------------------------------------------------------------- +// Context Store (INFERENCE.md § Context Management › Context Store, +// ARCHITECTURE.md § Change History) +// --------------------------------------------------------------------------- + +/** + * A named commit point in the context store. Corresponds to a git commit. + * + * (ARCHITECTURE.md § Change History › Named Checkpoints) + */ +export type ContextCommit = { + hash: string; + message: string; + timestamp: number; + parentHash?: string; +}; + +/** + * The state of an active connector thread. The connector is one durable + * thread per agent; participants accumulate as they speak. Persisted + * alongside the conversation context so the thread survives sidecar + * restarts. + * + * `replyTo` is the most recent speaker — the primary recipient (`to`) + * on the next outbound reply. `cc` is every other participant who has + * spoken on the thread, deduplicated, in arrival order — they ride as + * `cc` on the next outbound reply so everyone stays in the loop. + * `subject` is set when the thread starts and preserved for its life. + * + * Defined as an arktype so the wire layer (sidecar↔hub frames) and + * other parsing boundaries can validate snapshots without + * re-declaring the shape. + */ +export const ConnectorThreadState = type({ + "threadRoot?": "string", + "lastMessageId?": "string", + replyTo: "string", + cc: "string[]", + "subject?": "string", +}); +export type ConnectorThreadState = typeof ConnectorThreadState.infer; + +/** + * The context store interface. Implementations back the store with git + * (filesystem, in-memory, or virtual) depending on the execution environment. + * The reactor accepts any implementation that satisfies this interface. + * + * The store holds the turn history and reactor metadata. Forking creates + * a git branch. Compaction commits the compacted history. + * + * (INFERENCE.md § Context Management › Context Store) + */ +export interface ContextStore { + /** + * Load the current turn history and reactor metadata from the store. + * Called during reactor initialization. + */ + load(signal?: AbortSignal): Promise<{ + turns: ConversationTurn[]; + pendingOperations: PendingOperation[]; + tokenUsage: TokenUsage; + connectorState: ConnectorThreadState | null; + }>; + + /** + * Buffer connector thread state for the next commit. The harness calls + * this before each checkpoint so that connector state is persisted + * atomically with the conversation context. + */ + setConnectorState(state: ConnectorThreadState | null): void; + + /** + * Commit whatever currently lives in the working tree, using the supplied + * commit message. The reactor's per-cycle checkpoint routes through this + * overload after writing the per-cycle files via `writeTurns`, + * `writePrompt`, `writeResponse`, `writeManifest`, and any `writeBlob` + * calls produced by transforms. + */ + commit( + options: { message: string }, + signal?: AbortSignal, + ): Promise; + + /** + * Create a branch for a fork operation. The branch starts from the current + * HEAD commit. + */ + branch(name: string, signal?: AbortSignal): Promise; + + /** + * List recent commits. Used by the agent's history query tools. + */ + log(limit?: number, signal?: AbortSignal): Promise; + + /** + * Read the turn history at a specific commit hash. Used for history + * inspection and rollback. + */ + readAt(hash: string, signal?: AbortSignal): Promise; + + /** + * Write an opaque blob to the working tree under `tool-output/`. Used by + * `ToolResultTransform`s that spill oversized payloads out of the inline + * conversation. The file is staged at the next `commit({ message })`. + * + * `key` is sanitized for filesystem safety; callers should pass the tool + * call id. `contentType` selects a file extension when known. + */ + writeBlob( + key: string, + bytes: Uint8Array, + contentType?: string, + signal?: AbortSignal, + ): Promise; + + /** + * Read a blob previously written via `writeBlob`. Throws if no blob with + * that key exists. + */ + readBlob(key: string, signal?: AbortSignal): Promise; + + /** + * Overwrite `prompt.jsonl` with the materialized prompt for the current + * inference cycle. One `ConversationTurn` per line. Staged at the next + * `commit({ message })`. + */ + writePrompt(turns: ConversationTurn[], signal?: AbortSignal): Promise; + + /** + * Overwrite `response.jsonl` with the assistant turn returned for the + * current cycle. Single-line JSONL for consistency with the per-cycle file + * conventions. Staged at the next `commit({ message })`. + */ + writeResponse(turn: AssistantTurn, signal?: AbortSignal): Promise; + + /** + * Overwrite `manifest.jsonl` with the ordered transform records produced + * for the current cycle. One `TransformRecord` per line. Staged at the + * next `commit({ message })`. + */ + writeManifest( + records: TransformRecord[], + signal?: AbortSignal, + ): Promise; + + /** + * Overwrite `turns.jsonl` with the durable conversation history. One + * `ConversationTurn` per line. Staged at the next `commit({ message })`. + */ + writeTurns(turns: ConversationTurn[], signal?: AbortSignal): Promise; + + /** + * Overwrite `metadata.json` with non-turn-shaped reactor state needed for + * restart: pending async operations and cumulative token usage. The store + * combines this with the most recently buffered connector state (from + * `setConnectorState`) and writes the merged payload. Staged at the next + * `commit({ message })`. + */ + writeMetadata( + metadata: { + pendingOperations: PendingOperation[]; + tokenUsage: TokenUsage; + }, + signal?: AbortSignal, + ): Promise; + + /** + * Read manifest entries from the most recent `limit` commits that contain + * a `manifest.jsonl`. Newest commit first; records within a commit are + * returned in their natural in-file order (chronological per-cycle). + */ + readManifestHistory( + limit: number, + signal?: AbortSignal, + ): Promise; +} + +// --------------------------------------------------------------------------- +// Audit Store (INTR-4 § Audit Trail) +// --------------------------------------------------------------------------- + +/** + * Persistent store for tool invocation audit records. Separated from + * ContextStore so the audit capability is opt-in at the composition + * layer. The isogit implementation writes audit records as individual + * JSON files in the same git repo used for context storage. + */ +export interface AuditStore { + /** + * Persist a batch of audit records. Called at checkpoint boundaries + * with all records accumulated since the last checkpoint. + */ + commitAudit(records: AuditRecord[], signal?: AbortSignal): Promise; + + /** + * Load audit records for a session. Returns all records matching + * the given sessionId, ordered by seq. + */ + loadAudit(sessionId: string, signal?: AbortSignal): Promise; + + /** + * Persist a batch of error records. Called at checkpoint boundaries + * and shutdown with all error records accumulated since the last flush. + */ + commitErrors(records: ErrorRecord[], signal?: AbortSignal): Promise; + + /** + * Load error records for a session. Returns all records matching + * the given sessionId, ordered by seq. + * + * Locally patched — see vendor/intx-types/PATCHES.md#runtime-ts-audit-store-load-errors + */ + loadErrors(sessionId: string, signal?: AbortSignal): Promise; +} + +// --------------------------------------------------------------------------- +// Agent / Harness Configuration (ARCHITECTURE.md § Agent Harness) +// --------------------------------------------------------------------------- + +/** + * Configured tool definition exposed to the model. The harness registers + * available tools; the reactor passes this list to the inference provider as + * part of each request. + * + * (ARCHITECTURE.md § Agent Harness › Tools) + */ +export const ToolDefinition = type({ + name: "string", + description: "string", + inputSchema: "Record", +}); +export type ToolDefinition = typeof ToolDefinition.infer; + +/** + * Agent harness configuration. Assembled from the agent definition package + * and capability grants during harness initialization. + * + * `principalId` is the agent's principal in the hub's authorization model. + * The sidecar needs it to reconstruct the in-memory grant store on restart + * (the store's `collectGrants` filters by principal). + * + * `grants` uses `WireGrantRule` because this type arrives over JSON where + * `GrantRule.expiresAt` is serialized as a string. The wire validator + * coerces strings back to Date instances. + * + * (ARCHITECTURE.md § Agent Harness) + */ +export const HarnessConfig = type({ + sessionId: "string", + agentId: "string", + tenantId: "string", + principalId: "string", + agentAddress: "string", + systemPrompt: "string", + tools: ToolDefinition.array(), + grants: WireGrantRule.array(), + sources: InferenceSource.array(), + defaultSource: "string", + "sessionChannelEnabled?": "boolean", +}); +export type HarnessConfig = typeof HarnessConfig.infer; diff --git a/vendor/intx-types/src/runtime.test.ts b/vendor/intx-types/src/runtime.test.ts index c26ce7e28..25bd2bdce 100644 --- a/vendor/intx-types/src/runtime.test.ts +++ b/vendor/intx-types/src/runtime.test.ts @@ -5,8 +5,8 @@ import { APPROVAL_SNAPSHOT_MAX_BYTES, BoundedApprovalSnapshot, ContentBlock, - formatSafetyRatingText, InferenceEvent, + InterchangeType, MediaSource, TransformRecord, type ContextTransform, @@ -17,8 +17,12 @@ import { type BlobReader, type BlobSource, createBlobReader, + isConversationType, + isMessageTransportError, + MessageTransportError, parseToolOutputURI, } from "./runtime"; +import { formatSafetyRatingText } from "./format-safety-rating"; // --------------------------------------------------------------------------- // 1. TransformRecord validator @@ -1277,3 +1281,156 @@ describe("BoundedApprovalSnapshot", () => { expect(result instanceof type.errors).toBe(true); }); }); + +describe("isMessageTransportError", () => { + test("accepts a condition-bearing error the class did not construct", () => { + // The reason the guard is structural: a tool package is published as a + // bundle with its workspace imports inlined, so the consumer classifying a + // failure holds its own copy of MessageTransportError and an `instanceof` + // check against the host's copy answers false. This foreign class stands in + // for that second copy. + class ForeignTransportError extends Error { + readonly condition = "NONEXISTENT"; + } + const foreign = new ForeignTransportError("mailbox is not here"); + + expect(isMessageTransportError(foreign)).toBe(true); + expect(foreign instanceof MessageTransportError).toBe(false); + }); + + test("accepts the error the class constructs and reads back its condition", () => { + const failure = new MessageTransportError("SERVERBUG", "no reader wired"); + + expect(isMessageTransportError(failure)).toBe(true); + expect(failure.condition).toBe("SERVERBUG"); + expect(failure.name).toBe("MessageTransportError"); + }); + + test("refuses an error carrying no condition, or one RFC 5530 does not name", () => { + expect(isMessageTransportError(new Error("the socket went away"))).toBe( + false, + ); + expect( + isMessageTransportError( + Object.assign(new Error("invented"), { condition: "MAILBOX_MISSING" }), + ), + ).toBe(false); + expect(isMessageTransportError({ condition: "NONEXISTENT" })).toBe(false); + expect(isMessageTransportError(undefined)).toBe(false); + }); +}); + +// isConversationType answers from a set of members written out by hand, and +// nothing in InterchangeType forces an entry there: a member added to the union +// is not a conversation type until someone lists it in that set, which is what +// the predicate's own docstring says. The consequence of the missing edit is +// silent -- a conversational message would route as structured mail -- so the +// classification below is held to the union at runtime. +// +// The members come from the validator rather than from a second list written +// here: `type.enumerated` compiles to unit nodes, and `select("unit")` yields +// them. A member added to the union therefore reaches this comparison with no +// entry and is reported, and answering the report is the same decision the +// enumerated set exists to force. Deriving membership from the `conversation.` +// name prefix here would make that decision instead of asking for it. + +type ConversationMembership = ReadonlyMap; + +/** + * Whether each member of InterchangeType is a conversation type. The set behind + * isConversationType is not exported, so this is a second description of the + * same fact rather than a copy of it, and the predicate is the only handle the + * comparison below has on the first one. + */ +const CONVERSATION_MEMBERSHIP: ConversationMembership = new Map([ + ["conversation.message", true], + ["conversation.join", true], + ["conversation.leave", true], + ["offering.request", false], + ["offering.response", false], + ["offering.error", false], + ["offering.discover", false], + ["offering.catalog", false], + ["payment.required", false], + ["payment.receipt", false], + ["payment.verified", false], + ["approval.request", false], + ["approval.granted", false], + ["approval.denied", false], + ["system.health", false], + ["system.register", false], + ["system.deregister", false], + ["system.credential.refresh", false], +]); + +// The members InterchangeType enumerates. Each unit is put back through the +// validator, which narrows it from the `unknown` a node carries and reports the +// case where the introspected members and the validated ones disagree. +function enumeratedTypes(): InterchangeType[] { + return InterchangeType.select("unit").map((unit) => { + const member = InterchangeType(unit.unit); + if (member instanceof type.errors) { + throw new Error( + `InterchangeType enumerates ${String(unit.unit)} and then refuses it: ${member.summary}`, + ); + } + return member; + }); +} + +// Every disagreement between a classification and the union: a member the +// classification does not name, a member it names with an answer +// isConversationType does not give, and a name the union does not enumerate. +// Reported as one line per member, so a failure names the member. +function membershipDrift(membership: ConversationMembership): string[] { + const enumerated = enumeratedTypes(); + const findings: string[] = []; + + for (const member of enumerated) { + const expected = membership.get(member); + if (expected === undefined) { + findings.push(`${member} is not classified`); + continue; + } + const answered = isConversationType(member); + if (answered !== expected) { + findings.push( + `${member} is classified as ${expected ? "" : "not "}conversational and isConversationType answers ${String(answered)}`, + ); + } + } + + const known = new Set(enumerated); + for (const member of membership.keys()) { + if (!known.has(member)) { + findings.push(`${member} is classified and InterchangeType omits it`); + } + } + + return findings; +} + +describe("the conversation-type subset of InterchangeType", () => { + test("names every member of the union, and isConversationType agrees", () => { + expect(membershipDrift(CONVERSATION_MEMBERSHIP)).toEqual([]); + }); + + test("the comparison reports a member left out and a member answered wrongly", () => { + // The check is worth its line only if it fails when the two descriptions + // part, and the classification is the half a test can perturb without + // touching the union or the set. This one is local to the test: the dropped + // entry stands for a member added to the union and not classified, and the + // flipped entry for a member the two descriptions disagree about. + const perturbed = new Map(CONVERSATION_MEMBERSHIP); + perturbed.delete("conversation.leave"); + perturbed.set("system.health", true); + + const drift = membershipDrift(perturbed); + + expect(drift).toHaveLength(2); + expect(drift).toContain("conversation.leave is not classified"); + expect(drift).toContain( + "system.health is classified as conversational and isConversationType answers false", + ); + }); +}); diff --git a/vendor/intx-types/src/runtime.ts b/vendor/intx-types/src/runtime.ts index 318fefebd..6e4ebb27d 100644 --- a/vendor/intx-types/src/runtime.ts +++ b/vendor/intx-types/src/runtime.ts @@ -1,2979 +1,11 @@ -// Runtime definitions for the Interchange agent harness. -// -// Wire-facing data types (AbortReason, InferenceSource, ToolDefinition, -// HarnessConfig) are arktype validators so they can be composed into -// WebSocket frame validators and used for runtime validation at parse -// boundaries. Behavioral interfaces (ContextStore, MessageTransport, -// ToolRunner, etc.) remain plain TypeScript. - -import { type } from "arktype"; -import type { AuditRecord, ErrorRecord } from "./audit"; -import { WireGrantRule } from "./grant-wire"; -import type { SignalKind } from "./signals"; - -// --------------------------------------------------------------------------- -// Cryptographic Identity (ARCHITECTURE.md § Cryptographic Identity, -// IMPLEMENTATION.md § Cryptographic Identity: Key Formats) -// --------------------------------------------------------------------------- - -/** - * An Ed25519 key pair as raw bytes. The private key is 32 bytes; the public - * key is the corresponding 32-byte compressed point. - * - * Key material is represented as Uint8Array throughout so it stays - * runtime-agnostic (Bun, Node, browser) and never accidentally leaks through - * JSON serialization. - */ -export type KeyPair = { - privateKey: Uint8Array; - publicKey: Uint8Array; -}; - -/** - * A key-bound cryptographic provider. Each instance is constructed with a - * specific agent's Ed25519 key pair and holds the private key internally. - * - * `sign` uses the instance's own private key — no key parameter is accepted. - * `verify` accepts a public key parameter so the holder can verify messages - * from arbitrary senders without constructing a new provider instance. - * - * The in-memory transport stores one CryptoProvider per registered agent and - * calls `crypto.sign(content)` during `send()` without passing keys around. - * - * Key formats (IMPLEMENTATION.md): - * - Ed25519 in SSH format — control plane interactions - * - Ed25519 in PGP format — message-level signatures over SMTP/IMAP - * - Ed25519 in X.509 format — TLS mutual auth certificates - * - * `getPublicKey` returns the raw public key bytes so callers can publish - * them to the control plane or embed them in discovery metadata. - */ -export interface CryptoProvider { - /** - * Sign `content` with the instance's private key. Returns the Ed25519 - * detached signature as raw bytes. - */ - sign(content: Uint8Array): Promise; - - /** - * Sign `payload` with the instance's private key using the SSH signature - * envelope (sshsig). Returns an ASCII-armored SSH SIGNATURE block suitable - * for the `gpgsig` header of a git commit or any other site that consumes - * `git verify-commit`-compatible signatures. The framing differs from - * `sign`'s raw output; callers that need either format should pick the - * matching method rather than reframing the result themselves. - */ - signSSH(payload: string): Promise; - - /** - * Verify that `signature` over `content` was produced by `publicKey`. - * Returns true if the signature is valid; false otherwise. - */ - verify( - content: Uint8Array, - signature: Uint8Array, - publicKey: Uint8Array, - ): Promise; - - /** The public key for this instance, as raw bytes. */ - getPublicKey(): Uint8Array; -} - -/** - * Generate a fresh Ed25519 key pair. The returned pair is used to construct - * a CryptoProvider instance. - */ -export type GenerateKeyPair = () => Promise; - -// --------------------------------------------------------------------------- -// Message Transport (MESSAGE.md § Transport Interface) -// --------------------------------------------------------------------------- - -/** - * Opaque reference to a message in a specific mailbox. Carries the IMAP UID - * and the mailbox name. Passed to fetch, flag, and move operations without - * requiring re-search. - */ -export type MessageRef = { - uid: number; - mailbox: string; -}; - -/** - * Interchange payload types as defined in MESSAGE.md § Payload Types. - * The type field in structured messages matches the Interchange-Type header. - * - * Exposed as both an arktype validator (for runtime validation at parse - * boundaries and tool-argument schemas) and a derived TypeScript union. - */ -export const InterchangeType = type.enumerated( - "conversation.message", - "conversation.join", - "conversation.leave", - "offering.request", - "offering.response", - "offering.error", - "offering.discover", - "offering.catalog", - "payment.required", - "payment.receipt", - "payment.verified", - "approval.request", - "approval.granted", - "approval.denied", - "system.health", - "system.register", - "system.deregister", - "system.credential.refresh", -); -export type InterchangeType = typeof InterchangeType.infer; - -/** - * Attachment for an outbound message. Content is raw bytes; the transport - * handles Content-Transfer-Encoding (base64 for binary, quoted-printable - * for 8-bit text). - */ -export type MessageAttachment = { - name: string; - contentType: string; - data: Uint8Array; -}; - -/** - * A message the harness submits for delivery via SMTP. The transport - * assembles the PGP/MIME multipart structure, signs it with the agent's - * CryptoProvider, and submits it. - * - * Conversation types (conversation.*) carry `content` as text/plain. - * Structured types carry `payload` as application/vnd.interchange+json. - * Providing both is an error. - * - * (MESSAGE.md § Transport Interface › Outbound) - */ -export type OutboundMessage = { - to: string | string[]; - cc?: string | string[]; - subject?: string; - - type: InterchangeType; - - /** Plain text body — used when type is a conversation.* type. */ - content?: string; - - /** Structured JSON body — used when type is a non-conversation type. */ - payload?: Record; - - /** Human-readable summary for structured messages (the text/plain part). */ - summary?: string; - - attachments?: MessageAttachment[]; - - /** Message-ID of the message being replied to. */ - inReplyTo?: string; - - /** - * The RFC 5322 References chain for a threaded reply: the parent's own - * References plus the parent's Message-ID, in order. When present the - * transport ships it verbatim (after appending `inReplyTo` if it is not - * already the tail) rather than deriving a single-element `[inReplyTo]` - * chain, so a reply carries the full conversational ancestry. Absent for a - * non-reply or a reply whose parent could not be located. - */ - references?: string[]; - - /** Correlation ID linking this message to a pending async request. */ - correlationId?: string; - - /** Reactor session ID from the Interchange-Session-ID header. */ - sessionId?: string; - - /** Tenant ID for the Interchange-Tenant-ID header. */ - tenantId?: string; -}; - -/** - * Receipt returned by `send()`. Contains the assigned Message-ID and - * delivery status. - * - * (MESSAGE.md § Transport Interface › Outbound) - */ -export type SendReceipt = { - messageId: string; - status: "delivered" | "queued"; -}; - -/** - * Parsed headers from an inbound message. Field names follow RFC 5322 and - * the Interchange-specific header conventions from MESSAGE.md § Headers. - */ -export type MessageHeaders = { - from: string; - to: string[]; - cc?: string[]; - date: string; - messageId: string; - inReplyTo?: string; - references?: string[]; - subject?: string; - listId?: string; - - interchangeType?: InterchangeType; - interchangeCorrelationId?: string; - interchangeTenantId?: string; - interchangeAgentId?: string; - interchangeSessionId?: string; - interchangeOfferingId?: string; - interchangeSchemaVersion?: string; - - traceparent?: string; - tracestate?: string; -}; - -/** - * Signature verification status of an inbound message. - * - * - `valid` — signature verified against the sender's public key - * - `invalid` — signature check failed (tampering or wrong key) - * - `unknown` — public key not available for verification - * - `missing` — message was not signed - * - * (MESSAGE.md § Transport Interface › fetchFull) - */ -export const SignatureStatus = type.enumerated( - "valid", - "invalid", - "unknown", - "missing", -); -export type SignatureStatus = typeof SignatureStatus.infer; - -/** - * The admission outcome of an inbound message. This is the single vocabulary a - * delivery decision keys on, distinct from the two-axis signature verdict that - * produces it. - * - * - `clean` — nothing suspect; always admitted - * - `untrustedFrom` — the visible `From` cannot be trusted, either because it is - * present but unparseable or because a valid signature is worn under a - * mismatched sender identity - * - `invalid` — the signature check failed (tampering or the wrong key) - * - `missing` — the message carried no signature - * - `unknown` — no key was available to verify against - * - `error` — a fault stopped the check from running at all; always rejected - */ -export const InboundMailOutcome = type.enumerated( - "clean", - "untrustedFrom", - "invalid", - "missing", - "unknown", - "error", -); -export type InboundMailOutcome = typeof InboundMailOutcome.infer; - -/** - * The subset of {@link InboundMailOutcome} a workflow author may relax to admit - * a message that would otherwise be rejected. It omits `clean` (which always - * admits, so there is nothing to relax) and `error` (pinned to reject, since a - * fault we could not check through is never something an author should be able - * to wave past). A per-workflow policy keys on exactly these outcomes. - */ -export const AuthorControllableOutcome = type.enumerated( - "untrustedFrom", - "invalid", - "missing", - "unknown", -); -export type AuthorControllableOutcome = typeof AuthorControllableOutcome.infer; - -/** - * A per-workflow inbound-mail admission policy: for each admission outcome the - * author may control, whether a message that resolved to that outcome is - * `reject`ed or `admit`ted. The key set is exactly the - * {@link AuthorControllableOutcome} values -- `clean` (always admitted) and - * `error` (pinned to reject) are deliberately not keys. - * - * The object is SPARSE: every key is optional, and an omitted key is NOT a - * default of any kind here. It is left for a later resolution step to interpret - * an absent outcome. Keeping it sparse means the content hash covers only what - * the author actually declared, so a definition that omits the policy hashes - * identically to one authored before the field existed. Undeclared keys are - * rejected so a typo such as `clean` or `errror` fails at the wire boundary - * rather than riding through as an inert unknown key. - */ -export const InboundMailPolicy = type({ - "untrustedFrom?": "'reject' | 'admit'", - "invalid?": "'reject' | 'admit'", - "missing?": "'reject' | 'admit'", - "unknown?": "'reject' | 'admit'", -}).onUndeclaredKey("reject"); -export type InboundMailPolicy = typeof InboundMailPolicy.infer; - -/** - * A parsed MIME part. `content` is the DECODED bytes in memory (the - * transfer-encoding has already been undone). `filename` and `disposition` are - * surfaced from the part's `Content-Disposition` / `Content-Type` so a consumer - * can distinguish an inline part from a named attachment without re-parsing - * headers. - */ -export type MessagePart = { - contentType: string; - content: Uint8Array; - filename?: string; - disposition?: "inline" | "attachment"; - /** - * Original Content-Transfer-Encoding, when a producer chooses to record it. - * Not set for a decoded mail part -- `content` is already decoded, so the - * wire encoding is spent transport metadata. - */ - encoding?: string; -}; - -/** - * A single part of a persisted `Mail`. The bytes live in the durable store; - * this descriptor carries the part's metadata plus an opaque, relocation-stable - * `ref` a `MailPartReader` resolves to the part's bytes. Small UTF-8 text parts - * also carry their decoded `text` inline so a selector can read them without - * resolving. - */ -export type MailPart = { - contentType: string; - filename?: string; - disposition?: "inline" | "attachment"; - ref: string; - text?: string; -}; - -/** - * The single, environment-agnostic interface for reading a persisted mail - * part's bytes. Modeled on `BlobReader`: a consumer -- a workflow step, an - * agent tool, the agent's content-block projection -- resolves a - * `MailPart.ref` to its bytes without knowing or caring where the bytes live - * (a committed file on the sidecar, a blob in a browser runtime). The `ref` - * is opaque; the reader owns its scheme. Threaded to consumers through the - * runtime so a browser runtime can supply its own implementation. - */ -export interface MailPartReader { - /** Resolve a `MailPart.ref` to the part's decoded bytes. Throws if the ref - * is unrecognized or its bytes are missing. */ - read(ref: string): Promise; -} - -/** - * A fully decoded mail message: every header (both the typed, ergonomic subset - * and a raw catch-all with nothing dropped) plus the flat list of decoded leaf - * parts. This is the lossless representation a deployed workflow receives as - * its trigger input; a workflow programs against it directly (select headers, - * walk parts, route on content type), and an agent step projects the parts into - * model content blocks. It is JSON-safe: each part carries a `ref` (not raw - * bytes), so binary content never enters the event log; the runtime resolves a - * `ref` into a loadable `MessagePart` on demand. - */ -export type Mail = { - headers: MessageHeaders; - /** Every header, lowercased name to its ordered values; nothing dropped. */ - rawHeaders: Record; - parts: MailPart[]; -}; - -const MailShape = type({ - // Require the header fields a consumer dereferences unconditionally (the - // sender/recipient a projection reads); other header fields stay optional - // and are carried losslessly in `rawHeaders`. - headers: { - from: "string", - to: "string[]", - }, - rawHeaders: "object", - parts: type({ - contentType: "string", - ref: "string", - "filename?": "string", - "disposition?": "'inline' | 'attachment'", - "text?": "string", - }) - .onUndeclaredKey("reject") - .array(), -}).onUndeclaredKey("reject"); - -/** - * Narrow an opaque value (a workflow step input) to a `Mail`. Used at the - * `agent.send` boundary to decide whether the input is a mail-derived message - * whose parts must be projected into content blocks, or an arbitrary value - * delivered as synthesized text. The strict undeclared-key rejection keeps an - * arbitrary step value that merely carries a `parts` field from matching. - */ -export function isMail(value: unknown): value is Mail { - return !(MailShape(value) instanceof type.errors); -} - -/** - * MIME tree metadata returned by `fetchStructure()`. Describes content types, - * sizes, and dispositions without transferring content. - * - * (MESSAGE.md § Partial Fetch) - */ -export type BodyStructure = { - contentType: string; - size?: number; - disposition?: string; - parts?: BodyStructure[]; -}; - -/** - * A fully parsed inbound message including structured payload, headers, - * attachments, and signature verification status. - * - * (MESSAGE.md § Transport Interface › fetchFull) - */ -export type InboundMessage = { - ref: MessageRef; - headers: MessageHeaders; - flags: string[]; - - /** Plain text body for conversation.* types. */ - content?: string; - - /** Parsed JSON payload for structured types. */ - payload?: { - type: InterchangeType; - version: string; - body: Record; - }; - - attachments?: MessageAttachment[]; - signatureStatus: SignatureStatus; -}; - -/** - * IMAP mailbox descriptor. - * - * (MESSAGE.md § Inbox Management) - */ -export type Mailbox = { - name: string; - role?: string; - delimiter?: string; -}; - -/** - * Current status of an IMAP mailbox, including QRESYNC identifiers. - * - * (MESSAGE.md § Inbox Management) - */ -export type MailboxStatus = { - total: number; - unseen: number; - recent: number; - uidNext: number; - uidValidity: number; - highestModSeq: number; -}; - -/** - * Structured IMAP search query. Maps the IMAP SEARCH grammar to a typed - * object. Supports recursive boolean composition via `and`, `or`, `not`. - * - * (MESSAGE.md § Search) - */ -export type SearchQuery = { - from?: string; - to?: string; - cc?: string; - bcc?: string; - header?: { field: string; contains: string }; - before?: Date; - after?: Date; - on?: Date; - sentBefore?: Date; - sentAfter?: Date; - sentOn?: Date; - hasFlags?: string[]; - missingFlags?: string[]; - body?: string; - text?: string; - largerThan?: number; - smallerThan?: number; - and?: SearchQuery[]; - or?: SearchQuery[]; - not?: SearchQuery; -}; - -/** - * A thread node returned by `thread()`. Carries a message reference and - * child threads representing replies. Implements the RFC 5256 REFERENCES - * threading algorithm. - * - * (MESSAGE.md § Thread Retrieval) - */ -export type Thread = { - ref: MessageRef; - children: Thread[]; -}; - -/** - * QRESYNC state the harness provides when reconnecting to the transport. - * - * (MESSAGE.md § Synchronization) - */ -export type SyncState = { - uidValidity: number; - uidNext: number; - highestModSeq: number; - knownUids?: number[]; -}; - -/** - * Result of a QRESYNC-style sync operation. - * - * (MESSAGE.md § Synchronization) - */ -export type SyncResult = { - vanished: number[]; - changed: { uid: number; flags: string[] }[]; - newMessages: MessageRef[]; - fullResyncRequired: boolean; -}; - -/** - * Distribution list metadata returned by `createList()`. - * - * (MESSAGE.md § Message Topologies) - */ -export type ListInfo = { - address: string; - name: string; - memberCount: number; - createdAt: string; -}; - -/** - * Event emitted by the mailbox watcher callback. Corresponds to IMAP IDLE - * notifications. - * - * (MESSAGE.md § Real-Time Notification) - */ -export type MailboxEvent = - | { type: "exists"; uid: number; headers: MessageHeaders } - | { type: "flagsChanged"; uid: number; flags: string[] } - | { type: "expunged"; uid: number }; - -/** Unsubscribe function returned by `watch()`. */ -export type Unsubscribe = () => void; - -/** - * The message transport interface. Abstracts SMTP and IMAP behind a - * TypeScript API. Implementations range from real SMTP/IMAP servers to - * in-process stubs that route messages through memory. - * - * All long-running operations accept an AbortSignal for cooperative - * cancellation. - * - * (MESSAGE.md § Transport Interface) - */ -export interface MessageTransport { - // --- Outbound --- - - /** Compose, sign, and deliver a message via SMTP. */ - send(message: OutboundMessage, signal?: AbortSignal): Promise; - - /** Append a raw message to a mailbox (IMAP APPEND). */ - append( - mailbox: string, - message: InboundMessage, - flags?: string[], - signal?: AbortSignal, - ): Promise; - - // --- Mailbox management --- - - listMailboxes(signal?: AbortSignal): Promise; - createMailbox(name: string, signal?: AbortSignal): Promise; - deleteMailbox(name: string, signal?: AbortSignal): Promise; - getMailboxStatus(name: string, signal?: AbortSignal): Promise; - - // --- Message search and retrieval --- - - search( - mailbox: string, - query: SearchQuery, - signal?: AbortSignal, - ): Promise; - - thread( - mailbox: string, - algorithm: "references" | "orderedsubject", - query?: SearchQuery, - signal?: AbortSignal, - ): Promise; - - fetchHeaders(ref: MessageRef, signal?: AbortSignal): Promise; - fetchStructure(ref: MessageRef, signal?: AbortSignal): Promise; - fetchPart( - ref: MessageRef, - partPath: string, - signal?: AbortSignal, - ): Promise; - fetchFull(ref: MessageRef, signal?: AbortSignal): Promise; - - // --- Flag management --- - - setFlags( - ref: MessageRef, - flags: string[], - signal?: AbortSignal, - ): Promise; - - clearFlags( - ref: MessageRef, - flags: string[], - signal?: AbortSignal, - ): Promise; - - // --- Message organization --- - - move(ref: MessageRef, toMailbox: string, signal?: AbortSignal): Promise; - - copy(ref: MessageRef, toMailbox: string, signal?: AbortSignal): Promise; - - /** - * Permanently remove every `\Deleted` message from the mailbox. Returns the - * uids that were expunged, so a caller can report how many messages it - * consumed and which ones. - */ - expunge( - mailbox: string, - signal?: AbortSignal, - ): Promise<{ expungedUids: number[] }>; - - // --- Real-time notification --- - - /** Monitor a mailbox for new messages and flag changes (IMAP IDLE). */ - watch(mailbox: string, callback: (event: MailboxEvent) => void): Unsubscribe; - - // --- Synchronization --- - - /** Efficient reconnection using QRESYNC semantics. */ - sync( - mailbox: string, - knownState: SyncState, - signal?: AbortSignal, - ): Promise; - - // --- Distribution lists --- - - createList( - address: string, - name: string, - signal?: AbortSignal, - ): Promise; - - listMembers(address: string, signal?: AbortSignal): Promise; - - subscribe( - listAddress: string, - subscriberAddress: string, - signal?: AbortSignal, - ): Promise; - - unsubscribe( - listAddress: string, - subscriberAddress: string, - signal?: AbortSignal, - ): Promise; -} - -// --------------------------------------------------------------------------- -// Tool Execution (ARCHITECTURE.md § Tools, INFERENCE.md § Tool Execution) -// --------------------------------------------------------------------------- - -/** - * A tool call as requested by the model. Carries the provider-assigned call - * ID, the tool name, and the parsed arguments. - * - * (INFERENCE.md § Message Format › Content Types) - */ -export const ToolCall = type({ - id: "string", - name: "string", - arguments: "Record", -}); -export type ToolCall = typeof ToolCall.infer; - -/** - * Approver-facing snapshot of the tool call awaiting approval. Built at the - * authz `ask` branch from the tool's definition and the live call, then - * threaded unchanged from the reactor's pending operation through every - * suspend hop to the hub co-write that records it on the approval row. - * - * `name`, `description`, and `inputSchema` mirror the {@link ToolDefinition}; - * `arguments` is the live call's arguments. Carried as a sibling of the pending - * operation's `suspendedCall`, never folded into {@link ToolCall}, so the - * re-dispatch artifact and the approval snapshot stay separate concerns. - */ -export const ApprovalSnapshot = type({ - name: "string", - description: "string", - inputSchema: "Record", - arguments: "Record", -}); -export type ApprovalSnapshot = typeof ApprovalSnapshot.infer; - -/** - * The kind of a control-plane park: a step suspended awaiting an external - * event. `"approval"` and `"input"` park on a reserved - * `signalName(correlationId)` channel; `"signal-relay"` parks on an - * author-chosen name. - * - * - `"approval"` -- the step parked on a tool/authz gate and REQUIRES an - * {@link ApprovalSnapshot}; the runtime notifies the host (`env.onPark`) so - * the sidecar co-writes the approval/correlation rows the hub registers. - * - `"input"` -- the step parked awaiting its next input (e.g. a long-lived - * agent run awaiting the next mail so it can take another turn). It carries - * NO snapshot and does NOT notify the host: the run's owner delivers the - * input on the same channel and the step re-arms. It is a runtime-local - * concept -- deliberately NOT a {@link SignalKind}, so it never touches the - * approval-routing machinery (IPC register frames, the hub co-write, the - * approval columns). - * - `"signal-relay"` -- an onTrigger section container parked on an - * author-named signal so a body child's `awaitSignal` on that name is - * serviced through the deployment run: the external signal is delivered to - * the parent run and the runtime relays it down into the live body child. - * The channel name is the author's free-form signal name, NOT a reserved - * `signalName(correlationId)`, so recovery must branch on this kind BEFORE - * assuming the awaited name is a reserved control-plane channel. Carries no - * snapshot and is not hub-registered. - * - * The kinds are distinguished by an EXPLICIT discriminant everywhere the kind - * flows -- never inferred from the presence or absence of a snapshot, which - * would silently reclassify a malformed snapshot-less approval as another - * park kind rather than failing loud. - */ -export const ControlParkKind = type.enumerated( - "approval", - "input", - "signal-relay", -); -export type ControlParkKind = typeof ControlParkKind.infer; - -/** - * Maximum serialized size, in UTF-8 bytes, of an {@link ApprovalSnapshot} that - * crosses a trust boundary. A tool `inputSchema` is normally single-digit KB; - * a snapshot approaching this bound is malformed or hostile and is rejected at - * the parse boundary rather than co-written onto an approval row. - */ -export const APPROVAL_SNAPSHOT_MAX_BYTES = 131072; - -/** - * {@link ApprovalSnapshot} bounded to {@link APPROVAL_SNAPSHOT_MAX_BYTES}. - * Parse the snapshot through this validator where it crosses a trust boundary - * (the `park.notify` IPC frame, the `parked-correlations.response` IPC frame, - * and the sidecar→hub register frame); internal hops use the unbounded - * {@link ApprovalSnapshot}. `.narrow` bounds the runtime check only — its - * inferred type is identical to {@link ApprovalSnapshot} — so the cap holds only - * where a frame is actually parsed, not merely typed. - */ -export const BoundedApprovalSnapshot = ApprovalSnapshot.narrow( - (snapshot, ctx) => { - const bytes = Buffer.byteLength(JSON.stringify(snapshot), "utf8"); - return ( - bytes <= APPROVAL_SNAPSHOT_MAX_BYTES || - ctx.mustBe(`at most ${APPROVAL_SNAPSHOT_MAX_BYTES} bytes when serialized`) - ); - }, -); -export type BoundedApprovalSnapshot = typeof BoundedApprovalSnapshot.infer; - -/** - * Result of a tool execution. `content` is text or structured data the model - * sees as the tool result. `detail` is additional data that the harness may - * use (e.g., for validation or audit) but that is not shown to the model. - * - * When `isError` is true the model sees the result as an error. When - * `pendingMarker` is present the tool is async — the reactor registers the - * correlation ID and waits for a matching inbound message. - * - * (INFERENCE.md § Tool Execution Semantics) - */ -export const ToolResult = type({ - callId: "string", - content: "string | Record", - "detail?": "unknown", - "isError?": "boolean", - "pendingMarker?": { - status: "'pending'", - correlationId: "string", - "expectedFrom?": "string", - }, -}); -export type ToolResult = typeof ToolResult.infer; - -/** - * The tool runner interface. The harness implements this; the reactor calls - * it when the director requests tool execution. - * - * Parallel execution is modeled by calling `run` concurrently for each call - * in a batch — the interface is per-call, not per-batch. - * - * (ARCHITECTURE.md § Agent Harness › Tools) - */ -export interface ToolRunner { - /** - * Execute a single tool call. Resolves with the result. Must not throw — - * errors are returned as `ToolResult` with `isError: true`. - */ - run(call: ToolCall, signal: AbortSignal): Promise; -} - -// --------------------------------------------------------------------------- -// Inference Event Building Blocks (INFERENCE.md § Event Protocol) -// --------------------------------------------------------------------------- - -/** - * Partial assistant message accumulated during streaming. Carries all - * content blocks seen so far so late-joining subscribers receive current - * state without replaying deltas. - * - * `text` and `thinking` are cumulative across every emitted delta of - * that kind in the current turn — intentionally flat, even when the - * harness's per-index block tracking has split the stream into - * multiple ThinkingBlocks or TextBlocks. Consumers that need per-block - * structure walk the finalized inference.done turn's content[]; this - * snapshot is the live "what bytes has the assistant streamed so - * far" view. - * - * (INFERENCE.md § Event Protocol › Partial State) - */ -export const PartialMessage = type({ - text: "string", - "thinking?": "string", - "toolCalls?": type({ - id: "string", - name: "string", - partialArguments: "string", - }).array(), -}); -export type PartialMessage = typeof PartialMessage.infer; - -/** - * Token usage for a single inference call. Cache read/write counts are - * provider-specific and may be zero when the provider does not report them. - * - * (INFERENCE.md § Token Accounting) - */ -export const TokenUsage = type({ - input: "number", - output: "number", - cacheRead: "number", - cacheWrite: "number", - thinking: "number", -}); -export type TokenUsage = typeof TokenUsage.infer; - -/** - * Slim source descriptor stamped onto `inference.usage` / `inference.done` - * events and onto `ReactorState.lastCycleSource`. - * - * Carries enough identity for state-aware policies (cost gating, budget - * caps, governance triggers, audit) to attribute usage to a specific - * inference source without re-reading the live, mutable `InferenceSource` - * the harness owns. - * - * Deliberately a strict subset of `InferenceSource` — `apiKey` and - * `baseURL` are intentionally excluded. Credentials and endpoints must - * not leak to director-side policy code or to external event consumers. - * Any code path that needs the full source obtains it through the - * harness's source registry, not through this descriptor. - * - * `sourceId` aliases `InferenceSource.id` to disambiguate from message - * ids, turn ids, and session ids in director-side code where `id` alone - * would be ambiguous. - */ -export const LastCycleSource = type({ - sourceId: "string", - provider: "string", - model: "string", -}); -export type LastCycleSource = typeof LastCycleSource.infer; - -// --------------------------------------------------------------------------- -// Internal Turn Format (INFERENCE.md § Message Format) -// --------------------------------------------------------------------------- - -/** - * A single content block within a conversation turn. Provider-agnostic. - * - * (INFERENCE.md § Message Format › Content Types) - */ -const TextBlock = type({ - type: "'text'", - text: "string", - // Opaque provider signature authenticating this block, echoed back - // verbatim on follow-up turns. Gemini attaches a `thoughtSignature` to - // output parts (including plain text); absent for providers that do not - // sign this block kind. - "signature?": "string", -}); - -/** - * How a media payload is carried by a content block. One of three - * variants: inline as a base64-encoded string, by reference to an - * opaque provider-native handle (e.g. a Gemini fileUri, an Anthropic - * file_id), or by public URL the provider fetches itself. The wire - * shape each provider expects is built by the provider adapter; - * MediaSource is the internal, provider-agnostic representation. - * - * (INFERENCE.md § Generalized Multimodal Taxonomy) - */ -const MediaSourceBase64 = type({ - kind: "'base64'", - mimeType: "string", - data: "string", -}); - -const MediaSourceFileReference = type({ - kind: "'file-reference'", - mimeType: "string", - reference: "string", -}); - -const MediaSourceUrl = type({ - kind: "'url'", - mimeType: "string", - url: "string", -}); - -export const MediaSource = type.or( - MediaSourceBase64, - MediaSourceFileReference, - MediaSourceUrl, -); -export type MediaSource = typeof MediaSource.infer; - -// Exported because `inference.image_output` events reference it by -// name, following the same pattern as `CitationBlock`, -// `CodeExecutionRequestBlock`, and `RedactedThinkingBlock`. -export const ImageBlock = type({ - type: "'image'", - source: MediaSource, - // Opaque provider signature authenticating this block, echoed back - // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the - // inlineData part; absent otherwise. - "signature?": "string", -}); -export type ImageBlock = typeof ImageBlock.infer; - -const AudioBlock = type({ - type: "'audio'", - source: MediaSource, -}); - -const VideoBlock = type({ - type: "'video'", - source: MediaSource, -}); - -const DocumentBlock = type({ - type: "'document'", - source: MediaSource, - "title?": "string", - "context?": "string", -}); - -const ThinkingBlock = type({ - type: "'thinking'", - thinking: "string", - "signature?": "string", -}); - -/** - * A thinking block whose content the provider has filtered. The - * opaque `data` blob must echo back verbatim on every follow-up turn - * — Anthropic 400s the request if it changes or goes missing. Treat - * the bytes as opaque: do not log them and do not render them to - * users. - * - * Exported because `inference.thinking.redacted` events reference it - * by name. - */ -export const RedactedThinkingBlock = type({ - type: "'redacted_thinking'", - data: "string", -}); -export type RedactedThinkingBlock = typeof RedactedThinkingBlock.infer; - -/** - * A model-emitted refusal. Produced when a provider's strict-mode - * structured-outputs path declines to satisfy the requested schema — - * OpenAI's `delta.refusal` / `message.refusal` field is the canonical - * wire shape. The `reason` is the accumulated human-readable text the - * model emitted in lieu of conformant output. - * - * Refusal is semantically distinct from `inference.error`: the HTTP - * call succeeded and the model produced a coherent response, but that - * response is "I will not satisfy this schema" rather than schema- - * conformant content. Callers that distinguish policy declines from - * transport/protocol failures should branch on the block type rather - * than treat the assistant turn as an error. - * - * Exported because `inference.refusal.delta` events reference it by - * name and adapters construct RefusalBlocks in the finalized - * AssistantTurn from accumulated delta fragments. - */ -export const RefusalBlock = type({ - type: "'refusal'", - // Refusals must carry text — a zero-length reason corrupts the - // "human-readable text the model emitted in lieu of conformant - // output" contract and would round-trip indistinguishably from a - // refusal block whose payload was lost. The arktype constraint is - // belt-and-braces alongside the adapter's wire-boundary filter on - // empty `delta.refusal` chunks: synthetic fixtures or future - // adapters without that filter still cannot construct a vacuous - // refusal. - reason: "string > 0", -}); -export type RefusalBlock = typeof RefusalBlock.infer; -const ToolCallBlock = type({ - type: "'tool_call'", - id: "string", - name: "string", - arguments: "Record", - // Opaque provider signature authenticating this block, echoed back - // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the - // functionCall part; absent otherwise. - "signature?": "string", -}); -/** - * Location of a citation's cited span within its source document. - * The unit of `start` and `end` varies by `kind`: - * - "page": 1-indexed page numbers (Anthropic `page_location`). - * - "char": UTF-16 character offsets, matching JS string semantics - * (Anthropic `char_location`; Gemini `groundingSupports[].segment`). - * - "content-block": index into a structured source's content blocks - * (Anthropic `content_block_location`). - */ -const CitationLocation = type({ - kind: "'page' | 'char' | 'content-block'", - start: "number", - end: "number", -}); - -const CitationSource = type({ - "title?": "string", - // Self-contained dereferenceable URL — populated by providers whose - // citations carry URLs directly (Gemini `groundingChunks[].web.uri`). - "uri?": "string", - // Back-pointer into the request's `documents` array, populated by - // providers that cite uploaded documents by position (Anthropic - // `document_index`). - "documentRef?": type({ index: "number" }), -}); - -/** - * A citation that supports a span of assistant text. Consumers - * receiving a CitationBlock without a paired source-block index MUST - * attribute it by adjacency to the nearest preceding TextBlock in the - * same turn. - * - * Citations are deliberately excluded from ToolResultBlock.content - * — they annotate model output, not tool output. - * - * Exported because `inference.citation` events reference it by name, - * following the same pattern as `AssistantTurn`, `ToolCall`, and - * `ToolResult`. See the `inference.citation` event docstring for how - * a paired source-block index is carried on the wire and consumed by - * the harness. - */ -export const CitationBlock = type({ - type: "'citation'", - // The exact substring of the preceding TextBlock this citation - // supports. Both providers emit it; required for inspection and - // for fallback offset reconstruction. - citedText: "string", - source: CitationSource, - "location?": CitationLocation, - // UTF-16 character offsets into the preceding TextBlock's text. - // Providers that emit offsets natively populate these directly; - // adapters that derive offsets from a cited substring populate - // them only when the substring appears unambiguously in the - // preceding text. Omitted when the offset cannot be determined. - "textOffset?": type({ start: "number", end: "number" }), -}); -export type CitationBlock = typeof CitationBlock.infer; - -/** - * A structured safety signal on model output or request filtering. - * - * The name `SafetyRatingBlock` follows the issue vocabulary; the - * payload is derived from the first real Gemini capture that engaged - * the structured classifier (2026-07-28). That wire shape is - * prompt-level only: - * - * `promptFeedback: { blockReason: "PROHIBITED_CONTENT" }` - * - * with no candidates and no per-category `safetyRatings` arrays. So - * this block carries `blockReason` and does **not** invent category / - * probability / blocked fields. When a future capture surfaces - * candidate-level ratings, extend the type from those bytes rather - * than from the API reference. - * - * Deliberately excluded from ToolResultBlock.content — safety - * signals annotate model/request filtering, not tool output. - * - * Exported because `inference.safety_rating` events reference it by - * name. - */ -export const SafetyRatingBlock = type({ - type: "'safety_rating'", - // Provider-native block reason string (observed: "PROHIBITED_CONTENT"). - // Open string so a new reason token does not force a type bump. - blockReason: "string > 0", -}); -export type SafetyRatingBlock = typeof SafetyRatingBlock.infer; - -/** - * Human-readable rendering of a SafetyRatingBlock for reply text, - * timeline summaries, and request-history rewrites when a provider - * has no input wire shape for safety_rating. Single owner of the - * display string so reply / history / transform stay in lockstep. - */ -export function formatSafetyRatingText(block: SafetyRatingBlock): string { - return `Request blocked: ${block.blockReason}`; -} - -/** - * The model's request to execute code via a server-side execution tool. - * Paired with a CodeExecutionResultBlock carrying the same `id` as the - * result's `requestId`. Streaming order within a single execution is - * `inference.code_execution.start` → zero or more - * `inference.code_execution.delta` → `inference.code_execution.result`, - * uninterrupted by other events that share the same `requestId`; events - * with different `requestId`s or for other block kinds at distinct - * `index`es may interleave. - * - * Exported because `inference.code_execution.start` references it by - * name. - */ -export const CodeExecutionRequestBlock = type({ - type: "'code_execution_request'", - // Identifier for the execution request. Populated from the - // provider's call id where one exists (Anthropic - // `srvtoolu_...`); synthesized by the adapter for providers that - // don't emit one (Gemini), using a deterministic per-response - // position-based scheme so replays match. - id: "string", - // Source code the model is asking to execute. - code: "string", - // Language hint. Absent when the provider does not emit one; - // adapters MUST NOT default this — callers narrow on its - // presence rather than fall through to a guessed language. - "language?": "string", - // Opaque provider signature authenticating this block, echoed back - // verbatim on follow-up turns. Gemini rides a `thoughtSignature` on the - // executableCode part; absent otherwise. - "signature?": "string", -}); -export type CodeExecutionRequestBlock = typeof CodeExecutionRequestBlock.infer; - -/** - * The result of executing a CodeExecutionRequestBlock. The `requestId` - * back-points to the request block's `id`. Status is normalized across - * providers; raw provider signals (return code, native outcome string, - * abort reason) are preserved on optional fields for callers that need - * them. - * - * File outputs from code execution (e.g. generated plots that - * Anthropic returns in `code_execution_tool_result.content`) are NOT - * modeled by this block today. The block carries no field for them; - * surfacing file outputs is a separate concern. - * - * Exported because `inference.code_execution.result` references it by - * name. - */ -export const CodeExecutionResultBlock = type({ - type: "'code_execution_result'", - // Back-pointer to the originating CodeExecutionRequestBlock.id. - requestId: "string", - // Normalized outcome. Translated from provider-specific signals: - // - Anthropic: derived from `return_code` (0 → "ok", non-zero → - // "error") and `abort_reason` (non-null → "aborted" or - // "timeout" per the reason). - // - Gemini: derived from the `outcome` enum - // (OUTCOME_OK → "ok", OUTCOME_FAILED → "error", - // OUTCOME_DEADLINE_EXCEEDED → "timeout", etc.). - status: "'ok' | 'error' | 'aborted' | 'timeout'", - // Standard output. Providers that don't split stdout from stderr - // (Gemini) map their combined `output` here and leave `stderr` empty. - "stdout?": "string", - // Standard error. Empty for providers that don't split. - "stderr?": "string", - // Provider-native numeric return code when available - // (Anthropic `return_code`). Absent for providers whose outcome - // is enum-only (Gemini). - "returnCode?": "number", - // Provider-native outcome string preserved verbatim for callers - // that need the raw signal (Gemini `OUTCOME_OK` / - // `OUTCOME_FAILED` / `OUTCOME_DEADLINE_EXCEEDED` / ...). Absent - // when the provider does not emit one (Anthropic). - "providerOutcome?": "string", - // Human-readable reason populated when status is "aborted" - // (Anthropic `abort_reason`). Absent otherwise. - "abortReason?": "string", -}); -export type CodeExecutionResultBlock = typeof CodeExecutionResultBlock.infer; - -const ToolResultBlock = type({ - type: "'tool_result'", - callId: "string", - // Deliberately narrow: tool results carry user-facing media, not - // CitationBlocks (citations annotate the model's text output), not - // SafetyRatingBlocks (safety signals annotate model/request - // filtering), and not CodeExecution blocks (server-side code - // execution is a distinct lifecycle from the user-tool round-trip). - content: type - .or(TextBlock, ImageBlock, AudioBlock, VideoBlock, DocumentBlock) - .array(), - "detail?": "unknown", - "isError?": "boolean", -}); - -export const ContentBlock = type.or( - TextBlock, - ThinkingBlock, - RedactedThinkingBlock, - RefusalBlock, - ImageBlock, - AudioBlock, - VideoBlock, - DocumentBlock, - CitationBlock, - SafetyRatingBlock, - CodeExecutionRequestBlock, - CodeExecutionResultBlock, - ToolCallBlock, - ToolResultBlock, -); -export type ContentBlock = typeof ContentBlock.infer; - -/** - * A turn in the internal conversation history. The `model` field records - * which provider model produced this turn (present only on assistant - * turns). Used by cross-provider transformation to strip or preserve - * thinking blocks. - * - * (INFERENCE.md § Message Format) - */ -export type ConversationTurn = { - role: "user" | "assistant" | "system"; - content: ContentBlock[]; - model?: string; - timestamp: number; -}; - -/** - * A completed assistant turn returned in `inference.done`. Narrower type - * than ConversationTurn to make the inference boundary explicit. - */ -export const AssistantTurn = type({ - role: "'assistant'", - content: ContentBlock.array(), - model: "string", - timestamp: "number", -}); -export type AssistantTurn = typeof AssistantTurn.infer; - -// --------------------------------------------------------------------------- -// Error Classification (INFERENCE.md § Error Classification) -// --------------------------------------------------------------------------- - -/** - * Classified inference error. The category determines the reactor's default - * response; the director can override per its policy. - * - * (INFERENCE.md § Error Classification) - */ -export const InferenceError = type({ - category: type.enumerated( - "retryable", - "context_overflow", - "credential_failure", - "quota_exhausted", - "fatal", - "aborted", - "timeout", - "protocol_mismatch", - ), - message: "string", - "statusCode?": "number", - "retryAfterMs?": "number", - "raw?": "unknown", -}); -export type InferenceError = typeof InferenceError.infer; - -// --------------------------------------------------------------------------- -// Agent Reactor (INFERENCE.md § Agent Reactor) -// --------------------------------------------------------------------------- - -/** - * Gate types that can block the reactor. - * - * (INFERENCE.md § Gates) - */ -export const GateType = type.enumerated( - "approval", - "payment", - "credential", - "budget", - "child_completion", - "message_response", -); -export type GateType = typeof GateType.infer; - -/** - * Fork mode. `independent` creates a divergent reactor with its own context. - * `child` creates a reactor that reports results back to the parent. - * - * (INFERENCE.md § Forking) - */ -export const ForkMode = type.enumerated("independent", "child"); -export type ForkMode = typeof ForkMode.infer; - -// --------------------------------------------------------------------------- -// Inference Event Protocol (INFERENCE.md § Event Protocol) -// --------------------------------------------------------------------------- - -/** - * Wire-safe representation of InboundMessage for use in InferenceEvent - * variants. The runtime InboundMessage type contains Uint8Array fields - * (MessageAttachment.data) that cannot survive JSON serialization, so the - * wire validator uses `unknown` for attachment data and accepts whatever - * JSON.parse produces. - */ -const WireInboundMessage = type({ - ref: { uid: "number", mailbox: "string" }, - headers: "Record", - flags: "string[]", - "content?": "string", - "payload?": "object", - "attachments?": "unknown[]", - signatureStatus: type.enumerated("valid", "invalid", "unknown", "missing"), -}); - -/** - * A single event in the inference event protocol. Every event carries a - * monotonic session-scoped sequence number. - * - * Event types are namespaced: `inference.*`, `tool.*`, `reactor.*`, - * `fork.*`, `message.*`, `custom.*`. - * - * (INFERENCE.md § Event Protocol) - */ -export const InferenceEvent = type.or( - { - type: "'inference.start'", - seq: "number", - data: { model: "string" }, - }, - { - type: "'inference.thinking.delta'", - seq: "number", - data: { - token: "string", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.block.signature'", - seq: "number", - data: { signature: "string", "index?": "number" }, - }, - { - type: "'inference.thinking.redacted'", - seq: "number", - data: { redactedThinking: RedactedThinkingBlock, "index?": "number" }, - }, - { - type: "'inference.text.delta'", - seq: "number", - data: { - token: "string", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.refusal.delta'", - seq: "number", - data: { - token: "string", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.tool_call.start'", - seq: "number", - data: { - callId: "string", - name: "string", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.tool_call.delta'", - seq: "number", - data: { - callId: "string", - argumentFragment: "string", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.tool_call.end'", - seq: "number", - data: { - callId: "string", - name: "string", - arguments: "Record", - partial: PartialMessage, - "index?": "number", - }, - }, - { - type: "'inference.usage'", - seq: "number", - // Locally patched — see vendor/intx-types/PATCHES.md#types-ts-usage-stop-reason - data: { usage: TokenUsage, source: LastCycleSource, "stopReason?": "string" }, - }, - { - type: "'inference.done'", - seq: "number", - data: { - turn: AssistantTurn, - usage: TokenUsage, - source: LastCycleSource, - "pacingDelayMs?": "number", - }, - }, - { - type: "'inference.error'", - seq: "number", - data: { error: InferenceError, partial: PartialMessage }, - }, - { - type: "'inference.retry'", - seq: "number", - data: { - attempt: "number", - delayMs: "number", - previousError: InferenceError, - }, - }, - { - type: "'inference.citation'", - seq: "number", - // `index`, when present, names the source content block (typically - // a TextBlock) the citation annotates. The harness uses it to - // interleave the citation into the finalized turn's `content[]` - // immediately after the matching block. Adapters whose wire - // protocol does not carry per-citation block indices omit the - // field; the harness then appends those citations at the end of - // `content[]` and consumers attribute them to the nearest - // preceding TextBlock per the CitationBlock docstring. - data: { citation: CitationBlock, "index?": "number" }, - }, - { - type: "'inference.safety_rating'", - seq: "number", - // Prompt-level structured safety signal (observed Gemini - // `promptFeedback.blockReason`). No candidate index: the first - // capture has zero candidates. Harness appends the block to the - // finalized turn's `content[]`. - data: { safetyRating: SafetyRatingBlock }, - }, - { - type: "'inference.code_execution.start'", - seq: "number", - data: { request: CodeExecutionRequestBlock, "index?": "number" }, - }, - { - type: "'inference.code_execution.delta'", - seq: "number", - // requestId correlates fragments back to the originating - // CodeExecutionRequestBlock; index is the positional hint into - // the response's content-block stream. They are independent: a - // single response may stream code execution for multiple - // requests interleaved, distinguished by requestId; index lets - // the harness's per-block accumulator route the fragment to - // the correct block when the array isn't yet finalized. - data: { - requestId: "string", - codeFragment: "string", - "index?": "number", - }, - }, - { - type: "'inference.code_execution.result'", - seq: "number", - data: { result: CodeExecutionResultBlock, "index?": "number" }, - }, - { - type: "'inference.image_output'", - seq: "number", - // Fires mid-stream when an adapter finalizes an image-output - // block, signaling that the image is ready for downstream - // handoff before the full inference.done lands. The wrapped - // ImageBlock typically carries a base64 MediaSource — the - // payload can be large (Gemini's image-output captures show - // ~1MB inline blobs); consumers that subscribe to this event - // should treat it as a non-trivial transport size. - data: { image: ImageBlock, "index?": "number" }, - }, - { - type: "'tool.start'", - seq: "number", - data: { call: ToolCall }, - }, - { - type: "'tool.update'", - seq: "number", - data: { callId: "string", partial: "string" }, - }, - { - type: "'tool.done'", - seq: "number", - data: { result: ToolResult }, - }, - { - type: "'message.queued'", - seq: "number", - data: { message: WireInboundMessage }, - }, - { - type: "'message.run.started'", - seq: "number", - data: { - messageId: "string", - messageRunId: "string", - receivedAt: "number", - }, - }, - { - type: "'message.run.ended'", - seq: "number", - data: { - messageRunId: "string", - messageId: "string", - status: type.enumerated("completed", "failed"), - "error?": { - message: "string", - "kind?": "string", - }, - }, - }, - { - type: "'message.correlated'", - seq: "number", - data: { message: WireInboundMessage, correlationId: "string" }, - }, - { - type: "'connector.reply'", - seq: "number", - data: { content: "string", "checkpointHash?": "string" }, - }, - { - type: "'reactor.start'", - seq: "number", - data: "object", - }, - { - type: "'reactor.gate.blocked'", - seq: "number", - data: { - reason: GateType, - gateId: "string", - "correlationId?": "string", - "approvalSnapshot?": ApprovalSnapshot, - }, - }, - { - type: "'reactor.gate.cleared'", - seq: "number", - data: { - gateId: "string", - reason: type.enumerated("resolved", "timeout", "shutdown"), - }, - }, - { - type: "'reactor.done'", - seq: "number", - data: "object", - }, - { - type: "'reactor.error'", - seq: "number", - data: { error: "string", fatal: "boolean" }, - }, - { - type: "'fork.created'", - seq: "number", - data: { forkId: "string", parentId: "string", mode: ForkMode }, - }, - { - type: "'fork.done'", - seq: "number", - data: { forkId: "string", "result?": "unknown" }, - }, - { - type: "'fork.error'", - seq: "number", - data: { forkId: "string", error: "string" }, - }, - { - type: "'fork.aborted'", - seq: "number", - data: { forkId: "string" }, - }, - { - type: /^custom\./, - seq: "number", - data: "Record", - }, -); -// The TypeScript type is defined manually rather than inferred from the -// validator because the `custom.*` variant uses a regex pattern which -// arktype infers as `string`. A bare `string` in the discriminant position -// prevents TypeScript from narrowing the union in switch statements. -// The manually defined type uses a `custom.${string}` template literal -// for that variant, preserving the narrowing behavior downstream code -// relies on. -export type InferenceEvent = - | { type: "inference.start"; seq: number; data: { model: string } } - | { - type: "inference.thinking.delta"; - seq: number; - data: { token: string; partial: PartialMessage; index?: number }; - } - | { - type: "inference.block.signature"; - seq: number; - data: { signature: string; index?: number }; - } - | { - type: "inference.thinking.redacted"; - seq: number; - data: { redactedThinking: RedactedThinkingBlock; index?: number }; - } - | { - type: "inference.text.delta"; - seq: number; - data: { token: string; partial: PartialMessage; index?: number }; - } - | { - type: "inference.refusal.delta"; - seq: number; - data: { token: string; partial: PartialMessage; index?: number }; - } - | { - type: "inference.tool_call.start"; - seq: number; - data: { - callId: string; - name: string; - partial: PartialMessage; - index?: number; - }; - } - | { - type: "inference.tool_call.delta"; - seq: number; - data: { - callId: string; - argumentFragment: string; - partial: PartialMessage; - index?: number; - }; - } - | { - type: "inference.tool_call.end"; - seq: number; - data: { - callId: string; - name: string; - arguments: Record; - partial: PartialMessage; - index?: number; - }; - } - | { - type: "inference.usage"; - seq: number; - // Locally patched — see vendor/intx-types/PATCHES.md#types-ts-usage-stop-reason - data: { usage: TokenUsage; source: LastCycleSource; stopReason?: string }; - } - | { - type: "inference.done"; - seq: number; - data: { - turn: AssistantTurn; - usage: TokenUsage; - source: LastCycleSource; - pacingDelayMs?: number; - }; - } - | { - type: "inference.error"; - seq: number; - data: { error: InferenceError; partial: PartialMessage }; - } - | { - /** - * Emitted between attempts when the per-call retry policy decides - * to retry after an error. `attempt` is the 1-indexed number of - * the attempt that just **failed** — the same value the policy - * saw on its `RetrySituation.attempt` reading. `delayMs` is the - * delay the wrapper will apply before the next attempt starts; - * `previousError` carries the classified error that triggered - * the retry. The event is not emitted when the policy aborts. - */ - type: "inference.retry"; - seq: number; - data: { - attempt: number; - delayMs: number; - previousError: InferenceError; - }; - } - | { - type: "inference.citation"; - seq: number; - data: { citation: CitationBlock; index?: number }; - } - | { - type: "inference.safety_rating"; - seq: number; - data: { safetyRating: SafetyRatingBlock }; - } - | { - type: "inference.code_execution.start"; - seq: number; - data: { request: CodeExecutionRequestBlock; index?: number }; - } - | { - type: "inference.code_execution.delta"; - seq: number; - data: { requestId: string; codeFragment: string; index?: number }; - } - | { - type: "inference.code_execution.result"; - seq: number; - data: { result: CodeExecutionResultBlock; index?: number }; - } - | { - type: "inference.image_output"; - seq: number; - data: { image: ImageBlock; index?: number }; - } - | { type: "tool.start"; seq: number; data: { call: ToolCall } } - | { - type: "tool.update"; - seq: number; - data: { callId: string; partial: string }; - } - | { type: "tool.done"; seq: number; data: { result: ToolResult } } - | { - type: "message.queued"; - seq: number; - data: { message: InboundMessage }; - } - | { - /** - * Per-message run-bracket open. Emitted by the reactor when it - * dequeues an inbound mail message and begins per-message work. - * - * `messageRunId` is reactor-minted, unique per dequeue. It is - * non-negotiable for crash-replay correlation: the reactor can - * legitimately dequeue the same `messageId` more than once across - * a crash + replay cycle, so two bracket-open events with the - * same `messageId` and no run-id cannot be unambiguously paired - * with their `message.run.ended` counterparts. - */ - type: "message.run.started"; - seq: number; - data: { - messageId: string; - messageRunId: string; - receivedAt: number; - }; - } - | { - /** - * Per-message run-bracket close. Pairs with `message.run.started` - * by `messageRunId`. `messageId` is carried redundantly so log - * readers can correlate without a join against the open event. - * - * The `status` enum is `"completed" | "failed"` only. - * Cancellation lives in the workflow-runtime's - * `CancelRequested` -> `RunFailed` vocabulary, not on the - * reactor's bracket: the reactor does not run a state machine - * and what it observes when cancellation arrives is a harness - * abort, which is structurally `"failed"` with a specific - * `error.kind`. - * - * `error.kind` is documented as one of - * `"inference_error" | "tool_error" | "reactor_fatal" | - * "harness_aborted" | "doom_loop"` initially, extensible as new - * failure categories surface. `"doom_loop"` marks a protective - * break the reactor took on the agent's behalf when the agent - * repeated an identical tool batch past the configured threshold; - * unlike `"reactor_fatal"` it is not an internal fault. - */ - type: "message.run.ended"; - seq: number; - data: { - messageRunId: string; - messageId: string; - status: "completed" | "failed"; - error?: { - message: string; - kind?: string; - }; - }; - } - | { - type: "message.correlated"; - seq: number; - data: { message: InboundMessage; correlationId: string }; - } - | { - type: "connector.reply"; - seq: number; - data: { content: string; checkpointHash?: string }; - } - | { type: "reactor.start"; seq: number; data: Record } - | { - type: "reactor.gate.blocked"; - seq: number; - data: { - reason: GateType; - gateId: string; - correlationId?: string; - approvalSnapshot?: ApprovalSnapshot; - }; - } - | { - type: "reactor.gate.cleared"; - seq: number; - data: { - gateId: string; - reason: "resolved" | "timeout" | "shutdown"; - }; - } - | { type: "reactor.done"; seq: number; data: Record } - | { - type: "reactor.error"; - seq: number; - data: { error: string; fatal: boolean }; - } - | { - type: "fork.created"; - seq: number; - data: { forkId: string; parentId: string; mode: ForkMode }; - } - | { - type: "fork.done"; - seq: number; - data: { forkId: string; result?: unknown }; - } - | { - type: "fork.error"; - seq: number; - data: { forkId: string; error: string }; - } - | { type: "fork.aborted"; seq: number; data: { forkId: string } } - | { - type: `custom.${string}`; - seq: number; - data: Record; - }; - -// Load-bearing drift guards for the dual-maintained `reactor.gate.blocked` -// event. The arktype `InferenceEvent` validator and the hand-written -// `InferenceEvent` type are kept in lockstep by hand (the `custom.*` regex -// variant forces the manual mirror). arktype passes undeclared keys through at -// runtime, so a schema that dropped `approvalSnapshot` would not fail at -// runtime. Projecting the field off each inferred shape makes it load-bearing: -// `tsc` errors if either mirror stops carrying it, mirroring the -// `_persistedSuspendedCall` guard in storage-isogit. -const _arkGateBlockedApprovalSnapshot = ( - data: Extract< - typeof InferenceEvent.infer, - { type: "reactor.gate.blocked" } - >["data"], -): ApprovalSnapshot | undefined => data.approvalSnapshot; -void _arkGateBlockedApprovalSnapshot; - -const _tsGateBlockedApprovalSnapshot = ( - data: Extract["data"], -): ApprovalSnapshot | undefined => data.approvalSnapshot; -void _tsGateBlockedApprovalSnapshot; - -/** - * Validate unknown data as an InferenceEvent. ArkType's regex-based validator - * infers `custom.*` event types as `string`, but the manual InferenceEvent type - * uses a `custom.${string}` template literal for switch narrowing. This function - * centralizes that single unavoidable cast. - */ -export function parseInferenceEvent( - data: unknown, -): InferenceEvent | type.errors { - const result = InferenceEvent(data); - if (result instanceof type.errors) return result; - // eslint-disable-next-line @typescript-eslint/no-unsafe-type-assertion -- arktype regex infers as string; manual type uses template literal - return result as InferenceEvent; -} - -/** - * A pending async operation registered in the reactor's async state. - * Correlates an outbound message (or payment/approval request) to the - * expected inbound response. - * - * (INFERENCE.md § Correlation) - */ -export type PendingOperation = { - correlationId: string; - kind: SignalKind; - expectedFrom?: string; - registeredAt: number; - gateId: string; - /** - * Absolute deadline (epoch ms) for the gate that parks this operation. - * Persisted so that rehydration after a restart re-arms the gate with the - * remaining time against the original deadline rather than restarting the - * countdown. Absent for operations parked with no deadline. - */ - timeoutAt?: number; - /** - * The tool call that was suspended when this operation parked. Captured for - * `kind: "approval"` operations minted from the ask flow so the approved - * call can be re-run on resume. Absent for operations parked by the - * director path (async-tool pending markers), which carry no tool call. - */ - suspendedCall?: ToolCall; - /** - * Approver-facing snapshot of `suspendedCall`, built at the authz `ask` - * branch from the tool definition and the live arguments. A sibling of - * `suspendedCall`, not a widening of it: `suspendedCall` is the re-dispatch - * artifact, this is what the approver decides on. Present only for ask-rail - * operations that carry a `suspendedCall`; absent for async-tool pending - * markers. Threaded through the suspend hops to the hub co-write. - */ - approvalSnapshot?: ApprovalSnapshot; -}; - -/** - * Complete reactor state visible to the director decision function. - * - * `tokenUsage` is the cumulative usage across the session. - * - * `lastCycleUsage` and `lastCycleSource` describe the most recent - * *successful* inference call. They move together: both null before the - * first completion; every `inference.done` sets both atomically. - * `inference.error` does not clear either — the pair always reflects the - * last cycle that produced a well-defined turn and usage. The director's - * `afterInferenceDone` hook fires only on `inference.done`, so policy - * code never observes a torn or stale-vs-fresh window. - * - * The per-cycle values support compaction triggers that key off recent - * input cost rather than session totals, and state-aware policies (cost - * gating, budget caps, governance triggers) that need to attribute - * usage to the source that produced it. - * - * (INFERENCE.md § Agent Reactor › Director Decision Function) - */ -export type ReactorState = { - turns: ConversationTurn[]; - activeForks: { forkId: string; mode: ForkMode }[]; - pendingOperations: PendingOperation[]; - activeGates: { gateId: string; type: GateType; timeoutAt: number }[]; - tokenUsage: TokenUsage; - lastCycleUsage: TokenUsage | null; - lastCycleSource: LastCycleSource | null; - sessionId: string; -}; - -/** - * Actions the director can direct the reactor to take. - * - * (INFERENCE.md § Agent Reactor › Actions) - */ -export type ReactorAction = - | { - type: "infer"; - options?: InferenceOptions; - } - | { - type: "execute_tools"; - calls: ToolCall[]; - parallel?: boolean; - addToHistory?: boolean; - } - | { - type: "suspend"; - gate: { - type: GateType; - gateId: string; - timeoutMs: number; - correlationId?: string; - }; - } - | { - type: "fork"; - mode: ForkMode; - forkId: string; - } - | { - type: "emit"; - eventType: `custom.${string}`; - data: Record; - } - | { - type: "reply"; - content: string; - } - | { type: "checkpoint"; message: string } - | { type: "compact"; compactor: string; reason: string } - | { type: "wait" } - | { type: "done" }; - -/** - * The capabilities object passed to the director. Mirrors the `ReactorAction` - * union — provides a type-safe way for the director to construct actions. - * - * (INFERENCE.md § Agent Reactor › Director Decision Function) - */ -export type ReactorCapabilities = { - infer(options?: InferenceOptions): ReactorAction; - executeTools( - calls: ToolCall[], - parallel?: boolean, - addToHistory?: boolean, - ): ReactorAction; - suspend(gate: { - type: GateType; - gateId: string; - timeoutMs: number; - correlationId?: string; - }): ReactorAction; - fork(mode: ForkMode, forkId: string): ReactorAction; - emit( - eventType: `custom.${string}`, - data: Record, - ): ReactorAction; - reply(content: string): ReactorAction; - checkpoint(message?: string): ReactorAction; - compact(compactor: string, reason: string): ReactorAction; - wait(): ReactorAction; - done(): ReactorAction; -}; - -/** - * The inbound events delivered to the director decision function. - * - * `resume.execute_tools` is raised by the reactor when an approval resolves - * and a parked tool call must be re-run on resume. It carries the calls the - * reactor is about to dispatch so the director can seed its outstanding - * tool-result count before those calls' `tool.done` events arrive — the - * reactor drives the execution, the director counts the results. Without this - * seed the count would sit at zero and the first `tool.done` would drive an - * accidental re-inference off a negative count. - * - * `resume.tool_result` is raised by the reactor when a parked approval ends - * without running its tool — a rejected decision or a gate timeout. It carries - * a synthetic error tool result that answers the parked call so history stays - * well-formed; the director appends it and re-infers exactly once. No tool - * runs, so it seeds no outstanding-result count. - * - * (INFERENCE.md § Agent Reactor › Reactor Structure) - */ -export type ReactorInboundEvent = - | { type: "message.received"; message: InboundMessage } - | { - type: "inference.done"; - turn: AssistantTurn; - usage: TokenUsage; - source: LastCycleSource; - } - | { type: "inference.error"; error: InferenceError; partial: PartialMessage } - | { type: "tool.done"; result: ToolResult } - | { - type: "reactor.gate.cleared"; - gateId: string; - reason: "resolved" | "timeout" | "shutdown"; - } - | { type: "resume.execute_tools"; calls: ToolCall[] } - | { type: "resume.tool_result"; result: ToolResult } - | { type: "abort"; reason: AbortReason }; - -/** - * The core director is a single decision function: given an event and the - * current reactor state, return one or more actions. - * - * If the director throws, the reactor catches the exception, emits - * `reactor.error`, and initiates graceful shutdown. - * - * (INFERENCE.md § Reactor Director › Core Director) - */ -export interface ReactorDirector { - decide( - event: ReactorInboundEvent, - state: ReactorState, - capabilities: ReactorCapabilities, - ): Promise; -} - -// --------------------------------------------------------------------------- -// Director Extension Hooks (INFERENCE.md § Reactor Director › Extension Hooks) -// --------------------------------------------------------------------------- - -/** - * Decision returned by a `BeforeToolExtension`. - * - * - `allow` — the tool proceeds. - * - `block` — the tool is answered with an error result carrying `reason`; - * the call is done. - * - `suspend` — the call is parked awaiting an external decision. The reactor - * registers `gate`, persists `pendingOp`, and does not answer the call: it - * is neither run nor error-completed. `gate.timeoutAt` is the absolute - * deadline (epoch ms) so the reactor can compute the remaining time; the - * `correlationId` on both `gate` and `pendingOp` ties an inbound resolution - * back to the suspension. - */ -export type BeforeToolDecision = - | { type: "allow" } - | { type: "block"; reason: string } - | { - type: "suspend"; - gate: { - type: GateType; - gateId: string; - correlationId: string; - timeoutAt: number; - }; - pendingOp: PendingOperation; - }; - -/** - * Extension that runs before a tool call is executed. Returns a - * `BeforeToolDecision`: `allow` lets the call run, `block` answers it with an - * error result, `suspend` parks it awaiting an external decision. - * - * `grantOneShot` registers a within-cycle bypass token keyed on a - * `ToolCall.id`: the next `beforeTool` for that id skips a suspension it would - * otherwise raise, consuming the token as it does so. It is optional because - * only extensions that can suspend a call have anything to bypass; extensions - * that never suspend omit it. - */ -export interface BeforeToolExtension { - beforeTool( - call: ToolCall, - state: ReactorState, - signal: AbortSignal, - ): Promise; - grantOneShot?(id: string): void; -} - -/** - * Extension that runs after a tool result is produced. Can modify the result - * (redaction, enrichment, audit logging). Extensions run in order. - */ -export interface AfterToolExtension { - afterTool( - result: ToolResult, - call: ToolCall, - state: ReactorState, - signal: AbortSignal, - ): Promise; -} - -// --------------------------------------------------------------------------- -// Context Strategies: Transforms and Compactors -// (INFERENCE.md § Context Management, § Tool Result Lifecycle) -// --------------------------------------------------------------------------- - -/** - * Durable description of a single strategy invocation. Written to the - * per-cycle manifest in the context store so that future operators can - * reconstruct exactly which strategy made which change, with what - * parameters, and why. - * - * - `strategy` is the implementation name (e.g. `"size-cap"`). - * - `version` is the implementation version. Changes to the strategy's - * behavior bump the version so old manifest entries remain unambiguous. - * - `parameters` records the configuration the strategy ran with. - * - `reason` is a short machine-readable cause label - * (e.g. `"exceeded-cap"`, `"overflow-recovery"`). - * - `decisions` records strategy-specific details about what was actually - * done (e.g. the keep count, the spill key, the original byte size). - */ -export const TransformRecord = type({ - strategy: "string", - version: "string", - parameters: "Record", - reason: "string", - decisions: "Record", -}); -export type TransformRecord = typeof TransformRecord.infer; - -/** - * Per-invocation context passed to every `ContextStrategy.apply` call. - * `state` is the reactor's snapshot at the moment the strategy runs; - * `trigger` is a short label describing why the strategy was invoked - * (e.g. `"tool-result-ingest"`, `"pre-inference"`, `"director-request"`). - */ -export interface StrategyContext { - readonly state: ReactorState; - readonly trigger: string; -} - -/** - * Optional blob attachment emitted by a strategy. The reactor writes each - * blob to the context store's working tree via `ContextStore.writeBlob` - * (Phase 2) so the data is durable and migrates with the conversation. - */ -export type StrategyBlob = { - key: string; - bytes: Uint8Array; - contentType?: string; -}; - -/** - * Result returned by `ContextStrategy.apply`. Carries the transformed - * output, a `TransformRecord` describing what happened, and any blobs - * that should be persisted in the context store. - */ -export interface StrategyResult { - output: O; - record: TransformRecord; - blobs?: StrategyBlob[]; -} - -/** - * Generic base interface for content-mutating strategies. The role-specific - * aliases below specialize `I` and `O` for tool-result ingestion, pre- - * inference context shaping, and explicit compaction. - * - * Strategies are pure with respect to the context store: they describe what - * should change via their return value. The reactor decides where to write - * the result (history, prompt, manifest) and which blobs to persist. - */ -export interface ContextStrategy { - readonly name: string; - readonly version: string; - apply(input: I, ctx: StrategyContext): Promise>; -} - -/** - * Runs on each tool result entering history. Output is appended to the - * conversation; any emitted blobs are written to the context store's - * `tool-output/` directory. - */ -export type ToolResultTransform = ContextStrategy< - { call: ToolCall; result: ToolResult }, - ToolResult ->; - -/** - * Runs in order before every inference call, producing the materialized - * prompt. Output is written to `prompt.jsonl` for that cycle; the durable - * history in `turns.jsonl` is left untouched. - * - * (INFERENCE.md § Async State Awareness › Pending Status Injection) - */ -export type ContextTransform = ContextStrategy< - ConversationTurn[], - ConversationTurn[] ->; - -/** - * Named compaction strategy. Registered in a registry on the reactor and - * invoked explicitly via the director's `compact` action. Output overwrites - * `turns.jsonl`; a `TransformRecord` is appended to the manifest. - */ -export type Compactor = ContextStrategy; - -// --------------------------------------------------------------------------- -// Blob Reader (INFERENCE.md § Tool Result Lifecycle) -// --------------------------------------------------------------------------- - -/** - * Read-only capability for resolving `tool-output:///{callId}` URIs to the - * underlying blob bytes. A `ToolResultTransform` that spills oversized tool - * output writes a blob via `ContextStore.writeBlob` and returns a pointer of - * the form `tool-output:///{callId}`; the agent's read tool reaches the spill - * by calling `BlobReader.read(uri)`. - * - * The URI scheme is deliberately rigid: - * - * - Scheme: `tool-output` - * - Authority: empty (the `///` makes pathname carry the callId) - * - Path: `/{callId}` — preserves case so provider-assigned callIds with - * uppercase letters survive parsing - * - Query and fragment: rejected - * - * Any deviation (different scheme, missing or non-empty hostname, extra path - * segments, search string, or fragment) throws. Missing blobs throw. - * `BlobReader` never accepts a filesystem path; the agent has no direct view - * of the context store's working tree. - */ -export interface BlobReader { - /** - * Resolve `uri` to the underlying blob bytes. Throws if the URI is not a - * well-formed `tool-output:///{callId}` reference or if no blob exists for - * the extracted callId. - */ - read(uri: string): Promise; -} - -/** Source for blob bytes used by `createBlobReader`. */ -export interface BlobSource { - readBlob(key: string, signal?: AbortSignal): Promise; -} - -/** - * Parse a `tool-output:///{callId}` URI and return the callId. Throws on any - * deviation from the documented shape: wrong scheme, non-empty authority, - * missing or extra path components, search string, or fragment. - * - * The two-slash form `tool-output://abc` is rejected because the URL parser - * lowercases the hostname, which silently corrupts provider-assigned callIds - * that contain uppercase letters. The three-slash form puts the callId in - * `pathname`, where case is preserved. - */ -export function parseToolOutputURI(uri: string): string { - let parsed: URL; - try { - parsed = new URL(uri); - } catch (cause) { - throw new Error(`invalid tool-output URI: ${uri}`, { cause }); - } - if (parsed.protocol !== "tool-output:") { - throw new Error( - `invalid tool-output URI scheme: expected "tool-output:", got "${parsed.protocol}"`, - ); - } - if (parsed.hostname !== "") { - throw new Error( - `invalid tool-output URI: authority must be empty (use the form tool-output:///{callId}), got "${parsed.hostname}"`, - ); - } - if (parsed.search !== "") { - throw new Error( - `invalid tool-output URI: query string is not allowed, got "${parsed.search}"`, - ); - } - if (parsed.hash !== "") { - throw new Error( - `invalid tool-output URI: fragment is not allowed, got "${parsed.hash}"`, - ); - } - const path = parsed.pathname; - if (!path.startsWith("/")) { - throw new Error(`invalid tool-output URI: empty path: ${uri}`); - } - const callId = path.slice(1); - if (callId === "") { - throw new Error(`invalid tool-output URI: missing callId: ${uri}`); - } - if (callId.includes("/")) { - throw new Error( - `invalid tool-output URI: path must contain a single callId segment, got "${callId}"`, - ); - } - return callId; -} - -/** - * Construct a `BlobReader` that resolves `tool-output:///{callId}` URIs by - * delegating to `source.readBlob(callId)`. The most common source is a - * `ContextStore` (Phase 2 added `readBlob` to that interface), but any object - * implementing `BlobSource` works — this keeps tests trivial. - * - * URI parsing is performed in this layer; the source only ever sees the - * extracted callId. Missing blobs surface as whatever error the source - * raises (`ContextStore.readBlob` already throws for unknown keys). - */ -export function createBlobReader(source: BlobSource): BlobReader { - return { - async read(uri: string): Promise { - const callId = parseToolOutputURI(uri); - return source.readBlob(callId); - }, - }; -} - -// --------------------------------------------------------------------------- -// Abort Reasons (INFERENCE.md § Abort Handling) -// --------------------------------------------------------------------------- - -/** - * Reason codes for the `abort` reactor event. The reason determines the - * appropriate cleanup action. - * - * (INFERENCE.md § Abort Handling › Abort Reasons) - */ -export const AbortReason = type.enumerated( - "user_disconnect", - "wallet_exhaustion", - "admin_kill", - "session_timeout", - "credential_revocation", -); -export type AbortReason = typeof AbortReason.infer; - -// --------------------------------------------------------------------------- -// Inference Source (INFERENCE.md § Providers) -// --------------------------------------------------------------------------- - -/** - * Model-bound default knobs for an inference source. Per-call - * `InferenceOptions.X` overrides `defaults.X`; the merge happens once at - * the top of `runInference` before the adapter sees anything. New fields - * land here as separately-scoped issues. - */ -export const InferenceSourceDefaults = type({ - "maxTokens?": "number", - // A bag of provider-native knobs the caller wants merged into the - // outbound request body (Anthropic's `metadata.user_id`, - // OpenAI's `user`, Gemini's `safetySettings`, etc.). Adapters that - // recognize keys translate; unrecognized keys are passed through or - // dropped per the adapter's documented behavior. The merge into - // per-call `InferenceOptions.providerOptions` is shallow — a per- - // call providerOptions object wholesale replaces the source-bound - // one, it does not deep-merge per key. - "providerOptions?": "Record", -}); -export type InferenceSourceDefaults = typeof InferenceSourceDefaults.infer; - -/** - * A specific (provider, model) bundle the agent runtime can route to. - * Carries wire reachability, credentials, the model identity at the - * provider, and the model-bound default knobs. - * - * `id` is the catalog offering's primary key, set by the resolver from the - * matched offering. It is the routing key used by `AgentConfig.defaultSource` - * and `Agent.setSource`. - * - * Multi-model providers become multiple sources — `model` is part of the - * identity, not an optional override. - * - * `capabilities` is carried for the selection-policy layer (the model - * selector consumes it). The runtime ignores it; populating the field - * later is not a wire-format change. - * - * `quirks` is the opaque per-deployment bag of provider-specific adapter - * accommodations. The harness reads it once, at adapter instantiation, and - * passes it to `AdapterRegistry.resolve` as a sibling of the slim - * `LastCycleSource` — quirks are deliberately kept off `LastCycleSource`, - * which rides on every usage event. The field is present-and-populated or - * absent; it is never `null`. A source row with no quirks stores SQL `NULL`, - * and the catalog resolver translates that absence into an omitted key here, - * so downstream code sees `undefined`, never `null`. - * - * (INFERENCE.md § Providers) - */ -export const InferenceSource = type({ - id: "string", - provider: "string", - baseURL: "string", - // Reference into the run's credential-material cell. The provider's secret - // (formerly an inline `apiKey`) is resolved from that cell by `credentialId` - // at call time, so the source config carries no secret and the child never - // holds the key inline. The same cell backs tool credentials. - credentialId: "string", - model: "string", - "defaults?": InferenceSourceDefaults, - "capabilities?": "string[]", - "quirks?": "Record", -}); -export type InferenceSource = typeof InferenceSource.infer; - -/** - * Replace every field on `active` with the corresponding field from - * `next`, in place. Optional fields (`defaults`, `capabilities`, - * `quirks`) are `delete`d from `active` when absent on `next` so the - * swap is exact — no stale value from a previous rotation can survive. - * - * Used by both the agent's source registry and the harness's source - * hot-swap path to mutate the single shared `InferenceSource` object the - * reactor reads lazily at the start of each inference call. Putting the - * field list in one place means the next field added to - * `InferenceSource` only has to be remembered here. - */ -export function applyInferenceSourceFields( - active: InferenceSource, - next: InferenceSource, -): void { - active.id = next.id; - active.provider = next.provider; - active.baseURL = next.baseURL; - active.credentialId = next.credentialId; - active.model = next.model; - if (next.defaults !== undefined) { - active.defaults = next.defaults; - } else { - delete active.defaults; - } - if (next.capabilities !== undefined) { - active.capabilities = next.capabilities; - } else { - delete active.capabilities; - } - if (next.quirks !== undefined) { - active.quirks = next.quirks; - } else { - delete active.quirks; - } - - // Compile-time exhaustiveness check. `Required<>` forces optional - // keys to also be required in the guard — so a future optional field - // (e.g. `region?: string`) added to `InferenceSource` without being - // handled above is flagged by TypeScript, not silently dropped. - const _handled: { readonly [K in keyof Required]: true } = { - id: true, - provider: true, - baseURL: true, - credentialId: true, - model: true, - defaults: true, - capabilities: true, - quirks: true, - }; - void _handled; -} - -/** - * Outcome of a `RetryPolicy` consultation. Either abort the call - * (surface the most recent `inference.error` to the caller), or retry - * after `delayMs` milliseconds, measured against the harness Scheduler. - * - * (INFERENCE.md § Providers › Streaming Harness) - */ -export type RetryDecision = - | { kind: "abort"; error?: InferenceError } - | { kind: "retry"; delayMs: number }; - -/** - * Context supplied to a `RetryPolicy` each time an attempt produces an - * `inference.error`. - * - * (INFERENCE.md § Providers › Streaming Harness) - */ -export type RetrySituation = { - /** The classified error the most recent attempt produced. */ - readonly error: InferenceError; - /** - * 1-indexed attempt counter. The first failure has `attempt: 1`; - * the second failure (after one retry) has `attempt: 2`; and so on. - */ - readonly attempt: number; - /** - * Milliseconds since the *first* attempt of this call started, - * measured via the harness `Scheduler.now()`. The default Scheduler - * uses `performance.now()` (sub-millisecond resolution), so the - * value may be fractional; virtual-clock test schedulers report - * integer virtual time. Both are valid; policies that compare - * against integer thresholds should `Math.floor` if they need that. - */ - readonly elapsedMs: number; - /** Immutable shallow snapshot of the source selected when this call began. */ - readonly source?: Readonly; - /** 1-indexed within credential failures only; zero for other categories. */ - readonly credentialFailureOrdinal?: number; - /** Credential failures observed earlier in this call. */ - readonly credentialFailureHistory?: readonly InferenceError[]; -}; - -/** - * Per-call retry policy. The harness invokes the policy once per - * `inference.error` an attempt produces, in 1-indexed attempt order. - * Returning `{ kind: "abort" }` ends the call by surfacing the most - * recent error to the caller; returning `{ kind: "retry", delayMs }` - * causes the harness to discard the failed attempt's events, sleep - * `delayMs` milliseconds against the Scheduler, and re-issue the - * underlying HTTP request with the identical body. The policy may be - * async; the harness awaits the returned `Promise` if - * it is a thenable. - * - * (INFERENCE.md § Providers › Streaming Harness) - */ -export type RetryPolicy = (( - situation: RetrySituation, -) => RetryDecision | Promise) & { - readonly normalizeError?: ( - error: InferenceError, - source?: Readonly, - ) => InferenceError; -}; - -/** - * Options for a single inference call. Override the defaults from the agent - * configuration on a per-call basis. - * - * (INFERENCE.md § Providers › Streaming Harness) - */ -export type InferenceOptions = { - maxTokens?: number; - temperature?: number; - thinking?: { enabled: boolean; budgetTokens?: number }; - systemPrompt?: string; - tools?: ToolDefinition[]; - /** - * Modalities the caller wants the model to emit. Adapters translate - * to the provider-native shape (Gemini's - * `generationConfig.responseModalities` accepts `"TEXT"` / `"IMAGE"` - * uppercase; see `packages/inference-discovery-google-genai/sessions/ - * google-genai/gemini-2.5-flash-image/image-output/exchanges/0/request.json` - * for the captured shape). Providers that do not expose a modality - * switch ignore the - * field. When omitted the provider's default modalities apply. - */ - responseModalities?: ("text" | "image" | "audio")[]; - /** - * Structured-output constraint. Asks the model to produce text, free- - * form JSON, or JSON conforming to a specific schema. Adapters - * translate to the provider-native wire shape: - * - * - **OpenAI** (`response_format`): - * - `text` → `{ type: "text" }` - * - `json` → `{ type: "json_object" }` - * - `json-schema` → `{ type: "json_schema", json_schema: { name, schema, strict } }` - * When the model declines in strict mode, the wire emits - * `delta.refusal` chunks; the adapter surfaces them as - * `inference.refusal.delta` events and a final `RefusalBlock` in - * the assistant turn's `content[]`. - * - **Google GenAI** (`generationConfig`): - * - `text` → no constraint (default). - * - `json` → `{ responseMimeType: "application/json" }`. - * - `json-schema` → `{ responseMimeType: "application/json", responseSchema: }`. - * `name` and `strict` are OpenAI-specific and have no Gemini - * counterpart; adapters ignore them. Gemini enforces a subset of - * JSON Schema (no `oneOf`, limited `pattern`, no `$ref`, etc.) — - * the adapter forwards the schema verbatim and surfaces Gemini's - * HTTP error if the subset is violated. - * - **Anthropic**: no native structured-output API. - * - `text` is a no-op (the default). - * - `json` and `json-schema` throw at the adapter boundary; there - * is no shim that synthesizes a tool to extract structured - * output. - * - * When omitted the provider's default applies (typically free-form - * text). - */ - responseFormat?: - | { kind: "text" } - | { kind: "json" } - | { - kind: "json-schema"; - name: string; - schema: unknown; - strict?: boolean; - }; - /** - * A bag of provider-native knobs the adapter merges into the outbound - * request body. Primary home is `InferenceSourceDefaults.providerOptions` - * (model-bound); this field exists for per-call overrides through the - * standard merge precedence at the top of `runInference`. The merge is - * shallow: a per-call providerOptions object wholesale replaces the - * source-bound one, it does not deep-merge per key. - */ - providerOptions?: Record; - /** - * Per-call inactivity timeout in milliseconds. If the harness yields no - * event (other than `inference.start`) for this many ms, the underlying - * fetch is aborted and the call ends with `inference.error` of category - * `"timeout"`. Default 120_000 (2 min). Tune higher for reasoning models - * that exhibit long silent-thinking stretches between token bursts; tune - * lower to fail fast. `0` arms the timer to fire on the next tick (a - * "fail-fast even if the fetch is instant" mode useful in tests). - */ - inactivityTimeoutMs?: number; - /** - * Per-call total wall-clock cap in milliseconds. Starts at fetch. - * Default 600_000 (10 min). Backstop for streams that keep emitting - * forever without terminating. Same error category as `inactivityTimeoutMs`. - * `0` arms the timer to fire on the next tick. - */ - totalTimeoutMs?: number; - /** - * Per-call mechanical retry policy. Consulted once per attempt that - * ends in `inference.error`; see `RetryPolicy` for the contract. If - * omitted, a built-in default policy is applied. - */ - retryPolicy?: RetryPolicy; -}; - -// --------------------------------------------------------------------------- -// Context Store (INFERENCE.md § Context Management › Context Store, -// ARCHITECTURE.md § Change History) -// --------------------------------------------------------------------------- - -/** - * A named commit point in the context store. Corresponds to a git commit. - * - * (ARCHITECTURE.md § Change History › Named Checkpoints) - */ -export type ContextCommit = { - hash: string; - message: string; - timestamp: number; - parentHash?: string; -}; - -/** - * The state of an active connector thread. The connector is one durable - * thread per agent; participants accumulate as they speak. Persisted - * alongside the conversation context so the thread survives sidecar - * restarts. - * - * `replyTo` is the most recent speaker — the primary recipient (`to`) - * on the next outbound reply. `cc` is every other participant who has - * spoken on the thread, deduplicated, in arrival order — they ride as - * `cc` on the next outbound reply so everyone stays in the loop. - * `subject` is set when the thread starts and preserved for its life. - * - * Defined as an arktype so the wire layer (sidecar↔hub frames) and - * other parsing boundaries can validate snapshots without - * re-declaring the shape. - */ -export const ConnectorThreadState = type({ - threadRoot: "string", - lastMessageId: "string", - replyTo: "string", - cc: "string[]", - "subject?": "string", -}); -export type ConnectorThreadState = typeof ConnectorThreadState.infer; - -/** - * The context store interface. Implementations back the store with git - * (filesystem, in-memory, or virtual) depending on the execution environment. - * The reactor accepts any implementation that satisfies this interface. - * - * The store holds the turn history and reactor metadata. Forking creates - * a git branch. Compaction commits the compacted history. - * - * (INFERENCE.md § Context Management › Context Store) - */ -export interface ContextStore { - /** - * Load the current turn history and reactor metadata from the store. - * Called during reactor initialization. - */ - load(signal?: AbortSignal): Promise<{ - turns: ConversationTurn[]; - pendingOperations: PendingOperation[]; - tokenUsage: TokenUsage; - connectorState: ConnectorThreadState | null; - }>; - - /** - * Buffer connector thread state for the next commit. The harness calls - * this before each checkpoint so that connector state is persisted - * atomically with the conversation context. - */ - setConnectorState(state: ConnectorThreadState | null): void; - - /** - * Commit whatever currently lives in the working tree, using the supplied - * commit message. The reactor's per-cycle checkpoint routes through this - * overload after writing the per-cycle files via `writeTurns`, - * `writePrompt`, `writeResponse`, `writeManifest`, and any `writeBlob` - * calls produced by transforms. - */ - commit( - options: { message: string }, - signal?: AbortSignal, - ): Promise; - - /** - * Create a branch for a fork operation. The branch starts from the current - * HEAD commit. - */ - branch(name: string, signal?: AbortSignal): Promise; - - /** - * List recent commits. Used by the agent's history query tools. - */ - log(limit?: number, signal?: AbortSignal): Promise; - - /** - * Read the turn history at a specific commit hash. Used for history - * inspection and rollback. - */ - readAt(hash: string, signal?: AbortSignal): Promise; - - /** - * Write an opaque blob to the working tree under `tool-output/`. Used by - * `ToolResultTransform`s that spill oversized payloads out of the inline - * conversation. The file is staged at the next `commit({ message })`. - * - * `key` is sanitized for filesystem safety; callers should pass the tool - * call id. `contentType` selects a file extension when known. - */ - writeBlob( - key: string, - bytes: Uint8Array, - contentType?: string, - signal?: AbortSignal, - ): Promise; - - /** - * Read a blob previously written via `writeBlob`. Throws if no blob with - * that key exists. - */ - readBlob(key: string, signal?: AbortSignal): Promise; - - /** - * Overwrite `prompt.jsonl` with the materialized prompt for the current - * inference cycle. One `ConversationTurn` per line. Staged at the next - * `commit({ message })`. - */ - writePrompt(turns: ConversationTurn[], signal?: AbortSignal): Promise; - - /** - * Overwrite `response.jsonl` with the assistant turn returned for the - * current cycle. Single-line JSONL for consistency with the per-cycle file - * conventions. Staged at the next `commit({ message })`. - */ - writeResponse(turn: AssistantTurn, signal?: AbortSignal): Promise; - - /** - * Overwrite `manifest.jsonl` with the ordered transform records produced - * for the current cycle. One `TransformRecord` per line. Staged at the - * next `commit({ message })`. - */ - writeManifest( - records: TransformRecord[], - signal?: AbortSignal, - ): Promise; - - /** - * Overwrite `turns.jsonl` with the durable conversation history. One - * `ConversationTurn` per line. Staged at the next `commit({ message })`. - */ - writeTurns(turns: ConversationTurn[], signal?: AbortSignal): Promise; - - /** - * Overwrite `metadata.json` with non-turn-shaped reactor state needed for - * restart: pending async operations and cumulative token usage. The store - * combines this with the most recently buffered connector state (from - * `setConnectorState`) and writes the merged payload. Staged at the next - * `commit({ message })`. - */ - writeMetadata( - metadata: { - pendingOperations: PendingOperation[]; - tokenUsage: TokenUsage; - }, - signal?: AbortSignal, - ): Promise; - - /** - * Read manifest entries from the most recent `limit` commits that contain - * a `manifest.jsonl`. Newest commit first; records within a commit are - * returned in their natural in-file order (chronological per-cycle). - */ - readManifestHistory( - limit: number, - signal?: AbortSignal, - ): Promise; -} - -// --------------------------------------------------------------------------- -// Audit Store (INTR-4 § Audit Trail) -// --------------------------------------------------------------------------- - -/** - * Persistent store for tool invocation audit records. Separated from - * ContextStore so the audit capability is opt-in at the composition - * layer. The isogit implementation writes audit records as individual - * JSON files in the same git repo used for context storage. - */ -export interface AuditStore { - /** - * Persist a batch of audit records. Called at checkpoint boundaries - * with all records accumulated since the last checkpoint. - */ - commitAudit(records: AuditRecord[], signal?: AbortSignal): Promise; - - /** - * Load audit records for a session. Returns all records matching - * the given sessionId, ordered by seq. - */ - loadAudit(sessionId: string, signal?: AbortSignal): Promise; - - /** - * Persist a batch of error records. Called at checkpoint boundaries - * and shutdown with all error records accumulated since the last flush. - */ - commitErrors(records: ErrorRecord[], signal?: AbortSignal): Promise; - - /** - * Load error records for a session. Returns all records matching - * the given sessionId, ordered by seq. - * - * Locally patched — see vendor/intx-types/PATCHES.md#runtime-ts-audit-store-load-errors - */ - loadErrors(sessionId: string, signal?: AbortSignal): Promise; -} - -// --------------------------------------------------------------------------- -// Agent / Harness Configuration (ARCHITECTURE.md § Agent Harness) -// --------------------------------------------------------------------------- - -/** - * Configured tool definition exposed to the model. The harness registers - * available tools; the reactor passes this list to the inference provider as - * part of each request. - * - * (ARCHITECTURE.md § Agent Harness › Tools) - */ -export const ToolDefinition = type({ - name: "string", - description: "string", - inputSchema: "Record", -}); -export type ToolDefinition = typeof ToolDefinition.infer; - -/** - * Agent harness configuration. Assembled from the agent definition package - * and capability grants during harness initialization. - * - * `principalId` is the agent's principal in the hub's authorization model. - * The sidecar needs it to reconstruct the in-memory grant store on restart - * (the store's `collectGrants` filters by principal). - * - * `grants` uses `WireGrantRule` because this type arrives over JSON where - * `GrantRule.expiresAt` is serialized as a string. The wire validator - * coerces strings back to Date instances. - * - * (ARCHITECTURE.md § Agent Harness) - */ -export const HarnessConfig = type({ - sessionId: "string", - agentId: "string", - tenantId: "string", - principalId: "string", - agentAddress: "string", - systemPrompt: "string", - tools: ToolDefinition.array(), - grants: WireGrantRule.array(), - sources: InferenceSource.array(), - defaultSource: "string", - "sessionChannelEnabled?": "boolean", -}); -export type HarnessConfig = typeof HarnessConfig.infer; +// Runtime contracts and the inference event wire protocol. Process-specific +// consumers can import runtime-core without constructing the event union. +export * from "./runtime-core"; +export * from "./inference-events"; +// Compat re-export. Locally patched — see vendor/intx-types/PATCHES.md#vendor-types-ts-runtime-format-safety-rating-compat +// Upstream 4cc6972 moved `formatSafetyRatingText` to `format-safety-rating`, +// but the published `@corbits/openai-responses@0.2.1` dist imports it from +// the `./runtime` subpath; npm has no newer release importing the new +// subpath. Remove when a republished @corbits/openai-responses compiles +// against `@intx/types/format-safety-rating`. +export { formatSafetyRatingText } from "./format-safety-rating"; diff --git a/vendor/intx-types/src/sessions.test.ts b/vendor/intx-types/src/sessions.test.ts index 6800d6c96..ec35fcc32 100644 --- a/vendor/intx-types/src/sessions.test.ts +++ b/vendor/intx-types/src/sessions.test.ts @@ -79,6 +79,11 @@ describe("AttachmentError schema", () => { mimeType: "image/tiff", }, { code: "malformed_base64", message: "bad", attachmentIndex: 2 }, + { + code: "invalid_encoding", + message: "not text", + attachmentIndex: 2, + }, { code: "invalid_attachment_name", message: "bad name", @@ -90,6 +95,12 @@ describe("AttachmentError schema", () => { totalBytes: 99, limitBytes: 30, }, + { + code: "too_many_attachments", + message: "too many", + count: 21, + limit: 20, + }, ]; for (const variant of variants) { expect(AttachmentError(variant) instanceof type.errors).toBe(false); diff --git a/vendor/intx-types/src/sessions.ts b/vendor/intx-types/src/sessions.ts index 8296b76ea..04be03c43 100644 --- a/vendor/intx-types/src/sessions.ts +++ b/vendor/intx-types/src/sessions.ts @@ -91,11 +91,12 @@ export const MailResponse = type({ }); export type MailResponse = typeof MailResponse.infer; -// Structured attachment-rejection errors returned by POST /:runId/mail. -// Each variant carries a machine-actionable `code` plus the fields a client -// needs to locate and explain the rejection, alongside a human-readable -// `message`. This is the wire contract for the route's attachment 400s; the -// route handler is the single producer. +// Structured attachment-rejection errors. Each variant carries a +// machine-actionable `code` plus the fields a client needs to locate and +// explain the rejection, alongside a human-readable `message`. The shared +// validator emits every variant but `invalid_encoding`, which only the +// mail tools produce; the mail routes return the union as their +// attachment 400 body and the mail tools return its `code` and `message`. export const AttachmentError = type.or( { code: "'oversize_attachment'", @@ -120,12 +121,23 @@ export const AttachmentError = type.or( message: "string", attachmentIndex: "number", }, + { + code: "'invalid_encoding'", + message: "string", + attachmentIndex: "number", + }, { code: "'oversize_total'", message: "string", totalBytes: "number", limitBytes: "number", }, + { + code: "'too_many_attachments'", + message: "string", + count: "number", + limit: "number", + }, ); export type AttachmentError = typeof AttachmentError.infer; diff --git a/vendor/intx-types/src/sidecar-allocation.ts b/vendor/intx-types/src/sidecar-allocation.ts index 6a5a4cf7e..f830617cf 100644 --- a/vendor/intx-types/src/sidecar-allocation.ts +++ b/vendor/intx-types/src/sidecar-allocation.ts @@ -12,9 +12,14 @@ export const sidecarAllocationStatuses = [ export type SidecarAllocationStatus = (typeof sidecarAllocationStatuses)[number]; +export type DispatchableSidecarAllocationStatus = Extract< + SidecarAllocationStatus, + "pending" | "provisioning" | "allocated" | "replacing" +>; + export function isSidecarAllocationDispatchable( status: SidecarAllocationStatus, -): boolean { +): status is DispatchableSidecarAllocationStatus { switch (status) { case "pending": case "provisioning": diff --git a/vendor/intx-types/src/sidecar.test.ts b/vendor/intx-types/src/sidecar.test.ts index 5b99553cd..f84d6855f 100644 --- a/vendor/intx-types/src/sidecar.test.ts +++ b/vendor/intx-types/src/sidecar.test.ts @@ -3,11 +3,16 @@ import { type } from "arktype"; import { APPROVAL_SNAPSHOT_MAX_BYTES } from "./runtime"; import { AgentDeployFrame, + fitDeploymentError, CredentialsUpdateFrame, DeployApplyErrorCategory, FrozenApprovalBundle, HubFrame, - MAX_AGENT_ADDRESSES_FRAME, + MAX_DEPLOYMENT_ADDRESS_LENGTH, + MAX_DEPLOYMENT_ERROR_LENGTH, + MAX_REF_TIP_ENTRY_LENGTH, + MAX_REF_TIPS_FRAME, + MAX_SIDECAR_INCARNATIONS, MAX_CACHED_SENDER_ADDRESSES_FRAME, MAX_CREDENTIAL_REVOCATIONS_FRAME, MAX_MAIL_ADDRESSES_FRAME, @@ -17,8 +22,7 @@ import { MailOutboundFrame, PackRejectFrame, PackRejectReason, - ReconnectFrame, - RegisterFrame, + HelloFrame, RunGrantsFrame, SenderKeyEvictFrame, SidecarFrame, @@ -31,6 +35,7 @@ describe("MailOutboundFrame sender ownership claim", () => { const frame = { type: "mail.outbound", senderAddress: "run_sender@example.test", + generation: 1, rawMessage: "bWFpbA==", recipients: ["recipient@example.test"], }; @@ -145,7 +150,9 @@ describe("AgentDeployFrame", () => { const trivialFrame = { type: "agent.deploy" as const, + requestId: "req_1", agentAddress: "agt_1@example.test", + generation: 1, agentId: "agt_1", config: baseConfig, hubPublicKey: "hub_pubkey_hex", @@ -230,6 +237,7 @@ describe("SourcesUpdateFrame", () => { type: "sources.update" as const, requestId: "req_1", agentAddress: "agt_1@example.test", + generation: 1, defaultSource: "src_a", }; @@ -264,6 +272,7 @@ describe("CredentialsUpdateFrame", () => { type: "credentials.update" as const, requestId: "req_1", agentAddress: "agt_1@example.test", + generation: 1, }; test("accepts a well-formed delivery", () => { @@ -307,6 +316,7 @@ describe("SignalCorrelationRegisterFrame snapshot requirement", () => { runId: "run-1", anchorRunId: "dep-1", agentAddress: "run_dep@integration.interchange", + generation: 1, kind: "approval", }; const snapshot = { @@ -365,6 +375,7 @@ describe("CredentialsUpdateFrame revoke", () => { type: "credentials.update", requestId: "req_1", agentAddress: "dep@integration.interchange", + generation: 1, delivery: { bindings: [], materials: [] }, revoke: ["cred_x"], }; @@ -393,6 +404,7 @@ describe("CredentialsUpdateFrame revoke", () => { type: "credentials.update", requestId: "req_1", agentAddress: "dep@integration.interchange", + generation: 1, delivery: { bindings: [], materials: [] }, }); if (out instanceof type.errors) { @@ -424,7 +436,11 @@ describe("RunGrantsFrame senderIdentities co-delivery", () => { test("the HubFrame union admits a run.grants frame carrying identities", () => { // The sidecar parses inbound frames through the HubFrame union, so the // co-delivered keys must reach the run.grants member and round-trip. - const out = HubFrame({ ...base, senderIdentities: identities }); + const out = HubFrame({ + ...base, + generation: 1, + senderIdentities: identities, + }); if (out instanceof type.errors) { throw new Error(`expected a valid HubFrame: ${out.summary}`); } @@ -434,6 +450,13 @@ describe("RunGrantsFrame senderIdentities co-delivery", () => { expect(out.senderIdentities).toEqual(identities); }); + test("the HubFrame union requires the incarnation a run.grants names", () => { + // The payload schema also validates grants away from the wire, where no + // generation applies, so only the union requires it. + expect(RunGrantsFrame(base) instanceof type.errors).toBe(false); + expect(HubFrame(base) instanceof type.errors).toBe(true); + }); + test("the HubFrame union rejects a malformed identity entry", () => { // arktype passes undeclared keys through unchanged, so a valid-input // round-trip alone cannot prove the field is declared on the wire path: @@ -485,43 +508,199 @@ describe("frame array-length ceilings", () => { const addresses = (n: number) => Array.from({ length: n }, (_, i) => `addr-${String(i)}@example.test`); - describe("RegisterFrame agentAddresses", () => { - const base = { type: "register", sidecarId: "sc-1", token: "tok" }; + const incarnations = (n: number) => + addresses(n).map((address) => ({ + address, + generation: 1, + state: "live", + })); + + describe("HelloFrame incarnations", () => { + const base = { type: "hello", sidecarId: "sc-1", token: "tok" }; test("accepts a frame at the ceiling", () => { const frame = { ...base, - agentAddresses: addresses(MAX_AGENT_ADDRESSES_FRAME), + incarnations: incarnations(MAX_SIDECAR_INCARNATIONS), }; - expect(RegisterFrame(frame) instanceof type.errors).toBe(false); + expect(HelloFrame(frame) instanceof type.errors).toBe(false); expect(SidecarFrame(frame) instanceof type.errors).toBe(false); }); test("rejects a frame past the ceiling through the union", () => { const frame = { ...base, - agentAddresses: addresses(MAX_AGENT_ADDRESSES_FRAME + 1), + incarnations: incarnations(MAX_SIDECAR_INCARNATIONS + 1), }; - expect(RegisterFrame(frame) instanceof type.errors).toBe(true); + expect(HelloFrame(frame) instanceof type.errors).toBe(true); expect(SidecarFrame(frame) instanceof type.errors).toBe(true); }); + + test("rejects an incarnation without a whole, non-negative generation", () => { + for (const generation of [-1, 1.5, "1"]) { + const frame = { + ...base, + incarnations: [ + { address: "wf@example.test", generation, state: "live" }, + ], + }; + expect(HelloFrame(frame) instanceof type.errors).toBe(true); + } + }); + + test("rejects a frame that names an address twice", () => { + const frame = { + ...base, + incarnations: [ + { address: "wf@example.test", generation: 1, state: "live" }, + { address: "wf@example.test", generation: 2, state: "deploying" }, + ], + }; + expect(HelloFrame(frame) instanceof type.errors).toBe(true); + expect(SidecarFrame(frame) instanceof type.errors).toBe(true); + }); + + test("rejects an incarnation in an unknown state", () => { + const frame = { + ...base, + incarnations: [ + { address: "wf@example.test", generation: 1, state: "paused" }, + ], + }; + expect(HelloFrame(frame) instanceof type.errors).toBe(true); + }); + + test("carries why a stopped incarnation stopped, when it says", () => { + const stopped = (error?: string) => ({ + ...base, + incarnations: [ + { + address: "wf@example.test", + generation: 1, + state: "stopped", + ...(error !== undefined ? { error } : {}), + }, + ], + }); + expect(HelloFrame(stopped()) instanceof type.errors).toBe(false); + expect( + HelloFrame(stopped("The child ended itself")) instanceof type.errors, + ).toBe(false); + expect(HelloFrame(stopped("")) instanceof type.errors).toBe(true); + }); }); - describe("RegisterFrame cachedSenderAddresses", () => { + describe("what a sidecar reports about a deployment", () => { + const address = (length: number) => + `run_${"a".repeat(length - "run_@example.test".length)}@example.test`; + const hello = (incarnation: Record) => ({ + type: "hello", + sidecarId: "sc-1", + token: "tok", + incarnations: [ + { address: "wf@example.test", generation: 1, state: "stopped" }, + ].map((base) => ({ ...base, ...incarnation })), + }); + const stopped = (fields: Record) => ({ + type: "deployment.stopped", + agentAddress: "wf@example.test", + generation: 1, + error: "The child ended itself", + ...fields, + }); + const tips = (refs: number, nameLength = 20) => + Object.fromEntries( + Array.from({ length: refs }, (_, i) => [ + `refs/heads/${String(i)}`.padEnd(nameLength, "x"), + "a".repeat(40), + ]), + ); + + test("bounds an incarnation's address and error", () => { + const rejects = (fields: Record) => + SidecarFrame(hello(fields)) instanceof type.errors; + expect(rejects({ address: address(MAX_DEPLOYMENT_ADDRESS_LENGTH) })).toBe( + false, + ); + expect( + rejects({ address: address(MAX_DEPLOYMENT_ADDRESS_LENGTH + 1) }), + ).toBe(true); + expect(rejects({ error: "x".repeat(MAX_DEPLOYMENT_ERROR_LENGTH) })).toBe( + false, + ); + expect( + rejects({ error: "x".repeat(MAX_DEPLOYMENT_ERROR_LENGTH + 1) }), + ).toBe(true); + }); + + test("bounds a stop report's address, error and ref tips", () => { + const rejects = (fields: Record) => + SidecarFrame(stopped(fields)) instanceof type.errors; + expect(rejects({ refTips: tips(MAX_REF_TIPS_FRAME) })).toBe(false); + expect( + rejects({ agentAddress: address(MAX_DEPLOYMENT_ADDRESS_LENGTH + 1) }), + ).toBe(true); + expect( + rejects({ error: "x".repeat(MAX_DEPLOYMENT_ERROR_LENGTH + 1) }), + ).toBe(true); + expect(rejects({ refTips: tips(MAX_REF_TIPS_FRAME + 1) })).toBe(true); + expect(rejects({ refTips: tips(1, MAX_REF_TIP_ENTRY_LENGTH + 1) })).toBe( + true, + ); + expect( + rejects({ + refTips: { + "refs/heads/events": "a".repeat(MAX_REF_TIP_ENTRY_LENGTH + 1), + }, + }), + ).toBe(true); + }); + + test.each(["agent.deploy.error", "agent.undeploy.error"])( + "bounds an %s's error", + (frameType) => { + const rejects = (error: string) => + SidecarFrame({ + type: frameType, + requestId: "req-1", + agentAddress: "wf@example.test", + generation: 1, + error, + }) instanceof type.errors; + expect(rejects("x".repeat(MAX_DEPLOYMENT_ERROR_LENGTH))).toBe(false); + expect(rejects("x".repeat(MAX_DEPLOYMENT_ERROR_LENGTH + 1))).toBe(true); + }, + ); + + test("fits a long error to the bound without splitting a surrogate pair", () => { + expect(fitDeploymentError("short")).toBe("short"); + const fitted = fitDeploymentError( + "x".repeat(MAX_DEPLOYMENT_ERROR_LENGTH + 100), + ); + expect(fitted).toHaveLength(MAX_DEPLOYMENT_ERROR_LENGTH); + expect(fitted.endsWith("...")).toBe(true); + const split = fitDeploymentError( + `${"x".repeat(MAX_DEPLOYMENT_ERROR_LENGTH - 4)}\u{1F600}${"y".repeat(10)}`, + ); + expect(split).toBe(`${"x".repeat(MAX_DEPLOYMENT_ERROR_LENGTH - 4)}...`); + }); + }); + + describe("HelloFrame cachedSenderAddresses", () => { const base = { - type: "register", + type: "hello", sidecarId: "sc-1", token: "tok", - agentAddresses: ["wf@example.test"], + incarnations: incarnations(1), }; test("accepts a count above the resync handler cap but within the ceiling", () => { // The ceiling sits far above the hub-sessions `MAX_RESYNC_SENDER_ADDRESSES` // handler cap (2048) so a report over that cap still parses and reaches the // handler's graceful "resync the first N, log the overflow" degrade rather - // than dropping the whole register frame and stalling the reconnect. + // than dropping the whole hello and stalling the reconnect. const frame = { ...base, cachedSenderAddresses: addresses(2049) }; - expect(RegisterFrame(frame) instanceof type.errors).toBe(false); + expect(HelloFrame(frame) instanceof type.errors).toBe(false); expect(SidecarFrame(frame) instanceof type.errors).toBe(false); }); @@ -530,7 +709,7 @@ describe("frame array-length ceilings", () => { ...base, cachedSenderAddresses: addresses(MAX_CACHED_SENDER_ADDRESSES_FRAME + 1), }; - expect(RegisterFrame(frame) instanceof type.errors).toBe(true); + expect(HelloFrame(frame) instanceof type.errors).toBe(true); expect(SidecarFrame(frame) instanceof type.errors).toBe(true); }); }); @@ -539,6 +718,7 @@ describe("frame array-length ceilings", () => { const base = { type: "mail.outbound", senderAddress: "sender@example.test", + generation: 1, rawMessage: "bWFpbA==", }; @@ -619,30 +799,16 @@ describe("frame array-length ceilings", () => { // regression that mechanical change risks is losing optionality (the key // becomes required) or gaining a lower bound (an empty array is rejected). describe("bounded optional fields stay optional", () => { - test("RegisterFrame validates with cachedSenderAddresses omitted or empty", () => { - const base = { - type: "register", - sidecarId: "sc-1", - token: "tok", - agentAddresses: ["wf@example.test"], - }; - expect(RegisterFrame(base) instanceof type.errors).toBe(false); - expect( - RegisterFrame({ ...base, cachedSenderAddresses: [] }) instanceof - type.errors, - ).toBe(false); - }); - - test("ReconnectFrame validates with cachedSenderAddresses omitted or empty", () => { + test("HelloFrame validates with cachedSenderAddresses omitted or empty", () => { const base = { - type: "reconnect", + type: "hello", sidecarId: "sc-1", token: "tok", - agentAddresses: ["wf@example.test"], + incarnations: [], }; - expect(ReconnectFrame(base) instanceof type.errors).toBe(false); + expect(HelloFrame(base) instanceof type.errors).toBe(false); expect( - ReconnectFrame({ ...base, cachedSenderAddresses: [] }) instanceof + HelloFrame({ ...base, cachedSenderAddresses: [] }) instanceof type.errors, ).toBe(false); }); @@ -651,6 +817,7 @@ describe("frame array-length ceilings", () => { const frame = { type: "mail.outbound", senderAddress: "sender@example.test", + generation: 1, rawMessage: "bWFpbA==", recipients: ["recipient@example.test"], to: [], @@ -664,6 +831,7 @@ describe("frame array-length ceilings", () => { type: "credentials.update", requestId: "req_1", agentAddress: "dep@example.test", + generation: 1, delivery: { bindings: [], materials: [] }, revoke: [], }; @@ -676,6 +844,7 @@ describe("frame array-length ceilings", () => { type: "credentials.update", requestId: "req_1", agentAddress: "dep@example.test", + generation: 1, delivery: { bindings: [], materials: [] }, }; @@ -717,6 +886,35 @@ describe("frame payload byte limits", () => { }); }); +describe("DeploymentStoppedFrame", () => { + const frame = { + type: "deployment.stopped" as const, + agentAddress: "wf@example.test", + generation: 1, + error: "Its workflow child ended itself", + refTips: { "refs/heads/main": "c".repeat(40), "refs/heads/events": null }, + }; + + test("the SidecarFrame union admits a stop report with its history tips", () => { + const out = SidecarFrame(frame); + if (out instanceof type.errors) { + throw new Error(`expected a valid SidecarFrame: ${out.summary}`); + } + expect(out).toEqual(frame); + }); + + test("admits a report whose tips could not be read", () => { + const { refTips: _refTips, ...withoutTips } = frame; + expect(SidecarFrame(withoutTips) instanceof type.errors).toBe(false); + }); + + test("refuses a report without why", () => { + expect(SidecarFrame({ ...frame, error: "" }) instanceof type.errors).toBe( + true, + ); + }); +}); + describe("SenderKeyEvictFrame", () => { const frame = { type: "sender.key.evict", diff --git a/vendor/intx-types/src/sidecar.ts b/vendor/intx-types/src/sidecar.ts index 65ace287d..8743d11f7 100644 --- a/vendor/intx-types/src/sidecar.ts +++ b/vendor/intx-types/src/sidecar.ts @@ -8,6 +8,8 @@ // efficient but JSON is simpler to debug and inspect. import { type } from "arktype"; +import { CredentialDelivery } from "./credential-delivery"; +import { RepoId } from "./repo"; import { GrantWalkSnapshot } from "./grant-snapshot"; import { ApprovalItem } from "./grants"; import { WireGrantRule } from "./grant-wire"; @@ -39,17 +41,42 @@ import { WorkflowDefinitionSource } from "./workflow-sources"; // and routes through the existing invalid-frame drop+log path; no handler change // is needed. -// A sidecar's reported agent addresses. The register/reconnect handler already -// gates each reported address against the allocation's single minted workflow -// address, so the legitimate count is ~1; this is a generous absurdity backstop. -export const MAX_AGENT_ADDRESSES_FRAME = 512; +// The most deployment incarnations one sidecar holds, which bounds those a +// `hello` reports and a `welcome` routes back. The Hub places no deployment on +// a sidecar that already hosts this many, and a sidecar refuses a deploy past +// it, so every hello it sends stays within the bound. +export const MAX_SIDECAR_INCARNATIONS = 128; + +// Bounds on what a sidecar reports about a deployment it holds. The Hub keeps +// a reported stop until it fails the deployment, and stores a deploy error on +// each of the deployment's runs, so these, not the frame-size limit, bound +// what one sidecar can make it hold. A deployment address is a mail address +// of at most 320 characters, and its history has a few refs. +export const MAX_DEPLOYMENT_ADDRESS_LENGTH = 320; +export const MAX_DEPLOYMENT_ERROR_LENGTH = 4096; +export const MAX_REF_TIPS_FRAME = 16; +export const MAX_REF_TIP_ENTRY_LENGTH = 255; + +/** + * Cuts an error a sidecar reports to `MAX_DEPLOYMENT_ERROR_LENGTH`, so a long + * error never makes the frame carrying it invalid. + */ +export function fitDeploymentError(error: string): string { + if (error.length <= MAX_DEPLOYMENT_ERROR_LENGTH) return error; + // A cut between the two halves of a surrogate pair would leave a code + // unit the Hub stores as a replacement character. + let end = MAX_DEPLOYMENT_ERROR_LENGTH - 3; + const last = error.charCodeAt(end - 1); + if (last >= 0xd800 && last <= 0xdbff) end -= 1; + return `${error.slice(0, end)}...`; +} // A sidecar's reported cached sender addresses. This MUST stay well above the // `MAX_RESYNC_SENDER_ADDRESSES` handler cap (currently 2048 in the hub-sessions // sidecar-handler): that cap drives a graceful "resync the first N, log the // overflow" degrade rather than dropping the frame, so a schema ceiling at or // below it would turn the degrade into a hard reconnect outage -- the whole -// register frame would fail this parse and drop, and the sidecar could not +// hello frame would fail this parse and drop, and the sidecar could not // reconnect. The `@intx/types` package must not import from `@intx/hub-sessions`, // so the coupling is a documented invariant guarded by a test in that package. export const MAX_CACHED_SENDER_ADDRESSES_FRAME = 65536; @@ -106,92 +133,134 @@ export const MAX_SIDECAR_FRAME_BYTES = MAX_MAIL_OUTBOUND_BODY_BYTES + FRAME_OVERHEAD_BYTES; // --------------------------------------------------------------------------- -// Sidecar → Hub +// Deployment incarnations // --------------------------------------------------------------------------- +// +// One connection can carry several deployments, and an allocation's +// generation advances while its sidecar may still hold or send for the earlier +// one, as when the Hub releases a deployment its sidecar keeps running. An +// incarnation is a deployment address plus the allocation generation that owns +// it. Every frame +// that concerns one deployment names its incarnation, except +// `connector.state.changed` and replies matched to their request by an id, so +// neither end infers it from the socket, the order of frames, or their timing, +// and each end drops or refuses a frame for an incarnation it does not hold. + +/** An allocation generation. An allocation's generations only increase. */ +export const Generation = type("number.integer >= 0"); +export type Generation = typeof Generation.infer; /** - * Sent on first connect when the sidecar has no existing agents in its data - * directory. Identifies the sidecar and declares it ready to receive - * agent.deploy frames. + * The incarnation a deployment frame belongs to. `run.grants` and + * `signal.deliver` payloads are also validated and persisted away from the + * wire, where no generation applies yet, so their schemas leave it out and the + * frame unions below add this stamp to them. */ -export const RegisterFrame = type({ - type: "'register'", - sidecarId: "string", - token: "string", - agentAddresses: type("string") - .array() - .atMostLength(MAX_AGENT_ADDRESSES_FRAME), - // The rotatable (non-run) sender addresses this sidecar holds cached keys - // for. The hub re-resolves each current key and re-pushes it on a - // `sender.key.refresh`, so a user-principal rotation that landed while the - // sidecar was disconnected reaches its cache. Additive-optional and omitted - // when empty: a sidecar with no cached senders (or a pre-upgrade one) sends - // no field, and the hub treats absence as "nothing to refresh". - "cachedSenderAddresses?": type("string") - .array() - .atMostLength(MAX_CACHED_SENDER_ADDRESSES_FRAME), +export const IncarnationStamp = type({ generation: Generation }); +export type IncarnationStamp = typeof IncarnationStamp.infer; + +/** + * What a sidecar is doing with an incarnation it holds. A `stopped` one no + * longer runs, and its local state is kept for inspection until the Hub + * undeploys it. + */ +export const IncarnationState = type.enumerated( + "deploying", + "live", + "stopped", + "tearing-down", +); +export type IncarnationState = typeof IncarnationState.infer; + +export const HostedIncarnation = type({ + address: type("string").atMostLength(MAX_DEPLOYMENT_ADDRESS_LENGTH), + generation: Generation, + state: IncarnationState, + // Why a stopped incarnation stopped when the Hub did not stop it: its + // workflow child ended itself, or the sidecar could not restore it. The Hub + // fails such a deployment. + "error?": type("string > 0").atMostLength(MAX_DEPLOYMENT_ERROR_LENGTH), }); -export type RegisterFrame = typeof RegisterFrame.infer; +export type HostedIncarnation = typeof HostedIncarnation.infer; + +// --------------------------------------------------------------------------- +// Sidecar → Hub +// --------------------------------------------------------------------------- /** - * Sent on connect after a provisioned sidecar restores its deployment. - * The bearer token binds the connection to one allocation generation, so the - * Hub accepts only that allocation's workflow address. + * The first frame on every connection. Authenticates the sidecar and reports + * every incarnation it holds, whether still deploying, live, stopped, or + * tearing down, so the Hub reconciles against what the sidecar actually runs: + * it routes the live incarnations it still owns, keeps unrouted the stopped + * ones it still owns and the live ones whose run ended on its own, and + * undeploys the rest, then answers `welcome`. The sidecar sends nothing that + * must be delivered until then. */ -export const ReconnectFrame = type({ - type: "'reconnect'", +export const HelloFrame = type({ + type: "'hello'", sidecarId: "string", token: "string", - agentAddresses: type("string") - .array() - .atMostLength(MAX_AGENT_ADDRESSES_FRAME), + // A sidecar holds one incarnation of an address, so a list that names one + // twice does not say which of them it holds. + incarnations: HostedIncarnation.array() + .atMostLength(MAX_SIDECAR_INCARNATIONS) + .narrow( + (incarnations, ctx) => + new Set(incarnations.map((incarnation) => incarnation.address)).size === + incarnations.length || + ctx.mustBe("a list that names each address once"), + ), // The rotatable (non-run) sender addresses this sidecar holds cached keys - // for; see `RegisterFrame`. Carried on both frames because the register vs - // reconnect choice turns on workflow-address presence, not sender-cache - // presence -- a sidecar that restored no workflow substrate still reports its - // cached senders on a register frame. Additive-optional, omitted when empty. + // for. The hub re-resolves each current key and re-pushes it on a + // `sender.key.refresh`, so a user-principal rotation that landed while the + // sidecar was disconnected reaches its cache. Omitted when empty, and the + // hub treats absence as "nothing to refresh". "cachedSenderAddresses?": type("string") .array() .atMostLength(MAX_CACHED_SENDER_ADDRESSES_FRAME), }); -export type ReconnectFrame = typeof ReconnectFrame.infer; +export type HelloFrame = typeof HelloFrame.infer; /** * Acknowledges a successful agent deployment. Includes the agent's Ed25519 * public key (hex-encoded) for published identity and content provenance. - * Reconnect authority comes from the allocation credential. + * Echoes the request id and incarnation of the `agent.deploy` it answers. */ export const AgentDeployAckFrame = type({ type: "'agent.deploy.ack'", + requestId: "string", agentAddress: "string", + generation: Generation, publicKey: "string", }); export type AgentDeployAckFrame = typeof AgentDeployAckFrame.infer; -/** - * Reports a failed agent deployment. - */ -export const AgentErrorFrame = type({ - type: "'agent.error'", +/** Reports that the `agent.deploy` with this request id failed. */ +export const AgentDeployErrorFrame = type({ + type: "'agent.deploy.error'", + requestId: "string", agentAddress: "string", - error: "string", + generation: Generation, + error: type("string").atMostLength(MAX_DEPLOYMENT_ERROR_LENGTH), }); -export type AgentErrorFrame = typeof AgentErrorFrame.infer; +export type AgentDeployErrorFrame = typeof AgentDeployErrorFrame.infer; /** * A message from a local agent. When `delivered` is absent or false the hub * should route the message to its recipients. When `delivered` is true the - * message was already delivered locally and is forwarded for audit/projection + * message already went out and this copy is forwarded for audit/projection * only — the hub must not re-route it. * * Structured metadata (senderAddress, messageId, to, cc) is available for * audit and projection purposes without parsing the raw MIME bytes. + * `generation` names the sender's incarnation. */ export const MailOutboundFrame = type({ type: "'mail.outbound'", rawMessage: "string", recipients: type("string").array().atMostLength(MAX_MAIL_ADDRESSES_FRAME), senderAddress: "string", + generation: Generation, "sessionId?": "string", "messageId?": "string", "to?": type("string").array().atMostLength(MAX_MAIL_ADDRESSES_FRAME), @@ -207,6 +276,7 @@ export type MailOutboundFrame = typeof MailOutboundFrame.infer; export const AgentEventFrame = type({ type: "'agent.event'", agentAddress: "string", + generation: Generation, sessionId: "string", event: InferenceEvent, }); @@ -258,16 +328,28 @@ export const SessionErrorFrame = type({ export type SessionErrorFrame = typeof SessionErrorFrame.infer; /** - * Acknowledges that an agent has been fully undeployed: the deployment's - * workflow child stopped, state pushed (best-effort), and directory deleted. + * Acknowledges that the sidecar no longer holds the incarnation an + * `agent.undeploy` named and that its teardown, if it ran one, succeeded. + * Echoes the request id and incarnation. */ export const AgentUndeployAckFrame = type({ type: "'agent.undeploy.ack'", + requestId: "string", agentAddress: "string", - statePushed: "boolean", + generation: Generation, }); export type AgentUndeployAckFrame = typeof AgentUndeployAckFrame.infer; +/** Reports that the `agent.undeploy` with this request id failed. */ +export const AgentUndeployErrorFrame = type({ + type: "'agent.undeploy.error'", + requestId: "string", + agentAddress: "string", + generation: Generation, + error: type("string").atMostLength(MAX_DEPLOYMENT_ERROR_LENGTH), +}); +export type AgentUndeployErrorFrame = typeof AgentUndeployErrorFrame.infer; + /** * Registers a control-signal correlation as a workflow agent step suspends. * The fields on this frame all converge at the sidecar's suspend emit point; @@ -287,6 +369,7 @@ export const SignalCorrelationRegisterFrame = type({ runId: "string", anchorRunId: "string", agentAddress: "string", + generation: Generation, kind: SignalKind, // Approver-facing snapshot of the suspended tool call, size-capped at this // trust boundary. Required: the ask rail is the only producer of this frame @@ -348,6 +431,7 @@ export type SignalCorrelationRegisterAckFrame = export const MailInboundFrame = type({ type: "'mail.inbound'", agentAddress: "string", + generation: Generation, rawMessage: "string", authenticatedSender: "string", "messageId?": "string", @@ -367,6 +451,7 @@ export type MailInboundFrame = typeof MailInboundFrame.infer; export const MailInboundAckFrame = type({ type: "'mail.inbound.ack'", agentAddress: "string", + generation: Generation, messageId: "string", }); export type MailInboundAckFrame = typeof MailInboundAckFrame.infer; @@ -395,6 +480,9 @@ export const SignalDeliverFrame = type({ signalName: "string", signalId: "string", payload: "unknown", + // Addresses one onTrigger relay park. Absent, the delivery stays + // name-FIFO and does not complete a correlated park. + "correlationId?": "string > 0", }); export type SignalDeliverFrame = typeof SignalDeliverFrame.infer; @@ -510,6 +598,7 @@ export type SenderKeyEvictFrame = typeof SenderKeyEvictFrame.infer; export const DrainDeliverFrame = type({ type: "'drain.deliver'", agentAddress: "string", + generation: Generation, deadlineMs: "number", }); export type DrainDeliverFrame = typeof DrainDeliverFrame.infer; @@ -525,42 +614,6 @@ import { export { WorkflowStep } from "./wire-workflow"; export { WorkflowProjectionDefinition, WorkflowProjectionWithSources }; -/** - * The decrypted credential material and per-handle binding descriptors - * delivered to a running agent so its tools can use provider-backed - * credentials. Secrets are decrypted hub-side and ride this payload on the - * live channel ONLY -- the deploy frame at launch, a `credentials.update` - * frame on rotation, and the child's in-memory cell. They are NEVER written to - * disk (they do not ride the git-committed grants file) and NEVER copied into - * any snapshot, event, or state -- redaction is by construction, mirroring how - * an `InferenceSource`'s `apiKey` stays off every egress type. - * - * `materials` is keyed by `credentialId` (a credential can back several handles, - * so its secret is stored once); `bindings` maps each declared tool handle to - * the credential that backs it and the consumer identity allowed to use it. - */ -export const CredentialMaterialEntry = type({ - credentialId: "string", - providerKey: "string", - origin: "string", - secret: "string", -}); -export type CredentialMaterialEntry = typeof CredentialMaterialEntry.infer; - -export const CredentialBindingDescriptor = type({ - handle: "string", - credentialId: "string", - consumer: "string", -}); -export type CredentialBindingDescriptor = - typeof CredentialBindingDescriptor.infer; - -export const CredentialDelivery = type({ - bindings: CredentialBindingDescriptor.array(), - materials: CredentialMaterialEntry.array(), -}); -export type CredentialDelivery = typeof CredentialDelivery.infer; - /** * The source-ref pin: where a code-sourced (npm) workflow definition's bytes * come from (`source`) plus the frozen dependency closure the hub resolved for @@ -699,10 +752,16 @@ export type AgentDeployWorkflow = typeof AgentDeployWorkflow.infer; * after every step is provisioned) spawns the child. * A frame carrying neither is rejected -- there is no in-process * fall-through. `workflow` and `provisionStep` are mutually exclusive. + * + * The sidecar answers with `agent.deploy.ack` or `agent.deploy.error` + * carrying the same `requestId`. A deploy of an address the sidecar already + * holds an incarnation of is refused. */ export const AgentDeployFrame = type({ type: "'agent.deploy'", + requestId: "string", agentAddress: "string", + generation: Generation, agentId: "string", config: HarnessConfig, hubPublicKey: "string", @@ -712,17 +771,100 @@ export const AgentDeployFrame = type({ export type AgentDeployFrame = typeof AgentDeployFrame.infer; /** - * Remove an agent from this sidecar. The sidecar shuts the deployment's - * supervisor down, pushes state to the hub (best-effort), deletes the agent - * directory, and responds with agent.undeploy.ack. + * Remove a deployment incarnation from this sidecar. The sidecar shuts the + * deployment's supervisor down, deletes the local state its teardown covers, + * and responds with `agent.undeploy.ack` or `agent.undeploy.error` carrying + * the same `requestId`. Every incarnation of the address up to `generation` + * goes. For one the sidecar does not hold, it still runs that teardown, and + * answers with an error if that fails. */ export const AgentUndeployFrame = type({ type: "'agent.undeploy'", + requestId: "string", agentAddress: "string", + generation: Generation, reason: "string", }); export type AgentUndeployFrame = typeof AgentUndeployFrame.infer; +/** + * Cancel the workflow run of the incarnation this names, or stop its process + * while keeping its local state for inspection. The sidecar refuses a command + * for a generation other than the one it holds. + */ +export const WorkflowControlFrame = type({ + type: "'workflow.control'", + requestId: "string", + agentAddress: "string", + generation: Generation, + runId: "string", + action: "'cancel' | 'stop'", + reason: "string", +}); +export type WorkflowControlFrame = typeof WorkflowControlFrame.infer; + +export const WORKFLOW_CONTROL_INITIALIZING_ERROR = "workflow_initializing"; + +/** The tip of each authoritative workflow-run ref, `null` for an absent ref. */ +export const WorkflowRunRefTips = type({ + "[string]": type("string").atMostLength(MAX_REF_TIP_ENTRY_LENGTH).or("null"), +}).narrow( + (tips, ctx) => + (Object.keys(tips).length <= MAX_REF_TIPS_FRAME && + Object.keys(tips).every( + (ref) => ref.length <= MAX_REF_TIP_ENTRY_LENGTH, + )) || + ctx.mustBe( + `at most ${String(MAX_REF_TIPS_FRAME)} refs whose names have at most ${String(MAX_REF_TIP_ENTRY_LENGTH)} characters`, + ), +); +export type WorkflowRunRefTips = typeof WorkflowRunRefTips.infer; + +export const WorkflowControlAckFrame = type({ + type: "'workflow.control.ack'", + requestId: "string", + "error?": "string", + /** + * A stopped worker's ref tips, read after its child exited. The Hub + * confirms the stop only once it holds the same tips. + */ + "refTips?": WorkflowRunRefTips, +}); +export type WorkflowControlAckFrame = typeof WorkflowControlAckFrame.infer; + +/** + * Reports a deployment that stopped though the Hub did not stop it: its + * workflow child ended itself, or the sidecar could not restore it. Sent when + * it happens and again after every `welcome` while the sidecar holds it, with + * the tips of its history branches, so the Hub can wait for that history + * before it fails the deployment. The tips are left out when the sidecar + * could not read them. + */ +export const DeploymentStoppedFrame = type({ + type: "'deployment.stopped'", + agentAddress: type("string").atMostLength(MAX_DEPLOYMENT_ADDRESS_LENGTH), + generation: Generation, + error: type("string > 0").atMostLength(MAX_DEPLOYMENT_ERROR_LENGTH), + "refTips?": WorkflowRunRefTips, +}); +export type DeploymentStoppedFrame = typeof DeploymentStoppedFrame.infer; + +/** + * The Hub's answer to `hello`, sent once it has reconciled the reported + * incarnations. `routed` lists the incarnations it routes on this connection; + * it has already asked the sidecar to undeploy every other one reported, but + * for the ones it keeps unrouted: stopped ones, and live ones whose run ended + * on its own. From here on the sidecar may send what must be delivered, and + * it re-drives what it owes the Hub for each routed incarnation. + */ +export const WelcomeFrame = type({ + type: "'welcome'", + routed: type({ address: "string", generation: Generation }) + .array() + .atMostLength(MAX_SIDECAR_INCARNATIONS), +}); +export type WelcomeFrame = typeof WelcomeFrame.infer; + /** * Keepalive pong sent by the hub in response to a ping frame. * If the sidecar stops receiving pongs, it considers the hub dead. @@ -743,6 +885,7 @@ export const SourcesUpdateFrame = type({ type: "'sources.update'", requestId: "string", agentAddress: "string", + generation: Generation, sources: InferenceSource.array().atLeastLength(1), defaultSource: "string", }); @@ -763,6 +906,7 @@ export const CredentialsUpdateFrame = type({ type: "'credentials.update'", requestId: "string", agentAddress: "string", + generation: Generation, delivery: CredentialDelivery, "revoke?": type("string") .array() @@ -795,50 +939,15 @@ export type CredentialsUpdateFrame = typeof CredentialsUpdateFrame.infer; // `repoId` to name a non-agent source while `agentAddress` continues // to address the destination agent. // +// `repo.pack.push` and `repo.pack.done` also carry the generation of the +// incarnation the transfer belongs to. The ack and reject answer one transfer +// on the connection that carried it, and transfer ids are never reused, so +// they need no generation. +// // Flow control: deferred. Agent deploy trees are small enough that the sender // can push all chunks without windowing. If this becomes a problem, a credit- // based mechanism can be added later. -/** - * Tag identifying a kind of repository in the hub's kind-keyed RepoStore. - * Lives in `@intx/types` because the wire-level pack frames reference it; - * the substrate package re-exports it for handler authors. - */ -export const RepoKind = type.enumerated( - "agent-state", - "skill", - "package-registry", - "workflow", - "workflow-run", -); -export type RepoKind = typeof RepoKind.infer; - -/** - * Operations a principal may invoke against a repo in the RepoStore. - * Lives in `@intx/types` so storage layers (e.g. `@intx/db`) can validate - * persisted action vocabularies without depending on the substrate - * package. The substrate re-exports it for handler authors. - */ -export const RepoAction = type.enumerated( - "init", - "writeTree", - "receivePack", - "createPack", - "resolveRef", -); -export type RepoAction = typeof RepoAction.infer; - -/** - * Hub-side identity of a repository in the RepoStore. Pack frames carry - * this alongside `agentAddress` so the hub can map a pack back to the - * originating repo independently of which sidecar/agent it is destined for. - */ -export const RepoId = type({ - kind: RepoKind, - id: "string", -}); -export type RepoId = typeof RepoId.infer; - /** * A chunk of git pack data. The sender splits the packfile into chunks of at * most 64 KiB (before base64 encoding) and sends them in order. @@ -849,6 +958,7 @@ export type RepoId = typeof RepoId.infer; export const PackPushFrame = type({ type: "'repo.pack.push'", agentAddress: "string", + generation: Generation, repoId: RepoId, transferId: "string", seq: "number", @@ -869,6 +979,7 @@ export type PackPushFrame = typeof PackPushFrame.infer; export const PackDoneFrame = type({ type: "'repo.pack.done'", agentAddress: "string", + generation: Generation, repoId: RepoId, transferId: "string", ref: "string", @@ -1031,18 +1142,6 @@ export const DeployApplyErrorCategory = type.enumerated( ); export type DeployApplyErrorCategory = typeof DeployApplyErrorCategory.infer; -/** - * Hub requests the sidecar to push its current agent state. The sidecar - * responds by sending pack.push frames followed by pack.done using the - * same transferId. - */ -export const SyncRequestFrame = type({ - type: "'sync.request'", - agentAddress: "string", - transferId: "string", -}); -export type SyncRequestFrame = typeof SyncRequestFrame.infer; - // --------------------------------------------------------------------------- // Workflow probe (bidirectional) // --------------------------------------------------------------------------- @@ -1137,10 +1236,9 @@ export type WorkflowProbeErrorFrame = typeof WorkflowProbeErrorFrame.infer; /** All frame types the sidecar sends to the hub. */ export const SidecarFrame = type.or( - RegisterFrame, - ReconnectFrame, + HelloFrame, AgentDeployAckFrame, - AgentErrorFrame, + AgentDeployErrorFrame, MailOutboundFrame, AgentEventFrame, ConnectorStateChangedFrame, @@ -1148,6 +1246,7 @@ export const SidecarFrame = type.or( SessionAckFrame, SessionErrorFrame, AgentUndeployAckFrame, + AgentUndeployErrorFrame, SignalCorrelationRegisterFrame, PackPushFrame, PackDoneFrame, @@ -1156,6 +1255,8 @@ export const SidecarFrame = type.or( MailInboundAckFrame, WorkflowProbeResultFrame, WorkflowProbeErrorFrame, + WorkflowControlAckFrame, + DeploymentStoppedFrame, ); export type SidecarFrame = typeof SidecarFrame.infer; @@ -1164,6 +1265,7 @@ export const HubFrame = type.or( MailInboundFrame, AgentDeployFrame, AgentUndeployFrame, + WelcomeFrame, PongFrame, SourcesUpdateFrame, CredentialsUpdateFrame, @@ -1171,14 +1273,14 @@ export const HubFrame = type.or( PackDoneFrame, PackAckFrame, PackRejectFrame, - SyncRequestFrame, - SignalDeliverFrame, - RunGrantsFrame, + SignalDeliverFrame.and(IncarnationStamp), + RunGrantsFrame.and(IncarnationStamp), SenderKeyRefreshFrame, SenderKeyEvictFrame, SignalCorrelationRegisterAckFrame, DrainDeliverFrame, WorkflowProbeRequestFrame, + WorkflowControlFrame, ); export type HubFrame = typeof HubFrame.infer; diff --git a/vendor/intx-types/src/tenants.ts b/vendor/intx-types/src/tenants.ts index bf1ffa783..da91f5487 100644 --- a/vendor/intx-types/src/tenants.ts +++ b/vendor/intx-types/src/tenants.ts @@ -1,22 +1,46 @@ import { type } from "arktype"; import { SidecarCapabilityPolicy } from "./sidecar-capabilities"; +import { WorkflowLifecyclePolicy } from "./workflow-lifecycle"; export const TenantConfig = type({ + "lifecycle?": WorkflowLifecyclePolicy, "sidecarPlacement?": SidecarCapabilityPolicy, "[string]": "unknown", }); export type TenantConfig = typeof TenantConfig.infer; +/** + * A tenant slug, constrained to a single DNS label: the create route derives + * `tenant.domain` from the lowercased slug, and that domain is both the sender + * stamp on outbound mail and the recipient run address. The grammar is RFC 1035 + * section 2.3.1 as relaxed by RFC 1123 section 2.1. + * + * https://www.rfc-editor.org/rfc/rfc1035#section-2.3.1 + * https://www.rfc-editor.org/rfc/rfc1123#section-2.1 + */ +export const TenantSlug = type( + /^[A-Za-z0-9](?:[A-Za-z0-9-]*[A-Za-z0-9])?$/, +).and("string<=63"); +export type TenantSlug = typeof TenantSlug.infer; + +/** Top-level `config` keys to change; `null` removes a key. */ +export const TenantConfigPatch = type({ + "lifecycle?": WorkflowLifecyclePolicy.or("null"), + "sidecarPlacement?": SidecarCapabilityPolicy.or("null"), + "[string]": "unknown", +}); +export type TenantConfigPatch = typeof TenantConfigPatch.infer; + export const CreateTenant = type({ name: "string", - slug: "string", + slug: TenantSlug, "parentId?": "string | null", }); export const UpdateTenant = type({ "name?": "string", - "config?": TenantConfig, + "config?": TenantConfigPatch, }); export const TenantResponse = type({ diff --git a/vendor/intx-types/src/wire-workflow.ts b/vendor/intx-types/src/wire-workflow.ts index 015cca33f..81bcbf2aa 100644 --- a/vendor/intx-types/src/wire-workflow.ts +++ b/vendor/intx-types/src/wire-workflow.ts @@ -7,7 +7,8 @@ import { type } from "arktype"; import { CredentialBinding } from "./credentials"; -import { InboundMailPolicy, InferenceSource } from "./runtime"; +import { InferenceSource } from "./runtime"; +import { InboundMailPolicy } from "./inbound-mail-policy"; import { SidecarCapabilityPolicy } from "./sidecar-capabilities"; /** @@ -78,7 +79,7 @@ const WorkflowSteps = type({ "[string]": "unknown" }).narrow((steps, ctx) => { * `definition` is the wire projection of `WorkflowDefinition` from * `@intx/workflow`. The arktype validator enforces the structural * envelope the workflow-process child re-parses on the sidecar after - * materialization (`packages/hub-sessions/src/workflow-kind.ts`'s + * materialization (`packages/types/src/workflow-definition.ts`'s * `workflowDefinitionEnvelopeSchema`): `id`, `triggers`, `steps`, * `stepOrder`, optional `state`. The wire validator MUST require every * field the envelope requires — this projection is the approved surface diff --git a/vendor/intx-types/src/workflow-definition.test.ts b/vendor/intx-types/src/workflow-definition.test.ts new file mode 100644 index 000000000..fa0f2cb1f --- /dev/null +++ b/vendor/intx-types/src/workflow-definition.test.ts @@ -0,0 +1,182 @@ +import { describe, test, expect } from "bun:test"; +import { type } from "arktype"; + +import { workflowDefinitionEnvelopeSchema } from "./workflow-definition"; + +describe("workflowDefinitionEnvelopeSchema", () => { + // What declaring `grantRequirements` on the envelope buys is VALIDATION, + // not field survival. `.onUndeclaredKey("ignore")` is passthrough, so an + // undeclared field would survive the read regardless; only the declaration + // makes a malformed requirement fail at the deploy boundary. These tests + // assert that validation property. The rejection test fails if the + // `grantRequirements?` line is removed from the schema; a survival test + // would pass either way and prove nothing. + test("rejects a declared grantRequirement carrying an unknown source", () => { + const validated = workflowDefinitionEnvelopeSchema({ + id: "my-workflow", + triggers: [{ type: "manual" }], + steps: { first: { kind: "step", id: "first" } }, + stepOrder: ["first"], + grantRequirements: [ + { resource: "tool:search", action: "invoke", source: "stranger" }, + ], + }); + expect(validated instanceof type.errors).toBe(true); + }); + + test("accepts and preserves a well-formed grantRequirement", () => { + const blob = { + id: "my-workflow", + triggers: [{ type: "manual" }], + steps: { first: { kind: "step", id: "first" } }, + stepOrder: ["first"], + grantRequirements: [ + { + resource: "credential:openai", + action: "use", + source: "creator" as const, + }, + { + resource: "tool:search", + action: "invoke", + effect: "ask" as const, + source: "invoker" as const, + }, + ], + }; + const validated = workflowDefinitionEnvelopeSchema(blob); + if (validated instanceof type.errors) { + throw new Error(`unexpected validation error: ${validated.summary}`); + } + expect(validated.grantRequirements).toEqual(blob.grantRequirements); + }); + + // Same property for credentialBindings: declaring it on the envelope makes a + // malformed binding fail at the deploy boundary. This is the path launch-time + // resolution reads bindings back from, so a bad locator/authority must not + // pass through unchecked. The rejection test fails if the + // `credentialBindings?` line is removed from the schema. + test("rejects a declared credentialBinding carrying an unknown authority", () => { + const validated = workflowDefinitionEnvelopeSchema({ + id: "my-workflow", + triggers: [{ type: "manual" }], + steps: { first: { kind: "step", id: "first" } }, + stepOrder: ["first"], + credentialBindings: [ + { + package: "@acme/tools", + handle: "gh", + provider: "github", + locator: "stranger", + }, + ], + }); + expect(validated instanceof type.errors).toBe(true); + }); + + test("accepts and preserves a well-formed credentialBinding", () => { + const blob = { + id: "my-workflow", + triggers: [{ type: "manual" }], + steps: { first: { kind: "step", id: "first" } }, + stepOrder: ["first"], + credentialBindings: [ + { + package: "@acme/tools", + handle: "gh", + provider: "github", + locator: "tenant" as const, + }, + ], + }; + const validated = workflowDefinitionEnvelopeSchema(blob); + if (validated instanceof type.errors) { + throw new Error(`unexpected validation error: ${validated.summary}`); + } + expect(validated.credentialBindings).toEqual(blob.credentialBindings); + }); + + // Same property for inboundMailPolicy: declaring it on the envelope makes a + // malformed policy fail at the deploy boundary. The policy keys on exactly + // the four author-controllable outcomes, so an unknown outcome key (a typo or + // a non-controllable outcome such as `clean`) must be rejected here rather + // than ride through to later admission resolution. The rejection test fails + // if the `inboundMailPolicy?` line is removed from the schema. + test("rejects a declared inboundMailPolicy carrying an unknown outcome key", () => { + const validated = workflowDefinitionEnvelopeSchema({ + id: "my-workflow", + triggers: [{ type: "mail", to: "wf@acme.test" }], + steps: { first: { kind: "step", id: "first" } }, + stepOrder: ["first"], + inboundMailPolicy: { clean: "admit" }, + }); + expect(validated instanceof type.errors).toBe(true); + }); + + test("rejects a declared inboundMailPolicy carrying a non reject/admit value", () => { + const validated = workflowDefinitionEnvelopeSchema({ + id: "my-workflow", + triggers: [{ type: "mail", to: "wf@acme.test" }], + steps: { first: { kind: "step", id: "first" } }, + stepOrder: ["first"], + inboundMailPolicy: { missing: "quarantine" }, + }); + expect(validated instanceof type.errors).toBe(true); + }); + + test("accepts and preserves a well-formed sparse inboundMailPolicy", () => { + const blob = { + id: "my-workflow", + triggers: [{ type: "mail", to: "wf@acme.test" }], + steps: { first: { kind: "step", id: "first" } }, + stepOrder: ["first"], + inboundMailPolicy: { + untrustedFrom: "admit" as const, + missing: "reject" as const, + }, + }; + const validated = workflowDefinitionEnvelopeSchema(blob); + if (validated instanceof type.errors) { + throw new Error(`unexpected validation error: ${validated.summary}`); + } + expect(validated.inboundMailPolicy).toEqual(blob.inboundMailPolicy); + // The two unset outcomes stay absent -- the envelope does not populate a + // default for an outcome the author omitted. + expect(validated.inboundMailPolicy).not.toHaveProperty("invalid"); + expect(validated.inboundMailPolicy).not.toHaveProperty("unknown"); + }); +}); + +// The static workflow.json envelope push path is retired: a workflow asset is +// now a codebase, and a bare envelope tree is rejected at the push boundary +// before any steps/state shape check runs. The structural guard that a +// definition's `steps` and `state` are JSON objects (not arrays) now lives in +// `workflowDefinitionEnvelopeSchema`, which the codebase ambiguity check and the +// hydrate-time definition loaders both reuse. These regressions pin that guard +// at the schema so a loosened narrow surfaces here. +describe("workflow-definition steps/state-as-array rejection (regression)", () => { + test("rejects a definition whose steps field is a JSON array", () => { + const result = workflowDefinitionEnvelopeSchema({ + id: "wf-1", + triggers: [], + steps: [{ name: "step1" }, { name: "step2" }], + stepOrder: ["step1", "step2"], + }); + expect(result instanceof type.errors).toBe(true); + if (!(result instanceof type.errors)) throw new Error("unreachable"); + expect(result.summary).toMatch(/array|object/); + }); + + test("rejects a definition whose state field is a JSON array", () => { + const result = workflowDefinitionEnvelopeSchema({ + id: "wf-1", + triggers: [], + steps: { s1: { name: "s1" } }, + stepOrder: ["s1"], + state: [], + }); + expect(result instanceof type.errors).toBe(true); + if (!(result instanceof type.errors)) throw new Error("unreachable"); + expect(result.summary).toMatch(/array|object/); + }); +}); diff --git a/vendor/intx-types/src/workflow-definition.ts b/vendor/intx-types/src/workflow-definition.ts new file mode 100644 index 000000000..a51fa8d6f --- /dev/null +++ b/vendor/intx-types/src/workflow-definition.ts @@ -0,0 +1,62 @@ +import { type } from "arktype"; + +import { CredentialBinding } from "./credentials"; +import { GrantRequirement } from "./grants"; +import { InboundMailPolicy } from "./inbound-mail-policy"; +import { SidecarCapabilityPolicy } from "./sidecar-capabilities"; + +/** + * Structural arktype validator for the `workflow.json` envelope. The + * substrate checks the cross-cutting shape of `WorkflowDefinition` + * (presence and primitive type of `id`, `triggers`, `steps`, + * `stepOrder`) but does not re-derive `defineWorkflow`'s DAG-level + * validation here — primitive-level shape, default-input application, + * and `after`-ref resolution belong to the runtime layer that hydrates + * the definition. The codebase push uses this validator to detect an + * ambiguous tree that also carries an envelope-valid `workflow.json`, + * and the hydrate-time definition loaders reuse it to validate a + * materialized definition before instantiation. + */ +const StepsObject = type("Record").narrow((value, ctx) => { + if (Array.isArray(value)) { + return ctx.mustBe("a JSON object, not an array"); + } + return true; +}); + +const StateObject = type("Record").narrow((value, ctx) => { + if (Array.isArray(value)) { + return ctx.mustBe("a JSON object, not an array"); + } + return true; +}); + +export const workflowDefinitionEnvelopeSchema = type({ + id: "string > 0", + triggers: "unknown[]", + steps: StepsObject, + stepOrder: "string[]", + "state?": StateObject, + // `grantRequirements` passes through the envelope whether or not it is + // declared here: arktype's `.onUndeclaredKey("ignore")` below is + // passthrough, not stripping (only `"delete"` strips), so the hydrate read + // sees the field either way. Declaring it here VALIDATES declared + // requirements at the deploy boundary — a malformed `source` is rejected + // rather than passed through unchecked — as defense in depth alongside the + // trigger route's own `GrantRequirements` re-validation. Compose the + // exported `GrantRequirement` arktype rather than restating its shape so the + // envelope and the definition stay in lockstep. + "grantRequirements?": GrantRequirement.array(), + // `credentialBindings` is validated here too -- same defense-in-depth + // rationale as grantRequirements above: a malformed binding (bad locator, + // authority, or handle) is rejected at the deploy boundary rather than + // passed through to launch-time resolution unchecked. + "credentialBindings?": CredentialBinding.array(), + "sidecarPlacement?": SidecarCapabilityPolicy, + // `inboundMailPolicy` is validated here too -- same defense-in-depth + // rationale as credentialBindings above: a malformed policy (an unknown + // outcome key or a value that is not reject/admit) is rejected at the deploy + // boundary rather than passed through to later admission resolution + // unchecked. + "inboundMailPolicy?": InboundMailPolicy, +}).onUndeclaredKey("ignore"); diff --git a/vendor/intx-types/src/workflow-lifecycle.test.ts b/vendor/intx-types/src/workflow-lifecycle.test.ts new file mode 100644 index 000000000..081b66eef --- /dev/null +++ b/vendor/intx-types/src/workflow-lifecycle.test.ts @@ -0,0 +1,178 @@ +import { describe, expect, test } from "bun:test"; +import { type } from "arktype"; + +import { + WorkflowLifecyclePolicy, + clampWorkflowLifecyclePolicy, + lifecycleDeadline, + resolveWorkflowLifecyclePolicy, +} from "./workflow-lifecycle"; + +describe("workflow lifecycle policy", () => { + test("bounds every duration unit equally and keeps the maximum deadline representable", () => { + const start = new Date("2026-09-21T00:00:00Z"); + const expected = new Date(start.getTime() + 36_500 * 86_400_000); + for (const duration of ["36500d", "876000h", "52560000m", "3153600000s"]) { + expect( + WorkflowLifecyclePolicy({ + maxLifetime: duration, + capacityRetention: { completed: duration }, + }), + ).not.toBeInstanceOf(type.errors); + expect(lifecycleDeadline(start, duration)).toEqual(expected); + } + for (const duration of [ + "36501d", + "876001h", + "52560001m", + "3153600001s", + "100000000d", + ]) { + expect(WorkflowLifecyclePolicy({ maxLifetime: duration })).toBeInstanceOf( + type.errors, + ); + for (const status of ["completed", "failed", "cancelled"] as const) { + expect( + WorkflowLifecyclePolicy({ + capacityRetention: { [status]: duration }, + }), + ).toBeInstanceOf(type.errors); + } + } + }); + + test("accepts immediate release but rejects a zero lifetime or disconnect limit and malformed durations", () => { + expect( + WorkflowLifecyclePolicy({ capacityRetention: { completed: "0s" } }), + ).toEqual({ capacityRetention: { completed: "0s" } }); + for (const duration of [ + "0s", + "-1h", + "1.5h", + "1hour", + "1e3s", + "999999999999999999d", + ]) { + expect(WorkflowLifecyclePolicy({ maxLifetime: duration })).toBeInstanceOf( + type.errors, + ); + expect( + WorkflowLifecyclePolicy({ maxDisconnected: duration }), + ).toBeInstanceOf(type.errors); + } + expect( + WorkflowLifecyclePolicy({ capacityRetention: { typo: "1h" } }), + ).toBeInstanceOf(type.errors); + }); + + test("inherits each field and permits shorter equivalent-unit limits", () => { + expect( + resolveWorkflowLifecyclePolicy([ + { + maxLifetime: "1d", + capacityRetention: { completed: "0s", failed: "1h" }, + }, + { maxLifetime: "24h" }, + { maxLifetime: "2h", capacityRetention: { failed: "15m" } }, + ]), + ).toEqual({ + ok: true, + policy: { + maxLifetime: "2h", + capacityRetention: { completed: "0s", failed: "15m" }, + }, + }); + }); + + test("omission cannot remove an ancestor limit and a descendant cannot extend it", () => { + expect( + resolveWorkflowLifecyclePolicy([ + { maxLifetime: "1h" }, + {}, + { maxLifetime: "2h" }, + ]), + ).toEqual({ + ok: false, + field: "maxLifetime", + requested: "2h", + limit: "1h", + }); + expect( + resolveWorkflowLifecyclePolicy([ + { capacityRetention: { completed: "0s" } }, + { capacityRetention: { completed: "1m" } }, + ]), + ).toEqual({ + ok: false, + field: "capacityRetention.completed", + requested: "1m", + limit: "0s", + }); + expect(resolveWorkflowLifecyclePolicy([{}, {}])).toEqual({ + ok: true, + policy: {}, + }); + }); + + test("clamping caps each value at its inherited limit and keeps valid shorter ones", () => { + expect( + clampWorkflowLifecyclePolicy([ + { + maxLifetime: "6h", + maxDisconnected: "30m", + capacityRetention: { completed: "0s" }, + }, + { + maxLifetime: "12h", + maxDisconnected: "1h", + capacityRetention: { completed: "1h", failed: "30m" }, + }, + { + maxLifetime: "2h", + maxDisconnected: "10m", + capacityRetention: { failed: "2h" }, + }, + ]), + ).toEqual({ + maxLifetime: "2h", + maxDisconnected: "10m", + capacityRetention: { completed: "0s", failed: "30m" }, + }); + }); + + test("a descendant cannot extend an inherited disconnect limit", () => { + expect( + resolveWorkflowLifecyclePolicy([ + { maxDisconnected: "15m" }, + {}, + { maxDisconnected: "1h" }, + ]), + ).toEqual({ + ok: false, + field: "maxDisconnected", + requested: "1h", + limit: "15m", + }); + }); + + test("validation reports the first conflict even when later levels also exceed", () => { + expect( + resolveWorkflowLifecyclePolicy([ + { maxLifetime: "1h" }, + { capacityRetention: { failed: "1h" } }, + { maxLifetime: "3h", capacityRetention: { failed: "2h" } }, + ]), + ).toEqual({ + ok: false, + field: "maxLifetime", + requested: "3h", + limit: "1h", + }); + }); + + test("computes deadlines from the supplied durable timestamp", () => { + expect( + lifecycleDeadline(new Date("2026-01-01T00:00:00Z"), "2h").toISOString(), + ).toBe("2026-01-01T02:00:00.000Z"); + }); +}); diff --git a/vendor/intx-types/src/workflow-lifecycle.ts b/vendor/intx-types/src/workflow-lifecycle.ts new file mode 100644 index 000000000..636cb10e3 --- /dev/null +++ b/vendor/intx-types/src/workflow-lifecycle.ts @@ -0,0 +1,167 @@ +import { type } from "arktype"; +import { sidecarAllocationStatuses } from "./sidecar-allocation"; + +// Fixed policy range with ample headroom for deployment and terminal timestamps. +const MAX_LIFECYCLE_DURATION_MS = 36_500 * 86_400_000; + +function parseDuration(value: string): number | null { + const match = /^(0|[1-9][0-9]*)(s|m|h|d)$/.exec(value); + if (match === null) return null; + const amount = Number(match[1]); + const unit = match[2]; + const multiplier = + unit === "d" + ? 86_400_000 + : unit === "h" + ? 3_600_000 + : unit === "m" + ? 60_000 + : 1_000; + const milliseconds = amount * multiplier; + return Number.isSafeInteger(milliseconds) && + milliseconds <= MAX_LIFECYCLE_DURATION_MS + ? milliseconds + : null; +} + +export const LifecycleDuration = type("string").narrow( + (value, ctx) => + parseDuration(value) !== null || + ctx.mustBe( + "a non-negative whole duration ending in s, m, h, or d, at most 36500d", + ), +); +export type LifecycleDuration = typeof LifecycleDuration.infer; + +export function lifecycleDurationMs(value: LifecycleDuration): number { + const milliseconds = parseDuration(value); + if (milliseconds === null) + throw new Error(`Invalid lifecycle duration: ${value}`); + return milliseconds; +} + +export function lifecycleDeadline( + start: Date, + duration: LifecycleDuration, +): Date { + const deadline = new Date(start.getTime() + lifecycleDurationMs(duration)); + if (!Number.isFinite(deadline.getTime())) { + throw new Error( + "Lifecycle deadline is outside the supported timestamp range", + ); + } + return deadline; +} + +const PositiveLifecycleDuration = LifecycleDuration.narrow( + (value, ctx) => + lifecycleDurationMs(value) > 0 || ctx.mustBe("greater than zero"), +); + +export const WorkflowLifecyclePolicy = type({ + "maxLifetime?": PositiveLifecycleDuration, + "maxDisconnected?": PositiveLifecycleDuration, + "capacityRetention?": type({ + "completed?": LifecycleDuration, + "failed?": LifecycleDuration, + "cancelled?": LifecycleDuration, + }).onUndeclaredKey("reject"), +}).onUndeclaredKey("reject"); +export type WorkflowLifecyclePolicy = typeof WorkflowLifecyclePolicy.infer; + +/** A policy with every field set, as a deployment's effective policy is. */ +export type ResolvedWorkflowLifecyclePolicy = { + maxLifetime: LifecycleDuration; + maxDisconnected: LifecycleDuration; + capacityRetention: Required< + NonNullable + >; +}; + +export const WorkflowLifecycleResponse = type({ + runId: "string", + status: "'deployed' | 'running' | 'completed' | 'failed' | 'cancelled'", + policy: WorkflowLifecyclePolicy, + expiresAt: "string | null", + cancellationRequestedAt: "string | null", + cancellationDeadline: "string | null", + cancellationReason: "string | null", + capacityReleaseAt: "string | null", + allocation: type({ + id: "string", + status: type.enumerated(...sidecarAllocationStatuses), + failureCode: "string | null", + failureMessage: "string | null", + }).or("null"), +}); + +export type WorkflowLifecyclePolicyResolution = + | { ok: true; policy: WorkflowLifecyclePolicy } + | { ok: false; field: string; requested: string; limit: string }; + +type LifecycleLimitConflict = Extract< + WorkflowLifecyclePolicyResolution, + { ok: false } +>; + +function foldWorkflowLifecyclePolicies( + policies: readonly WorkflowLifecyclePolicy[], +): { policy: WorkflowLifecyclePolicy; conflict?: LifecycleLimitConflict } { + const effective: WorkflowLifecyclePolicy = {}; + let conflict: LifecycleLimitConflict | undefined; + for (const policy of policies) { + for (const field of ["maxLifetime", "maxDisconnected"] as const) { + const requested = policy[field]; + if (requested === undefined) continue; + const limit = effective[field]; + if ( + limit !== undefined && + lifecycleDurationMs(requested) > lifecycleDurationMs(limit) + ) { + conflict ??= { ok: false, field, requested, limit }; + continue; + } + effective[field] = requested; + } + for (const status of ["completed", "failed", "cancelled"] as const) { + const requested = policy.capacityRetention?.[status]; + if (requested === undefined) continue; + const limit = effective.capacityRetention?.[status]; + if ( + limit !== undefined && + lifecycleDurationMs(requested) > lifecycleDurationMs(limit) + ) { + conflict ??= { + ok: false, + field: `capacityRetention.${status}`, + requested, + limit, + }; + continue; + } + effective.capacityRetention ??= {}; + effective.capacityRetention[status] = requested; + } + } + return conflict === undefined + ? { policy: effective } + : { policy: effective, conflict }; +} + +/** Resolve root-to-leaf policies; a descendant can only shorten an inherited limit. */ +export function resolveWorkflowLifecyclePolicy( + policies: readonly WorkflowLifecyclePolicy[], +): WorkflowLifecyclePolicyResolution { + const { policy, conflict } = foldWorkflowLifecyclePolicies(policies); + return conflict ?? { ok: true, policy }; +} + +/** + * Resolve root-to-leaf policies, capping any value above an inherited limit. + * An ancestor tightened after a descendant was saved must not strand it. + */ +export function clampWorkflowLifecyclePolicy( + policies: readonly WorkflowLifecyclePolicy[], +): WorkflowLifecyclePolicy { + return foldWorkflowLifecyclePolicies(policies).policy; +} diff --git a/vendor/intx-types/src/workflow-run-id.test.ts b/vendor/intx-types/src/workflow-run-id.test.ts index d8901c419..ec06966f0 100644 --- a/vendor/intx-types/src/workflow-run-id.test.ts +++ b/vendor/intx-types/src/workflow-run-id.test.ts @@ -9,19 +9,19 @@ describe("deriveWorkflowRunId", () => { test("throws when there is no @", () => { expect(() => deriveWorkflowRunId("run_abc123")).toThrow( - "Invalid run address", + 'Invalid run address: "run_abc123"', ); }); test("throws when the domain is empty", () => { expect(() => deriveWorkflowRunId("run_abc123@")).toThrow( - "Invalid run address", + 'Invalid run address: "run_abc123@"', ); }); test("throws when the local part lacks the run_ prefix", () => { expect(() => deriveWorkflowRunId("usr_abc123@tenant.example")).toThrow( - "Invalid run address", + 'Invalid run address: "usr_abc123@tenant.example"', ); }); @@ -30,4 +30,20 @@ describe("deriveWorkflowRunId", () => { "run_abc123", ); }); + + test("throws when the local part is empty", () => { + expect(() => deriveWorkflowRunId("@tenant.example")).toThrow( + 'Invalid run address: "@tenant.example"', + ); + }); + + test("throws on empty string", () => { + expect(() => deriveWorkflowRunId("")).toThrow('Invalid run address: ""'); + }); + + test("includes the invalid address in the error message", () => { + expect(() => deriveWorkflowRunId("not-an-address")).toThrow( + 'Invalid run address: "not-an-address"', + ); + }); }); diff --git a/vendor/intx-types/src/workflow-sources.ts b/vendor/intx-types/src/workflow-sources.ts index 4a9313787..484648f88 100644 --- a/vendor/intx-types/src/workflow-sources.ts +++ b/vendor/intx-types/src/workflow-sources.ts @@ -72,3 +72,13 @@ export const WorkflowDefinitionSource = WorkflowDefinitionRegistrySource.or( WorkflowDefinitionAssetSource, ); export type WorkflowDefinitionSource = typeof WorkflowDefinitionSource.infer; + +/** + * The workspace-relative mount path a workflow definition's source asset is + * delivered under, keyed by asset id. Both the probe (inline delivery) and the + * deploy (streamed fan-out) build their `assetId -> mountPath` maps through + * this one helper, so the same asset lands at the same path on both paths. + */ +export function workflowSourceAssetMountPath(assetId: string): string { + return `source-assets/${assetId}/`; +} diff --git a/vendor/intx-types/src/workflows.ts b/vendor/intx-types/src/workflows.ts index 014ed7d66..8d783dcff 100644 --- a/vendor/intx-types/src/workflows.ts +++ b/vendor/intx-types/src/workflows.ts @@ -13,6 +13,7 @@ export type WorkflowDefinitionVersionStatus = (typeof workflowDefinitionVersionStatuses)[number]; import { type } from "arktype"; +import { WorkflowLifecyclePolicy } from "./workflow-lifecycle"; const WorkflowDefinitionStatusType = type.enumerated( ...workflowDefinitionStatuses, @@ -35,6 +36,7 @@ export const WorkflowDefinitionResponse = type({ name: "string", "description?": "string | null", currentVersion: "string", + "lifecycle?": WorkflowLifecyclePolicy, status: WorkflowDefinitionStatusType.describe( "Lifecycle state of the definition: `deployed` (a launchable version is active) or `stopped` (deactivated).", ), @@ -47,6 +49,10 @@ export const WorkflowRollbackRequest = type({ version: "string", }); +export const UpdateWorkflowDefinitionLifecycle = type({ + lifecycle: WorkflowLifecyclePolicy, +}).onUndeclaredKey("reject"); + export const WorkflowDeploymentStatus = type.enumerated( "deployed", "pending", @@ -63,7 +69,7 @@ export const WorkflowDeploymentResponse = type({ tenantId: "string", definitionAssetId: "string", status: WorkflowDeploymentStatus.describe( - "Deployment lifecycle status. `failed` is a terminal failure with no infrastructure. `destroy_failed` is a permanent cleanup failure where infrastructure may remain and require operator cleanup.", + "Deployment lifecycle status. `recovering` means the Hub is replacing capacity whose provisioning failed before the deployment first ran. `failed` is a terminal failure with no infrastructure. `destroy_failed` is a permanent cleanup failure where infrastructure may remain and require operator cleanup.", ), createdAt: "string", }); diff --git a/vendor/intx-workflow-host/adapters/substrate-mailbox-store.test.ts b/vendor/intx-workflow-host/adapters/substrate-mailbox-store.test.ts index c11e159c1..2aea106d8 100644 --- a/vendor/intx-workflow-host/adapters/substrate-mailbox-store.test.ts +++ b/vendor/intx-workflow-host/adapters/substrate-mailbox-store.test.ts @@ -1,4 +1,5 @@ import { describe, test, expect, afterAll, beforeAll } from "bun:test"; +import { type } from "arktype"; import fs from "node:fs"; import os from "node:os"; import path from "node:path"; @@ -10,6 +11,7 @@ import { assembleSignedContent, createDetachedSignatureFromProvider, generateMessageId, + parseHeaderSection, } from "@intx/mime"; import type { MessageHeaders } from "@intx/mime"; import type { CryptoProvider } from "@intx/types/runtime"; @@ -23,7 +25,13 @@ import type { import { executeSearch, fetchFull } from "@intx/mailbox"; import type { StoredEnvelope } from "@intx/mailbox"; -import { createSubstrateMailboxStore } from "./substrate-mailbox-store"; +import { + createSubstrateMailboxStore, + MAILBOX_INBOX_DIR, + MAILBOX_INBOX_PREFIX, + MAILBOX_INDEX_FILE, + MAILBOX_PREFIX, +} from "./substrate-mailbox-store"; const REF = "refs/heads/main"; const tempDirs: string[] = []; @@ -205,6 +213,130 @@ async function senderCrypto(): Promise { return createEd25519Crypto(await generateKeyPair()); } +/** + * Drop one named header's line from a message's header section, leaving every + * other byte alone. The detached signature covers the signed-content part + * inside the body rather than the outer headers, so the message still verifies + * -- which makes this the raw counterpart of an envelope whose corresponding + * field is absent. + */ +function stripHeader(raw: Uint8Array, name: string): Uint8Array { + const { headerEnd } = parseHeaderSection(raw); + const prefix = `${name.toLowerCase()}:`; + const kept = new TextDecoder() + .decode(raw.subarray(0, headerEnd)) + .split("\r\n") + .filter((line) => !line.toLowerCase().startsWith(prefix)) + .join("\r\n"); + const keptBytes = new TextEncoder().encode(kept); + const rest = raw.subarray(headerEnd); + const out = new Uint8Array(keptBytes.length + rest.length); + out.set(keptBytes, 0); + out.set(rest, keptBytes.length); + return out; +} + +/** + * The nullable envelope fields each committed message holds, in index order. + * Reads `index.json` as JSON rather than through the store, so the assertion is + * about the durable on-disk encoding and not about a serialize/deserialize pair + * that could agree with each other on a value no other reader expects. + */ +const IndexEnvelopes = type({ + messages: type({ + envelope: { from: "string | null", date: "string | null" }, + }).array(), +}); + +async function readIndexEnvelopes( + handles: Handles, +): Promise<{ from: string | null; date: string | null }[]> { + const substrate = makeStore(handles.dataDir); + const reads = await substrate.openCommittedReads( + handles.principal, + handles.repoId, + REF, + ); + if (reads === null) throw new Error("no committed reads"); + const entries = await reads.listDir(`${MAILBOX_PREFIX}/${MAILBOX_INBOX_DIR}`); + const entry = entries.find((e) => e.name === MAILBOX_INDEX_FILE); + if (entry === undefined) { + throw new Error("the committed tree holds no index.json"); + } + const raw: unknown = JSON.parse( + new TextDecoder().decode(await reads.readBlobByOid(entry.oid)), + ); + const parsed = IndexEnvelopes(raw); + if (parsed instanceof type.errors) { + throw new Error(`index.json is not index-shaped: ${parsed.summary}`); + } + return parsed.messages.map((m) => m.envelope); +} + +/** + * The `version` every committed `index.json` carries. Written as a literal + * rather than imported from the module under test: the loader matches the + * version exactly and carries no migration arm, so importing the production + * constant would let both sides move together and assert nothing. + */ +const COMMITTED_INDEX_VERSION = 1; + +/** + * The committed `index.json`, validated only as far as its `version`. + * Undeclared keys are ignored rather than deleted, so the parsed object still + * carries every other index field for `rewriteIndexVersion` to spread. + */ +const CommittedIndex = type({ + version: "number", + "+": "ignore", +}); + +async function readCommittedIndex(handles: Handles) { + const substrate = makeStore(handles.dataDir); + const reads = await substrate.openCommittedReads( + handles.principal, + handles.repoId, + REF, + ); + if (reads === null) throw new Error("no committed reads"); + const entries = await reads.listDir(`${MAILBOX_PREFIX}/${MAILBOX_INBOX_DIR}`); + const entry = entries.find((e) => e.name === MAILBOX_INDEX_FILE); + if (entry === undefined) { + throw new Error("the committed tree holds no index.json"); + } + const raw: unknown = JSON.parse( + new TextDecoder().decode(await reads.readBlobByOid(entry.oid)), + ); + const parsed = CommittedIndex(raw); + if (parsed instanceof type.errors) { + throw new Error(`index.json carries no version: ${parsed.summary}`); + } + return parsed; +} + +/** + * Rewrite the committed `index.json` with a different `version`, carrying every + * other field through unchanged, so a load that fails afterwards can only be + * failing on the version. The delta puts `index.json` alone, so each + * `.eml` carries forward by object id and the loader's blob-presence + * check still passes. + */ +async function rewriteIndexVersion(handles: Handles, version: number) { + const next = { ...(await readCommittedIndex(handles)), version }; + const substrate = makeStore(handles.dataDir); + await substrate.writeTreeDelta(handles.principal, handles.repoId, REF, { + computeDelta: async () => ({ + puts: { + [`${MAILBOX_INBOX_PREFIX}${MAILBOX_INDEX_FILE}`]: + new TextEncoder().encode(JSON.stringify(next)), + }, + deletes: [], + }), + changedPathPrefixes: new Set([MAILBOX_INBOX_PREFIX]), + message: "rewrite index.json version", + }); +} + describe("substrate mailbox store", () => { test("append assigns monotonic UID and bumps uidNext / highestModSeq", async () => { const handles = await makeHandles("dep-append"); @@ -583,4 +715,142 @@ describe("substrate mailbox store", () => { expect(Array.from(reopened.find(uid1)?.flags ?? [])).toEqual(["\\Seen"]); expect(reopened.find(uid2)).toBeUndefined(); }); + + test("an absent originator survives the index round trip as an absence", async () => { + // JSON has no `undefined`, so an envelope that carried no originator is + // written as `null` and read back as `undefined`. Any string in either + // position -- an empty one or a stand-in address -- would present the + // message to every consumer as having named a sender it never named. + // Both encodings sit in the same version-1 index, which is what makes the + // widening safe with no INDEX_VERSION bump: an index written before it + // holds plain strings and is the same format. + const handles = await makeHandles("dep-no-originator"); + const store = await openStore(handles); + const crypto = await senderCrypto(); + + const named = await makeSignedMessage(crypto, { + from: "alice@example.com", + to: ["run@dep.example.com"], + subject: "named", + messageId: "", + text: "named body", + }); + const source = await makeSignedMessage(crypto, { + from: "placeholder@example.com", + to: ["run@dep.example.com"], + subject: "orphan", + messageId: "", + text: "orphan body", + }); + const orphanRaw = stripHeader(source.raw, "From"); + const orphanEnvelope: StoredEnvelope = { + ...source.envelope, + from: undefined, + }; + + const namedUid = store.append(named.raw, named.envelope, []); + const orphanUid = store.append(orphanRaw, orphanEnvelope, []); + await store.flush(); + + expect((await readIndexEnvelopes(handles)).map((e) => e.from)).toEqual([ + "alice@example.com", + null, + ]); + + const reopened = await openStore(handles); + expect(reopened.find(namedUid)?.envelope.from).toBe("alice@example.com"); + expect(reopened.find(orphanUid)?.envelope.from).toBeUndefined(); + }); + + test("an absent date survives the index round trip as an absence", async () => { + // The same widening the originator got, for the same reason: JSON has no + // `undefined`, so an envelope whose message named no date is written as + // `null` and read back as `undefined`. Any string in that position -- the + // arrival instant included -- would place the message in a search window on + // a date its sender never stated. Both encodings sit in the same version-1 + // index, which is what makes the widening safe with no INDEX_VERSION bump. + const handles = await makeHandles("dep-no-date"); + const store = await openStore(handles); + const crypto = await senderCrypto(); + + const dated = await makeSignedMessage(crypto, { + from: "alice@example.com", + to: ["run@dep.example.com"], + subject: "dated", + messageId: "", + text: "dated body", + }); + const source = await makeSignedMessage(crypto, { + from: "alice@example.com", + to: ["run@dep.example.com"], + subject: "undated", + messageId: "", + text: "undated body", + }); + const undatedRaw = stripHeader(source.raw, "Date"); + const undatedEnvelope: StoredEnvelope = { + ...source.envelope, + date: undefined, + }; + + const datedUid = store.append(dated.raw, dated.envelope, []); + const undatedUid = store.append(undatedRaw, undatedEnvelope, []); + await store.flush(); + + expect((await readIndexEnvelopes(handles)).map((e) => e.date)).toEqual([ + "2026-02-01T00:00:00.000Z", + null, + ]); + + const reopened = await openStore(handles); + expect(reopened.find(datedUid)?.envelope.date).toEqual( + new Date("2026-02-01T00:00:00Z"), + ); + expect(reopened.find(undatedUid)?.envelope.date).toBeUndefined(); + }); + + test("the loader matches the index version exactly", async () => { + // `index.json` is a durable on-disk format with no migration arm: the + // loader matches `version` as an exact literal, so a bump rejects every + // mailbox already on disk instead of upgrading it. That is what makes a + // backward-compatible widening of a field -- `from` to `string | null` -- + // safe while the version stays put, and why a change that genuinely needs + // a bump needs a migration first. + const handles = await makeHandles("dep-index-version"); + const store = await openStore(handles); + const crypto = await senderCrypto(); + const msg = await makeSignedMessage(crypto, { + from: "a@example.com", + to: ["run@dep.example.com"], + subject: "versioned", + messageId: generateMessageId("a@example.com"), + text: "versioned body", + }); + const uid = store.append(msg.raw, msg.envelope, []); + await store.flush(); + + // The writer commits the pinned version, so writer and loader agree on one + // literal rather than on whatever the constant happens to hold. + expect((await readCommittedIndex(handles)).version).toBe( + COMMITTED_INDEX_VERSION, + ); + + // Rewriting with the version already committed changes nothing, so the + // mailbox still loads. Without this arm a loader that refused every index + // would satisfy the rejection arms below. + await rewriteIndexVersion(handles, COMMITTED_INDEX_VERSION); + const accepted = await openStore(handles); + expect(accepted.messages.map((m) => m.uid)).toEqual([uid]); + expect(accepted.find(uid)?.envelope.from).toBe("a@example.com"); + + // Every other version is refused, in both directions: a forward bump and a + // rollback to an older format are equally unreadable. + for (const version of [ + COMMITTED_INDEX_VERSION - 1, + COMMITTED_INDEX_VERSION + 1, + ]) { + await rewriteIndexVersion(handles, version); + await expect(openStore(handles)).rejects.toThrow(/invalid .*index\.json/); + } + }); }); diff --git a/vendor/intx-workflow-host/adapters/substrate-mailbox-store.ts b/vendor/intx-workflow-host/adapters/substrate-mailbox-store.ts index 8e75a7df1..e0aa39436 100644 --- a/vendor/intx-workflow-host/adapters/substrate-mailbox-store.ts +++ b/vendor/intx-workflow-host/adapters/substrate-mailbox-store.ts @@ -81,15 +81,18 @@ const encoder = new TextEncoder(); /** * On-disk envelope shape. Mirrors `StoredEnvelope` but serializes `date` as an - * ISO string and the three nullable header fields as `string | null` (JSON has - * no `undefined`); the loader maps `null` back to `undefined`. + * ISO string and the nullable header fields as `string | null` (JSON has no + * `undefined`); the loader maps `null` back to `undefined`. + * + * Widening a field needs no `INDEX_VERSION` bump: the version is matched + * exactly, so a bump rejects every existing mailbox instead of migrating it. */ const StoredEnvelopeJson = type({ messageId: "string", - from: "string", + from: "string | null", to: "string[]", subject: "string", - date: "string", + date: "string | null", inReplyTo: "string | null", references: "string[]", interchangeType: "string | null", @@ -191,10 +194,10 @@ export type SubstrateMailboxStoreOpts = { function serializeEnvelope(envelope: StoredEnvelope) { return { messageId: envelope.messageId, - from: envelope.from, + from: envelope.from ?? null, to: envelope.to, subject: envelope.subject, - date: envelope.date.toISOString(), + date: envelope.date === undefined ? null : envelope.date.toISOString(), inReplyTo: envelope.inReplyTo ?? null, references: envelope.references, interchangeType: envelope.interchangeType ?? null, @@ -207,10 +210,10 @@ function deserializeEnvelope( ): StoredEnvelope { return { messageId: raw.messageId, - from: raw.from, + from: raw.from === null ? undefined : raw.from, to: raw.to, subject: raw.subject, - date: new Date(raw.date), + date: raw.date === null ? undefined : new Date(raw.date), inReplyTo: raw.inReplyTo === null ? undefined : raw.inReplyTo, references: raw.references, interchangeType: diff --git a/vendor/intx-workflow-host/workflow-definition-loader.ts b/vendor/intx-workflow-host/workflow-definition-loader.ts index 63619bb4b..a1953663e 100644 --- a/vendor/intx-workflow-host/workflow-definition-loader.ts +++ b/vendor/intx-workflow-host/workflow-definition-loader.ts @@ -11,8 +11,8 @@ // validated at this boundary before being returned. // // Materialization is deliberately NOT done here. `@intx/workflow-host` -// stays free of a `@intx/tool-packaging` dependency (the sidecar owns -// that layer, see `apps/sidecar/src/tool-materialization.ts`), so the +// stays free of a `@intx/tool-packaging` dependency (that layer lives +// in `packages/tool-packaging/src/tool-materialization.ts`), so the // caller runs the closure machinery and hands the resulting package // directory in. This module only performs the import + evaluate + // validate step, which is the part that must run inside the child's @@ -32,9 +32,9 @@ import { type AnnotatedPluginFactory, type DirectorRegistry, type ToolDeclaration, -} from "@intx/agent"; +} from "@intx/agent/authoring"; import { PackageJSON, isContainedEntryPath } from "@intx/types/package-json"; -import { workflowDefinitionEnvelopeSchema } from "@intx/hub-sessions/substrate"; +import { workflowDefinitionEnvelopeSchema } from "@intx/types/workflow-definition"; import type { WorkflowDefinition } from "@intx/workflow/definition"; import type { ActionHandler, LoopFn, LoopFnRegistry } from "@intx/workflow";