diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 05ed8373..e41ef0cf 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -45,6 +45,7 @@ jobs: run: npm run build env: DOCS_BASE_PATH: /donkey-development-kit + GITHUB_TOKEN: ${{ github.token }} - name: Disable Jekyll so Pages serves Next's _next/ directory run: touch out/.nojekyll - uses: actions/configure-pages@v6 diff --git a/website/README.md b/website/README.md index b936e79e..acdeda32 100644 --- a/website/README.md +++ b/website/README.md @@ -80,17 +80,43 @@ future custom domain serve at the root without the sub-path. ## Structure +The sidebar is grouped into capability pillars by separators in +`content/_meta.js`; page URLs are flat, so moving a page between pillars never +breaks a link. + ``` content/ - index.mdx Introduction — what the SDK is - quickstart.mdx First governed request - feature-overview.mdx The three pillars at a glance - frameworks/ Model access — one page per framework (Pillar 1) - tool-access.mdx Pillar 2 (roadmap) - provisioning.mdx Pillar 3 (roadmap) - concepts/ Verification policy, governance, attribution - errors.mdx Governed error taxonomy - reference/ Configuration, unsupported boundary + index.mdx, quickstart.mdx, + feature-overview.mdx Get started + frameworks/ Model access — one page per framework + errors, budget, identity, + hitl, policies Governance + telemetry Observability + simulator, testing, cli, + use-with-your-agent Developer tooling + tool-access/, a2a, publishing Registry & catalog + scenarios/ Scenarios + examples/ Examples (from donkey-development-kit-demos) + roadmap.mdx Roadmap — rendered from data/roadmap.json + community/ Team, Contribute + reference/ Configuration, unsupported boundary +``` + +Capability status uses one vocabulary: `Live` for +what is available now and `Roadmap` for what is +planned. Delivery detail (phases, milestones, issues) belongs on the Roadmap +page only. + +## Roadmap data + +`data/roadmap.json` is a snapshot of the phase milestones and their issues, +written by [`scripts/generate-roadmap.mjs`](scripts/generate-roadmap.mjs) from +the GitHub API. `npm run build` refreshes it first (via `prebuild`); if the API +is unreachable the committed snapshot is kept, so builds never depend on the +network. Set `GITHUB_TOKEN` to avoid the unauthenticated rate limit. + +```bash +npm run generate:roadmap ``` ## Editing rules (inherited from the SDK — verification discipline) diff --git a/website/app/layout.tsx b/website/app/layout.tsx index 8313efee..62abd172 100644 --- a/website/app/layout.tsx +++ b/website/app/layout.tsx @@ -1,5 +1,5 @@ import type { Metadata } from 'next' -import { Inter } from 'next/font/google' +import { Play } from 'next/font/google' import { Footer, Layout, Navbar } from 'nextra-theme-docs' import { Head } from 'nextra/components' import { getPageMap } from 'nextra/page-map' @@ -8,17 +8,18 @@ import '../styles/globals.css' const SITE_NAME = 'Donkey Development Kit' const SITE_DESCRIPTION = - 'An SDK for Agent Fabric — governed model access from your own agent framework, with budget, refusals, telemetry and simulation on one skeleton.' + 'Governed by the gateway, understood by your code. An open-source SDK that brings Agent Fabric and Omni Gateway awareness into your agent framework.' // next/image's unoptimized loader does not prefix basePath for /public assets, // so reference the served copy under website/public/img/ with the base path // applied manually — same convention as components/index.tsx. const BASE_PATH = process.env.NEXT_PUBLIC_BASE_PATH ?? '' -const inter = Inter({ +const play = Play({ subsets: ['latin'], + weight: ['400', '700'], display: 'swap', - variable: '--font-inter', + variable: '--font-play', }) // Replaces theme.config.tsx `head()`: the Metadata API builds the same @@ -61,15 +62,38 @@ const navbar = ( } projectLink="https://github.com/Donkey-Development-Kit/donkey-development-kit" - /> + > + + MuleSoft + + ) const footer = ( ) @@ -85,16 +109,16 @@ export default async function RootLayout({ lang="en" dir="ltr" suppressHydrationWarning - className={`${inter.variable} ${inter.className}`} + className={`${play.variable} ${play.className}`} > - {/* Violet accent, close to the reference docs look (was theme.config `color`). */} - + {/* #00B4FF accent, matching --af-accent in styles/globals.css. */} + diff --git a/website/components/adapter-error-taxonomy-note.mdx b/website/components/adapter-error-taxonomy-note.mdx index faa5b33b..b31a7663 100644 --- a/website/components/adapter-error-taxonomy-note.mdx +++ b/website/components/adapter-error-taxonomy-note.mdx @@ -8,10 +8,10 @@ Intentional overrides that keep their own inline wording (documented, not silent): • langgraph — prefixes a proxy-surface caveat before this sentence. - • adk, crewai — "surface through LiteLLM's error path" (LiteLLM owns transport). + • adk, crewai — "surface through ADK's `LiteLlm` model" / "CrewAI's LiteLLM layer". • agent-framework — references the PolicyViolation hierarchy / middleware shape. */} See the [error taxonomy](/errors) for how proxy rejections surface as typed -exceptions, and the [verification policy](/concepts/verification) page for +exceptions, and the [verification ledger](https://github.com/Donkey-Development-Kit/donkey-development-kit/blob/develop/docs/verified-apis.md) for the current status of every constructor signature this adapter depends on. diff --git a/website/components/index.tsx b/website/components/index.tsx index 99959f04..3f7f948b 100644 --- a/website/components/index.tsx +++ b/website/components/index.tsx @@ -1,6 +1,7 @@ import React from 'react' import Link from 'next/link' import NextImage from 'next/image' +import roadmap from '../data/roadmap.json' /* Presentation components for the docs pages. * @@ -117,6 +118,23 @@ export function Card({ ) } +export function Output({ + children, + label = 'Expected output', + wrap = false, +}: { + children: React.ReactNode + label?: string + wrap?: boolean +}) { + return ( +
+
{label}
+ {children} +
+ ) +} + export function Figure({ src, alt, @@ -147,3 +165,141 @@ export function Figure({ ) } + +/* ---------------------------------------------------------------- roadmap */ + +type RoadmapIssue = { + number: number + title: string + state: string + url: string + epic: boolean +} + +function IssueList({ issues }: { issues: RoadmapIssue[] }) { + return ( + + ) +} + +// Renders one milestone from data/roadmap.json (see scripts/generate-roadmap.mjs): +// a status pill, a progress bar, and its issues split into open and done. +export function RoadmapPhase({ + milestone, + children, +}: { + milestone: string + children?: React.ReactNode +}) { + const m = roadmap.milestones.find(x => x.title.startsWith(milestone)) + if (!m) return null + const total = m.open + m.closed + const pct = total ? Math.round((m.closed / total) * 100) : 0 + const status = + m.open === 0 && total > 0 + ? { label: 'Complete', tone: 'live' as Tone } + : m.closed > 0 + ? { label: 'In progress', tone: 'accent' as Tone } + : { label: 'Planned', tone: 'roadmap' as Tone } + const open = m.issues.filter(i => i.state === 'open') + const done = m.issues.filter(i => i.state !== 'open') + return ( +
+
+ + {m.title} + + {status.label} +
+ {children ?
{children}
: null} +
+ +
+

+ {pct}% complete · {m.closed} done · {m.open} open +

+ {open.length ? ( +
+ Open issues ({open.length}) + +
+ ) : null} + {done.length ? ( +
+ Done ({done.length}) + +
+ ) : null} +
+ ) +} + +export function RoadmapUpdated() { + const date = new Date(roadmap.generatedAt) + return ( + + Status synced from GitHub milestones on{' '} + {date.toLocaleDateString('en-GB', { day: 'numeric', month: 'long', year: 'numeric' })}. + + ) +} + +/* ------------------------------------------------------------------- team */ + +export function TeamGrid({ children }: { children: React.ReactNode }) { + return
{children}
+} + +export function TeamMember({ + name, + role, + linkedin, + photo, +}: { + name: string + role: 'Creator' | 'Contributor' + linkedin: string + photo?: string +}) { + const initials = name + .split(/[\s-]+/) + .filter(Boolean) + .slice(0, 2) + .map(part => part[0]) + .join('') + return ( + + {photo ? ( + + ) : ( + + )} + {name} + + {role} + LinkedIn + + + ) +} diff --git a/website/content/_meta.js b/website/content/_meta.js index 8beeb1a2..4a4a3d56 100644 --- a/website/content/_meta.js +++ b/website/content/_meta.js @@ -1,48 +1,65 @@ export default { + '-- get-started': { + type: 'separator', + title: 'Get started', + }, index: 'Introduction', quickstart: 'Quickstart', 'feature-overview': 'Feature overview', - 'use-with-your-agent': 'Use with your agent', - roadmap: 'Roadmap', - '-- live': { + '-- model-access': { type: 'separator', - title: 'Live today', + title: 'Model access', }, - frameworks: 'Model access', - '-- six-piece': { + frameworks: 'Frameworks', + '-- governance': { type: 'separator', - title: 'Phase 1 — the six-piece minimum', + title: 'Governance', }, errors: 'Typed refusals', budget: 'Budget & pacing', + identity: 'Identity', + hitl: 'Human-in-the-loop', + policies: 'Policy handshake', + '-- observability': { + type: 'separator', + title: 'Observability', + }, + telemetry: 'Telemetry & cost', + '-- developer-tooling': { + type: 'separator', + title: 'Developer tooling', + }, simulator: 'Local simulator', testing: 'Testing & conformance', - telemetry: 'Telemetry & cost', cli: 'CLI & decorators', + 'use-with-your-agent': 'Use with your coding agent', + '-- registry': { + type: 'separator', + title: 'Registry & catalog', + }, + 'tool-access': 'Tool access', + a2a: 'A2A agents', + publishing: 'Scan & publish', '-- scenarios': { type: 'separator', - title: 'Scenarios — the pieces together', + title: 'Scenarios', }, scenarios: 'Scenarios', - '-- phase-2': { + '-- examples': { type: 'separator', - title: 'Phase 2 — differentiate', + title: 'Examples', }, - 'tool-access': 'Tool access', - a2a: 'A2A agents', - identity: 'Identity', - hitl: 'Human-in-the-loop', - publishing: 'Scan & publish', - '-- phase-3': { + examples: 'Examples', + '-- roadmap': { type: 'separator', - title: 'Phase 3 — platform', + title: 'Roadmap', }, - policies: 'Policy handshake', - '-- concepts': { + roadmap: 'Roadmap', + '-- community': { type: 'separator', - title: 'Concepts', + title: 'Community', }, - concepts: 'Concepts', + community: 'Community', '-- reference': { type: 'separator', title: 'Reference', diff --git a/website/content/a2a.mdx b/website/content/a2a.mdx index a2ea3fbb..1f379cc4 100644 --- a/website/content/a2a.mdx +++ b/website/content/a2a.mdx @@ -1,47 +1,38 @@ --- -description: Make your agent callable by other agents — serve, expose, and dev, wrapping the official a2a-sdk. +description: Make your agent callable by other agents over A2A — serve it locally, expose it through the gateway, and develop against a gateway on your laptop. --- import { Callout } from 'nextra/components' +import { Badge } from '../components' # A2A agents - - **Phase 2 — designed, not yet shipped.** Command and API shapes below are a - proposal; the committed part is the behaviour. See [Roadmap](/roadmap) and - [Verification policy](/concepts/verification). - +Roadmap -Three commands that take an agent from "runs on my laptop" to "callable by -other agents, through the gateway." +This capability is on the [Roadmap](/roadmap); the API shown here is the planned design. -## First, an honest answer to the obvious question +Three steps take an agent from "runs on my laptop" to "callable by other agents, +through the gateway": **serve** it, **expose** it, and **develop** against a +gateway locally. The protocol is the official `a2a-sdk`, wrapped; the SDK adds +the governance around it. -The question that always comes up: *A2A needs an HTTP listener — we have the -gateway, so can the gateway ingress be embedded in the agent process, making -the agent A2A-ready without running its own server?* +## Why the agent still runs a listener -**No to the literal idea, yes to what it is really asking for.** +An A2A agent *is* a server: it serves an agent card at a well-known path and +answers JSON-RPC task calls. Something must accept the socket, so the SDK hides +that listener behind one line rather than removing it. -Omni Gateway is an Envoy-based data plane whose policies compile to WASM. It is -a separate process, not a library — you cannot `import` it any more than you -could import nginx. More importantly you would not want to: +What it can remove is everything painful around the listener — TLS, auth, rate +limits, public exposure, and registration. Those are the gateway's job. Omni +Gateway is an Envoy-based data plane whose policies compile to WASM; it runs as +a separate process in front of your agent, not as a library inside it. - **The moment the enforcement point lives inside the agent's process, the - agent's own code can bypass it.** The gateway's value depends on being - *outside* the thing it governs. + **Enforcement stays outside the agent's process.** If the enforcement point + lived inside the agent, the agent's own code could bypass it. The gateway's + value depends on being outside the thing it governs. -So be precise about what A2A actually requires: something must accept the -socket. An A2A agent *is* a server — it serves an agent card at a well-known -path and answers JSON-RPC task calls. That listener can be hidden behind one -line, but it cannot be removed. - -What *can* be removed is everything painful about it: TLS, auth, rate limits, -public exposure, and registration. Those are the gateway's job, and that is -where the SDK helps. - ## `donkey serve` — the listener, in one line ```python @@ -52,19 +43,14 @@ async def handle(task: A2ATask) -> A2AResult: donkey.serve(handle) # A2A server on 127.0.0.1:8000, card auto-generated ``` -The card is generated from the same `@donkey.tool` / `@donkey.agent` markers -the [scanner](/publishing) reads, so there is no second description of your -agent to keep in sync. It binds to **localhost by default** — your agent is -never the public face. - -The protocol itself is the official **`a2a-sdk`**, wrapped. This project does -not implement A2A; re-implementing a protocol someone else maintains is on the -[will-not-build list](/roadmap). - -What a plain A2A server does *not* give you is the inbound governance path: -every call arriving over A2A gets the same treatment as an outgoing one — -correlation ID, cost tags, an OTel span, and typed refusals when the agent's -own downstream calls are blocked. +- **The card is generated from your code** — the same `@donkey.tool` / + `@donkey.agent` markers the [scanner](/publishing) reads — so there is no + second description of your agent to keep in sync. +- **It binds to localhost by default**, so your agent is never the public face; + the gateway is. +- **Inbound calls are governed.** Every call arriving over A2A gets the same + treatment as an outgoing one: correlation ID, cost tags, an OTel span, and + typed refusals when the agent's own downstream calls are blocked. ## `donkey expose` — the ingress, registered from code @@ -76,24 +62,25 @@ $ donkey expose --env prod ``` The gateway does the work — provisioning the A2A proxy, attaching the policy -set, registering the card. The SDK's contribution is turning a console session -into one command that runs in CI. +set, and registering the card. `donkey expose` turns that console session into +one command that runs in CI. ## `donkey dev` — a gateway in front of your laptop -Two options, and which one ships depends on what is actually available rather -than on preference: +`donkey dev` puts a gateway in front of `donkey serve` on your machine, so you +can call your agent over A2A through governance before deploying. Two modes are +planned: -- **A real gateway.** If a self-managed Omni Gateway image is usable for local - development, `donkey dev` runs it alongside `donkey serve` so you hit A2A - *through real policies* on your machine. Gated on image availability and - licensing. -- **Simulated ingress.** If that is blocked, the [local simulator](/simulator) - gains an A2A ingress mode: a local fake gateway in front of `donkey serve`, - replaying the same rejection fixtures. Fully within this project's control, - and held to the same honesty test — a plain A2A client must see - byte-identical responses. +- **A real gateway.** Where a self-managed Omni Gateway image is available for + local development, `donkey dev` runs it alongside `donkey serve`, so A2A calls + pass through real policies. +- **Simulated ingress.** The [local simulator](/simulator) gains an A2A ingress + mode: a local fake gateway in front of `donkey serve`, replaying the same + rejection fixtures. A plain A2A client sees byte-identical responses, and + every response carries `x-donkey-simulator: true`. ---- +## Related -**Status: Phase 2 — not yet shipped.** +- [A2A agent tools](/tool-access/a2a) — call *other* A2A agents as tools. +- [Scan & publish](/publishing) — register your agent card with Agent Registry. +- [Local simulator](/simulator) — the fixture-replay server behind simulated ingress. diff --git a/website/content/budget.mdx b/website/content/budget.mdx index 1c35100f..cd706b81 100644 --- a/website/content/budget.mdx +++ b/website/content/budget.mdx @@ -3,26 +3,18 @@ description: Remaining token budget as a first-class object, with pacing that sl --- import { Callout } from 'nextra/components' +import { Badge } from '../components' # Budget & pacing - - **Phase 1 — the `Budget` object and its pacing helpers ship.** The object - below (`limit` / `remaining` / `reset_at` / `observed_at` / `fraction_used`), - updated in-band from every response, and the two pacing helpers — `pace()` and - `wait_for_reset()` — are implemented. The one deferred piece is the *end-to-end - test against the local gateway simulator* (`donkey mock`), which lands with the - simulator itself. See [Roadmap](/roadmap) and - [Verification policy](/concepts/verification). - +Live -The governed proxy tells you about your token budget on **every response**, in -`x-token-*` headers. Today you would parse those yourself, in every call site, -and most people don't — so the first time budget matters is the moment it runs -out. +The governed proxy reports your token budget on **every response**, in +`x-token-*` headers. Parsing those yourself in every call site is tedious, and +most code skips it — so the first time budget matters is the moment it runs out. -The point of this piece: **you never parse a header.** Every response updates a -`Budget` object on the `Donkey` instance. +With DDK **you never parse a header.** Every response updates a `Budget` object +on the `Donkey` instance. ## The object @@ -44,16 +36,19 @@ async with donkey.budget.pace(reserve=0.10): # raises BudgetReserveReached at 9 ``` `pace()` raises **before** issuing the request that would cross your reserve — -not after a `429` comes back. That distinction is the whole feature. +not after a `429` comes back. + +The budget object is per-`Donkey`, not global: two instances with different +credentials do not share state. -## The batch job that finishes by itself +## Example: a batch job that finishes by itself 50,000 product records, enriched overnight against a governed model, budget window resetting every hour, no human awake. Without a budget object, the script runs flat out, takes a `429` at record 31,000, crashes, and someone re-runs it from record 0 in the morning — -spending the budget twice to do the same work. +spending the budget twice to do the same work. With `pace()`: ```python for batch in chunks(records, 200): @@ -71,53 +66,43 @@ for batch in chunks(records, 200): ``` Once `reset_at` has elapsed, the old observation is stale and `pace()` no longer -refuses. A response carrying a budget signal updates the observed fields; a -fresh future `reset_at` makes the guard active again. A response that does not -supply a fresh future `reset_at` leaves the stale pass-through open. The job can -continue unattended without a manual budget observation. A partial observation -can report usage without reporting `reset_at`; in that defensive case the loop +refuses, so the job continues unattended without a manual budget observation. A +later response updates the observed fields only when it carries a recognised +budget signal, and a fresh future `reset_at` makes the guard active again. + +If a partial observation reports usage without a `reset_at`, the loop above re-raises `BudgetReserveReached` after one attempt instead of calling `wait_for_reset()` and spinning at zero delay. Preserve the last checkpoint and -escalate the incomplete signal rather than crossing the reserve. +escalate rather than crossing the reserve. -## The dashboard that prevents the outage +## Example: a dashboard that prevents the outage `fraction_used` is per-agent, so it can be graphed. The owner of a support agent sees it climbing at 14:00 and asks for an increase before the 16:00 peak — rather than explaining an outage afterwards. -## The honest limitation +## Budget is observed in-band - **Budget is only visible in-band.** The gateway reports it on response - headers; there is **no endpoint that answers "what is my remaining - budget?"**. So `remaining` is only as fresh as your last call, and a - brand-new process knows nothing at all until its first request completes. + The gateway reports budget on response headers; there is **no endpoint that + answers "what is my remaining budget?"**. So `remaining` is only as fresh as + your last call, and a brand-new process knows nothing until its first request + completes. -This is why `observed_at` exists and is part of the public surface: so nobody -mistakes stale data for live data. A dashboard reading `remaining` without -checking `observed_at` is reporting history, not state. - -It is also filed as an upstream gap against the gateway — a budget-query -endpoint is the thing that would make this object live rather than -last-known-good. See [Roadmap](/roadmap). +This is why `observed_at` is part of the public surface: a dashboard reading +`remaining` without checking `observed_at` is reporting history, not state. +A budget-query endpoint on the gateway would make this object live rather than +last-known-good — see [Roadmap](/roadmap). -## Acceptance bar +## Behaviour guarantees -- `x-token-reset` in milliseconds converts correctly — asserted to the second - against a fixture with a known value. +- `x-token-reset` (milliseconds) is converted to a `datetime`, accurate to the + second. - `pace()` raises before the request that would cross the reserve, never after a `429`. -- After `reset_at`, `pace()` no longer refuses, so the wait-and-retry loop can - continue without a manual budget observation. A later response updates the - budget only when it carries a recognised signal, and the guard becomes active - again when that update supplies a future `reset_at`. +- After `reset_at`, `pace()` no longer refuses, so a wait-and-retry loop can + continue without a manual budget observation. - If the reserve is reached without a known `reset_at`, the retry loop raises once instead of spinning at zero delay. -- The budget object is per-`Donkey`, not global: two instances with different - credentials do not share state. - ---- - -**Status: Phase 1 — the `Budget` object, `pace()`, and `wait_for_reset()` are shipped. The end-to-end simulator test is deferred to the local gateway simulator (`donkey mock`).** +- Budget state is per-`Donkey` instance. diff --git a/website/content/cli.mdx b/website/content/cli.mdx index 975eca9f..dd4104ef 100644 --- a/website/content/cli.mdx +++ b/website/content/cli.mdx @@ -1,39 +1,40 @@ --- -description: Decorators and a four-command CLI so the six pieces are reachable in one line. +description: The @donkey.governed and @donkey.tool decorators, and the donkey CLI — init, doctor, mock, and test. --- import { Callout } from 'nextra/components' +import { Badge, Output } from '../components' # CLI & decorators - - **Phase 1 — the decorators and the four-command CLI are shipped.** Both - decorators below (`@donkey.governed` / `@donkey.tool`) have landed (#200), and - the CLI now exposes its four commands — `init`, `doctor`, `mock`, and `test` - (#201, #202) — with consistent global flags and exit codes. See - [Roadmap](/roadmap) and [Verification policy](/concepts/verification). - +Live -Two on-ramps. Everything on the [Roadmap](/roadmap)'s six-piece list attaches -to the same transport hooks, and these are how you reach all of it without -wiring each piece by hand. +Two on-ramps to the SDK: decorators that govern a function in one line, and a +four-command CLI for setup, diagnosis, local simulation, and conformance +testing. ## Decorators -`@donkey.governed` runs a function inside a [`donkey.run()`](/telemetry) scope: +### `@donkey.governed` + +Runs a function inside a [`donkey.run()`](/telemetry) scope: ```python @donkey.governed(team="support") async def handle_ticket(ticket): ... ``` -One decorator gives that function a run/correlation ID, cost tags, an OTel span, +One decorator gives the function a run/correlation ID, cost tags, an OTel span, and typed refusals — the four things you would otherwise set up per call site. -It wraps **both sync and async** callables, works bare (`@donkey.governed`) or -parametrised (`@donkey.governed(team=…, project=…, env=…, enduser_id=…)`), and -deliberately takes **no `id=`**: each call opens its own run (a "run of one"), -and a fixed id pinned across calls would collapse unrelated runs into one -correlation. When you need a specific id, use `donkey.run(id=…)` directly. + +- Wraps **both sync and async** callables. +- Works bare (`@donkey.governed`) or parametrised + (`@donkey.governed(team=…, project=…, env=…, enduser_id=…)`). +- Takes **no `id=`**: each call opens its own run, so unrelated calls are never + collapsed into one correlation. When you need a specific id, use + `donkey.run(id=…)` directly. + +### `@donkey.tool` ```python @donkey.tool @@ -42,21 +43,24 @@ async def lookup_crm(customer_id: str) -> dict: ... ``` -`@donkey.tool` marks a function as a governed tool **without changing how it's -called** — it returns the same function with a `__donkey_tool__` marker and -records a `ToolSpec` (name, qualname, signature, docstring, `is_async`) in a -process-global registry you read with `registered_tools()`. A tool with **no -docstring is rejected at decoration time** (`ValueError`) — an undescribed tool -is useless to a model and to the registry. `ToolSpec` and `registered_tools` -are exported from `donkey_kit`. +Marks a function as a governed tool **without changing how it's called**. It +returns the same function with a `__donkey_tool__` marker and records a +`ToolSpec` (name, qualname, signature, docstring, `is_async`) in a +process-global registry you read with `registered_tools()`. Both `ToolSpec` and +`registered_tools` are exported from `donkey_kit`. -The registry exists in Phase 1; its **consumers** are Phase 2 — the -[scanner](/publishing) reads these markers to derive a manifest from your code, -and the [A2A](/a2a) agent-card generator reads the same markers to build a card. -Marking tools early costs nothing and saves the migration later. +A tool with **no docstring is rejected at decoration time** (`ValueError`) — an +undescribed tool is useless to a model and to a registry. + +The same markers are what the planned [scanner](/publishing) and +[A2A](/a2a) agent-card generator read, so marking tools now carries forward. ## The CLI +```bash +pip install "donkey-kit[cli]" +``` + ```bash donkey init # writes a commented .donkey-kit.toml, names every missing env var at once donkey doctor # checks creds, reaches the gateway, reports budget state @@ -64,78 +68,118 @@ donkey mock # the local simulator — --scenario scripts failures donkey test # a thin front end to pytest --donkey-conformance ``` -All four share three global flags, which precede the subcommand: +### Global flags + +Three global flags precede the subcommand: ```bash donkey --config ./cfg.toml init # write/read a config file at a non-default path -donkey --env Production doctor # override the Anypoint environment -donkey --json init # machine-readable output where a command supports it +donkey --env Sandbox doctor # override the Anypoint environment +donkey --json init # machine-readable output where a command supports it ``` +### Exit codes + Every command exits non-zero on failure, so any of them drops into CI as a preflight. A command that needs an optional extra (`donkey mock` → `[local]`, `donkey test` → `[test]`, `donkey doctor` → `[llm]`) prints the exact `pip install` line and exits `1` — never a stack trace. -### `donkey init` bootstraps your config +### `donkey init` -`donkey init` resolves your current configuration (kwargs → env vars → -`.donkey-kit.toml` → defaults) and writes a **commented** `.donkey-kit.toml` -with the values it found. Three properties matter: +Resolves your current configuration (kwargs → env vars → `.donkey-kit.toml` → +defaults) and writes a **commented** `.donkey-kit.toml` with the values it +found. -- **It names every missing required field at once** — control-plane *and* proxy - — reusing the same validation the SDK runs at a live call, so `init` and a +- **Names every missing required field at once** — control plane *and* LLM + proxy — using the same validation the SDK runs at call time, so `init` and a real request never disagree about what is required. -- **It never writes a secret.** `client_secret`, `llm_proxy_client_secret`, and - `llm_proxy_key` are emitted as commented `env` pointers, not values — a - committed config file is the wrong home for a credential. -- **It is idempotent.** An existing file is left untouched unless you pass - `--force`. +- **Never writes a secret.** `client_secret`, `llm_proxy_client_secret`, and + `llm_proxy_key` are emitted as commented `env` pointers, not values. +- **Idempotent.** An existing file is left untouched unless you pass `--force`. + +```bash +donkey init --force +donkey --json init # {"path": "...", "written": true, "missing": [...]} +``` + +### `donkey doctor` + +A governed call can fail for several reasons that look identical from the +outside. `doctor` makes one real governed call and tells them apart: + +- **Wrong credentials** — the `client_id`/`client_secret` pair is rejected. +- **Wrong URL** — the credentials are fine but the base URL is not a proxy + instance. (A trailing `/v1` lands here; the governed proxy has no `/v1` + segment.) +- **Credentials fine, model not in the allow-list** — nothing is misconfigured; + your platform team has not granted that model. + +Each verdict prints the same remediation string the matching typed exception +carries, so the fix is in the output rather than in a runbook. The budget line +states how old the reading is, since the proxy has no budget endpoint. -### `donkey test` runs the conformance suite against your agent +```bash +donkey doctor +``` + + +```text +[ok] config env (3 fields) +[ok] gateway reachable, responded +[ok] credentials client_id accepted +[ok] model accepted by the proxy +[i] budget 99,000 / 100,000 remaining, resets in 59s, observed 0s ago +``` + + +```bash +donkey doctor --model gpt-4o # model to test against the allow-list (default gpt-4o) +donkey doctor --json # machine-readable checks +``` + +### `donkey mock` + +Runs the [local simulator](/simulator), which replays captured gateway +rejections. Needs the `[local]` extra. + +```bash +donkey mock --port 8080 --host 127.0.0.1 \ + --scenario pii_block:every=5 \ + --scenario budget:limit=20000,window=60s +``` + +| Flag | Default | Meaning | +|---|---|---| +| `--port` | `8080` | TCP port to bind. | +| `--host` | `127.0.0.1` | Host/interface to bind. | +| `--scenario` | none | Fault-injection rule, repeatable. See [Scenario scripting](/simulator#scenario-scripting). | + +An invalid `--scenario` exits with code `2`. + +### `donkey test` -`donkey test` is a thin front end to `pytest --donkey-conformance` — it does -not re-implement the runner. Point it at your agent factory and pass any -trailing pytest arguments straight through; pytest's exit code becomes -`donkey test`'s own. +A thin front end to `pytest --donkey-conformance` — it does not re-implement the +runner. Point it at your agent factory and pass any trailing pytest arguments +straight through; pytest's exit code becomes `donkey test`'s own. Needs the +`[test]` extra. ```bash donkey test --agent my.pkg:make_agent -k governance -x ``` -### `donkey doctor` is the one that pays for itself +See [Testing & conformance](/testing) for what the suite checks. -A governed call can fail for several reasons that all look identical from the -outside. `doctor` exists to tell them apart: +## Planned commands Roadmap -- **Wrong credentials** — the `client_id`/`client_secret` pair is rejected. -- **Wrong URL** — the credentials are fine but the base URL is not a proxy - instance. (The trailing `/v1` mistake lands here; the governed proxy has no - `/v1` segment.) -- **Credentials fine, model not in the allow-list** — nothing is misconfigured - at all; your platform team has not granted that model. - -Each answer comes with the same remediation string the matching typed -exception carries, so the fix is in the output rather than in a runbook. This -is the difference between an afternoon of guessing and thirty seconds. - -## Acceptance bar - -- `donkey --help` lists exactly the four commands: `init`, `doctor`, `mock`, - `test`. -- Global `--config` / `--env` / `--json` are consistent across the commands - that support them, and every command exits non-zero on failure. -- `donkey init` writes a commented `.donkey-kit.toml` with the resolved values, - lists every missing required field in one run, never writes a secret, and does - not clobber an existing file without `--force`. -- `donkey doctor` distinguishes those three failures from each other — not one - generic "could not connect." Each verdict prints the remediation string from - the corresponding typed refusal, so the CLI and the exception never disagree. -- `donkey test` forwards to `pytest --donkey-conformance` and propagates its - exit code. +These commands are part of the [Roadmap](/roadmap) and are not available in the +CLI yet: ---- +- `donkey scan` and `donkey publish` — derive a manifest and agent card from + your code and register them with Exchange. See [Scan & publish](/publishing). +- `donkey serve`, `donkey expose`, and `donkey dev` — serve your agent over + A2A and expose it through the gateway. See [A2A agents](/a2a). -**Status: Phase 1 — the `@donkey.governed` / `@donkey.tool` decorators (#200) -and the four-command CLI (`init`, `doctor`, `mock`, `test`) are shipped (#201, -#202).** + + Run `donkey --help` to see the commands available in your installed version. + diff --git a/website/content/community/_meta.js b/website/content/community/_meta.js new file mode 100644 index 00000000..7375756a --- /dev/null +++ b/website/content/community/_meta.js @@ -0,0 +1,4 @@ +export default { + team: 'Team', + contribute: 'Contribute', +} diff --git a/website/content/community/contribute.mdx b/website/content/community/contribute.mdx new file mode 100644 index 00000000..a5dd5af8 --- /dev/null +++ b/website/content/community/contribute.mdx @@ -0,0 +1,111 @@ +--- +description: How to contribute to DDK — issues, docs, examples and code — from filing an issue to a merged pull request. +--- + +import { Callout, Steps } from 'nextra/components' +import { Card, CardGrid } from '../../components' + +# Contribute + +DDK is open source under the Apache-2.0 licence, and contributions of every +size are welcome. This page is the short version; the full runbook is +[`CONTRIBUTING.md`](https://github.com/Donkey-Development-Kit/donkey-development-kit/blob/develop/CONTRIBUTING.md) +in the repository. + +## Ways to help + + + + Every change starts as an issue. Search first, then file one with what you + expected and what happened. + + + Small, well-scoped issues that are a good way into the codebase. + + + This site lives in `website/` (Nextra). Every page has an "Edit this page" + link. + + + Runnable demos live in the companion `donkey-development-kit-demos` + repository. + + + +## From issue to merge + + + +### File or find the issue + +No change lands without a GitHub issue — the issue is the plan. Each issue +carries exactly one milestone, which is the release it targets (see the +[Roadmap](/roadmap)). + +### Cut a branch from `develop` + +Branch names follow `/-`, for example +`fix/42-proxy-url-trailing-slash` or `docs/13-verified-apis-update`. Always +branch from `develop`; never from or into `main`. + +```bash +git checkout develop && git pull --ff-only +git checkout -b docs/13-verified-apis-update +``` + +### Run the pre-PR gate + +Run the same checks CI runs, from `python/`: + +```bash +pip install -e ".[dev,llm,cli]" +pytest -q # tests +mypy # mypy --strict +ruff check . # lint +lint-imports # the framework-free core contract +``` + +If you touched an adapter, also run `python scripts/verify_frameworks.py`. + +### Open a pull request into `develop` + +The PR body includes `Closes #`, a `## Summary`, a `## Test plan` and +a `## Post-deploy steps` section (write `None.` when nothing applies). PRs are +squash-merged, so `develop` reads as one commit per issue. + + + +## Contributing from a fork + +Not a member of the `Donkey-Development-Kit` organisation? The flow is the +same, from a fork: + +```bash +git clone https://github.com//donkey-development-kit.git +cd donkey-development-kit +git remote add upstream https://github.com/Donkey-Development-Kit/donkey-development-kit.git +git fetch upstream +git checkout -b docs/13-verified-apis-update upstream/develop +``` + +Open the PR from your fork into `Donkey-Development-Kit:develop` and tick +**Allow edits by maintainers**. A maintainer sets the milestone and labels, +runs the secret-gated checks that GitHub does not run on fork PRs, and merges. + +## Keep the docs in sync + +When code changes what DDK does, the docs change in the same pull request — +or a `documentation` follow-up issue is filed and linked. After editing pages +here, regenerate the AI-readable docs and commit the result: + +```bash +cd website +npm run generate:llms +``` + + + **Verification discipline.** DDK never documents or codes against an + endpoint, header or class name that has not been confirmed against the real + platform. If you can't confirm one, say so in the issue rather than guessing + — see the [verification ledger](https://github.com/Donkey-Development-Kit/donkey-development-kit/blob/develop/docs/verified-apis.md). + diff --git a/website/content/community/team.mdx b/website/content/community/team.mdx new file mode 100644 index 00000000..e50f411d --- /dev/null +++ b/website/content/community/team.mdx @@ -0,0 +1,45 @@ +--- +description: The people behind the Donkey Development Kit — its creators and contributors. +--- + +import { Callout } from 'nextra/components' +import { Figure, TeamGrid, TeamMember } from '../../components' + +# The DDK team + +DDK is built in the open by its two creators and a growing group of +contributors, with plenty of help from AI coding agents along the way. The +goal we share: agents that are as aware of the gateway as the gateway is of +them. + +
+ +## Creators + + + + + + +## Contributors + + + + + + + + + + + + + DDK is an open-source community project, not an official Salesforce or + MuleSoft product. Team members contribute in a personal capacity. Want to + join them? Start with [Contribute](/community/contribute). + diff --git a/website/content/concepts/_meta.js b/website/content/concepts/_meta.js deleted file mode 100644 index 96c16122..00000000 --- a/website/content/concepts/_meta.js +++ /dev/null @@ -1,7 +0,0 @@ -export default { - verification: 'Verification policy', - governance: 'Governance', - environments: 'Governance & environments', - attribution: 'Attribution & cost', - observability: 'Observability', -} diff --git a/website/content/concepts/attribution.mdx b/website/content/concepts/attribution.mdx deleted file mode 100644 index 85aa451a..00000000 --- a/website/content/concepts/attribution.mdx +++ /dev/null @@ -1,66 +0,0 @@ -import { Callout } from 'nextra/components' - -# Attribution & cost - -The core value of governed model access is **per-agent attribution**: knowing -which agent/app spent which tokens, without every team rolling their own -bookkeeping. - -## How attribution works (live-verified) - -For a directly-called ingress LLM proxy, the **attribution unit is the -`client_id` credential**. The calling agent/app is a registered Anypoint client -application, enforced by the `client-id-enforcement` policy — it is *not* a -bespoke header the SDK invents. - -- **Identity** rides on the verified `client_id` / `client_secret` request-header - pair (also your auth — see [Governance](/concepts/governance)). -- **The gateway emits identity/telemetry on the response**: an - `x-envoy-decorator-operation` carrying the API-instance id + environment id, an - `x-correlation-id`, and `x-llm-proxy-*` routing headers. -- **Token usage** for cost attribution comes from the response `usage` block - (`input_tokens`, `output_tokens`, `total_tokens`, plus the detail counts - `cached_tokens` / `cache_write_tokens` / `reasoning_tokens`), passed through - verbatim from the provider and surfaced per-call on `donkey.last_call` when - the response passes through the SDK's shared HTTP client. See - [when `last_call` is unavailable](/telemetry#when-last_call-is-unavailable). - -## Optional attribution metadata - -You can attach human-readable attribution that the SDK surfaces on telemetry: - -```bash -export DONKEY_APP_NAME="checkout-agent" -export DONKEY_BUSINESS_GROUP="payments" -``` - -## Correlation IDs across a run - -Bind a **run** id once and every governed call inside the block shares it — on -the wire, on the span, and on any exception — so a run's calls trace together: - -```python -async with donkey.run(id="order-1234"): - ... # calls in here share the run id -``` - -The run id becomes the `X-Correlation-Id` request header, the span's -`donkey.correlation_id`, and `DonkeyError.correlation_id`; each individual -request additionally carries a per-call `X-Donkey-Request-Id` that surfaces on -`DonkeyError.call_id`. See [Observability](/concepts/observability#grouping-a-run-the-run-id-on-the-span-the-call-id-on-the-error) -for the full two-id model. - - - **Header-only adapter caveat.** Google ADK and CrewAI route through LiteLLM, - while LlamaIndex and Microsoft Agent Framework receive only a static - `default_headers` snapshot. None receives the SDK's HTTP client, so they - cannot propagate the correlation id bound by `donkey.run(id=...)` per run. - These are documented, asserted conformance exemptions, not silent gaps. - - - - The **agent→agent egress** attribution header (`x-anypoint-api-instance-id`) is - a separate telemetry path for component-to-component traffic, not needed for - direct LLM proxy calls. The business-group *request* header shape is not - surfaced in the direct-proxy path and remains unverified. - diff --git a/website/content/concepts/environments.mdx b/website/content/concepts/environments.mdx deleted file mode 100644 index 7280c036..00000000 --- a/website/content/concepts/environments.mdx +++ /dev/null @@ -1,227 +0,0 @@ -import { Callout } from 'nextra/components' - -# Governance & environments - - - **Planned design.** This documents SDK behaviour as designed in the build plan; platform contracts it depends on are being live-verified, and unverified paths raise `NotImplementedError("blocked on verification: …")` rather than guess. See [Verification policy](/concepts/verification). - - -The [`Governance`](/concepts/governance) object is the single thing an agent -declares once. What changes across local, sandbox, and production is not the -object — it's which of its three verbs you call, and what a gateway can -actually enforce on your laptop versus a shared control plane. - -## The `Governance` object - -```python -from donkey_kit.governance import Governance, GatewayTarget, PolicyBinding - -gov = Governance( - name="hr-agent", - gateway=GatewayTarget.from_env(), # resolves local | sandbox | prod - policies=[ - PolicyBinding("rate-limiting-sla-based", "1.4.0", config={ - "rateLimits": [{"maximumRequests": 100, "timePeriodInMilliseconds": 60000}], - }), - PolicyBinding("openai-token-policy", "1.0.0", config={ - "maxTokensPerMinute": 50_000, - }), - PolicyBinding("prompt-injection-protection", "1.0.0"), - ], -) -``` - -The same object attaches wherever a model or a tool is constructed: - -```python -model = donkey.langgraph.chat_model("gpt-4o", governance=gov) -tools = await donkey.tools.discover(domain="hr", governed_only=True, governance=gov) -``` - -### `GatewayTarget` — the environment-varying part - -```python -@dataclass(frozen=True) -class GatewayTarget: - mode: Literal["local", "managed", "self-managed"] - base_url: str # http://localhost:8081 | https://.gw... | https:// - environment: str | None = None # None for local - gateway_name: str | None = None # control-plane gateway/registration name - connected: bool = True # local mode gateways are unconnected - - @classmethod - def from_env(cls) -> "GatewayTarget": - """DONKEY_TARGET=local|sandbox|production selects a profile from - .donkey-kit.toml. Local requires no Anypoint credentials beyond - whatever registration the local gateway itself needs.""" -``` - -Profiles live in `.donkey-kit.toml`, committed to the repo — the same -pattern as Terraform workspaces or Kustomize overlays: - -```toml -[targets.local] -mode = "local" -base_url = "http://localhost:8081" - -[targets.sandbox] -mode = "managed" -base_url = "https://hr-agent.sandbox.eu1.gw.mulesoft.com" -environment = "Sandbox" -gateway_name = "managed-omni-eu-1" - -[targets.production] -mode = "self-managed" -base_url = "https://gw.internal.acme.com" -environment = "Production" -gateway_name = "prod-k8s-omni" -``` - -`DONKEY_TARGET` selects the profile; `GatewayTarget.from_env()` resolves it -into the object above. - -## One object, three verbs — not one deploy - -Application code that applies policies directly to a shared gateway has taken -the platform team's job, and that's the review comment that gets an SDK -rejected. So the `Governance` object does not have a runtime `apply()`. It has -three verbs, at three different trust levels: - -| Verb | Target | Who runs it | What it does | -|---|---|---|---| -| `simulate()` | local | developer, laptop | Starts an ephemeral local Omni Gateway, applies the config, returns a live `base_url`. Nothing shared is touched. | -| `export()` | sandbox, prod | developer, laptop | Compiles the object into a `donkey.yaml` fragment (see [Scan & publish](/publishing)). Writes a file. Touches nothing. | -| `resolve()` | sandbox, prod | agent process, runtime | **Read-only.** Looks up the already-provisioned gateway route, verifies the expected policies are actually applied, returns the `base_url`. Raises `GovernanceDrift` if reality does not match the declaration. | - -There is deliberately **no `apply()` on the runtime object.** Applying to -sandbox and production goes through `donkey apply` in CI, against the -reviewed spec, under platform-controlled credentials, filtered by a policy -allow-list. The lifecycle is linear: - -```text -agent code declares Governance - → export() → donkey.yaml → PR review - → CI: donkey plan → posted as PR comment - → merge: donkey apply → gateway provisioned - → runtime: resolve() → verifies + returns URL -``` - -`resolve()` is what makes this feel like one object rather than a bureaucratic -downgrade: the developer writes one `Governance` object, and at runtime it -*checks* rather than *mutates*. A mismatch fails loudly at startup instead of -silently running ungoverned — a genuinely better property than blind -application. - - - **Escape hatch.** `Governance.apply(target, i_am_the_platform_team=True)` - exists for platform teams automating their own gateways. The kwarg name is - deliberately explicit, the connected app must hold write scopes it does not - have by default, and every use is logged at `WARNING`. - - -## Local ≠ sandbox (this matters) - -The difference between local and sandbox/prod is not just the URL. Local Mode -gateways are configured by declarative YAML on disk and do not talk to the -control plane — anything control-plane-dependent is unavailable locally. - -| Capability | Local Mode | Connected Mode | -|---|---|---| -| Routing, CORS, headers, TLS | yes | yes | -| Basic rate limiting / spike control | yes | yes | -| Custom WASM (PDK) policies | yes | yes | -| SLA-based rate limiting, client-ID enforcement, contracts | **no** (needs API Manager client apps) | yes | -| **LLM Proxy** | **probably not — verify** | yes | -| **MCP Bridge** | **probably not — verify** (API Manager guided experience) | yes | -| Control-plane analytics, token usage reporting | no | yes | - -If LLM Proxy and MCP Bridge turn out to be Connected-Mode-only — the single -most important thing verification has to confirm — `simulate()` still validates routing, -auth, rate limiting, custom policies, and request/response shape. Real value -for the dev loop. But LLM traffic is served by a **local mock proxy** the SDK -ships: an OpenAI-compatible stub that replays fixtures and simulates policy -rejections. It is labelled unambiguously in logs: - -```text -LOCAL SIMULATION — LLM proxy is mocked, not a real gateway -``` - -Never let a developer believe they tested the real thing. - -### Policies are not portable across modes - -```python -class PolicyPortability(Enum): - BOTH = "both" - CONNECTED_ONLY = "connected_only" - LOCAL_ONLY = "local_only" - UNKNOWN = "unknown" -``` - -Before it starts, `simulate()` prints exactly which declared policies are -being skipped and why — loud, non-suppressible by default, and repeated in the -teardown summary: - -```text -Starting local Omni Gateway (target: local) - applied cors 1.0.0 - applied rate-limiting 1.3.0 - applied acme-custom-redaction 0.2.0 (WASM) - SKIPPED rate-limiting-sla-based 1.4.0 connected-mode only (needs API Manager client apps) - SKIPPED prompt-injection-protection 1.0.0 connected-mode only - ! 2 of 5 policies are not exercised locally. Local pass does not imply sandbox pass. -``` - -A silent skip here is the worst possible failure mode for this feature. - -### Identity and secrets differ too - -There is no connected app, no client credentials, and no Secrets Manager -locally. A `LocalCredentialShim` generates throwaway credentials and injects -them into both the gateway config and the SDK client, so `${secret:...}` -references in the spec resolve to dev values from a gitignored -`.donkey-kit.local.toml`. - -## `simulate()` — the local harness - -```python -async with gov.simulate() as env: - model = donkey.langgraph.chat_model("gpt-4o", gateway=env.gateway) - tools = await donkey.tools.discover(domain="hr", gateway=env.gateway) - agent = create_react_agent(model, tools.langgraph()) - result = await agent.ainvoke({"messages": [("user", "…")]}) -assert env.policy_events("rate-limiting").count == 1 -``` - -`env` exposes `gateway` (a `GatewayTarget` pointing at a dynamically-allocated -localhost port), `logs()`, and `policy_events(policy_name)` parsed from -gateway logs, so a test can assert a policy actually fired. - - - **Docker is a hard dependency for this feature only.** It ships behind the - optional `[local]` extra — the rest of the SDK works without it. Whether - Local Mode itself requires a control-plane-issued registration/licence - artifact (which would gate whether CI runners without an Anypoint org can run - `simulate()` at all) is a verification item — see - [Verification policy](/concepts/verification). - - -## MCP servers written in Python or TypeScript - -Worth stating plainly: if you write an MCP server in your agent framework and -attach a `Governance` object, the SDK can put a gateway in front of it -locally, and can generate the gateway configuration for sandbox and -production. **It cannot deploy the server itself to sandbox or production.** -CloudHub 2.0 runs Mule applications; there is no MuleSoft-hosted runtime for a -Python or TypeScript process. See [Tool access](/tool-access) for the -discovery/binding side of the same server. - -| | Local | Sandbox / Prod | -|---|---|---| -| The MCP server process | SDK runs it in docker-compose | **Your own platform** — K8s, Cloud Run, Lambda, ECS | -| The gateway in front of it | SDK spins up local Omni Gateway | SDK exports config; CI applies it | -| Upstream URL in the config | container DNS name | user-supplied, from your deployment | - -To narrow that gap, `donkey-kit export --with-runtime` may emit a Dockerfile -and a plain Kubernetes manifest as a starting point — a convenience, never a -deployment platform. The SDK does not become a deployment orchestrator. diff --git a/website/content/concepts/governance.mdx b/website/content/concepts/governance.mdx deleted file mode 100644 index 7bbe1a6f..00000000 --- a/website/content/concepts/governance.mdx +++ /dev/null @@ -1,51 +0,0 @@ -import { Callout } from 'nextra/components' - -# Governance - -"Governance" is what the Omni Gateway proxy adds on top of a plain model call: -authentication, attribution, routing, and policy enforcement — applied at the -gateway, not in your code. - -## What the gateway enforces - -The live-verified `openai-sdk` proxy carried this policy stack, all enabled: - -- **client-id-enforcement** — the `client_id` + `client_secret` header pair is - required; this is also the per-agent identity (see [Attribution](/concepts/attribution)). -- **model-based-routing** — routes on the `model` field in the request body to a - provider; response carries `x-llm-proxy-*` routing headers. -- **dataweave-headers-transformation** — a gateway convenience that splits a - colon-joined credential in a single `authorization: Bearer :` or - `apikey: :` header into the `client_id` / `client_secret` pair, but - **only** when neither header is already present. DDK always emits the two-header - pair itself, so this policy is a **no-op** for DDK traffic (see - [Configuration](/reference/configuration)). -- **llm-proxy-core**, **cors**, **openai-transcoding-policy**. -- Optionally applied for capture: **llm-token-rate-limit** and - **llm-pii-detection** — these produce the `429` and `403` rejection shapes in - the [Error taxonomy](/errors). - -## What the SDK does with it - -The SDK's job is to make governance **invisible when it passes and typed when it -blocks**: - -- Injects the verified consumer-auth headers and attribution on every call. -- Retries transient failures in its own transport (frameworks are told - `max_retries=0` so retries happen in one place). -- Maps policy rejections to typed exceptions via `classify()`. - -## The `Governance` object (roadmap) - -The plan defines a `Governance` object with three verbs — inspect a governed API -instance, `resolve()` its policy state with drift detection, and `simulate()` -locally — plus a policy-portability table (which policies are Connected-Mode-only -vs. Local-Mode-capable). - - - These are **structural today, not wired**. Reading applied policies per API - instance is confirmed via the Anypoint CLI, but the governed-state join and - local-mode viability are still unverified. The object raises - `blocked on verification` rather than report a state it can't confirm. See - [Verification policy](/concepts/verification). - diff --git a/website/content/concepts/observability.mdx b/website/content/concepts/observability.mdx deleted file mode 100644 index a0bd5ae9..00000000 --- a/website/content/concepts/observability.mdx +++ /dev/null @@ -1,181 +0,0 @@ ---- -description: OTel GenAI spans at a pinned semconv version, dual-emitted with a stable donkey.* namespace. ---- - -import { Callout } from 'nextra/components' - -# Observability - -Every governed model call the SDK makes opens **one OpenTelemetry span**, and -that span carries **two namespaces at once**: - -- **`gen_ai.*`** — the OpenTelemetry [GenAI semantic - conventions](https://opentelemetry.io/docs/specs/semconv/gen-ai/), so your - existing OTel tooling recognises the call as a model call with no custom - mapping. -- **`donkey.*`** — a stable Agent Fabric namespace for the governance facts OTel - has no key for yet: the policy decision, the refusal type, the remaining token - budget, the run correlation id. - -Both land on the **same span** — never two spans for one call — so a trace reads -as one governed model call, not a generic call plus a separate governance event. - - - Telemetry is **opt-in and never a hard dependency.** Install it with - `pip install "donkey-kit[otel]"`. With OpenTelemetry absent, or with - `telemetry=False` in your config, the span code is an inert no-op — it never - raises and never becomes a required import. - - -## The pinned semantic-convention version - -The OTel GenAI conventions are still evolving upstream (they live under -`_incubating`), and the installed `opentelemetry.semconv` package tracks the -latest schema — its default drifts release to release. So the SDK does **not** -re-export attribute names from that package. Instead: - -- The convention version is pinned in one constant, - `donkey_kit.core.telemetry.GEN_AI_SEMCONV_VERSION` (currently `1.30.0`). -- The `gen_ai.*` keys are **transcribed literals** at that version. - -The payoff: upgrading `opentelemetry-semantic-conventions` in your environment -never silently changes what the SDK emits. Bumping the pin is a deliberate, -single-file edit called out in the [release notes](https://github.com/Donkey-Development-Kit/donkey-development-kit/releases) -— not something a transitive dependency bump does behind your back. - -## What lands on the span - -| Key | Namespace | Source | -| --- | --- | --- | -| `gen_ai.system` | `gen_ai.*` (pinned) | The verified `x-llm-proxy-llm-provider` response header (see [Attribution](/concepts/attribution)). Omitted — never guessed — when the header is absent. | -| `gen_ai.request.model` | `gen_ai.*` (pinned) | The `model` field of the request body. | -| `gen_ai.usage.input_tokens` | `gen_ai.*` (pinned) | The `usage` block of a buffered response, or the terminal `usage` event of a stream (`input_tokens`, falling back to `prompt_tokens`). | -| `gen_ai.usage.output_tokens` | `gen_ai.*` (pinned) | The `usage` block of a buffered response, or the terminal `usage` event of a stream (`output_tokens`, falling back to `completion_tokens`). | -| `donkey.usage.cached_tokens` | `donkey.*` (stable) | Prompt tokens served from cache, from `usage.input_tokens_details.cached_tokens`. The semconv pins no key for this at the pinned version, so it lives in `donkey.*`. Omitted — never `0` — when the provider reports no detail counts. | -| `donkey.usage.cache_write_tokens` | `donkey.*` (stable) | Prompt tokens written to cache, from `usage.input_tokens_details.cache_write_tokens`. Omitted when absent. | -| `donkey.usage.reasoning_tokens` | `donkey.*` (stable) | Reasoning-model thinking tokens, from `usage.output_tokens_details.reasoning_tokens`. Omitted when absent. | -| `donkey.policy.decision` | `donkey.*` (stable) | `allow` on success; `refuse` on a classified policy refusal (which also sets the span's OTel status to `ERROR`). | -| `donkey.policy.type` | `donkey.*` (stable) | The refusal type slug (e.g. `token_budget`, `pii_detected`) — see [Errors](/errors). | -| `donkey.budget.remaining` | `donkey.*` (stable) | The remaining token budget after this call, from the `x-token-*` headers. | -| `donkey.correlation_id` | `donkey.*` (stable) | The **run** correlation id actually sent on the request (the `X-Correlation-Id` header) — shared by every call in a `donkey.run()` block. The per-call id is deliberately *not* on the span; it lives on `DonkeyError.call_id` (see below). | - - - **The `donkey.*` keys are public API.** Renaming one is a breaking change, - independent of any `gen_ai.*` version bump. Build dashboards and alerts on them - with the same confidence you'd give a documented field. - - -Note what is **not** in that table: prompt and completion **text**. The span -carries metadata only. Message content (`gen_ai.prompt` / `gen_ai.completion`) -is emitted **only** when you set `telemetry_capture_content=true`, because spans -are created upstream of the gateway's PII masking — defaulting it on would -re-export the content the platform just masked. The emitter is allowlist-driven, -so a content-shaped attribute handed in from any call site is dropped unless that -opt-in is set. See [Telemetry & cost](/telemetry) for the obligation you take on -by enabling it. - -### `gen_ai.system` is never guessed - -The proxy routes to several providers (OpenAI, Azure OpenAI, Gemini, Bedrock, -Anthropic), so `gen_ai.system` is taken **only** from the verified -`x-llm-proxy-llm-provider` response header. If that header is absent on a given -response, the attribute is omitted rather than defaulted — a wrong provider -label is worse than a missing one ([Verification policy](/concepts/verification)). - -## Grouping a run: the run id on the span, the call id on the error - -Bind a **run** id once and every governed call inside the block carries it — on -the wire, on the span, and on any exception — with nothing threaded through your -framework state: - -```python -async with donkey.run(id=ticket.id): - await triage_agent.run(ticket) -``` - -`donkey.run(id=…)` returns a **dual sync/async** context manager, so plain -`with donkey.run(...)` works too. Inside the block: - -- The **run id** becomes the `X-Correlation-Id` request header, the span's - `donkey.correlation_id`, and `DonkeyError.correlation_id`. It is shared by - every call in the block, so a Slack log line joins to the gateway's own record. -- Each individual request also gets a fresh **per-call id** (the - `X-Donkey-Request-Id` header), unique per logical request and stable across - that request's retries. It surfaces on `DonkeyError.call_id` — **not** on the - span, because the span is already the per-call unit; the run id is what a span - needs to correlate *across* calls. - -Propagation is contextvar-based, so a run id set here reaches framework-spawned -`asyncio` tasks (every LangGraph node, for instance) without you passing an -argument through each function, and concurrent runs never leak into each other. -Nested `donkey.run()` blocks rebind then restore; omitting `id` binds a -generated "run of one". Correlation works with or without OpenTelemetry — the -ids are pure headers and contextvars; the span only decorates when OTel is -present. - - - **Verified (#522).** The gateway echoes `x-correlation-id` on its **response** - *and* reads the inbound `X-Correlation-Id` — a probe confirmed the response - echoes the client-sent value verbatim, so it is the real join key. - `X-Donkey-Request-Id` is a client-owned per-call id the gateway does not read - (it lives on `DonkeyError.call_id`). Both names stay overridable - (`correlation_header` / `call_id_header`) if your gateway expects different - names ([Verification policy](/concepts/verification)). - - -## Streaming, refusals, and exceptions - -The span is a lifecycle, not just a bag of attributes. Whatever shape a call -takes, it produces **exactly one span**, and that span closes: - -- **A refusal is a failed operation.** When the proxy rejects a call and - `classify()` maps it to a policy refusal, the span carries `donkey.policy.decision - = refuse` and the `donkey.policy.type` slug *and* its OTel status is set to - `ERROR` — so a trace reads a refused call as a failure, not a - successful-looking span with a `refuse` label buried in its attributes. -- **An exception still closes the span.** A transport error (or a cancelled - request) that escapes before a response settles marks the span `ERROR` and - ends it — the lifecycle never leaks an open span on the error path. -- **A stream produces one span, closed when the stream is done.** A streaming - (SSE) response has no usage on its envelope — the token counts arrive in a - terminal `usage` event the caller reads long after the request returns. The - span stays open until the stream finishes and is closed **exactly once**, - whether you drain it fully, abandon it mid-iteration, or it raises partway - through. `gen_ai.usage.*` and the `donkey.usage.*` detail counts are populated - from that terminal event (and merged onto `donkey.last_call` at the same time - when the response passes through the SDK's shared HTTP client; see - [when `last_call` is unavailable](/telemetry#when-last_call-is-unavailable)). - - - Streaming token counts appear **only if the stream actually carries a usage - event.** For OpenAI-style Chat Completions that means requesting it with - `stream_options={"include_usage": true}`; without it the provider emits no - usage event, so the counts are omitted rather than guessed - ([Verification policy](/concepts/verification)). - - -## What this piece does *not* do yet - -This is deliberately the span **contract** — the pinned keys, the dual -namespace, and the span lifecycle — not the whole observability story. Tracked, -not forgotten: - -- **Exporter configuration.** Wiring an OTLP exporter/endpoint is - [#194](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/194); the - SDK emits spans through whatever `TracerProvider` you have configured. -- **Cost tags.** The fixed `team` / `project` / `env` / `enduser.id` dimensions - now populate both `donkey.cost.*` span attributes and request headers, set once - via `Donkey.from_env(team=…)` and overridable per run via `donkey.run(team=…)` - ([#196](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/196), - shipped) — see [Telemetry & cost](/telemetry). -- **Prompt/completion content stays off the span.** Message bodies are never - attached — that privacy guarantee is - [#306](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/306). - -## Where to go next - -- [Attribution & cost](/concepts/attribution) — the `client_id` identity and the - response headers the spans draw from. -- [Errors](/errors) — the refusal taxonomy behind `donkey.policy.type`. -- [Verification policy](/concepts/verification) — why an unverified value is - omitted, not guessed. diff --git a/website/content/concepts/verification.mdx b/website/content/concepts/verification.mdx deleted file mode 100644 index 4fa6cf62..00000000 --- a/website/content/concepts/verification.mdx +++ /dev/null @@ -1,59 +0,0 @@ -import { Callout } from 'nextra/components' - -# Verification policy - -This is the most important thing to understand about the SDK's trustworthiness. - - - **Never invent an endpoint, header name, or class name.** A fabricated - endpoint that 404s in a customer sandbox destroys trust in the whole package. - - -Every platform fact the SDK relies on is either **verified against a real -Anypoint sandbox / an installed framework**, or it is **not used** — the code -path that would need it raises `NotImplementedError("blocked on verification: …")` -or a clear `ConfigError` instead of guessing. How each value is confirmed — and -how a blocked value flips to verified — is documented for contributors in -[ARCHITECTURE.md → Verification discipline](https://github.com/Donkey-Development-Kit/donkey-development-kit/blob/main/ARCHITECTURE.md#verification-discipline). - -## What is verified today - -- **LLM proxy data plane** (live-verified): the **OpenAI-format ingress** — - base-URL shape (`https:////`, no `/v1`), the `client_id` + - `client_secret` consumer-auth header pair, model-based routing, streaming, - token accounting, and the absence of a `/models` endpoint (`404`). The proxy's - ingress **Format** (OpenAI / Anthropic / Gemini) is chosen at provisioning - time; OpenAI is the primary verified path and the **Gemini**-native ingress is - also live-verified (#540) — see the row below and - [Which wire format does your proxy speak?](/frameworks#which-wire-format-does-your-proxy-speak). -- **Attribution** (live-verified): the `client_id` credential is the per-agent - attribution unit; the gateway emits identity/telemetry on the response. -- **Six policy rejection shapes** (live-verified): auth `401`, PII `403`, - token-budget `429`, upstream passthrough, plus the **regex prompt-guard** and - **Azure content-safety** `403` blocks (the last two confirmed 2026-09-22 - against the deployed provisioning proxies, #253). See [Error taxonomy](/errors). - -## What is still blocked — and why that's a feature - -| Area | Status | What the SDK does about it | -|---|---|---| -| Non-OpenAI proxy **ingress Format** (Anthropic / Gemini) | Both native ingress Formats now **VERIFIED (LIVE)**: Anthropic Messages route (#304) and Gemini `:generateContent` route (#540); no `donkey.gemini` SDK adapter | The default data-plane path is **Format=OpenAI**. MuleSoft Model Proxy [documents](https://docs.mulesoft.com/general/model-proxy) three selectable ingress Formats (OpenAI / Anthropic / Gemini) fixed at proxy creation, and both non-OpenAI Formats are now live-confirmed. The **Anthropic**-native Messages route is **VERIFIED (LIVE)**: a `Format=Anthropic` proxy serves `POST //v1/messages` natively (200, native Anthropic body; OpenAI-shape `/chat/completions` → 404), captured in `tests/fixtures/anypoint/anthropic_inbound/` (#304). The **Gemini**-native ingress is likewise **VERIFIED (LIVE)**: a `Format=Gemini` proxy serves a native `POST …/models/:generateContent` passthrough (200, native Gemini body; OpenAI-shape → 400), captured in `tests/fixtures/anypoint/gemini_inbound/` (#540). `donkey.anthropic.client()` points `AsyncAnthropic` at the native route — but it requires a `Format=Anthropic` proxy (default DDK proxies are `Format=OpenAI`). The SDK ships **no** `donkey.gemini` adapter — a native `google-genai` adapter is demand-driven (#244); Gemini and Claude models are also reachable as **upstream providers** behind an OpenAI-format ingress via model-based routing, which works today. Ingress Format is a property of the provisioned proxy, not an SDK setting (no `llm_proxy_format` field). See [Which wire format does your proxy speak?](/frameworks#which-wire-format-does-your-proxy-speak). | -| Framework **constructor signatures** (docs/verified-apis.md §8) | Unverified per framework | Adapters target recorded class names; a mismatch surfaces via the verification harness / nightly matrix rather than silently misbehaving. Agent Framework's client raises `blocked on verification` today. | -| MCP tool discovery + binding ([Phase 2](/tool-access)) | Unverified | `donkey.tools.discover` and the binding classes raise `blocked on verification`. | -| Exchange publish API ([Phase 2](/publishing)) | Partially verified | First-class MCP/agent asset types exist, but the exact publication and discovery token vocabulary is unverified. `donkey publish` and Exchange search remain blocked rather than sending the SDK's assumed `mcp` / `a2a-agent` values. | -| Token-exchange endpoint for [on-behalf-of](/identity) (Phase 2) | Unverified | The exact endpoint and header the gateway expects are not guessed. | -| [Policy discovery](/policies) (Phase 3) | **Missing upstream** | No such endpoint exists on the gateway today. Filed as an upstream gap — the handshake cannot be built until it ships. | -| Budget query endpoint ([Phase 1](/budget)) | **Missing upstream** | Budget is reported in-band on response headers only, so `remaining` is last-known-good; `observed_at` exposes the staleness. | -| Whether Anypoint Monitoring ingests OTLP GenAI spans ([Phase 1](/telemetry)) | Unverified | The SDK promises "exports OTLP" and lets the sink be your choice. | -| A provisioning control plane | **Cut** | Not built. It would compete with API Manager and Terraform — see [Roadmap](/roadmap). | -| Injection-protection / fall-through bodies | Not captured | The **regex prompt-guard** (`matched_patterns` list) and **content-safety** (`…-action: reject` header) bodies are now **live-captured** — Azure Content Safety 2026-09-22 (#253) and its **Bedrock Guardrails** sibling 2026-09-24 (#568). Still uncaptured: the distinct **Injection Protection** policy's body (typed by the `x-injection-protection: blocked` header; no such proxy is deployed) and the undiscriminated content-moderation fall-through to a generic `PolicyViolation` — pinned from the policy pages, and no verification row flips until a live round-trip confirms them. No invented discriminator — tracked in #253. | -| First-party **TypeScript SDK** | Planned | TypeScript examples reach the proxy over its OpenAI-compatible API via the `openai` / `@anthropic-ai/sdk` clients; no first-party TS package is implied until it ships. | - -## What this means for you - -As a consumer you don't have to track any of this. Any platform fact the SDK -hasn't confirmed raises `blocked on verification` rather than guessing, so a -governed call either works against the verified contract or fails loudly — it -never silently reaches an unconfirmed endpoint. The mechanism behind that and -the full verified-versus-blocked ledger are documented for contributors in -[ARCHITECTURE.md → Verification discipline](https://github.com/Donkey-Development-Kit/donkey-development-kit/blob/main/ARCHITECTURE.md#verification-discipline). diff --git a/website/content/errors.mdx b/website/content/errors.mdx index 545b2863..0837284c 100644 --- a/website/content/errors.mdx +++ b/website/content/errors.mdx @@ -1,50 +1,50 @@ +--- +description: Typed exceptions for every governance refusal, each with a discriminator, a retry rule, and a next step. +--- + import { Callout } from 'nextra/components' +import { Badge } from '../components' # Governed error taxonomy +Live + The proxy doesn't just pass model calls through — it enforces policy. When it -rejects a call, the SDK turns the response into a **typed exception** so you +rejects a call, DDK turns the response into a **typed exception** so you branch on the governance outcome instead of parsing bodies. ## The rejection shapes `classify()` types -`classify()` types eight rejection shapes. **Six are live-verified** against the -sandbox proxies (auth, PII, token rate limit, upstream, **regex prompt guard**, -and **content safety** — the last two confirmed 2026-09-22, [#253], with content -safety's **Bedrock Guardrails** vendor added 2026-09-24, [#568]). -Still pending live capture: **injection protection** (typed by its header -discriminator, body uncaptured — a distinct policy from regex prompt guard, with -no proxy deployed) and the **undiscriminated content-moderation** fall-through — -both tracked in [#253]. The critical lesson: -**neither the status code nor the shape of the `error` value alone is a -sufficient discriminator** — a `403` can be PII, a regex-guard block, a -content-safety block (all policy blocks) *or* auth, and the same nested-object -envelope is emitted by both the upstream provider and a gateway policy. The -authoritative discriminator is the error **`type`** plus specific headers. - -| Rejection | HTTP | Discriminator | Maps to | Verified? | -|---|---|---|---|---| -| Client-ID enforcement (auth) | `401` | flat `{"error":"…"}` + `www-authenticate: Client-ID-Enforcement` | `AuthError` | live | -| PII detected | `403` | nested `{"error":{type:"pii_detected"}}`, **no** `www-authenticate` | `PIIDetected` (parses `entities`) | live | -| Injection protection | `400` | header `x-injection-protection: blocked` (**not** the status) | `PromptInjectionBlocked` | pending (#253) | -| Regex prompt guard | `403` | top-level `matched_patterns` list (flat `error`) | `PromptInjectionBlocked` (`policy="regex-prompt-guard"`) | live (#253) | -| Content safety / guardrails | `403` | header `x-llm-proxy--…-action: reject` (Azure Content Safety / Bedrock Guardrails) | `ContentSafetyBlocked` (parses `categories`) | live — Azure (#253) + Bedrock (#568) | -| Token rate limit | `429` | **empty body**; `x-token-limit`/`-remaining`/`-reset` headers (ms) | `TokenBudgetExceeded` (`retry_after` derived) | live | -| Content moderation (undiscriminated) | `4xx` | falls through — no nested `error`, no injection/guard/safety discriminator | generic `PolicyViolation` | pending (#253) | -| Upstream provider 4xx | `4xx` | nested `error` object **with** `code`/`type`/`param` — in an OpenAI-style object envelope `{"error":{…}}` **or** a Gemini-style list envelope `[{"error":{…}}]` (`status`→`error_type`, #548) | `UpstreamRequestError` | live | -| Upstream 5xx | `5xx` | status range (no competing discriminator) | `UpstreamModelError` (retryable) | live | +`classify()` types eight rejection shapes. **Neither the status code nor the +shape of the `error` value alone is a sufficient discriminator** — a `403` can be +PII, a regex-guard block, a content-safety block (all policy blocks) *or* auth, +and the same nested-object envelope is emitted by both the upstream provider and +a gateway policy. The authoritative discriminator is the error **`type`** plus +specific headers. + +| Rejection | HTTP | Discriminator | Maps to | +|---|---|---|---| +| Client-ID enforcement (auth) | `401` | flat `{"error":"…"}` + `www-authenticate: Client-ID-Enforcement` | `AuthError` | +| PII detected | `403` | nested `{"error":{type:"pii_detected"}}`, **no** `www-authenticate` | `PIIDetected` (parses `entities`) | +| Injection protection | `400` | header `x-injection-protection: blocked` (**not** the status) | `PromptInjectionBlocked` | +| Regex prompt guard | `403` | top-level `matched_patterns` list (flat `error`) | `PromptInjectionBlocked` (`policy="regex-prompt-guard"`) | +| Content safety / guardrails | `403` | header `x-llm-proxy--…-action: reject` (Azure Content Safety / Bedrock Guardrails) | `ContentSafetyBlocked` (parses `categories`) | +| Token rate limit | `429` | **empty body**; `x-token-limit`/`-remaining`/`-reset` headers (ms) | `TokenBudgetExceeded` (`retry_after` derived) | +| Content moderation (undiscriminated) | `4xx` | falls through — no nested `error`, no injection/guard/safety discriminator | generic `PolicyViolation` | +| Upstream provider 4xx | `4xx` | nested `error` object **with** `code`/`type`/`param` — in an OpenAI-style object envelope `{"error":{…}}` **or** a Gemini-style list envelope `[{"error":{…}}]` (`status`→`error_type`) | `UpstreamRequestError` | +| Upstream 5xx | `5xx` | status range (no competing discriminator) | `UpstreamModelError` (retryable) | `PIIDetected`, the regex-prompt-guard check, and the content-safety check are all -evaluated **before** the generic 401/403→auth rule, precisely because each is a -`403` (or `4xx`) that is *not* an auth failure. Likewise the injection check gates -on the `x-injection-protection` header, so an ordinary malformed `400` stays an +evaluated **before** the generic 401/403→auth rule, because each is a `403` (or +`4xx`) that is *not* an auth failure. Likewise the injection check gates on the +`x-injection-protection` header, so an ordinary malformed `400` stays an ordinary refusal. Client-ID enforcement (`401`) is a **consumer-auth** case, not one of the eight policy-rejection rows. -[#253]: https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/253 -[#568]: https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/568 +The Injection Protection header is typed from the documented response shape; +`classify()` keys on the header discriminator alone for it. ## The exception tree @@ -57,7 +57,7 @@ DonkeyError # base of the whole tree ├─ PolicyViolation # base for every governance rejection │ ├─ PIIDetected # 403, type=pii_detected; .entities │ ├─ TokenBudgetExceeded # 429; .retry_after (seconds) -│ ├─ PromptInjectionBlocked # x-injection-protection: blocked, or regex matched_patterns (pending #253) +│ ├─ PromptInjectionBlocked # x-injection-protection: blocked, or regex matched_patterns │ └─ ContentSafetyBlocked # Azure Content Safety / Bedrock Guardrails vendor reject header; .categories ├─ GatewayUnavailable # transport failure — gateway unreachable, NO response; .base_url/.cause (ungoverned) ├─ UpstreamRequestError # upstream 4xx; .code/.error_type/.param @@ -84,19 +84,19 @@ never burn an exhausted budget or replay a blocked prompt. | `PIIDetected` | `403`, nested `type: "pii_detected"`, **no** `www-authenticate` | **No** — a `PolicyViolation`, never retried. | Remove or redact the flagged values (`.entities`), or relax the policy's entity list in API Manager. | | `TokenBudgetExceeded` | `429`, empty body, `x-token-*` headers | **Not immediately** — never auto-retried; only worth retrying *after* the window resets. | Wait for `.retry_after` (seconds) / the reset, then retry — or request an increase in API Manager. | | `PromptInjectionBlocked` | header `x-injection-protection: blocked`, **or** a top-level `matched_patterns` list (regex prompt guard) | **No** — a `PolicyViolation`, never retried. | Review and sanitise the untrusted input, or adjust the policy's sensitivity / deny-list in API Manager. | -| `ContentSafetyBlocked` | `403` + `x-llm-proxy--…-action: reject` (Azure Content Safety / Bedrock Guardrails) — *live-verified [#253]/[#568]* | **No** — a `PolicyViolation`, never retried. | Revise the flagged content (`.categories`), or adjust the policy's categories / severity thresholds in API Manager. | -| `PolicyViolation` (generic) | a `4xx` matching **no** documented rejection shape | **No** — terminal; the shape is simply not yet typed. | Inspect `.response`; file an issue with the status/headers/body so the shape can be typed ([#184]/[#253]). | -| `UpstreamRequestError` | non-`429` `4xx`, nested `error` with `code`/`type`/`param` (object **or** Gemini list envelope, [#548]) | **No** — a client-side request mistake passed through the gateway, terminal. | Fix the flagged model or parameter (`.code` / `.param`); if `model_not_found`, request the model in API Manager. | +| `ContentSafetyBlocked` | `403` + `x-llm-proxy--…-action: reject` (Azure Content Safety / Bedrock Guardrails) | **No** — a `PolicyViolation`, never retried. | Revise the flagged content (`.categories`), or adjust the policy's categories / severity thresholds in API Manager. | +| `PolicyViolation` (generic) | a `4xx` matching **no** known rejection shape | **No** — terminal. | Inspect `.response`; file an issue with the status/headers/body so the shape can be typed. | +| `UpstreamRequestError` | non-`429` `4xx`, nested `error` with `code`/`type`/`param` (object **or** Gemini list envelope) | **No** — a client-side request mistake passed through the gateway, terminal. | Fix the flagged model or parameter (`.code` / `.param`); if `model_not_found`, request the model in API Manager. | | `UpstreamModelError` | `5xx` | **Yes** — the transport already retries `502` / `503` / `504`; a persistent `5xx` is safe for you to retry too. | Transient provider failure — retry, then escalate if it persists. | -| `GatewayUnavailable` | transport failure — DNS, refused connection, TLS, timeout — with **no** HTTP response | **Not automatically** — terminal here (failover is separate); you may retry or fall back. | Check host reachability, `.base_url`, and network egress; run [`donkey doctor`](/cli). | +| `GatewayUnavailable` | transport failure — DNS, refused connection, TLS, timeout — with **no** HTTP response | **Not automatically** — terminal here; you may retry or fall back. | Check host reachability, `.base_url`, and network egress; run [`donkey doctor`](/cli). | Two more `DonkeyError`s are **client-side signals**, not gateway refusals, so -they sit outside the retry question: `BudgetReserveReached` is raised *before* a +they sit outside the retry question. `BudgetReserveReached` is raised *before* a call by [`donkey.budget.pace()`](/budget) and is meant to be recovered from (`await donkey.budget.wait_for_reset()`, then continue) when its `.reset_at` is known. If `.reset_at` is `None`, propagate or handle it instead — waiting returns immediately and an unconditional retry would spin. Its `.remediation` carries -that branch as an inspectable next step, rather than requiring you to parse the +that branch as an inspectable next step, so you don't have to parse the exception message. `ModelSubstituted` reports that a call *succeeded* against a different model than requested (opt-in via `on_model_substitution="raise"`). `ConfigError` is raised locally, pre-flight, and reports every missing field at @@ -105,12 +105,9 @@ once — fix the config and re-run. `AuthError.remediation` follows the plane that failed. Errors classified from an LLM-proxy response use the canonical consumer-credential guidance that [`donkey doctor`](/cli) also prints. Control-plane token failures override that -default with canonical guidance for the provider that failed: connected-app -errors point to the Anypoint credentials and scopes, while an exhausted -`ChainedAuth` points to each configured provider's credential or token source. - -[#184]: https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/184 -[#548]: https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/548 +default with guidance for the provider that failed: connected-app errors point +to the Anypoint credentials and scopes, while an exhausted `ChainedAuth` points +to each configured provider's credential or token source. ## When the gateway can't be reached at all @@ -123,8 +120,7 @@ queue, shed load, or fall back to a non-AI path — instead of pattern-matching raw `httpx` exception. `DonkeyAsyncClient` and its blocking twin both raise it, so the async and sync -surfaces behave identically. It is terminal here and **not retried** — this issue -only *names* the failure; failover is separate. It carries: +surfaces behave identically. It is terminal and **not retried**. It carries: - `.base_url` — the origin that failed, on the exception, not only in the message. - `.cause` — the underlying `httpx` exception (also chained via `raise … from`). @@ -132,15 +128,15 @@ only *names* the failure; failover is separate. It carries: - `.correlation_id` / `.call_id` — the run and per-call ids the client sent, carried even though no response came back, so the failure joins your logs like any other. -Its `.remediation` is a single canonical string that names the three real causes — -an unreachable host, a wrong base URL, or blocked network egress — and points at -`donkey doctor` for connectivity diagnosis. +Its `.remediation` names the three real causes — an unreachable host, a wrong +base URL, or blocked network egress — and points at `donkey doctor` for +connectivity diagnosis. ## Every refusal names a next step -`remediation` is not optional. Every `PolicyViolation` carries a non-empty, -human-readable next step — the constructor **raises** if you try to build one -without it — and each concrete subclass ships a canonical default: +Every `PolicyViolation` carries a non-empty, human-readable `remediation` — the +constructor **raises** if you try to build one without it — and each concrete +subclass ships a canonical default: - `PIIDetected` → remove or redact the flagged values, or relax the policy's entity list in API Manager. @@ -151,20 +147,19 @@ without it — and each concrete subclass ships a canonical default: - `ContentSafetyBlocked` → revise the flagged content, or adjust the policy's categories / severity thresholds. -The text names the **action you can take**, not the policy that fired — a typed -refusal without a next step is just a renamed exception. Because each default -lives on the exception class, it is a single source of wording that +The text names the **action you can take**, not the policy that fired. Because +each default lives on the exception class, it is a single source of wording that [`donkey doctor`](/cli) reuses for its own failure output, so the CLI and the -exception can never disagree. +exception never disagree. ## The ids every `DonkeyError` carries Every exception in the tree carries three ids so you can join a failure to your -logs and to the gateway's own record — **three ids, three provenances**: +logs and to the gateway's own record: | Attribute | What it is | Provenance | | --- | --- | --- | -| `.correlation_id` | The **run** id, shared by every call in a [`donkey.run()`](/concepts/observability#grouping-a-run-the-run-id-on-the-span-the-call-id-on-the-error) block | The `X-Correlation-Id` request header the client sent — always equals what went on the wire. | +| `.correlation_id` | The **run** id, shared by every call in a [`donkey.run()`](/telemetry#correlation-ids) block | The `X-Correlation-Id` request header the client sent — always equals what went on the wire. | | `.call_id` | The **per-call** id, unique per logical request and stable across that request's retries | The `X-Donkey-Request-Id` request header the client sent. Present **even when the request fails before any response** (a transport error). | | `.request_id` | The **upstream provider's own** id, passed through by the gateway | Read back from a **response** header whose name varies by provider (`x-request-id` for OpenAI, `x-amzn-requestid` for Bedrock, `apim-request-id` for Azure). Quote it to the provider's support team. Absent on a transport error, or on a route where the provider forwarded none. | @@ -204,31 +199,24 @@ except openai.APIConnectionError as e: The blocking client from `donkey.llm.client(sync=True)` behaves identically here — drop the `await`. It is the same OpenAI SDK raising the same -`openai.APIStatusError`, and `classify()` reads the response the same way, so the -taxonomy is not an async-only feature. +`openai.APIStatusError`, and `classify()` reads the response the same way. -One deliberate difference sits below the taxonomy, in the retry policy. Both -clients retry only transient upstream/gateway failures (502/503/504) and treat -every 4xx as terminal — **including a 429**: on this proxy a 429 is a +## Retry behaviour + +Both clients retry only transient upstream/gateway failures (502/503/504) and +treat every 4xx as terminal — **including a 429**: on this proxy a 429 is a token-budget refusal (`TokenBudgetExceeded`), so retrying it would only burn the same already-exhausted window. `retry_after` is still surfaced for you to pace -against, but the transport never silently retries it. The async client -additionally refreshes its token and retries **once** on a 401, because it may -carry an Anypoint control-plane credential. The blocking client holds no such -credential — that refresh protocol is async-only — so a 401 there is terminal and -surfaces immediately as `AuthError`. - -## Documented-but-not-live-captured shapes - -The regex `matched_patterns` list and the **Azure Content Safety** -`…-action: reject` header are now **live-captured** (2026-09-22, [#253]) against -the deployed provisioning proxies. Two documented discriminators remain pinned -from the policy pages but **not yet confirmed against a live sandbox**: the -`x-injection-protection` header of the distinct **Injection Protection** policy -(no such proxy is deployed to capture it) and the **Bedrock Guardrails** sibling -of the content-safety header. `classify()` types those from the header -discriminator only, and no verification row flips to `verified` until a live -round-trip confirms the shape. Any *other* content-moderation / -federated-guardrail shape still falls through to a generic `PolicyViolation` -rather than an invented type — this is deliberate (verification discipline): we don't invent a -discriminator we haven't observed. See [Verification policy](/concepts/verification). +against, but the transport never silently retries it. + +The async client additionally refreshes its token and retries **once** on a 401, +because it may carry an Anypoint control-plane credential. The blocking client +holds no such credential, so a 401 there is terminal and surfaces immediately as +`AuthError`. + +## Unrecognised shapes + +Any content-moderation or federated-guardrail response that matches none of the +discriminators above falls through to a generic `PolicyViolation` rather than an +invented type. DDK only types a refusal by a discriminator it can identify +reliably; everything else stays inspectable via `.response`. diff --git a/website/content/examples/_meta.js b/website/content/examples/_meta.js new file mode 100644 index 00000000..89db1886 --- /dev/null +++ b/website/content/examples/_meta.js @@ -0,0 +1,12 @@ +export default { + index: 'Overview', + 'governed-client': 'Governed client', + 'typed-refusals': 'Typed refusals', + 'budget-and-pacing': 'Budget & pacing', + 'simulating-refusals': 'Simulating refusals', + conformance: 'Conformance suite', + telemetry: 'Telemetry', + 'framework-objects': 'Framework objects & model handles', + 'langgraph-agent': 'LangGraph agent', + 'gateway-identity': 'Gateway identity (last_call)', +} diff --git a/website/content/examples/budget-and-pacing.mdx b/website/content/examples/budget-and-pacing.mdx new file mode 100644 index 00000000..b5aaebe5 --- /dev/null +++ b/website/content/examples/budget-and-pacing.mdx @@ -0,0 +1,167 @@ +--- +description: The gateway's token window as an object, pace(reserve=) refusing locally before a request would cross your reserve, and wait_for_reset(). +--- + +import { Callout } from 'nextra/components' +import { Output } from '../../components' + +# Budget & pacing + +The gateway reports your token budget only in-band, on response headers — +there is no endpoint to ask how much is left. `donkey.budget` reads those +headers for you, so the window is an object rather than a header you parse. +The useful half is pacing: `pace(reserve=…)` refuses locally *before* issuing a +request that would cross your reserve, turning a 429 you would have to recover +from into an exception you chose to raise, and `wait_for_reset()` sleeps once +until the window rolls over. + +| Example | Shows | Needs | +| --- | --- | --- | +| Narrative demo 03 | A cold process knowing nothing, the window updating per response, `pace(reserve=0.05)` raising `BudgetReserveReached`, `wait_for_reset()`, and the terminal 429 | Nothing (simulator) | +| OpenAI script 05 | `pace(reserve=)` letting the first call through, stopping the second, then `wait_for_reset()` | Proxy credentials | + +## Run it + +```bash +make demo N=03 +``` + + +```text +════════════════════════════════════════════════════════════════════════════════════════ +Demo 03 — budget and pacing +The token window as an object, and refusing to cross it before the gateway does. +════════════════════════════════════════════════════════════════════════════════════════ + +Run context +─────────── + target mock + proxy base_url http://127.0.0.1:8080/ + output masking on + credentials fake — the simulator enforces no auth + +[1] A new process knows nothing until its first call returns + before any call remaining=None limit=None used=unobserved + Every field is None rather than zero. An unobserved budget reports 'I don't know', + because reporting 0 remaining would be a lie that stops an agent that could have run. + +[2] Each response updates the window, with no code from you + + await client.responses.create(model=..., input=...) + donkey.budget.remaining # already up to date + + after call 1 remaining=97500 limit=100000 used=2.5% + after call 2 remaining=97000 limit=100000 used=3.0% + after call 3 remaining=96500 limit=100000 used=3.5% + observed_at 2026-09-24 09:27:23.806252+00:00 + reset_at 2026-09-24 09:28:23.806252+00:00 + + A live 200 carries this window as the prose header x-llm-proxy-ratelimit — that + sentence is live-verified. The numeric x-token-* trio is verified on the 429. The + simulator synthesises a decreasing window in the same prose shape, so the numbers + above are illustrative; the parse path is not. + +[3] pace(reserve=…) refuses before the request goes out + Rather than issue the 200 calls it would take to drain the simulator's window, we let + the budget observe a response that says we are already at 96% — the same code path a + real near-exhausted window takes. + + async with donkey.budget.pace(reserve=0.05): + await enrich(batch) # never runs if the reserve is crossed + + observed remaining=4000 limit=100000 used=96.0% + reset_at 2026-09-24 09:27:24.806704+00:00 + + PASS BudgetReserveReached — the request was never issued + fraction_used 96.0% + reserve 5.0% + reset_at 2026-09-24 09:27:24.806704+00:00 + + BudgetReserveReached is deliberately NOT a PolicyViolation. A refusal is the gateway + saying no and is terminal; this is your own client-side signal, raised locally, that + you are expected to recover from. + + try: + async with donkey.budget.pace(reserve=0.05): + await enrich(batch) + except BudgetReserveReached: + await donkey.budget.wait_for_reset() # one sleep, never a spin loop + + after wait_for_reset remaining=4000 limit=100000 used=96.0% + + PASS wait_for_reset() slept until reset_at — one sleep, never a spin loop + The local object is still the last observation. Waiting does not invent a fresh + window; the next call is what refreshes remaining / limit / reset_at. That is the same + in-band rule as act 1. + +[4] And if you do cross it, the 429 is terminal + classified as TokenBudgetExceeded + retry_after 41.728 + PASS The transport never retried it — retrying only burns the same window. + + This is the scenario the conformance suite checks other people's agents for, because + retrying a budget refusal is the single most common way an agent turns one refusal + into a rate-limit spiral. See demo 05. + +──────────────────────────────────────────────────────────────────────────────────────── +``` + + +```bash +python "demos/human-made/openai/05 - budget_and_pacing.py" # needs proxy credentials +``` + + + On a live gateway the window arrives only when the token rate limit policy + is applied to the proxy. The simulator synthesises a decreasing window so + pacing can run locally; its happy-path numbers are illustrative, the parse + path is not. + + +## Key code + +Pacing, and recovering from it (narrative demo 03, act 3): + +```python +try: + async with donkey.budget.pace(reserve=0.05): + await enrich(batch) +except BudgetReserveReached: + await donkey.budget.wait_for_reset() # one sleep, never a spin loop +``` + +Against a live proxy, the first call is unobserved so `pace()` lets it through +and the window arrives in-band; the second trips the reserve (OpenAI script 05): + +```python +async with donkey.budget.pace(reserve=0.99999): + response = await client.responses.create( + model="gpt-4o", + input="Say hello in exactly three words.", + ) + budget = donkey.budget + print("after request 1, budget remaining is", budget.remaining) + print("after request 1, budget fraction_used is", budget.fraction_used) + +try: + async with donkey.budget.pace(reserve=0.99999): + response = await client.responses.create( + model="gpt-4o", + input="Say hello in exactly three words.", + ) +except BudgetReserveReached as exc: + print("stopped locally [in-script]", exc.fraction_used, exc.reserve) +``` + +An unobserved budget reports `None` for every field, never `0` — reporting `0` +remaining would stop an agent that could have run. `BudgetReserveReached` is +deliberately not a `PolicyViolation`: a refusal is the gateway saying no and is +terminal, while this is a local signal you are expected to recover from. If you +do cross the window, the resulting `TokenBudgetExceeded` is not retried by the +transport — retrying only burns the same window. + +**Learn more:** [Budget & pacing](/budget) + +**Source:** +[narrative demo 03](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/claude-made/03_budget_and_pacing) · +[script 05](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/blob/main/demos/human-made/openai/05%20-%20budget_and_pacing.py) diff --git a/website/content/examples/conformance.mdx b/website/content/examples/conformance.mdx new file mode 100644 index 00000000..91d959f5 --- /dev/null +++ b/website/content/examples/conformance.mdx @@ -0,0 +1,239 @@ +--- +description: pytest --donkey-conformance grading an agent's refusal handling, budget retries and correlation logging — first a naive agent, then the fixed one, then exemptions. +--- + +import { Callout } from 'nextra/components' +import { Output } from '../../components' + +# Conformance suite + +Four questions a team usually cannot answer about its own agent: does it retry +a budget refusal (it must not)? Does a typed refusal survive its error +handling? Does the run's correlation id reach its logs? Does it still work when +the gateway sends no budget headers? The conformance suite answers them without +reading your code — it swaps a fixture-serving transport underneath, calls +`agent.run(...)`, and watches the wire and the logs. So it grades behaviour, in +any framework, and it runs in your CI as a pytest plugin with no gateway. + +| Example | Shows | Needs | +| --- | --- | --- | +| Narrative demo 05 | The suite failing a naive agent, what each finding means, the fixed agent passing, a correct exemption, and two broken exemptions failing at collection time | Nothing (`[test]` + `[llm]`) | + +## Run it + +```bash +make demo N=05 +``` + + +```text +════════════════════════════════════════════════════════════════════════════════════════ +Demo 05 — the conformance suite +Four questions about your agent that you cannot currently answer. +════════════════════════════════════════════════════════════════════════════════════════ + +Run context +─────────── + target offline — no gateway, no simulator, no credentials + output masking on + +[1] An agent written the way people actually write them + + for attempt in range(3): # retry on failure + try: + return await client.responses.create(...) + except openai.APIStatusError as exc: + if attempt == 2: + raise RuntimeError(...) from exc # friendly error + + Nothing there is obviously wrong. Retrying is a sane default, wrapping errors keeps + stack traces out of the caller's face, and it logs what it is doing. Run the suite + against it. + + pytest --donkey-conformance --agent=…:build_naive + + + FFF. [100%] + Donkey conformance + ================== + FAIL Retries a budget refusal — retried a TokenBudgetExceeded 3× — a budget refusal is terminal; retrying only burns the same exhausted window + FAIL Swallows PII as a generic error — raised a bare RuntimeError — the PII refusal was swallowed as a generic error (bridge it with classify()) + FAIL Propagates the correlation id — did not emit the correlation id in any log record — read it from current_correlation_id() and include it when you log + PASS Works without budget headers — completed a run when the gateway returned no budget headers + Donkey conformance: 1 passed, 3 failed + =========================== short test summary info ============================ + FAILED ::donkey-conformance::retries_token_budget - Retries a budget refusal:... + FAILED ::donkey-conformance::swallows_pii_as_generic - Swallows PII as a gene... + FAILED ::donkey-conformance::correlation_id_propagated - Propagates the corre... + 3 failed, 1 passed in 0.39s + + exit status 1 + +What each failure actually means +──────────────────────────────── + • Retried a TokenBudgetExceeded 3× — the window is already exhausted, so the retries + cannot succeed and the extra calls make the rate-limit situation worse for everyone + else on the same budget. + • Raised a bare RuntimeError for a PII block — the caller wanted to catch PIIDetected + and redact the flagged entities. It cannot, because the type was thrown away in the + name of a friendlier message. + • Never logged the correlation id — so when the platform team asks which gateway + request corresponds to this run, there is no answer. + +[2] The same agent, after the findings + + try: + return await client.responses.create(...) + except openai.APIStatusError as exc: + error = classify(exc.response) + log.warning("governed refusal", extra={ + "correlation_id": current_correlation_id(), + "refusal": type(error).__name__, + }) + raise error from exc # typed, and not retried + + + .... [100%] + Donkey conformance + ================== + PASS Retries a budget refusal — issued one call and did not retry the budget refusal + PASS Swallows PII as a generic error — surfaced the refusal as a typed PIIDetected + PASS Propagates the correlation id — emitted the run's correlation id in its own logs + PASS Works without budget headers — completed a run when the gateway returned no budget headers + Donkey conformance: 4 passed, 0 failed + 4 passed in 0.36s + + exit status 0 + +[3] When an agent genuinely cannot pass, it says so out loud + Suppose the correlation finding is not fixable: your framework owns the HTTP transport + and gives you no per-request hook. That is a real limitation, so you assert it — with + a reason — and it becomes an `exempt` row rather than a failure. It is never a silent + skip, and the reason is meant to be published. + + # exemptions.py + FRAMEWORK_LIMITS = { + "correlation_id_propagated": "This agent's framework owns the HTTP …", + } + + pytest --donkey-conformance --agent=shipping_agent:build_naive \ + --donkey-known-limitations=exemptions:FRAMEWORK_LIMITS + + + FF.. [100%] + Donkey conformance + ================== + FAIL Retries a budget refusal — retried a TokenBudgetExceeded 3× — a budget refusal is terminal; retrying only burns the same exhausted window + FAIL Swallows PII as a generic error — raised a bare RuntimeError — the PII refusal was swallowed as a generic error (bridge it with classify()) + EXEMPT Propagates the correlation id — This agent's framework owns the HTTP transport and offers no per-request context hook, so a run-scoped correlation id cannot reach the agent's logs. + PASS Works without budget headers — completed a run when the gateway returned no budget headers + Donkey conformance: 1 passed, 2 failed, 1 exempt + =========================== short test summary info ============================ + FAILED ::donkey-conformance::retries_token_budget - Retries a budget refusal:... + FAILED ::donkey-conformance::swallows_pii_as_generic - Swallows PII as a gene... + 2 failed, 2 passed in 0.35s + + exit status 1 + One row moved to EXEMPT with its reason attached. The other two findings are untouched + — an exemption excuses exactly what it names. + +[4] And an exemption you get wrong fails the run, loudly + The mapping is validated at collection time, before any scenario runs. A typo'd + scenario name would otherwise exempt nothing while looking like it exempted something, + and an empty reason is a skip wearing a costume. + + case a misspelled scenario name + ERROR: KNOWN_LIMITATIONS names unknown scenario 'retries_tokn_budget'; valid scenarios are ['correlation_id_propagated', 'retries_token_budget', 'swallows_pii_as_generic', 'works_without_budget_headers'] + exit status 4 + + case an exemption with an empty reason + ERROR: KNOWN_LIMITATIONS['retries_token_budget'] must be a non-empty reason string — an asserted exemption, never a silent skip + exit status 4 + + +The point +───────── + This is the deliverable, not our internal adapter matrix. It ships as a pytest plugin + so it runs in your CI, against your agent, in whatever framework you chose — and it + needs no gateway to do it. + +──────────────────────────────────────────────────────────────────────────────────────── +``` + + +Against your own agent, the suite is a plain pytest invocation: + +```bash +pip install "donkey-kit[test]" +pytest --donkey-conformance --agent=my_app.agent:build +``` + +## Key code + +The agent under test is a factory the suite calls; it never reads the agent's +source. This is the fixed agent from `shipping_agent.py` — a refusal is +classified and escapes as its own type, it is never retried, and the +correlation id goes into the logs: + +```python +class GovernedAgent: + def __init__(self, donkey: Donkey) -> None: + self._donkey = donkey + self._client = donkey.openai() + + async def run(self, user_input: str) -> str: + log.info( + "handling request", + extra={"correlation_id": current_correlation_id(), "input": user_input[:40]}, + ) + try: + response = await self._client.responses.create(model="gpt-4o", input=user_input) + except openai.APIStatusError as exc: + error = classify(exc.response) + log.warning( + "governed refusal", + extra={ + "correlation_id": current_correlation_id(), + "refusal": type(error).__name__, + }, + ) + raise error from exc + return getattr(response, "output_text", "") + + +def build_governed(donkey: Donkey) -> GovernedAgent: + return GovernedAgent(donkey) +``` + +When an agent genuinely cannot pass a scenario, you assert an exemption with a +reason (`exemptions.py`) and pass it on the command line: + +```python +FRAMEWORK_LIMITS = { + "correlation_id_propagated": ( + "This agent's framework owns the HTTP transport and offers no per-request " + "context hook, so a run-scoped correlation id cannot reach the agent's logs." + ), +} +``` + +```bash +pytest --donkey-conformance --agent=shipping_agent:build_naive \ + --donkey-known-limitations=exemptions:FRAMEWORK_LIMITS +``` + + + An exemption becomes an `EXEMPT` row with its reason attached — never a + silent skip — and it excuses exactly the scenario it names. The mapping is + validated at collection time, so a misspelled scenario name or an empty + reason fails the run before any scenario executes. + + +The naive agent in the same file retries three times and re-raises a bare +`RuntimeError`. The suite flags the retried `TokenBudgetExceeded`, the lost +`PIIDetected` type, and the missing correlation id in its logs. + +**Learn more:** [Testing & conformance](/testing) + +**Source:** +[narrative demo 05](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/claude-made/05_conformance) diff --git a/website/content/examples/framework-objects.mdx b/website/content/examples/framework-objects.mdx new file mode 100644 index 00000000..92dc6a51 --- /dev/null +++ b/website/content/examples/framework-objects.mdx @@ -0,0 +1,244 @@ +--- +description: Framework adapters that return your framework's own objects, connection_kwargs() for the rest, local model capability handles, and config validation that reports everything at once. +--- + +import { Callout } from 'nextra/components' +import { Output } from '../../components' + +# Framework objects & model handles + +No adapter returns a wrapper. `donkey.langgraph.chat_model(...)` hands back a +real `langchain_openai.ChatOpenAI`, so everything your framework can do with a +model still works and nothing new appears in your stack traces. LangGraph is +the one deep, conformance-gated adapter; the other seven are supported at +`connection_kwargs()` — the SDK gives you the base URL, headers and client +configuration, and you pass them to the framework's own constructor. The +companion example covers model handles: `resolve()` gives a local capability +handle, and `list_models(live=True)` raises a `ConfigError` explaining that the +proxy has no catalog endpoint rather than guessing one. + +| Example | Shows | Needs | +| --- | --- | --- | +| Narrative demo 07 | `resolve()` capability handles, `list_models(live=True)` raising `ConfigError`, and config validation listing every missing field at once | Nothing | +| Narrative demo 08 | One factory call per framework and what came back, then `connection_kwargs()` for the shallow adapters | Nothing — objects are constructed, no network calls | + +## Run it + +```bash +make demo N=07 +make demo N=08 +``` + + +```text +════════════════════════════════════════════════════════════════════════════════════════ +Demo 07 — model handles and honest gaps +What the SDK does when the platform has no endpoint for what you asked. +════════════════════════════════════════════════════════════════════════════════════════ + +Run context +─────────── + target offline — no gateway, no simulator, no credentials + output masking on + +[1] resolve() — a local capability handle for a known model id + + handle = donkey.llm.resolve("gpt-4o") + handle.capabilities + + gpt-4o ModelCapabilities(function_calling=True, vision=True, json_output=True, is_heuristic=True) + gpt-4o-mini ModelCapabilities(function_calling=True, vision=True, json_output=True, is_heuristic=True) + o3 ModelCapabilities(function_calling=True, vision=False, json_output=False, is_heuristic=True) + claude-3-5-sonnet ModelCapabilities(function_calling=True, vision=False, json_output=False, is_heuristic=True) + something-unknown-9 ModelCapabilities(function_calling=True, vision=False, json_output=False, is_heuristic=True) + + These are heuristics derived from the model id, and the SDK says so rather than + implying it asked the gateway. They are useful for routing decisions in your own code; + they are not a governed catalog. + +[2] list_models(live=True) — the honest failure + + await donkey.llm.list_models(live=True) + + raised ConfigError + + The governed LLM proxy exposes no /models endpoint (GET /models → 404, verified + docs/verified-apis.md §2): it only routes requests carrying `model` in the body. Live + model listing is not available from the proxy. Use resolve(model_id) or source the + catalog from Exchange/provider config. + + PASS It names the verified absence and points at the alternative, instead of guessing a /models path that would 404 in your sandbox. + +[3] The same discipline applied to configuration + The most common reason someone abandons an SDK in the first five minutes is the one- + missing-variable-per-run loop: fix a variable, re-run, discover the next one. So + validation reports everything at once. + + DonkeyConfig(llm_proxy_url="https://…").validated(need="llm") + + + Configuration for 'llm' is incomplete. Missing: + - llm_proxy_client_id (env DONKEY_LLM_PROXY_CLIENT_ID) + - llm_proxy_client_secret (env DONKEY_LLM_PROXY_CLIENT_SECRET) + Set them via kwargs, environment variables, or .donkey-kit.toml. + + Two missing fields, one error, each naming the environment variable that sets it. And + note the LLM proxy credential is validated separately from the Anypoint control-plane + one — a developer may legitimately have proxy access and no Exchange access. + + When the failure is live rather than a missing variable — wrong URL, wrong + credentials, or a model the allow-list does not include — `donkey doctor` is the CLI + that distinguishes those three. It reuses the same remediation strings the typed + errors carry (demo 02). + +──────────────────────────────────────────────────────────────────────────────────────── +``` + + + +```text +════════════════════════════════════════════════════════════════════════════════════════ +Demo 08 — native framework objects +One deep adapter, seven at connection_kwargs(), and no wrappers anywhere. +════════════════════════════════════════════════════════════════════════════════════════ + +Run context +─────────── + target offline — no gateway, no simulator, no credentials + output masking on + +[1] One call per framework, and what came back + + langgraph deep — the conformance-gated adapter + donkey.langgraph.chat_model(…) + PASS returned langchain_openai.chat_models.base.ChatOpenAI + + adk connection_kwargs() + donkey.adk.model(…) + not installed: pip install "donkey-kit[adk]" + + strands connection_kwargs() + donkey.strands.model(…) + not installed: pip install "donkey-kit[strands]" + + agent_framework connection_kwargs() + donkey.agent_framework.chat_client(…) + not installed: pip install "donkey-kit[agent_framework]" + + openai_agents no connection_kwargs() — builds its own client + donkey.openai_agents.model(…) + not installed: pip install "donkey-kit[openai-agents]" + + anthropic connection_kwargs() + donkey.anthropic.client() + not installed: pip install "donkey-kit[anthropic]" + + crewai connection_kwargs() + donkey.crewai.llm(…) + not installed: pip install "donkey-kit[crewai]" + + llamaindex connection_kwargs() + donkey.llamaindex.llm(…) + not installed: pip install "donkey-kit[llamaindex]" + +[2] connection_kwargs() — the surface that actually carries the roster + + kwargs = donkey.strands.connection_kwargs() + SomeFrameworkModel(model="gpt-4o", **kwargs) + + + langgraph.connection_kwargs() + base_url https://demo-gateway.example.invalid/openai-sdk/ + api_key client-id-enforced + default_headers.client_id (36 chars) + default_headers.client_secret (40 chars) + http_async_client + max_retries 0 + use_responses_api True + + Same base URL, same verified client_id / client_secret pair, handed to the framework's + own constructor. Bringing a framework up to the deep bar is demand-driven and happens + one at a time, so this is not a stepping stone that everything is queued behind — it + is the supported surface. + LangGraph is the only adapter held to the conformance bar, and it sets + use_responses_api=True so ChatOpenAI calls the live-verified /responses route rather + than the unverified /chat/completions default. + +What is and is not verified here +──────────────────────────────── + The proxy contract these objects are configured against is live-verified: the base URL + shape, the credential header pair, the rejection shapes. The exact framework class + names and constructor kwargs are not — they are checked against installed packages by + a nightly matrix rather than asserted from documentation. + Where a class name cannot be confirmed, the adapter raises 'blocked on verification' + rather than guessing. A guessed class name that fails on a developer's first import + costs more than the missing adapter. + +──────────────────────────────────────────────────────────────────────────────────────── +``` + + +Narrative demo 08 uses obviously-fake config, so it needs no credentials. +Frameworks that are not installed are reported with their exact `pip install` +line. + +## Key code + +The roster narrative demo 08 walks — attribute on `Donkey`, factory method, and +depth: + +```python +ROSTER = [ + ("langgraph", "chat_model", True, "deep — the conformance-gated adapter"), + ("adk", "model", True, "connection_kwargs()"), + ("strands", "model", True, "connection_kwargs()"), + ("agent_framework", "chat_client", True, "connection_kwargs()"), + ("openai_agents", "model", True, "no connection_kwargs() — builds its own client"), + ("anthropic", "client", False, "connection_kwargs()"), + ("crewai", "llm", True, "connection_kwargs()"), + ("llamaindex", "llm", True, "connection_kwargs()"), +] +``` + +For the shallow adapters, `connection_kwargs()` is the whole supported surface: + +```python +kwargs = donkey.strands.connection_kwargs() +SomeFrameworkModel(model="gpt-4o", **kwargs) +``` + +Model handles and the missing catalog (narrative demo 07): + +```python +handle = donkey.llm.resolve("gpt-4o") +handle.capabilities + +await donkey.llm.list_models(live=True) # raises ConfigError +``` + +And config validation reports every missing field in one error, each naming +the environment variable that sets it: + +```python +DonkeyConfig(llm_proxy_url="https://…").validated(need="llm") +``` + + + `donkey.openai_agents` is the OpenAI Agents SDK adapter; `donkey.openai()` is + the raw OpenAI client factory. The LangGraph adapter sets + `use_responses_api=True`, so `ChatOpenAI` calls the `/responses` route. Where + an adapter cannot confirm a framework's class name or constructor, it raises + "blocked on verification" rather than guessing. + + +`resolve()` capabilities are heuristics derived from the model id, not a +governed catalog. The gateway returns 404 for `GET /models` because +model-based routing only routes requests that already carry `model` in the +body. When a live call fails — wrong URL, wrong credentials, or a model the +allow-list does not include — [`donkey doctor`](/cli) tells those apart. + +**Learn more:** [Model access](/frameworks) · [LangGraph](/frameworks/langgraph) · [CLI & decorators](/cli) + +**Source:** +[narrative demo 07](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/claude-made/07_model_handles) · +[narrative demo 08](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/claude-made/08_framework_objects) diff --git a/website/content/examples/gateway-identity.mdx b/website/content/examples/gateway-identity.mdx new file mode 100644 index 00000000..9428fc6b --- /dev/null +++ b/website/content/examples/gateway-identity.mdx @@ -0,0 +1,226 @@ +--- +description: donkey.last_call on a successful call — which gateway served it, what it routed to, per-call token usage — and on_model_substitution="raise" turning a silent model swap into ModelSubstituted. +--- + +import { Callout } from 'nextra/components' +import { Output } from '../../components' + +# Gateway identity (last_call) + +A refusal already tells you which gateway said no. `donkey.last_call` is the +success-path counterpart: after every governed call it records which gateway +served the request, what it actually routed to, and what the call cost — +without parsing headers or running a span backend. That matters because a +silent model substitution is otherwise invisible to your cost model and your +evals, and reading only `total_tokens` misses cached and reasoning tokens. If +your assumptions are pinned to one model, `on_model_substitution="raise"` +turns a swap into a hard `ModelSubstituted` error. + +| Example | Shows | Needs | +| --- | --- | --- | +| Narrative demo 10 | A cold `UNOBSERVED` record, one call populating identity, routing and usage, the substitution flag, and `on_model_substitution="raise"` | Nothing (simulator) | +| OpenAI script 08 | The full `last_call` record on a live call, then `on_model_substitution="raise"` | Proxy credentials | + +## Run it + +```bash +make demo N=10 +``` + + +```text +════════════════════════════════════════════════════════════════════════════════════════ +Demo 10 — last_call, routing, and per-call usage +The success-path counterpart to a typed refusal: who served this, what they served, and +what it cost. +════════════════════════════════════════════════════════════════════════════════════════ + +Run context +─────────── + target mock + proxy base_url http://127.0.0.1:8080/ + output masking on + credentials fake — the simulator enforces no auth + +[1] A new process has not observed a call yet — and it says so + status unobserved + observed False + available True + request_id None + + PASS UNOBSERVED — not None, not 0, not 'unknown'. A cold read is a named state. + A bare None would be a lie of omission: you could not tell 'the gateway sent no id' + from 'we never saw a response'. Budget uses the same honesty rule for an unobserved + window (demo 03). UNAVAILABLE is the third state, for adapters that never route + through our transport — LiteLLM-backed ADK and CrewAI, or default_headers-only + LlamaIndex. Those surfaces report UNAVAILABLE by name rather than looking like a cold + read. + +[2] One governed call, and the record is the success-path counterpart + + await client.responses.create(model=..., input=...) + donkey.last_call.served_model + donkey.last_call.total_tokens + donkey.last_call.substituted + + status observed + observed True + request_id req_f85003861d5348c9a1d152c276082b07 + requested_model gpt-4o + served_model gpt-5.1 + served_provider openai + routing_type ModelBased + fallback False + substituted True + input_tokens 17 + output_tokens 51 + total_tokens 68 + cached_tokens 0 + cache_write_tokens 0 + reasoning_tokens 0 + + PASS OBSERVED — the SDK saw the response, even if some fields stayed None + request_id is the gateway's own id (x-request-id) — quote it in a ticket. It is the + same field classify() puts on a DonkeyError after a refusal, now present on the 200 as + well. api_instance_id and environment_id are parsed from x-envoy-decorator-operation; + they are masked in this output. + +[3] Routing, fallback, and the cost-relevant token counts + + +What the gateway did with the request +───────────────────────────────────── + requested gpt-4o + served openai/gpt-5.1 + routing_type ModelBased + fallback False + substituted True + + PASS substituted — asked for gpt-4o, gateway served gpt-5.1 + Against the simulator this is the captured happy-path fixture talking: it was recorded + against gpt-5.1, and we asked for a different id. That is not a live failover — and it + is exactly the mismatch last_call is for. A silent substitution is otherwise invisible + to your cost model, your eval, and your latency dashboard. + + +What this call cost +─────────────────── + input / output / total 17 / 51 / 68 + cached_tokens 0 + cache_write_tokens 0 + reasoning_tokens 0 + + cached_tokens are billed at the cached rate; reasoning_tokens are output the developer + never sees. Reading only total_tokens draws the wrong conclusion about both cost and + latency. An absent count is None, never 0 — 0 here means the gateway reported zero, + which is a different statement. These are per-call; donkey.budget is the shared window + (demo 03). + The SDK never double-retries a fallback. It retries 502/503/504 with backoff, but a + 503 the gateway already marked as a failover is left alone — a second recovery layer + stacked on a working first one just multiplies latency against an outage the gateway + already handled. + +[4] Opt in, and a substitution is a hard error instead of a flag + + donkey = Donkey.from_env(on_model_substitution="raise") + # raises ModelSubstituted when served_model != requested_model + + Off by default: the call succeeds and last_call.substituted is True. Raise is for + callers whose eval, cost model and token assumptions are pinned to one model. + ModelSubstituted is deliberately not a PolicyViolation — the request was neither + refused nor failed, it succeeded against a model you did not choose. Same shape as + BudgetReserveReached: a client-side signal you opted into. + + PASS ModelSubstituted — the 200 never reached the caller + requested_model gpt-4o + served_model gpt-5.1 + served_provider openai + request_id req_f85003861d5348c9a1d152c276082b07 + last_call.substituted True + The record still populated — observe happens before the raise — so a handler that + decides to accept the served completion can read last_call the same way. The exception + also carries the response. + + +The point +───────── + The refusal path already told you which gateway said no. The success path now tells + you which gateway said yes, what it actually served, and what that call cost — without + a span backend, without parsing headers, and without a second accessor for routing or + usage. + +──────────────────────────────────────────────────────────────────────────────────────── +``` + + +```bash +python "demos/human-made/openai/08 - last-call.py" # needs proxy credentials +``` + +## Key code + +Reading the record after a call (OpenAI script 08): + +```python +donkey = Donkey.from_env() +client = donkey.openai(sync=True) + +print("before any call ", donkey.last_call.status.value) + +reply = client.responses.create(model="gpt-4o", input="Say hello in exactly three words.") + +last = donkey.last_call +print("status ", last.status.value) +print("request_id ", last.request_id) +print("requested_model ", last.requested_model) +print("served_model ", last.served_model) +print("served_provider ", last.served_provider) +print("routing_type ", last.routing_type) +print("fallback ", last.fallback) +print("substituted ", last.substituted) +print("total_tokens ", last.total_tokens) +print("cached_tokens ", last.cached_tokens) +print("reasoning_tokens", last.reasoning_tokens) +``` + +Opting in to a hard error. The OpenAI client wraps the transport error, so the +typed `ModelSubstituted` is on `__cause__`: + +```python +strict = Donkey.from_env(on_model_substitution="raise") +strict_client = strict.openai(sync=True) +try: + strict_client.responses.create(model="gpt-4o", input="Say hello in exactly three words.") + print("NO RAISE substituted", strict.last_call.substituted) +except Exception as err: + hit = err if isinstance(err, ModelSubstituted) else err.__cause__ + if not isinstance(hit, ModelSubstituted): + raise + print("requested_model ", hit.requested_model) + print("served_model ", hit.served_model) +``` + +- **Three named states, never a bare `None`.** `UNOBSERVED` is a cold read; + `OBSERVED` means the SDK saw a response (fields may still be `None` if the + gateway said nothing); `UNAVAILABLE` marks adapter surfaces that never route + through the SDK's transport. +- **`request_id` is the gateway's own id** (`x-request-id`) — the same field + `classify()` puts on a refusal, now present on a 200 too. +- **Absent counts are `None`, never `0`.** `cached_tokens` are billed at the + cached rate; `reasoning_tokens` are output you never see. +- **`ModelSubstituted` is not a `PolicyViolation`.** The request succeeded, + against a model you did not choose. `last_call` is still populated before the + raise. + + + The simulator's captured success response was recorded against `gpt-5.1`, so + asking for any other model id shows up as a substitution in narrative demo + 10. That is the fixture, not a live failover — and it is exactly the + mismatch `last_call` exists to surface. + + +**Learn more:** [Feature overview](/feature-overview) · [Telemetry & cost](/telemetry) + +**Source:** +[narrative demo 10](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/claude-made/10_last_call) · +[script 08](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/blob/main/demos/human-made/openai/08%20-%20last-call.py) diff --git a/website/content/examples/governed-client.mdx b/website/content/examples/governed-client.mdx new file mode 100644 index 00000000..8de5aed4 --- /dev/null +++ b/website/content/examples/governed-client.mdx @@ -0,0 +1,254 @@ +--- +description: A stock OpenAI client next to a governed one, the same PII refusal through both, and the @donkey.governed / @donkey.tool decorators. +--- + +import { Callout } from 'nextra/components' +import { Output } from '../../components' + +# Governed client + +You do not need the SDK to reach the gateway — a stock OpenAI client with a +`base_url` and two headers gets there. These examples show that first, then +show what the stock client leaves you holding. Both clients hit the same +gateway and get the same PII refusal back; the stock one gives you an +`openai.APIStatusError` and a JSON body to parse, the governed one gives you a +`PIIDetected` with the flagged entities and a remediation string. And because +every request leaves through one client, that same call also updated the +budget window, stamped a correlation id and opened a GenAI span, with no wiring +from you. + +| Example | Shows | Needs | +| --- | --- | --- | +| Narrative demo 01 | `donkey.openai()` and `sync=True`, the injected headers, one call and what it updated, stock vs governed on the same PII refusal, `@donkey.governed` / `@donkey.tool` | Nothing (simulator) | +| OpenAI script 01 | Stock OpenAI with no gateway — the baseline | `OPENAI_API_KEY` | +| OpenAI script 02 | `donkey.openai()` plus `last_call` on a successful call | Proxy credentials | +| OpenAI script 09 | `@donkey.governed` and `@donkey.tool` on their own | Nothing | + +## Run it + +```bash +make demo N=01 # offline, against the local simulator +make demo N=01 ARGS="--target live" # against your gateway +``` + + +```text +════════════════════════════════════════════════════════════════════════════════════════ +Demo 01 — the governed client +Reaching the gateway is easy. Everything that hangs off the client is the product. +════════════════════════════════════════════════════════════════════════════════════════ + +Run context +─────────── + target mock + proxy base_url http://127.0.0.1:8080/ + output masking on + credentials fake — the simulator enforces no auth + +[1] donkey.openai() returns a real OpenAI client, already governed + + donkey = Donkey.from_env() + client = donkey.openai() # -> openai.AsyncOpenAI + blocking = donkey.openai(sync=True) # -> openai.OpenAI, same governance + + type openai.AsyncOpenAI + sync=True openai.OpenAI + base_url http://127.0.0.1:8080/ + + headers the SDK injects: + Accept application/json + Content-Type application/json + User-Agent AsyncOpenAI/Python 3.19.2 + client_id (41 chars) + client_secret (45 chars) + OpenAI-Organization + OpenAI-Project + + Note the base URL has no /v1 — the ingress is https://// and the + OpenAI SDK appends /responses itself. Auth is the client_id / client_secret header + pair, not a bearer token. sync=True is the same client without asyncio — useful for a + straight-line script. + +[2] One call. Nothing new to learn — it is the OpenAI SDK. + + response = await client.responses.create( + model='gpt-4o', + input='Say hello in exactly three words.', + ) + + WARN The simulator replays a captured success response, so the reply below answers the prompt that was recorded, not the one just sent. Run with --target live for a real completion. + reply A sleepy unicorn named Luma painted soft silver stars across the night sky with her glowing horn, then curled up on a moonbeam so all the children below could fall asleep beneath her gentle, sparkling light. + input tokens 17 + output tokens 51 + + That call also did these things nobody asked for, because every request leaves through + one client: + budget.remaining 98000 + budget.limit 100000 + budget.fraction_used 2.0% + budget.observed_at 2026-09-24 09:27:22.710265+00:00 + last_call.status observed + last_call.served_model gpt-5.1 + last_call.total_tokens 68 + • a correlation id went out on the request + • a gen_ai.* span opened and closed around it (demo 06) + donkey.last_call is the success-path counterpart to a typed refusal: who served this, + what they actually routed to, and what the call cost. Demo 10 walks the whole record — + routing, fallback, cached/reasoning tokens, and the opt-in ModelSubstituted error. + +[3] The same refusal, through both clients + The simulator serves a real captured PII rejection when the model id is the sentinel + below. This is the byte-identical body a live gateway sent. + + # A: stock OpenAI client, no SDK — just base_url + headers + raw = openai.AsyncOpenAI(base_url=..., api_key=..., default_headers=...) + await raw.responses.create(model='donkey-sim/pii-detected', input='My email is a@b.com') + + A: raised openai.PermissionDeniedError + A: status 403 + A: you get a JSON body to parse, and a status code to guess from + + + # B: the same request, through the governed client + try: + await client.responses.create(model=..., input="...") + except openai.APIStatusError as exc: + raise classify(exc.response) from exc + + B: raised PIIDetected + B: policy pii-detection + B: entities ['Email'] + B: remediation The PII-detection policy blocked this request because the prompt (or completion) contained personally identifiable information. Remove or redact the flagged values, or relax the policy's entity list / action in API Manager. + + PASS A 403 that is a policy refusal, not an auth failure — and it says so. + classify() is the bridge, because the raw client raises openai.* errors and the SDK + does not silently re-map them. Demo 02 walks the full taxonomy. + +[4] @donkey.governed and @donkey.tool — the one-line on-ramps + + @donkey.governed(team="support") + async def handle_ticket(ticket): + ... # every model call inside shares one run id + + There is deliberately no id= on the decorator: a fixed id pinned across every call + would collapse unrelated tickets into one correlation. When you need to pin a business + id, use donkey.run(id=...) directly (demo 06). + run id inside 7512d15e0ea94fe4b6f321aae903a781 + cost tags team=support project=triage + run id inside 491f95bfc2cc4d88b7b0ae1f200de6b5 + cost tags team=support project=triage + PASS each invocation opened a fresh run + run id after 235d9e98804b480092a2738f58fd7997 + PASS restored to the enclosing context — nested run() rebinds, then restores + + + @donkey.tool + def lookup_sku(sku: str) -> str: + """Return stock for a product SKU.""" + ... + + registered lookup_sku + signature (sku: 'str') -> 'str' + docstring Return stock for a product SKU. + PASS same function object — the decorator records it, it does not wrap it + PASS ValueError — an undescribed tool is rejected at decoration time + The same marker is what a Phase 2 scanner and an A2A agent-card generator will both + read. Neither consumer is built yet; this is the annotation they will look for, not a + wrapper around the tool. + + +The point +───────── + The wrapper is not sold as a way to reach the gateway. It is the one place every + request enters and every response leaves — which is why budget, last_call, typed + refusals, correlation ids, spans and simulation can all attach without the developer + wiring each one. @donkey.governed is that attachment as a function decorator; + @donkey.tool is the marker a scanner can find without executing it. + +──────────────────────────────────────────────────────────────────────────────────────── +``` + + +```bash +python "demos/human-made/openai/01 - basic-responses-no-gw.py" # needs OPENAI_API_KEY +python "demos/human-made/openai/02 - basic-responses-gw.py" # needs proxy credentials +python "demos/human-made/openai/09 - governed-and-tool.py" # no gateway +``` + + + Against the simulator the reply in act 2 is a captured response, so it + answers the recorded prompt rather than the one just sent. The demo prints a + warning saying so. The PII act against `--target live` needs the PII + detection policy applied with `Email` among its entities and its action set + to `Reject` — the default action, `Log`, does not block. + + +## Key code + +The governed client is a real OpenAI client, already pointed at the proxy +(OpenAI script 02): + +```python +async with Donkey.from_env() as donkey: + client = donkey.openai() # THIS is returning the native openai + + response = await client.responses.create( + model="gpt-4o", + input="Say hello in exactly three words.", + ) + + print(response.output_text) + last = donkey.last_call + print("last_call.status ", last.status.value) + print("last_call.served_model", last.served_model) +``` + +The same refusal through the governed client, typed with `classify()` +(narrative demo 01, act 3): + +```python +client = donkey.openai() +try: + await client.responses.create(model=pii_model, input=PII_PROMPT) +except openai.APIStatusError as exc: + governed = classify(exc.response) + say.field("B: raised", type(governed).__name__) + say.field("B: policy", getattr(governed, "policy", "—")) + say.field("B: entities", getattr(governed, "entities", []), raw=True) + say.field("B: remediation", getattr(governed, "remediation", "—")) +``` + +The one-line on-ramps (OpenAI script 09). `@donkey.governed` opens a fresh run +per invocation; `@donkey.tool` records the function without wrapping it, and +rejects a tool with no docstring at decoration time: + +```python +@donkey.governed(team="support", project="triage") +def handle_ticket(ticket: str) -> str: + print("run id inside", current_correlation_id()) + print("cost tags ", current_cost_tags()) + return ticket + + +@donkey.tool +def lookup_sku(sku: str) -> str: + """Return stock for a product SKU.""" + return "42" + + +print("same function object ", lookup_sku is registered_tools()[-1].func) +``` + +Note the base URL has no `/v1`, and auth is a `client_id` / `client_secret` +header pair rather than a bearer token. There is deliberately no `id=` on +`@donkey.governed`: a fixed id would collapse unrelated calls into one +correlation. Use `donkey.run(id=...)` when you need to pin a business id (see +[Telemetry](/examples/telemetry)). + +**Learn more:** [Introduction](/) · [Model access](/frameworks) · [CLI & decorators](/cli) + +**Source:** +[narrative demo 01](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/claude-made/01_governed_client) · +[script 01](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/blob/main/demos/human-made/openai/01%20-%20basic-responses-no-gw.py) · +[script 02](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/blob/main/demos/human-made/openai/02%20-%20basic-responses-gw.py) · +[script 09](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/blob/main/demos/human-made/openai/09%20-%20governed-and-tool.py) diff --git a/website/content/examples/index.mdx b/website/content/examples/index.mdx new file mode 100644 index 00000000..d5349439 --- /dev/null +++ b/website/content/examples/index.mdx @@ -0,0 +1,198 @@ +--- +description: Runnable DDK examples from the companion demos repo — narrative demos that run offline against the local simulator, and short OpenAI scripts for a live gateway. +--- + +import { Callout, Steps } from 'nextra/components' +import { Card, CardGrid } from '../../components' + +# Examples + +The [DDK demos repo](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos) +holds runnable examples for every piece of the SDK: the governed client, typed +refusals, budget pacing, simulation, the conformance suite, telemetry, +framework objects and `last_call`. Each page in this section covers one of +those, pairs the examples that show it, and gives you the command to run them. + +The repo ships **two suites on purpose**. They cover the same SDK, but they are +not interchangeable: + +| | Narrative demos | OpenAI scripts | +| --- | --- | --- | +| **Where** | [`demos/claude-made/`](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/claude-made) | [`demos/human-made/openai/`](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/human-made/openai) | +| **Best for** | A room, a recording, or CI that must stay offline | A terminal you type in, or paste from | +| **Shape** | Numbered demos (`01`–`10`), each a `demo.py` + README told in acts | Straight-line OpenAI scripts (`01`–`11`), one file each | +| **Runner** | `make demo N=03`, `make offline` | `python "demos/human-made/openai/.py"` | +| **Network** | Nine of ten run offline against the local simulator; demo 09 needs a live gateway | Most need gateway credentials; 09 and 11 need no gateway | +| **Output** | Masked by default | Not masked — use on a private terminal | + +Every narrative demo takes `--target mock` (the default, pointing at the local +simulator) or `--target live` (your real credentials). The OpenAI scripts have +no harness: they use the same environment you already use for the SDK. + +## Setup + + + +### Create a virtual environment + +```bash +git clone https://github.com/Donkey-Development-Kit/donkey-development-kit-demos.git +cd donkey-development-kit-demos +python3 -m venv .venv # .venv/ is git-ignored +source .venv/bin/activate # once per terminal +``` + +Homebrew's `python3` and most Linux distro Pythons are marked *externally +managed* (PEP 668), so a global `pip3 install` fails with +`externally-managed-environment`. Install into `.venv` instead of reaching for +`--break-system-packages`. + +### Install the demo harness + +```bash +python -m pip install -e . +``` + +This adds the shared harness to the path. It does not pin an SDK. + +### Install the SDK + +Pick one: + +```bash +# from git, with the extras every offline demo needs +python -m pip install -e ".[sdk]" + +# from git, with everything including OpenTelemetry and LangGraph +python -m pip install -e ".[full]" + +# from your own SDK checkout +python -m pip install -e "../donkey-development-kit/python[llm,local,test,otel,langgraph,cli]" + +# a published dev build from TestPyPI (its dependencies come from PyPI) +python -m pip install -i https://test.pypi.org/simple/ \ + --extra-index-url https://pypi.org/simple/ "donkey-kit[llm,local,test,otel,langgraph,cli]" +``` + + + **Quote the `[...]` extras.** zsh, the macOS default shell, treats unquoted + brackets as a glob and fails with `no matches found`. + + +`uv` users can replace the venv step with `uv venv` and `pip install` with +`uv pip install`. + + + +## Live credentials + +The offline narrative demos need none of this. Live runs, the LangGraph agent +example and most OpenAI scripts need three variables for the governed LLM +proxy, plus the model to ask for: + +| Variable | What it is | +| --- | --- | +| `DONKEY_LLM_PROXY_URL` | The proxy's base URL, **with a trailing `/`** and no `/v1` — the OpenAI SDK appends `/responses` itself | +| `DONKEY_LLM_PROXY_CLIENT_ID` | The client id the proxy authenticates on (a header, not a bearer token) | +| `DONKEY_LLM_PROXY_CLIENT_SECRET` | The matching client secret | +| `DEMO_MODEL` | The model id the narrative demos ask for; it must be one your proxy routes | + +Copy the template and fill it in: + +```bash +cp .env.example .env.local # .env.local is git-ignored +``` + +A shell `export` always wins over a file value, so you can skip the file and +export the variables directly. The narrative demos load `.env.local` for you; +the OpenAI scripts do not (the SDK never reads dotenv files on its own), so +export the file into your shell first: + +```bash +set -a; source .env.local; set +a +python "demos/human-made/openai/02 - basic-responses-gw.py" +``` + + + The OpenAI scripts hardcode their model ids (mostly `gpt-4o`). If your proxy + routes a different model, expect a routing refusal or a `ModelSubstituted` + until the script's model matches the proxy's. + + +## Commands + +```bash +make list # the narrative demos and what each one needs +make demo N=03 # one narrative demo +make demo N=01 ARGS="--target live" # the same demo against your gateway +make offline # every narrative demo that needs no credentials +make doctor # what is installed, and what will therefore run +make mock # the local simulator in the foreground, for a second pane + +DEMO_PAUSE=1 make demo N=03 # pause between acts — use this when presenting +``` + +Flags for the demo go in `ARGS`, not on the end of the `make` line. `python +run.py 03` works too, and each narrative demo is a plain script +(`python demos/claude-made/03_budget_and_pacing/demo.py`) once the repo is +installed. `make doctor` reports what it found without printing any values. + +## Credential safety + +The narrative demos assume they will be screen-shared and recorded: + +- **Output is masked by default.** Every value a narrative demo prints is + scrubbed: gateway hostnames, the client id and secret, and the tenant + identifiers that ride along in captured responses. Header *names* are shown; + their values are not. +- **Turning masking off announces itself.** `DEMO_REDACT=0` prints a warning + banner in every narrative demo's run context. Use it only when debugging + privately. +- **No captured traffic is vendored.** Fixtures load from the installed SDK, + not from copies in the demos repo. +- **`make scan` reads content, not filenames.** It fails on assigned credential + values, bearer tokens, instance ids, UUIDs and non-allowlisted hostnames. + `make hooks` installs it as a pre-commit hook. + +```bash +make scan # everything tracked +make hooks # then it runs on every commit +``` + + + **The OpenAI scripts do not mask anything.** They print completions and + error strings exactly as the SDK returned them. Run them on a private + terminal, not on a shared screen or recording. + + +## Browse by purpose + + + + Stock client versus governed client, then `@donkey.governed` and `@donkey.tool`. + + + Every captured rejection shape through `classify()`, plus `GatewayUnavailable`. + + + The token window as an object; `pace(reserve=)` and `wait_for_reset()`. + + + Run your refusal branch with `donkey.simulate()` and `donkey mock --scenario`. + + + `pytest --donkey-conformance` grading a naive agent, then the fixed one. + + + GenAI spans, `donkey.run(id=…)` correlation, and zero-config OTLP. + + + Native framework objects, `connection_kwargs()`, and `resolve()`. + + + A real tool-calling loop, governed end to end. + + + Who served the call, what it routed to, what it cost — and `ModelSubstituted`. + + diff --git a/website/content/examples/langgraph-agent.mdx b/website/content/examples/langgraph-agent.mdx new file mode 100644 index 00000000..7c2c992b --- /dev/null +++ b/website/content/examples/langgraph-agent.mdx @@ -0,0 +1,86 @@ +--- +description: A real LangGraph tool-calling agent governed end to end — LangChain's own ChatOpenAI, one run id around the loop, and typed refusals out of astream. +--- + +import { Callout } from 'nextra/components' + +# LangGraph agent + +A real multi-step agent: the model decides to call two tools, the tools +return, and the model composes an answer. Every model call in that loop goes +through the governed proxy, and the object driving it is LangChain's own +`ChatOpenAI`, not a wrapper. The only DDK lines are the one that builds the +model, `donkey.run(id=…)` around the loop, `typed_refusals()` so a proxy 403 +comes out of `astream` as `PIIDetected` rather than a framework-wrapped error, +and `@donkey.tool` on the two functions. Governance sits at the boundary, not +in the agent's control flow. + +| Example | Shows | Needs | +| --- | --- | --- | +| Narrative demo 09 | `donkey.langgraph.chat_model()`, a `create_agent` loop calling two tools, then the run's budget, `last_call` and the registered tools | Live credentials + `[langgraph]` | + +## Run it + +```bash +make demo N=09 # needs live credentials +``` + + + This example needs a live gateway. The local simulator replays a captured + `/responses` completion and will not decide to call tools, so there is no + offline version. Without credentials it exits cleanly with setup guidance. + The refusal path *can* run offline: [Simulating refusals](/examples/simulating-refusals) + drives the same `ChatOpenAI` through `donkey.simulate()`. + + +## Key code + +The tools are plain LangChain tools, marked for the SDK's registry: + +```python +@tool +@Donkey.tool +def check_inventory(sku: str) -> str: + """Return the units in stock and warehouse for a product SKU.""" + return INVENTORY.get(sku, "unknown SKU") + + +@tool +@Donkey.tool +def get_price(sku: str) -> str: + """Return the list price for a product SKU.""" + return PRICES.get(sku, "unknown SKU") +``` + +The model and the governed loop: + +```python +async with Donkey.from_env() as donkey: + model = donkey.langgraph.chat_model(MODEL, temperature=0) + agent = create_agent(model, tools=[check_inventory, get_price]) + + async with donkey.run(id="sku-lookup"): + with donkey.langgraph.typed_refusals(): + async for chunk in agent.astream( + {"messages": [("user", QUESTION)]}, stream_mode="updates" + ): + ... + + budget = donkey.budget + last = donkey.last_call +``` + +After the loop, `donkey.budget` reflects the run's real consumption across +every model call, and `donkey.last_call` describes the most recent one — who +served it, what they served and what it cost. The adapter targets the +`/responses` route (`use_responses_api=True`), the same one `donkey.openai()` +uses. `DEMO_MODEL` defaults to `gpt-4o-mini` in this example; set it to a model +your proxy routes. + +If the gateway is unavailable, [Framework objects](/examples/framework-objects) +constructs the same real framework objects with no network. + +**Learn more:** [LangGraph](/frameworks/langgraph) · [Model access](/frameworks) + +**Source:** +[narrative demo 09](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/claude-made/09_langgraph_agent) diff --git a/website/content/examples/simulating-refusals.mdx b/website/content/examples/simulating-refusals.mdx new file mode 100644 index 00000000..dc3fcc87 --- /dev/null +++ b/website/content/examples/simulating-refusals.mdx @@ -0,0 +1,210 @@ +--- +description: Run the refusal branch of your agent without a gateway — donkey.simulate() in-process, and donkey mock --scenario for a stock client. +--- + +import { Callout } from 'nextra/components' +import { Output } from '../../components' + +# Simulating refusals + +Every agent has an `except PIIDetected:` branch that has never executed. +Getting a real gateway to refuse on demand means finding a prompt that trips a +live policy — slow, flaky, and not something you can put in CI. +`donkey.simulate()` swaps a fixture-returning transport onto the client for the +next N calls, so the branch runs against the same captured refusal a real +gateway sent, with no network. For a client that does not use the SDK at all, +`donkey mock --scenario` scripts the running simulator the same way. + +| Example | Shows | Needs | +| --- | --- | --- | +| Narrative demo 04 | `simulate()` for each refusal, `times=`, what it refuses to fake, the same injection through LangChain's `ChatOpenAI`, and `donkey mock --scenario` parsing | Nothing; the LangChain act runs only if `[langgraph]` is installed | +| OpenAI script 03 | A `simulate()` loop over five refusal types, including `ContentSafetyBlocked` | Proxy credentials in the environment (no network calls) | + +## Run it + +```bash +make demo N=04 +``` + + +```text +════════════════════════════════════════════════════════════════════════════════════════ +Demo 04 — simulating refusals in-process +Run the except branch that has never executed. No network, no server, no credentials. +════════════════════════════════════════════════════════════════════════════════════════ + +Run context +─────────── + target mock + proxy base_url http://127.0.0.1:8080/ + output masking on + credentials fake — the simulator enforces no auth + +[1] Normally, the happy path is all you ever exercise + agent returned A sleepy unicorn named Luma painted soft silver stars across the night sky with her glowing horn, then curled up on a moonbeam so all the children below could fall asleep beneath her gentle, sparkling light. + refusal branch ran False + +[2] One context manager, and the branch runs + + with donkey.simulate(PIIDetected): + await agent.run("...") # fails as a real PIIDetected + + agent returned redacted ['Email'] and asked the user to rephrase + handled ['PIIDetected'] + PASS The branch executed against the real captured 403 body. + +[3] Every refusal you need to handle, one line each + TokenBudgetExceeded queued for retry after 42s — did NOT retry now + PIIDetected redacted ['Email'] and asked the user to rephrase + ContentSafetyBlocked revised ['severity_hate', 'severity_violence'] and did not retry + +[4] `times` counts calls, and normal service resumes after + + with donkey.simulate(TokenBudgetExceeded, times=2): + await agent.run("a") # refused + await agent.run("b") # refused + await agent.run("c") # succeeds — the injection is spent + + call 1 queued for retry after 42s — did NOT retry now + call 2 queued for retry after 42s — did NOT retry now + call 3 A sleepy unicorn named Luma painted soft silver stars across the night sky with her glowing horn, then curled up on a moonbeam so all the children below could fall asleep beneath her gentle, sparkling light. + +[5] It will inject a documented shape, and refuse to invent the rest + + with donkey.simulate(ContentSafetyBlocked): + await agent.run("...") # a real ContentSafetyBlocked + + ContentSafetyBlocked revised ['severity_hate', 'severity_violence'] and did not retry + PASS The documented content-safety fixture classifies and injects. + + + with donkey.simulate(ToolInvocationError): + ... + + PASS ValueError — no captured fixture maps back to it + message simulate() cannot inject ToolInvocationError: no captured fixture maps back to it via classify(). Supported: AuthError, ContentSafetyBlocked, PIIDetected, PolicyViolation, PromptInjectionBlocked, TokenBudgetExceeded, UpstreamModelError, UpstreamRequestError. + + + with donkey.simulate(GatewayUnavailable): + ... + + PASS ValueError — a transport failure has no captured body to inject + message simulate() cannot inject GatewayUnavailable: no captured fixture maps back to it via classify(). Supported: AuthError, ContentSafetyBlocked, PIIDetected, PolicyViolation, PromptInjectionBlocked, TokenBudgetExceeded, UpstreamModelError, UpstreamRequestError. + + Tool invocation, registry, and provisioning errors are not gateway refusals, and they + have no captured wire shape. GatewayUnavailable is the same kind of gap for a + different reason: there is no HTTP response at all, so there is nothing to replay. + Injecting a plausible body would let you write a handler against a body that does not + exist — so simulate() refuses instead. Provoke it by pointing at a dead origin (demo + 02 act 5). + +[6] It works through a framework too, because it is on the transport + + model = donkey.langgraph.chat_model("gpt-4o") # a real ChatOpenAI + with donkey.simulate(PIIDetected): + await model.ainvoke("...") + + LangChain raised OpenAIPermissionDeniedError + classifies as PIIDetected + PASS Same fixture, same taxonomy, through the framework's own object. + +[7] The same idea, as a running simulator a stock client can hit + simulate() swaps the transport on a Donkey you already own — that is the unit-test + form. When the client is a stock OpenAI SDK pointed at donkey mock, you script the + server instead: + + donkey mock --scenario pii_block:every=2 \ + --scenario 'injection:on-pattern=ignore previous' \ + --scenario budget:limit=200,window=5s,cost=80 + + pii_block:every=2 ['pass', 'pii-detected', 'pass', 'pii-detected'] + injection('hello') pass + injection('ignore previous') injection-protection + budget spec BudgetScenario + + PASS Three specs, three stateful rules — the same captured fixtures classify() is tested against. + pii_block fails every Nth call. injection matches request text. budget is a real wall- + clock window: passing 200s carry the prose x-llm-proxy-ratelimit header; exhaustion + serves the token-rate-limit 429 with live x-token-* until the window rolls over. A + stock client pointed at that mock sees the refusal with no SDK in the process — which + is how you test an agent that does not use this SDK at all. + + +The point +───────── + This needs no gateway, so it belongs in your unit tests. Demo 05 is the same idea + turned into a suite someone else can run against your agent without reading your code. + +──────────────────────────────────────────────────────────────────────────────────────── +``` + + +```bash +python "demos/human-made/openai/03 - typed-refusals-simulated.py" +``` + +## Key code + +One context manager, and the branch runs; `times=` counts calls and normal +service resumes after (narrative demo 04): + +```python +with donkey.simulate(PIIDetected): + await agent.run("...") # fails as a real PIIDetected + +with donkey.simulate(TokenBudgetExceeded, times=2): + await agent.run("a") # refused + await agent.run("b") # refused + await agent.run("c") # succeeds — the injection is spent +``` + +Looping over refusal types on a plain OpenAI client (OpenAI script 03): + +```python +async with Donkey.from_env() as donkey: + client = donkey.openai() + + for refusal in REFUSALS: + async with donkey.run(id=f"typed-refusals-{refusal.__name__}"): + # simulate() replays the captured gateway fixture in-process, so + # the refusal branch runs with no network and nothing to provoke. + with donkey.simulate(refusal): + try: + await client.responses.create( + model="gpt-4o", + input="Say hello in exactly three words.", + ) + except openai.APIStatusError as err: + report(classify(err.response)) +``` + +Scripting the running simulator instead, for a stock client (narrative demo 04, +act 7): + +```bash +donkey mock --scenario pii_block:every=2 \ + --scenario 'injection:on-pattern=ignore previous' \ + --scenario budget:limit=200,window=5s,cost=80 +``` + +`pii_block` fails every Nth call, `injection` matches request text, and +`budget` is a real wall-clock window that serves the token-rate-limit 429 on +exhaustion until the window rolls over. + + + `simulate()` only injects shapes that have a captured wire body. + `simulate(ToolInvocationError)` and `simulate(GatewayUnavailable)` raise + `ValueError` instead of inventing one — a transport failure has no HTTP + response to replay. To provoke `GatewayUnavailable`, point at a dead origin + (see [Typed refusals](/examples/typed-refusals)). + + +Because the injection sits on the transport, it also reaches framework objects +the SDK does not wrap — demo 04 drives `donkey.langgraph.chat_model("gpt-4o")` +through `simulate(PIIDetected)` and gets the same taxonomy back. + +**Learn more:** [Local simulator](/simulator) · [Testing & conformance](/testing) + +**Source:** +[narrative demo 04](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/claude-made/04_simulate_refusals) · +[script 03](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/blob/main/demos/human-made/openai/03%20-%20typed-refusals-simulated.py) diff --git a/website/content/examples/telemetry.mdx b/website/content/examples/telemetry.mdx new file mode 100644 index 00000000..a916a56d --- /dev/null +++ b/website/content/examples/telemetry.mdx @@ -0,0 +1,284 @@ +--- +description: OpenTelemetry GenAI spans with no instrumentation code, one correlation id per donkey.run(), refusal spans, and zero-config OTLP export. +--- + +import { Callout } from 'nextra/components' +import { Output } from '../../components' + +# Telemetry + +Platform teams ask for two things agent teams rarely deliver: a trace that +follows one logical run across every model call it fans out into, and spans in +the standard GenAI vocabulary so they land in existing dashboards. Both come +from the same client every request leaves through. Each governed call emits a +span carrying `gen_ai.*` and `donkey.*` attributes, `donkey.run(id=…)` ties a +whole run to one correlation id and one set of cost tags, and setting +`OTEL_EXPORTER_OTLP_ENDPOINT` is enough for `Donkey.from_env()` to install an +exporter. + +| Example | Shows | Needs | +| --- | --- | --- | +| Narrative demo 06 | One span per call with both namespaces and routing/usage, three calls under one run id and cost tags, a refused call as an `ERROR` span, and zero-config OTLP | `[otel]` (simulator) | +| OpenAI script 06 | A host-owned `TracerProvider` exporting over OTLP; Donkey rides it | Proxy credentials + an OTLP endpoint | +| OpenAI script 07 | Several `donkey.run(team=…, project=…)` blocks, including a refusal span | Proxy credentials + an OTLP endpoint | +| OpenAI script 10 | `Donkey.from_env()` installing OTLP itself when the env var is set | Proxy credentials | + +## Run it + +```bash +pip install "donkey-kit[otel]" +make demo N=06 +``` + + +```text +════════════════════════════════════════════════════════════════════════════════════════ +Demo 06 — OTel GenAI spans and correlation ids +Standard GenAI telemetry and one trace per run, without instrumentation code. +════════════════════════════════════════════════════════════════════════════════════════ + +Run context +─────────── + target mock + proxy base_url http://127.0.0.1:8080/ + output masking on + credentials fake — the simulator enforces no auth + +[1] One governed call, one GenAI span — with no instrumentation code + + # your usual OTel setup, then: + await client.responses.create(model=..., input=...) + + + span donkey.llm.chat + status UNSET + gen_ai.* + gen_ai.request.model gpt-4o + gen_ai.system openai + gen_ai.response.model gpt-5.1 + gen_ai.usage.input_tokens 17 + gen_ai.usage.output_tokens 51 + donkey.* + donkey.routing.type ModelBased + donkey.routing.fallback False + donkey.usage.cached_tokens 0 + donkey.usage.cache_write_tokens 0 + donkey.usage.reasoning_tokens 0 + donkey.policy.decision allow + donkey.budget.remaining 95000 + donkey.cost.team platform + donkey.cost.env dev + donkey.correlation_id a8fdd39a06f145f3b20520cd9ee129fc + + PASS gen_ai.prompt / gen_ai.completion are absent — capture is opt-in + + The gen_ai.* keys are pinned to semantic-convention version 1.30.0. They are + transcribed in the SDK rather than imported from the semconv package, whose default + version drifts release to release — so what lands on your span is decided by a + reviewable edit, not by a transitive upgrade. + Prompt and completion stay off the span unless you set telemetry_capture_content=True + (or DONKEY_TELEMETRY_CAPTURE_CONTENT=1). The gateway masks PII in its logs; spans are + emitted upstream of that, so defaulting capture on would re-export the content the + platform just masked. + + +Routing and usage, on the same span +─────────────────────────────────── + routing + gen_ai.request.model gpt-4o + gen_ai.response.model gpt-5.1 + donkey.routing.type ModelBased + donkey.routing.fallback False + donkey.usage.* + donkey.usage.cached_tokens 0 + donkey.usage.cache_write_tokens 0 + donkey.usage.reasoning_tokens 0 + + gen_ai.response.model is what the gateway actually served. When it differs from + gen_ai.request.model, a failover happened — the fastest read on a latency spike. + donkey.routing.fallback is emitted even when False: 'we routed normally' is a signal, + not the absence of one. The same facts live on donkey.last_call without a span backend + (demo 10). + +[2] One correlation id for a whole run, however many calls it makes + + async with donkey.run(id=ticket.id, team="support", project="triage"): + await client.responses.create(...) # all three calls share + await client.responses.create(...) # one id, and the cost + await client.responses.create(...) # tags, on wire and spans + + run id ticket-4417 + spans emitted 3 + distinct correlation ids 1 + PASS all 3 spans carry the one run id + PASS run() overrode team/project; env inherited from from_env() + + Nothing was threaded through the agent. The id is bound to the async context, and + tasks the framework spawns copy that context — so a LangGraph node running the model + on a child task is inside the same run without knowing the run exists. Concurrent runs + do not leak into each other, and nested run() blocks rebind then restore. Cost tags + ride the same context: run(team=..., project=...) overrides those dimensions for the + block and inherits the rest from from_env(). + + +Two ids, two questions +────────────────────── + run header X-Correlation-Id + call header X-Donkey-Request-Id + The run id answers 'show me everything this ticket did'. The per-call id answers + 'which one of those calls was this'. Both go out on every request, which is what lets + a line in your log join to the gateway's own record of the same call. + +[3] A refusal is a failed span, not a successful-looking one + A span that ends OK on a request the gateway refused is worse than no span: it makes a + dashboard say everything is fine. So a refusal sets the span status to ERROR and + records what refused it. + + span donkey.llm.chat + status ERROR + gen_ai.* + gen_ai.request.model donkey-sim/pii-detected + gen_ai.system openai + gen_ai.response.model gpt-5.1 + donkey.* + donkey.routing.type ModelBased + donkey.routing.fallback False + donkey.policy.decision refuse + donkey.policy.type pii_detected + donkey.budget.remaining 1 + donkey.cost.team platform + donkey.cost.env dev + donkey.correlation_id ticket-4417 + + donkey.policy.decision is 'refuse' and donkey.policy.type names the specific policy — + so a dashboard can separate 'the model failed' from 'governance said no', which are + very different operational stories. + + +The same two ids, on the exception +────────────────────────────────── + type PIIDetected + .correlation_id ticket-4417 + .call_id c37c6199f07c43fe9f0555fd88cc4cdd + classify() read those back off the request the failed response came from, so the + exception you catch already carries the ids without you passing them in. Put + .correlation_id in the alert and the platform team can pull the gateway's record of + the same refusal. + +[4] Zero-config OTLP: set the standard env var, or stay silent + + # no Donkey-specific variable + export OTEL_EXPORTER_OTLP_ENDPOINT=https://… + donkey = Donkey.from_env() # installs OTLP behind a BatchSpanProcessor + # no endpoint → inert, silent, nothing connects + # DONKEY_TELEMETRY=false → opt out even if an endpoint is set + + PASS no OTEL_EXPORTER_OTLP_ENDPOINT — export stayed inert and silent + Donkey.from_env() installs OTLP only when that standard env var is set. It will not + clobber a TracerProvider the host already installed — which is why this demo's in- + memory table still works. Opt out with DONKEY_TELEMETRY=false (or telemetry = false in + .donkey-kit.toml). Cost tags on donkey.run(team=..., project=...) are what let a + backend slice refusals, budget and latency by agent without another attribute + convention. + + +The point +───────── + Span name is 'donkey.llm.chat'. Nothing in the agent code above mentions OpenTelemetry + — the instrumentation hangs off the same transport hooks as the budget and the typed + refusals, which is why they all landed in one milestone rather than three. + Cost tags are the fixed four — team / project / env / enduser.id — set on from_env() + and overridable per donkey.run(). They land on donkey.cost.* whether or not the + gateway-side header names are verified yet. Routing (donkey.routing.*) and + cached/reasoning usage (donkey.usage.*) land on the same span. Zero-config OTLP is + shipped: OTEL_EXPORTER_OTLP_ENDPOINT, otherwise silent. + +──────────────────────────────────────────────────────────────────────────────────────── +``` + + +```bash +python "demos/human-made/openai/06 - otel exporter simple.py" # proxy + OTEL_EXPORTER_OTLP_ENDPOINT / _HEADERS +python "demos/human-made/openai/07 - otel exporter advanced.py" # proxy + OTEL_EXPORTER_OTLP_ENDPOINT / _HEADERS +python "demos/human-made/openai/10 - zero-config-otlp.py" # proxy; set OTEL_EXPORTER_OTLP_ENDPOINT to export +``` + +Without `[otel]` installed, narrative demo 06 prints the install command and +exits cleanly. It installs an in-memory exporter so it can print the spans as a +table. + +## Key code + +One correlation id and one set of cost tags for a whole run (narrative demo 06, +act 2): + +```python +async with donkey.run(id=ticket.id, team="support", project="triage"): + await client.responses.create(...) # all three calls share + await client.responses.create(...) # one id, and the cost + await client.responses.create(...) # tags, on wire and spans +``` + +Your own `TracerProvider`, with several runs and a refusal (OpenAI script 07): + +```python +provider = TracerProvider(resource=Resource.create({"service.name": "donkey-dev-kit"})) +provider.add_span_processor(SimpleSpanProcessor(OTLPSpanExporter())) +trace.set_tracer_provider(provider) + +donkey = Donkey.from_env() +client = donkey.openai(sync=True) + +with donkey.run(id="agent-greeter", team="cx", project="welcome"): + reply = client.responses.create(model=MODEL, input="Say hello in exactly three words.") + print("greeter:", reply.output_text) + +with donkey.run(id="agent-support", team="cx", project="tickets"): + try: + client.responses.create(model=MODEL, input=PII_PROMPT) + print("support: no refusal") + except openai.APIStatusError as err: + error = classify(err.response) + print("support:", type(error).__name__, getattr(error, "entities", None)) + +provider.force_flush() +donkey.close() +``` + +Zero-config export (OpenAI script 10) needs no provider setup at all: + +```python +donkey = Donkey.from_env() +client = donkey.openai(sync=True) + +with donkey.run(id="otel-zero-config", team="cx", project="welcome"): + reply = client.responses.create(model="gpt-4o", input="Say hello in exactly three words.") + print(reply.output_text) +``` + + + `gen_ai.prompt` and `gen_ai.completion` stay off the span unless you set + `telemetry_capture_content=True` (or `DONKEY_TELEMETRY_CAPTURE_CONTENT=1`). + Spans are emitted upstream of the gateway's PII mask, so capturing by default + would re-export content the platform just masked. + + +- **Two ids per request.** The run id (`X-Correlation-Id`) answers "everything + this ticket did"; the per-call id (`X-Donkey-Request-Id`) answers "which call + was this". A caught refusal carries both as `.correlation_id` and `.call_id`. +- **Refusals are failed spans.** A refused call sets the span status to + `ERROR` and records `donkey.policy.decision=refuse` with the specific + `donkey.policy.type`. +- **Cost tags are a fixed four** — `team`, `project`, `env`, `enduser.id` — + set on `from_env()` and overridable per `run()`, landing on `donkey.cost.*`. +- **Export is opt-in.** No endpoint means inert and silent; + `DONKEY_TELEMETRY=false` opts out even when one is set, and a host + `TracerProvider` is never replaced. + +**Learn more:** [Telemetry & cost](/telemetry) + +**Source:** +[narrative demo 06](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/claude-made/06_telemetry) · +[script 06](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/blob/main/demos/human-made/openai/06%20-%20otel%20exporter%20simple.py) · +[script 07](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/blob/main/demos/human-made/openai/07%20-%20otel%20exporter%20advanced.py) · +[script 10](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/blob/main/demos/human-made/openai/10%20-%20zero-config-otlp.py) diff --git a/website/content/examples/typed-refusals.mdx b/website/content/examples/typed-refusals.mdx new file mode 100644 index 00000000..9f0bbc14 --- /dev/null +++ b/website/content/examples/typed-refusals.mdx @@ -0,0 +1,282 @@ +--- +description: Every captured gateway rejection shape mapped to a typed exception with classify(), live refusals on a blocking client, and GatewayUnavailable when nothing answers. +--- + +import { Callout } from 'nextra/components' +import { Output } from '../../components' + +# Typed refusals + +Governance outcomes should be something you branch on, not something you +parse. These examples run the gateway's rejection shapes through `classify()` +and show the exception hierarchy you write `except` clauses against. The +discriminator is deliberately not the status code: a PII block is a 403 but is +not an auth failure, and an injection block is identified by a header. They +also show the one failure `classify()` cannot produce — `GatewayUnavailable`, +raised when there is no HTTP response at all. + +| Example | Shows | Needs | +| --- | --- | --- | +| Narrative demo 02 | Nine captured shapes through `classify()`, what the hierarchy buys, the handler you write, and a dead origin raising `GatewayUnavailable` | Nothing — no gateway, no simulator | +| OpenAI script 04 | Live `UpstreamRequestError`, `PIIDetected`, `TokenBudgetExceeded` and `AuthError` on a blocking client, no `async` | Proxy credentials, plus policies for the PII and budget cases | +| OpenAI script 11 | A dead origin surfacing `GatewayUnavailable` as `__cause__` | Nothing | + +## Run it + +```bash +make demo N=02 +``` + + +```text +════════════════════════════════════════════════════════════════════════════════════════ +Demo 02 — typed refusals +The gateway's rejection shapes, mapped to exceptions you can branch on. +════════════════════════════════════════════════════════════════════════════════════════ + +Run context +─────────── + target offline — no gateway, no simulator, no credentials + output masking on + +Nine captured responses through classify() +────────────────────────────────────────── + + from donkey_kit.core.errors import classify + + governed = classify(response) # -> a typed DonkeyError subclass + + + client-id-missing consumer auth — a genuinely missing/wrong client id + HTTP 401 + classified as AuthError + + pii-detected PII policy — a 403 that is NOT an auth failure + HTTP 403 + classified as PIIDetected + .policy pii-detection + .entities ['Email'] + + token-rate-limit token budget — a 429 with an EMPTY body; state is header-only + HTTP 429 + classified as TokenBudgetExceeded + .policy token-rate-limit + .retry_after 41.728 + + injection-protection prompt injection — identified by a header, not a status + HTTP 400 + classified as PromptInjectionBlocked + .policy prompt-injection-protection + + regex-prompt-guard regex prompt guard — 403 keyed on matched_patterns, not auth + HTTP 403 + classified as PromptInjectionBlocked + .policy regex-prompt-guard + + content-safety content safety / guardrails — 403 keyed on a vendor reject header + HTTP 403 + classified as ContentSafetyBlocked + .policy content-safety + .categories ['severity_hate', 'severity_violence'] + + content-moderation undiscriminated moderation — no live capture, left unnamed + HTTP 400 + classified as PolicyViolation + .policy unknown + + model-not-found upstream passthrough — the provider's own error, not a policy + HTTP 400 + classified as UpstreamRequestError + .code model_not_found + .error_type invalid_request_error + .param model + + upstream-5xx provider failure — retryable, unlike every refusal above + HTTP 503 + classified as UpstreamModelError + +Why the hierarchy is shaped this way +──────────────────────────────────── + PASS a PII block is a policy refusal, not an auth error + PASS a token-budget 429 is also a policy refusal — so one `except PolicyViolation` catches both + PASS content-safety is ContentSafetyBlocked, still a PolicyViolation + PASS regex-prompt-guard is PromptInjectionBlocked with its own policy name + PASS an upstream 400 is NOT a policy refusal — it is your request that is wrong, not the gateway saying no + PASS the budget refusal carries retry_after, parsed from x-token-reset (milliseconds, not an epoch) + PASS GatewayUnavailable is not a PolicyViolation — nothing was refused, the request never arrived + PASS a transport failure has no request_id — there was no response to read the gateway's id from + +What that looks like in your agent +────────────────────────────────── + + try: + response = await client.responses.create(model=..., input=...) + except openai.APIStatusError as exc: + raise classify(exc.response) from exc + + except PIIDetected as e: # 403, and e.entities says what tripped + redact_and_retry(e.entities) + except ContentSafetyBlocked as e: # 403, e.categories is the moderation analog + revise(e.categories) + except TokenBudgetExceeded as e: # 429, terminal — never retry it + await donkey.budget.wait_for_reset() + except PolicyViolation as e: # any other gateway refusal + escalate(e.remediation) + except GatewayUnavailable as e: # NO response — not a refusal + diagnose(e.base_url, e.cause) # checkpoint / shed / donkey doctor + except ModelSubstituted as e: # NOT classify() — you opted in (demo 10) + pin_or_accept(e.served_model) + except UpstreamRequestError as e: # your request was wrong (e.code) + fix(e.code) + except UpstreamModelError: # provider 5xx — this one IS retryable + retry_with_backoff() + + +What is typed from docs, and what is still unnamed +────────────────────────────────────────────────── + Four of these shapes are live-verified against a real proxy: consumer auth, PII, token + rate limit, and upstream passthrough. Injection, regex prompt guard, and content- + safety are typed from the documented wire shapes — classify() produces + PromptInjectionBlocked / ContentSafetyBlocked — and are pending a live sandbox + capture. That is the same posture as header-based injection: named because the shape + is specified, not because a capture has landed yet. + + content-moderation PolicyViolation + remediation This refusal matched no documented rejection shape, so its contract is unconfirmed (#184, #253). It is terminal and was NOT retried. Please file an issue on the donkey-development-kit repo with the response status, headers and body (all carried on this exception's .response) so the shape can be typed. + + An undiscriminated content-moderation 4xx still falls through to a generic + PolicyViolation. That leftover shape has never been captured from a live gateway, so + it is left unnamed rather than given a class that would imply more certainty than + exists. + + ModelSubstituted is not in the table above because it is not a gateway refusal and + classify() never produces it. It is raised by the transport when you opt into + on_model_substitution='raise' and the gateway serves a different model than you asked + for. Demo 10. + + GatewayUnavailable is the other type classify() never produces: there is no HTTP + response to classify. DNS, connection refused, TLS, timeout — the transport wraps + those as a typed DonkeyError so a long-running agent can tell 'lost the gateway' from + a policy refusal without matching raw httpx exceptions. It is not retried. Act 5 + actually raises it. + +A refused connection, typed — not a raw httpx error +─────────────────────────────────────────────────── + + donkey = Donkey(DonkeyConfig(llm_proxy_url="http://127.0.0.1:9/", ...)) + client.responses.create(...) # nothing is listening + # -> GatewayUnavailable, not ConnectError + + PASS GatewayUnavailable — the request never left the building + base_url http://127.0.0.1:9 + cause ConnectError + request_id None + call_id c46d490e7aef47b18a391a2764087952 + + The gateway could not be reached and no HTTP response came back. The three usual + causes: (1) the host is unreachable — DNS failure or the gateway is down; (2) the + configured base URL is wrong; or (3) network egress to the gateway is blocked — a + firewall or air-gapped environment. Run `donkey doctor` to diagnose connectivity, and + check `base_url` on this error against your gateway's address. + + PASS not a PolicyViolation — nothing was refused, because nothing arrived + +──────────────────────────────────────────────────────────────────────────────────────── +``` + + +```bash +python "demos/human-made/openai/04 - typed-refusals-live.py" # needs proxy credentials +python "demos/human-made/openai/11 - gateway-unavailable.py" # no gateway +``` + + + Narrative demo 02 loads its fixtures from the installed SDK + (`donkey_kit.simulator.fixtures`) — the same bytes `classify()` is tested + against and `donkey mock` serves. OpenAI script 04 provokes the upstream and + auth cases with nothing extra; the PII case needs the PII detection policy + with `Email` and action `Reject`, and the budget case needs the token rate + limit policy with a small `maximumTokens`. + + +## Key code + +The handler shape the hierarchy is designed for (narrative demo 02, act 3): + +```python +try: + response = await client.responses.create(model=..., input=...) +except openai.APIStatusError as exc: + raise classify(exc.response) from exc + +except PIIDetected as e: # 403, and e.entities says what tripped + redact_and_retry(e.entities) +except ContentSafetyBlocked as e: # 403, e.categories is the moderation analog + revise(e.categories) +except TokenBudgetExceeded as e: # 429, terminal — never retry it + await donkey.budget.wait_for_reset() +except PolicyViolation as e: # any other gateway refusal + escalate(e.remediation) +except GatewayUnavailable as e: # NO response — not a refusal + diagnose(e.base_url, e.cause) # checkpoint / shed / donkey doctor +except UpstreamRequestError as e: # your request was wrong (e.code) + fix(e.code) +except UpstreamModelError: # provider 5xx — this one IS retryable + retry_with_backoff() +``` + +A live refusal on a blocking client (OpenAI script 04): + +```python +cfg = DonkeyConfig.from_env() +donkey = Donkey(cfg) +client = donkey.openai(sync=True) + +with donkey.run(id="live-refusals-PIIDetected"): + try: + raw = client.responses.with_raw_response.create(model=MODEL, input=PII_PROMPT) + except openai.APIStatusError as err: + error = classify(err.response) + print(f" REFUSED {type(error).__name__} (HTTP {err.response.status_code})") + print(f" entities {getattr(error, 'entities', None)}") + print(f" remediation {getattr(error, 'remediation', None)}") + print(f" correlation_id {getattr(error, 'correlation_id', None)}") +``` + +When nothing is listening, the OpenAI client wraps the transport error and the +typed `GatewayUnavailable` sits on `__cause__` (OpenAI script 11): + +```python +donkey = Donkey( + DonkeyConfig( + llm_proxy_url="http://127.0.0.1:9/", + llm_proxy_client_id="demo-client-id-not-a-real-credential", + llm_proxy_client_secret="demo-client-secret-not-a-real-credential", + timeout_s=2.0, + max_retries=0, + ) +) +client = donkey.openai(sync=True) + +try: + client.responses.create(model="gpt-4o", input="hello") +except Exception as err: + hit = err if isinstance(err, GatewayUnavailable) else err.__cause__ + if isinstance(hit, GatewayUnavailable): + print("base_url ", hit.base_url) + print(hit.remediation) +``` + +A token-budget 429 and a PII 403 are both `PolicyViolation`s, so one `except +PolicyViolation` catches either. An upstream 400 is not — your request was +wrong, the gateway did not say no. `GatewayUnavailable` is not a +`PolicyViolation` either: nothing was refused, because nothing arrived. An +undiscriminated `content-moderation` 4xx falls through to a generic +`PolicyViolation`. + +**Learn more:** [Typed refusals](/errors) + +**Source:** +[narrative demo 02](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/tree/main/demos/claude-made/02_typed_refusals) · +[script 04](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/blob/main/demos/human-made/openai/04%20-%20typed-refusals-live.py) · +[script 11](https://github.com/Donkey-Development-Kit/donkey-development-kit-demos/blob/main/demos/human-made/openai/11%20-%20gateway-unavailable.py) diff --git a/website/content/feature-overview.mdx b/website/content/feature-overview.mdx index e21d17e9..7663a4ea 100644 --- a/website/content/feature-overview.mdx +++ b/website/content/feature-overview.mdx @@ -1,183 +1,166 @@ --- -description: A tour of everything governed model access gives you today. +description: Every DDK capability at a glance — its purpose, its goal, and what it saves developers — grouped by pillar. --- -import { Callout } from 'nextra/components' -import { Figure } from '../components' +import { Badge, Card, CardGrid, Figure } from '../components' # Feature overview -What the SDK gives you **today**, wired to the live-verified proxy contract. -For what is designed but not yet shipped, see the [Roadmap](/roadmap) — nothing -on this page is speculative. +DDK is **gateway-aware**: control stays at the proxy, efficiency moves into the +agent. The gateway enforces policy; DDK makes each decision visible and +actionable in your code, so the agent reacts to a refusal, paces its budget and +reports what it did.
-Everything the SDK adds attaches at stage 2 of that diagram, on a single shared -transport. That is the **skeleton**: one place where every request enters and -every response leaves, which is why budget, spans, correlation IDs and -simulation can all be added later without you rewiring a call site. - -## Native framework objects, not wrappers - -One call per framework returns the framework's **own** object, already pointed at -the proxy with the verified `client_id`/`client_secret` headers, attribution, and -retry policy injected: +Everything below hangs off a single shared transport inside your process — the +one place where every request enters and every response leaves. Each capability +attaches there once, so you never wire it call by call. + +## Model access + + + + **Purpose:** point any of eight agent frameworks at your governed Omni + Gateway proxy. **Advantage:** you get your framework's own object — + `ChatOpenAI`, `LiteLlm`, `OpenAIModel`, `crewai.LLM` … — with credentials, + correlation, attribution and retry policy injected. No wrapper to code + around, three lines to eject. + + + **Purpose:** use the OpenAI SDK directly. **Advantage:** + `donkey.llm.client()` returns a native `AsyncOpenAI` (or `OpenAI` with + `sync=True`) — Chat Completions and Responses, streaming included — governed + on identical terms. + + | Framework | Call | Returns | |---|---|---| | LangGraph | `donkey.langgraph.chat_model("gpt-4o")` | `langchain_openai.ChatOpenAI` | -| Google ADK | `donkey.adk.model("gpt-4o")` | `google.adk … LiteLlm` | -| Strands | `donkey.strands.model("gpt-4o")` | `strands … OpenAIModel` | +| Google ADK | `donkey.adk.model("gpt-4o")` | `LiteLlm` | +| Strands | `donkey.strands.model("gpt-4o")` | `OpenAIModel` | | MS Agent Framework | `donkey.agent_framework.chat_client("gpt-4o")` | Agent Framework chat client | -| LlamaIndex | `donkey.llamaindex.llm("gpt-4o")` | `OpenAILike` (`is_chat_model=True`) | -| OpenAI Agents SDK | `donkey.openai_agents.model("gpt-4o")` | `agents.OpenAIChatCompletionsModel` | -| Anthropic SDK | `donkey.anthropic.client()` | `anthropic.AsyncAnthropic` (client, not model-bound) | -| CrewAI | `donkey.crewai.llm("gpt-4o")` | `crewai.LLM` (LiteLLM-backed) | - -See [Model access](/frameworks) for each framework's page. Support **depth** -differs even though the shape does not: LangGraph is the one deep, -conformance-gated adapter, and the other seven are supported at the -`connection_kwargs()` level. - -## Three ways to construct — pick your ergonomics - -Every adapter offers the same governed connection three ways, from one source of -truth: - -```python -# 1. Factory method — shared client + explicit lifecycle -async with Donkey.from_env() as donkey: - model = donkey.langgraph.chat_model("gpt-4o", temperature=0.2) - -# 2. Module-level factory — shortest; cached default from the environment -from donkey_kit.integrations.langgraph import chat_model -model = chat_model("gpt-4o", temperature=0.2) - -# 3. Governed-kwargs accessor — you build the native object yourself -from langchain_openai import ChatOpenAI -model = ChatOpenAI(model="gpt-4o", **donkey.langgraph.connection_kwargs()) -``` - -Prefer the factory when you want shared-client reuse and lifecycle -(`async with`); reach for the accessor when you want full control of the -constructor. - -## Framework-free client - -`donkey.llm.client()` returns a native `AsyncOpenAI` aimed at the proxy — use the -OpenAI SDK exactly as you normally would (Chat Completions or the Responses API, -streaming included); the SDK adds the governance headers and retry policy. - -Pass `sync=True` for the blocking `OpenAI` instead, governed on identical terms — -same base URL, same verified `client_id` / `client_secret` headers, same -correlation ID and retry policy: - -```python -with Donkey.from_env() as donkey: - client = donkey.llm.client(sync=True) # openai.OpenAI - reply = client.chat.completions.create( - model="gpt-4o", - messages=[{"role": "user", "content": "Say hi in three words."}], - ) - print(reply.choices[0].message.content) -``` - -The two forms are declared as typed overloads, so your editor narrows the result -to `OpenAI` or `AsyncOpenAI` and keeps completing on it. Note the boundary: the -blocking transport takes no `AuthProvider`, because that protocol is async-only. -That costs nothing here — the proxy authenticates on the header pair, not a -fetched token — but it does mean the control-plane surfaces (`registry`, -`tools`) remain async. - -## One-line on-ramps: decorators - -Two decorators fold the run scope and the tool registry into a single line, so -the six-piece minimum is reachable without threading anything through your -framework. - -`@donkey.governed` runs a callable inside a [`donkey.run()`](/telemetry) scope — -a fresh run/correlation id per call (a "run of one"), the optional per-run cost -tags, the OTel span, and typed refusals — with nothing to wire up: - -```python -@donkey.governed(team="support") -async def handle_ticket(ticket): ... -``` - -It wraps **both sync and async** callables, works bare (`@donkey.governed`) or -parametrised (`@donkey.governed(team=…, project=…, env=…, enduser_id=…)`), and -deliberately takes **no `id=`**: each call opens its own run, and a fixed id -pinned across calls would collapse unrelated runs into one correlation. When you -need a specific id, use `donkey.run(id=…)` directly. - -`@donkey.tool` marks a callable as a governed tool **without changing how it's -called** — it returns the same function with a `__donkey_tool__` marker and -records a `ToolSpec` (name, qualname, signature, docstring, `is_async`) in a -process-global registry you read with `registered_tools()`: - -```python -@donkey.tool -async def lookup_crm(customer_id: str) -> dict: - """Look up a customer record by id.""" - ... -``` - -A tool with **no docstring is rejected at decoration time** (`ValueError`) — an -undescribed tool is useless to a model and to the registry. That one marker is -what the Phase 2 [scanner](/publishing) and the [A2A](/a2a) agent-card generator -will both read, so the single annotation pays off across surfaces. `ToolSpec` -and `registered_tools` are exported from `donkey_kit`. - -## A typed governed error taxonomy - -`classify()` maps the proxy's eight rejection shapes to typed exceptions — -`PIIDetected` (403), `TokenBudgetExceeded` (429), `PromptInjectionBlocked` -(`x-injection-protection` **or** the regex prompt-guard's `matched_patterns`), -`ContentSafetyBlocked` (Azure Content Safety / Bedrock Guardrails vendor reject -header; parses `categories`), a generic `PolicyViolation` fall-through for -undiscriminated content-moderation, `UpstreamRequestError` (4xx) and -`UpstreamModelError` (5xx), plus `AuthError` (401) for the separate consumer-auth -case — so you branch on governance outcomes instead of parsing bodies. Six are -live-verified (including the regex-prompt-guard and Azure content-safety `403` -blocks, confirmed 2026-09-22); the injection-protection, Bedrock-guardrails and -content-moderation fall-through bodies are pinned from the policy pages, pending -sandbox capture (#253). See [Error taxonomy](/errors). - -Those exception types are the client-side mirror of the policies the gateway -enforces. The **LLM** lane below is the one this SDK targets today; the same -control plane applies equivalent policy sets to APIs, MCP tools, and A2A -agents. Those two columns are not decoration — **MCP** is what -[tool access](/tool-access) reaches in Phase 2, and **Agents** is what -[A2A](/a2a) reaches in the same phase. - -
- -## Model handles without a `/models` endpoint - -The governed proxy has **no** catalog endpoint (`GET /models` → `404`, verified). -`donkey.llm.resolve("gpt-4o")` gives you a heuristic capability handle -(function-calling / vision / json-output) from a known id, and -`list_models(live=True)` raises a clear `ConfigError` explaining the absence -rather than fabricating a path. - - - **Honesty note (verification discipline).** The proxy *contract* the adapters target is - live-verified, but the exact framework *constructor signatures* are still being - confirmed against installed versions. Where a - name can't be resolved, the adapter raises `blocked on verification` rather - than guess. The [Verification policy](/concepts/verification) page tracks - current status. - +| LlamaIndex | `donkey.llamaindex.llm("gpt-4o")` | `OpenAILike` | +| OpenAI Agents SDK | `donkey.openai_agents.model("gpt-4o")` | `OpenAIChatCompletionsModel` | +| Anthropic SDK | `donkey.anthropic.client()` | `anthropic.AsyncAnthropic` | +| CrewAI | `donkey.crewai.llm("gpt-4o")` | `crewai.LLM` | + +Every adapter offers the same governed connection three ways — a factory on a +shared `Donkey` (`donkey.langgraph.chat_model(...)`), a module-level factory +(`from donkey_kit.integrations.langgraph import chat_model`), or +`connection_kwargs()` when you want to build the native object yourself. + +## Governance + + + + **Purpose:** turn every gateway rejection into a typed exception — + `PIIDetected`, `TokenBudgetExceeded`, `PromptInjectionBlocked`, + `ContentSafetyBlocked`, `AuthError`, `GatewayUnavailable` and more. + **Goal:** branch on the governance outcome, not on a parsed error body. + **Advantage:** a PII block is never mistaken for an auth failure, and a + policy `429` is never retried. + + + **Purpose:** expose the gateway's token window as a `Budget` object. + **Goal:** stop *before* the limit, not after it. **Advantage:** + `pace(reserve=)` and `wait_for_reset()` let an overnight batch slow down + instead of dying at 2am. + + + **Purpose:** on-behalf-of token exchange. **Goal:** per-user policy + reaches the gateway. **Advantage:** requests never silently fall back to + the service identity. + + + **Purpose:** one vocabulary for "pause and ask a human". **Advantage:** + mapped onto each framework's native interrupt, so approval flows look the + same everywhere. + + + **Purpose:** read the policy set in force. **Advantage:** skip calls that + are certain to be refused. Advisory only — the gateway always wins. + + + +## Observability + + + + **Purpose:** one span per governed call, using the GenAI semantic + conventions plus a stable `donkey.*` namespace — policy decision, policy + type, budget, routing and token usage. **Advantage:** refused calls still + produce a span, streaming produces exactly one, and prompt content stays + out by default. Zero-config OTLP export. + + + **Purpose:** `donkey.run(id=…, team=…, project=…)` binds one correlation + ID and validated cost tags to every call in a task. **Advantage:** your + log line, your span and the gateway's audit record join on the same ID. + + + **Purpose:** `donkey.last_call` records which gateway served the call, + how it was routed and what it used. **Advantage:** detect a model + substitution — or make it raise — instead of discovering it in a bill. + + + +## Developer tooling + + + + **Purpose:** `donkey mock` replays real gateway responses and refusal + shapes on `127.0.0.1`. **Advantage:** build and demo against governance + without an Anypoint account or credentials. + + + **Purpose:** `donkey.simulate()` injects a refusal in-process, and a + pytest plugin grades *your* agent against every refusal shape. + **Advantage:** the PII branch is tested before production, not in it. + + + **Purpose:** `donkey init`, `doctor`, `mock` and `test`, plus + `@donkey.governed` and `@donkey.tool`. **Advantage:** `doctor` tells wrong + credentials from wrong URL from model-not-allowed; one decorator gives a + function a run scope, span and typed refusals. + + + **Purpose:** the docs are published as `llms.txt` and per-page markdown. + **Advantage:** Cursor, Claude Code and other assistants write correct DDK + code from the source. + + + +## Registry & catalog + + + + **Purpose:** discover governed MCP tools from the catalog and bind them as + native framework tools. **Advantage:** allow/deny filtering, pinning and a + lockfile — only governed tools reach your agent. + + + **Purpose:** `serve`, `expose` and `dev` make your agent callable by other + agents, on the official `a2a-sdk`. **Advantage:** inbound tasks are + governed with the same correlation, spans and refusals. + + + **Purpose:** derive a manifest and agent card from your code and register + them in the Agent Fabric registry. **Advantage:** the catalog stays in sync + from CI, not by hand. + + + +## What DDK leaves to the platform + +DDK makes the platform's capabilities reachable and typed; it does not +reproduce them. Policy enforcement, semantic caching, provisioning, agent +scanners, kill switch, trusted agent identity, approval UIs and evaluation all +stay with Agent Fabric and Omni Gateway. See the [Roadmap](/roadmap#what-ddk-will-not-build) +for the full list. diff --git a/website/content/frameworks/adk.mdx b/website/content/frameworks/adk.mdx index 2ae14bc0..c7316826 100644 --- a/website/content/frameworks/adk.mdx +++ b/website/content/frameworks/adk.mdx @@ -1,16 +1,24 @@ +--- +description: Use a governed LiteLlm model in Google's Agent Development Kit (ADK), pointed at your Omni Gateway LLM proxy. +--- + import { Tabs } from 'nextra/components' import OpenAiProxyTs from '../../components/openai-proxy-ts.mdx' -# Google ADK — governed model access +# Google ADK + +Google's Agent Development Kit (ADK) reaches the Agent Fabric LLM proxy through +ADK's `LiteLlm` model wrapper. The adapter translates the governed connection +into LiteLLM's own model-string and kwarg conventions for you. -Google's Agent Development Kit (ADK) is Gemini-first; it reaches the -Agent Fabric LLM proxy through ADK's `LiteLlm` model wrapper, which speaks -LiteLLM's own model-string and kwarg conventions rather than raw OpenAI ones. +**What you get** -> **Supported at `connection_kwargs()` — not conformance-tested (`BG §1.8`)**, with two documented exemptions: LiteLLM -> owns its own transport, so the SDK's shared httpx client is not injected -> here. Per-run correlation and `donkey.last_call` response observation are -> unavailable as a result (see below). +- A native `google.adk.models.lite_llm.LiteLlm`, with the proxy auth and + attribution headers set. +- The `openai/` model prefix and LiteLLM kwarg names handled automatically. +- Supported at `connection_kwargs()`. ADK's `LiteLlm` model makes the HTTP + calls itself, so correlation is per client and `donkey.last_call` is not + populated (see [Notes](#notes)). ## Install @@ -29,19 +37,17 @@ from donkey_kit.integrations.adk import model llm = model("gpt-4o") ``` -`llm` is a real, native **`google.adk.models.lite_llm.LiteLlm`** instance. The -model string is auto-prefixed with `openai/` before it reaches LiteLLM -(`openai/gpt-4o`), because that's the prefix LiteLLM's OpenAI-compatible route -expects — you don't need to add it yourself. +`llm` is a real `google.adk.models.lite_llm.LiteLlm` instance. The model string +is prefixed with `openai/` before it reaches LiteLLM (`openai/gpt-4o`), which +is the prefix LiteLLM's OpenAI-compatible route expects — you don't add it +yourself. -There's no first-party Agent Fabric TypeScript SDK yet (it's on the -[roadmap](/concepts/verification)). The proxy is OpenAI-compatible, so point the -official `openai` npm client at it — the same base URL and -`client_id`/`client_secret` headers also drop into **ADK for TypeScript** -(`@google/adk`). +Call the proxy's OpenAI-compatible API with the official `openai` npm client. +The same base URL and `client_id`/`client_secret` headers also work with +**ADK for TypeScript** (`@google/adk`). @@ -77,7 +83,7 @@ async with Donkey.from_env() as donkey: llm = LiteLlm(model="openai/gpt-4o", **donkey.adk.connection_kwargs()) ``` -## The manual equivalent (eject at any time) +## Manual equivalent ```python from google.adk.models.lite_llm import LiteLlm @@ -90,31 +96,23 @@ llm = LiteLlm( ) ``` -Note the kwarg names: LiteLLM uses `api_base` and `extra_headers`, not -`base_url` / `default_headers` — `connection_kwargs()` already translates for -you. - -## Notes & limitations - -> **Correlation IDs are per-client, not per-run, for this adapter.** LiteLLM -> owns its own HTTP transport, so the SDK's shared httpx client — and the -> per-run correlation ID it stamps on every request — cannot be injected into -> it. This is a documented, asserted conformance exemption, not an oversight: -> ADK is the one adapter in the set where header injection is full but -> transport injection is not possible. +LiteLLM uses `api_base` and `extra_headers`, not `base_url` / +`default_headers` — `connection_kwargs()` already translates for you. -> **This adapter cannot populate `donkey.last_call`.** The same LiteLLM-owned -> transport means no response reaches the SDK's `_on_response` hook, so gateway -> identity, routing, and usage fields cannot be observed. When every adapter -> resolved on a `Donkey` is non-observing, the record reports -> `status == LastCallStatus.UNAVAILABLE`, `available == False`, and names the -> resolved adapters in `surface`. This is a -> documented, asserted `gateway_identity_observed` conformance exemption. +## Notes -> `google-adk` requires `litellm>=1.84` as a floor (not a ceiling) — pin your -> own upper bound if you need one. +- **Correlation IDs are per-client, not per-run.** ADK sends requests through + its built-in LiteLLM model layer rather than the SDK's shared HTTP client, so + the correlation ID is set once per client instead of per `donkey.run()`. Every + governance header is still sent on every request. The conformance suite + checks this as a documented behaviour. +- **`donkey.last_call` is unavailable.** Because the response is handled by LiteLLM, + gateway identity, routing, and usage fields can't be observed. When every + adapter resolved on a `Donkey` is like this one, `donkey.last_call` reports + `status == LastCallStatus.UNAVAILABLE` and `available == False`, and names the + resolved adapters in `surface`. +- `google-adk` requires `litellm>=1.84` as a floor, not a ceiling — pin your + own upper bound if you need one. See the [error taxonomy](/errors) for how proxy rejections surface through -LiteLLM's error path, and the [verification policy](/concepts/verification) -page for the current status of every constructor signature this adapter -depends on. +ADK's `LiteLlm` model. diff --git a/website/content/frameworks/agent-framework.mdx b/website/content/frameworks/agent-framework.mdx index b45e2f22..0da345ce 100644 --- a/website/content/frameworks/agent-framework.mdx +++ b/website/content/frameworks/agent-framework.mdx @@ -1,19 +1,25 @@ +--- +description: Use a governed OpenAIChatClient in Microsoft Agent Framework, pointed at your Omni Gateway LLM proxy, with policy middleware that ends a run cleanly on a governance rejection. +--- + import { Tabs } from 'nextra/components' import OpenAiProxyTs from '../../components/openai-proxy-ts.mdx' +import { Badge } from '../../components' + +# Microsoft Agent Framework -# Microsoft Agent Framework — governed model access +Microsoft Agent Framework gets a governed chat client pointed at the Omni +Gateway LLM proxy, plus policy middleware that stops a run on a governance +rejection instead of letting the agent loop retry. -Microsoft Agent Framework gets a governed chat client at the Agent Fabric LLM -proxy, plus first-class policy middleware for terminating a run cleanly on a -governance rejection instead of letting the agent loop retry. +**What you get** -> **Supported at `connection_kwargs()` — not conformance-tested (`BG §1.8`).** -> This adapter receives a static `default_headers` snapshot, not the SDK's shared -> httpx client, so per-run correlation and `donkey.last_call` response -> observation are documented exemptions. It also ships the strongest -> policy-integration story of the eight frameworks -> (`policy_middleware()`), but its native client class is young and -> **UNVERIFIED** — read the callouts below before depending on it. +- A native `agent_framework.openai.OpenAIChatClient`, checked against + agent-framework 1.19.0. +- `policy_middleware()` for terminating a run on a `PolicyViolation`. +- Supported at `connection_kwargs()`. The client receives a static + `default_headers` snapshot, so per-run correlation and `donkey.last_call` are + not available (see [Notes](#notes)). ## Install @@ -32,18 +38,14 @@ from donkey_kit.integrations.agent_framework import chat_client llm = chat_client("gpt-4o") ``` -`llm` is intended to be a native **`agent_framework.openai.OpenAIChatClient`** -instance. If that import fails on your installed version, the factory raises -`NotImplementedError("blocked on verification: ...")` rather than guessing — -see the UNVERIFIED callout below. +`llm` is a real `agent_framework.openai.OpenAIChatClient` instance. -There's no first-party Agent Fabric TypeScript SDK yet, and **Microsoft Agent -Framework has no TypeScript SDK** (it ships for .NET, Python, and Go). You can -still make the same governed call from TypeScript via the OpenAI-compatible -proxy with the official `openai` npm client: +Microsoft Agent Framework ships for .NET, Python, and Go, not TypeScript. From +TypeScript, call the proxy's OpenAI-compatible API directly with the official +`openai` npm client: @@ -82,50 +84,44 @@ async with Donkey.from_env() as donkey: ) ``` -## The manual equivalent (eject at any time) +## Manual equivalent ```python from agent_framework.openai import OpenAIChatClient llm = OpenAIChatClient( - model=..., # verified against agent-framework 1.19.0 - base_url=..., # from DONKEY_LLM_PROXY_URL, no /v1 suffix + model=..., # `model`, not `model_id` + base_url=..., # from DONKEY_LLM_PROXY_URL, no /v1 suffix api_key=..., default_headers=..., # client_id / client_secret header pair ) ``` -## Notes & limitations - -> **VERIFIED** against agent-framework 1.19.0. -> `agent_framework.openai.OpenAIChatClient` takes `model`, `base_url`, -> `api_key` and `default_headers` (`model_id` is **not** accepted). The -> `chat_client()` factory stays guarded on both ends: if the import fails or -> the constructor signature changes upstream, it raises -> `NotImplementedError("blocked on verification")` instead of leaking a raw -> `ImportError`/`TypeError` — so a future upstream rename surfaces as a clear -> verification refusal rather than a cryptic error. - -> Agent Framework has first-class middleware for intercepting agent actions. -> The adapter ships `policy_middleware()`, which catches a `PolicyViolation` -> and re-raises it so the host terminates the run cleanly rather than -> retrying — the best policy-integration story of any of the eight -> frameworks. The exact middleware signature Agent Framework expects is also -> **UNVERIFIED**; the shipped middleware is a plain async wrapper pending -> confirmation of the framework's middleware protocol. - -> **This adapter cannot propagate per-run correlation or populate -> `donkey.last_call`.** Agent Framework receives a static `default_headers` -> snapshot, which deliberately excludes the correlation ID bound later by -> `donkey.run(id=...)`, and not the SDK's httpx client. No response reaches -> `_on_response`, so gateway identity, routing, and usage fields cannot be -> observed either. When every adapter resolved on a `Donkey` is non-observing, -> the record reports `status == LastCallStatus.UNAVAILABLE`, `available == False`, -> and names the resolved adapters in `surface`. These are documented, asserted -> `correlation_id_propagated` and `gateway_identity_observed` conformance -> exemptions. +## Policy middleware + +`donkey.agent_framework.policy_middleware()` returns an async +`(context, next)` middleware that lets a `PolicyViolation` propagate, so the +host ends the run instead of retrying. It is a plain async wrapper whose +signature has not been confirmed against Agent Framework's middleware +protocol, so check it in your host before relying on it. Setting Agent +Framework's explicit "terminate run" signal instead of re-raising is planned +Roadmap. + +## Notes + +- **Constructor signature.** `OpenAIChatClient` takes `model`, `base_url`, + `api_key`, and `default_headers` (agent-framework 1.19.0; `model_id` is not + accepted). If the import fails or an upstream release renames a kwarg, + `chat_client()` raises a `NotImplementedError` naming the class path or + signature to check, rather than a raw `ImportError` or `TypeError`. +- **No per-run correlation or `donkey.last_call`.** The client receives a + static `default_headers` snapshot, which excludes the correlation ID bound + later by `donkey.run(id=...)`, and the SDK's httpx client is not used. No + response reaches the SDK, so gateway identity, routing, and usage fields + can't be observed. When every adapter resolved on a `Donkey` is like this + one, `donkey.last_call` reports `status == LastCallStatus.UNAVAILABLE` and + `available == False`, and names the resolved adapters in `surface`. The + conformance suite asserts both as documented exemptions. See the [error taxonomy](/errors) for the full `PolicyViolation` hierarchy -that `policy_middleware()` catches, and the -[verification policy](/concepts/verification) page for the current -verification status of this adapter's constructor and middleware shape. +that `policy_middleware()` lets through. diff --git a/website/content/frameworks/anthropic.mdx b/website/content/frameworks/anthropic.mdx index d52d96b6..fd2955b9 100644 --- a/website/content/frameworks/anthropic.mdx +++ b/website/content/frameworks/anthropic.mdx @@ -1,15 +1,30 @@ -import { Tabs } from 'nextra/components' +--- +description: Use a governed AsyncAnthropic client with the Anthropic SDK, pointed at a Format=Anthropic Omni Gateway LLM proxy with full header and transport injection. +--- + +import { Callout, Tabs } from 'nextra/components' import AdapterErrorTaxonomyNote from '../../components/adapter-error-taxonomy-note.mdx' -# Anthropic SDK — governed model access +# Anthropic SDK + +The Anthropic SDK gets a governed `AsyncAnthropic` client pointed at the Omni +Gateway LLM proxy, with the SDK's shared transport and proxy headers passed +straight into the client constructor. + +**What you get** -The Anthropic SDK gets a governed `AsyncAnthropic` client pointed at the -Agent Fabric LLM proxy, with the SDK's shared transport and verified headers -injected directly into the client constructor. +- A native `anthropic.AsyncAnthropic` client. +- Full header **and** transport injection. +- Supported at `connection_kwargs()`. -> **Supported at `connection_kwargs()` — not conformance-tested (`BG §1.8`)**, with one important divergence from -> every other adapter in this roster: this factory returns a bare **client**, -> not a model-bound object — see below before you reach for it. + + **Requires a `Format=Anthropic` proxy.** The native Anthropic Messages route + (`POST //v1/messages`) is only served by a proxy provisioned with + the Anthropic ingress Format. Default DDK proxies are `Format=OpenAI`: there, + `/v1/messages` returns 404 and Claude is reachable only as an upstream + provider through the OpenAI-compatible adapters. See + [Model access](/frameworks) for how ingress Format works. + ## Install @@ -28,9 +43,9 @@ from donkey_kit.integrations.anthropic import client llm = client() ``` -`llm` is a real, native **`anthropic.AsyncAnthropic`** instance. Unlike every -other framework in this roster, there is no `model` argument on the factory — -pass the model id per call, exactly as the native Anthropic SDK expects: +`llm` is a real `anthropic.AsyncAnthropic` instance. Unlike the other +adapters, the factory takes no `model` argument — pass the model ID per call, +as the Anthropic SDK expects: ```python reply = await llm.messages.create( @@ -43,20 +58,15 @@ reply = await llm.messages.create( -There's no first-party Agent Fabric TypeScript SDK yet (it's on the -[roadmap](/concepts/verification)). The native match here is the official -**`@anthropic-ai/sdk`** client pointed at the proxy — same `client_id` / -`client_secret` header pair, model id passed per call. **Caveat (proxy Format):** -the native Anthropic ingress route is LIVE-verified at `POST //v1/messages`, -but it requires a proxy provisioned `Format=Anthropic`; every default DDK proxy is -`Format=OpenAI`, so point `baseURL` at a `Format=Anthropic` proxy to use the native -surface (#304). +Use the official **`@anthropic-ai/sdk`** client pointed at a `Format=Anthropic` +proxy, with the same `client_id` / `client_secret` header pair and the model ID +passed per call: ```typescript import Anthropic from "@anthropic-ai/sdk"; const client = new Anthropic({ - baseURL: process.env.DONKEY_LLM_PROXY_URL, // native /v1/messages route LIVE-verified: use a Format=Anthropic proxy + baseURL: process.env.DONKEY_LLM_PROXY_URL, // a Format=Anthropic proxy apiKey: "unused", // required slot; proxy uses the headers below defaultHeaders: { client_id: process.env.DONKEY_LLM_PROXY_CLIENT_ID!, @@ -104,7 +114,7 @@ async with Donkey.from_env() as donkey: llm = AsyncAnthropic(**donkey.anthropic.connection_kwargs()) ``` -## The manual equivalent (eject at any time) +## Manual equivalent ```python from anthropic import AsyncAnthropic @@ -114,32 +124,26 @@ llm = AsyncAnthropic( api_key=..., default_headers=..., # client_id / client_secret header pair http_client=..., # the SDK's shared httpx client + max_retries=0, # the SDK retries in its own transport layer ) ``` -Nothing here is hidden — `connection_kwargs()` returns exactly these keys, so -you can always drop the factory and construct `AsyncAnthropic` by hand. - -## Notes & limitations - -> **Divergence: `client()`, not `model(...)`.** Every other adapter in this -> roster returns a framework object already bound to a model id, because the -> underlying native constructor accepts `model` as a kwarg. `AsyncAnthropic` -> doesn't work that way — it's a bare client, and the model id is an argument -> to `.messages.create()`, not to the constructor. `donkey.anthropic.client()` -> therefore takes no model argument at all; you supply the model id yourself -> on every call. - -> **The proxy's Anthropic-native ingress is LIVE-verified — but requires a -> `Format=Anthropic` proxy.** MuleSoft Model Proxy offers a **native Anthropic -> ingress Format** — one of three selectable Formats (OpenAI / Gemini / Anthropic) -> fixed at proxy creation ([MuleSoft docs](https://docs.mulesoft.com/general/model-proxy)). -> A live capture against a `Format=Anthropic` proxy confirms the native route at -> `POST //v1/messages` (200 with a native Anthropic body; an -> OpenAI-shaped `/chat/completions` request 404s), on the same `client_id` / -> `client_secret` CIE auth (#304). The one remaining caveat is provisioning: every -> default DDK proxy is `Format=OpenAI`, so pointing `AsyncAnthropic` at them -> reaches Claude only as an *upstream provider*, not natively (`/v1/messages` 404s -> there). Point `base_url` at a `Format=Anthropic` proxy to use the native surface. +`connection_kwargs()` returns exactly these keys, so you can drop the factory +and construct `AsyncAnthropic` by hand at any time. + +## Notes + +- **`client()`, not `model(...)`.** The other adapters return a framework + object already bound to a model ID, because their native constructors accept + `model`. `AsyncAnthropic` is a bare client and the model ID is an argument to + `.messages.create()`, so `donkey.anthropic.client()` takes no model argument. +- **Proxy Format.** MuleSoft Model Proxy offers three ingress Formats (OpenAI / + Gemini / Anthropic), fixed when the proxy is created + ([MuleSoft docs](https://docs.mulesoft.com/general/model-proxy)). A + `Format=Anthropic` proxy returns a native Anthropic body from + `/v1/messages` and 404s an OpenAI-shaped `/chat/completions` request. Auth is + the same `client_id` / `client_secret` header pair as every other proxy. + To use the native surface, set `DONKEY_LLM_PROXY_URL` (or `llm_proxy_url`) + to a `Format=Anthropic` proxy. diff --git a/website/content/frameworks/crewai.mdx b/website/content/frameworks/crewai.mdx index b64f425b..f7c14fba 100644 --- a/website/content/frameworks/crewai.mdx +++ b/website/content/frameworks/crewai.mdx @@ -1,17 +1,23 @@ +--- +description: Use a governed, LiteLLM-backed crewai.LLM in CrewAI, pointed at your Omni Gateway LLM proxy. +--- + import { Tabs } from 'nextra/components' import OpenAiProxyTs from '../../components/openai-proxy-ts.mdx' -# CrewAI — governed model access +# CrewAI + +CrewAI gets a governed `LLM`, backed by LiteLLM, pointed at the Agent Fabric +LLM proxy. The adapter translates the governed connection into LiteLLM's own +model-string and kwarg conventions for you. -CrewAI gets a governed `LLM`, backed by LiteLLM under the hood, pointed at -the Agent Fabric LLM proxy through LiteLLM's own model-string and kwarg -conventions rather than raw OpenAI ones. +**What you get** -> **Supported at `connection_kwargs()` — not conformance-tested (`BG §1.8`)**, with two documented exemptions: LiteLLM -> owns its own transport, so the SDK's shared httpx client is not injected -> here. Per-run correlation and `donkey.last_call` response observation are -> unavailable as a result (see below) — the same exemptions as the -> [Google ADK](/frameworks/adk) adapter. +- A native `crewai.LLM`, with the proxy auth and attribution headers set. +- The `openai/` model prefix and LiteLLM kwarg names handled automatically. +- Supported at `connection_kwargs()`. CrewAI's model calls go through + LiteLLM, so correlation is per client and `donkey.last_call` is not + populated — the same as [Google ADK](/frameworks/adk) (see [Notes](#notes)). ## Install @@ -30,18 +36,15 @@ from donkey_kit.integrations.crewai import llm model = llm("gpt-4o") ``` -`model` is a real, native **`crewai.LLM`** instance. The model string is -auto-prefixed with `openai/` before it reaches LiteLLM (`openai/gpt-4o`), -because that's the prefix LiteLLM's OpenAI-compatible route expects — you -don't need to add it yourself. +`model` is a real `crewai.LLM` instance. The model string is prefixed with +`openai/` before it reaches LiteLLM (`openai/gpt-4o`), which is the prefix +LiteLLM's OpenAI-compatible route expects — you don't add it yourself. -There's no first-party Agent Fabric TypeScript SDK yet, and **CrewAI itself is -Python-only** — it has no TypeScript SDK. You can still make the same -governed call from TypeScript via the OpenAI-compatible proxy with the -official `openai` npm client: +CrewAI is Python-only. From TypeScript, call the proxy's OpenAI-compatible API +directly with the official `openai` npm client: @@ -77,7 +80,7 @@ async with Donkey.from_env() as donkey: model = LLM(model="openai/gpt-4o", **donkey.crewai.connection_kwargs()) ``` -## The manual equivalent (eject at any time) +## Manual equivalent ```python from crewai import LLM @@ -90,28 +93,21 @@ model = LLM( ) ``` -Note the kwarg names: LiteLLM uses `api_base` and `extra_headers`, not -`base_url` / `default_headers` — `connection_kwargs()` already translates for -you. - -## Notes & limitations +LiteLLM uses `api_base` and `extra_headers`, not `base_url` / +`default_headers` — `connection_kwargs()` already translates for you. -> **Correlation IDs are per-client, not per-run, for this adapter.** LiteLLM -> owns its own HTTP transport, so the SDK's shared httpx client — and the -> per-run correlation ID it stamps on every request — cannot be injected into -> it. This is a documented, asserted conformance exemption, not an oversight: -> header injection is full, but transport injection is not possible, exactly -> as with Google ADK. +## Notes -> **This adapter cannot populate `donkey.last_call`.** The same LiteLLM-owned -> transport means no response reaches the SDK's `_on_response` hook, so gateway -> identity, routing, and usage fields cannot be observed. When every adapter -> resolved on a `Donkey` is non-observing, the record reports -> `status == LastCallStatus.UNAVAILABLE`, `available == False`, and names the -> resolved adapters in `surface`. This is a -> documented, asserted `gateway_identity_observed` conformance exemption. +- **Correlation IDs are per-client, not per-run.** CrewAI sends requests through + its built-in LiteLLM model layer rather than the SDK's shared HTTP client, so + the correlation ID is set once per client instead of per `donkey.run()`. Every + governance header is still sent on every request. The conformance suite + checks this as a documented behaviour. +- **`donkey.last_call` is unavailable.** Because the response is handled by LiteLLM, + gateway identity, routing, and usage fields can't be observed. When every + adapter resolved on a `Donkey` is like this one, `donkey.last_call` reports + `status == LastCallStatus.UNAVAILABLE` and `available == False`, and names the + resolved adapters in `surface`. See the [error taxonomy](/errors) for how proxy rejections surface through -LiteLLM's error path, and the [verification policy](/concepts/verification) -page for the current status of every constructor signature this adapter -depends on. +CrewAI's LiteLLM layer. diff --git a/website/content/frameworks/index.mdx b/website/content/frameworks/index.mdx index 8fe1927d..08469995 100644 --- a/website/content/frameworks/index.mdx +++ b/website/content/frameworks/index.mdx @@ -1,47 +1,34 @@ --- -description: Governed model access from eight agent frameworks, each returning its own native object — LangGraph deep, the rest at connection_kwargs(). +description: Governed model access from eight agent frameworks. Each adapter returns the framework's own native model or client object, pointed at your Omni Gateway LLM proxy. --- import { Callout } from 'nextra/components' -import { Card, CardGrid } from '../../components' +import { Badge, Card, CardGrid } from '../../components' # Model access +Live + Governed model access from eight agent frameworks. Each adapter returns the -framework's **own native object**, pointed at your Omni Gateway proxy with -verified auth, attribution, and retries injected. +framework's **own native object**, pointed at your Omni Gateway LLM proxy with +consumer auth and attribution headers already set. Nothing wraps the object you +get back, and every page shows the plain-framework code you can switch to at +any time. - **Language support.** Python is the only first-party SDK today; a TypeScript - SDK is planned. Until then, every framework - page's **TypeScript** tab reaches the same governed proxy over its - OpenAI-compatible API with the official `openai` npm client (or - `@anthropic-ai/sdk` for Anthropic) — a real, typed native integration, with no - first-party TS package implied. Native TypeScript SDKs exist for LangGraph, - ADK, Strands, the OpenAI Agents SDK, the Anthropic SDK, and LlamaIndex; - **CrewAI (Python-only)** and **MS Agent Framework (.NET/Python/Go)** have none. + **Using TypeScript or another language?** The SDK is Python. From TypeScript + or any other language, call the proxy's OpenAI-compatible HTTP API directly — + every framework page has a **TypeScript** tab showing the official `openai` + npm client (or `@anthropic-ai/sdk` for Anthropic) pointed at the same proxy. -## One deep adapter, seven supported - -Support depth is about **how much is guaranteed by tests**, not about quality -or how well a framework works: +## Supported frameworks -- **Deep** — conformance-gated in blocking CI, with the full scenario suite run - against it. -- **Supported at `connection_kwargs()`** — the governed connection is verified - at the kwargs level; the factory methods exist and return native objects, but - the constructor call itself is not conformance-gated. - - - **Why one deep adapter and not eight.** Eight conformance-gated adapters cost - more to maintain than they returned, and the maintenance came out of the - budget for the things that actually differentiate the SDK — budget pacing, the - simulator, the conformance plugin. A second framework is promoted to deep - support based on **real demand**, one at a time, rather than guessed up front. - This makes `connection_kwargs()` the most load-bearing surface here: it is the - entire supported contract for seven of the eight. - +LangGraph is the **deep** adapter: it runs the full conformance suite in CI, +including graph-level scenarios against a compiled `StateGraph`. The other +seven are **supported at `connection_kwargs()`**: the governed connection +settings are tested, and each exposes factory methods that return the native +object. @@ -73,40 +60,6 @@ or how well a framework works: -## Which wire format does your proxy speak? - -Every adapter on this page — and the raw `donkey.llm.client()` — speaks the -**OpenAI wire format**. Whether that matches your gateway depends on the -**Format** (OpenAI / Anthropic / Gemini) you chose when the proxy was -provisioned, which fixes the wire protocol the proxy's *ingress* accepts. - -The wire format is a property of the **provisioned proxy**, not an SDK -setting — there is no `llm_proxy_format` config field, by design. You match the -adapter to the proxy, not the other way round. - -| Proxy ingress **Format** | Point it at | Status | -|---|---|---| -| **OpenAI** | `donkey.llm.client()` or any framework adapter | **VERIFIED (LIVE)** — the SDK's verified data-plane path | -| **Anthropic** | `donkey.anthropic.client()` (native `AsyncAnthropic`) | **VERIFIED (LIVE)** — a `Format=Anthropic` proxy serves the native Anthropic Messages route at `POST //v1/messages` (OpenAI-shape `/chat/completions` → 404), captured in `tests/fixtures/anypoint/anthropic_inbound/` (#304). Requires a `Format=Anthropic` proxy — every default DDK proxy is `Format=OpenAI`, where Claude is reachable only as an upstream provider | -| **Gemini** | *No SDK adapter* — point a native `google-genai` client at `…/models/:generateContent` yourself | **VERIFIED (LIVE)** — a `Format=Gemini` proxy serves a native Gemini `:generateContent` passthrough (#540). The SDK ships **no** `donkey.gemini` adapter (demand-driven, #244); reach Gemini as an **upstream provider** behind an OpenAI-format ingress instead — see below | - - - **Ingress Format is not upstream provider routing.** "Anthropic" and "Gemini" - appear in two different places and mean different things: - - - **Ingress Format** — the wire protocol *your request* speaks to the proxy. - The verified path is OpenAI. - - **Upstream provider** — which model the proxy routes *to* after accepting - your request. A model-based-routing proxy with an OpenAI ingress already - fans out to OpenAI, Gemini, Azure OpenAI, Bedrock Anthropic, and NVIDIA - upstreams today, selected by the `model` value in your request body — no - new SDK code needed. - - So to *use* Gemini or Claude models you don't need a Gemini- or - Anthropic-format ingress at all: send an OpenAI-format request naming that - model to an OpenAI-format proxy. See [Verification policy](/concepts/verification). - - ## The shape is the same everywhere ```bash @@ -120,27 +73,55 @@ async with Donkey.from_env() as donkey: model = donkey..("gpt-4o") # native object at the proxy ``` -Each page shows the factory name, the native class you get back, the three ways -to construct it, and — importantly — **the manual equivalent** so you can eject -to plain framework code whenever you want. +Each framework page shows the factory name, the native class you get back, the +three ways to construct it, and **the manual equivalent** — the plain framework +constructor call the factory makes for you. + +## Match the adapter to your proxy's wire format + +Every adapter on this page except Anthropic — and the raw `donkey.llm.client()` +— speaks the **OpenAI wire format**. The format your proxy accepts is the +**Format** (OpenAI / Anthropic / Gemini) chosen when the proxy was provisioned. +It is a property of the proxy, not an SDK setting, so there is no config field +for it: pick the adapter that matches your proxy. + +| Proxy ingress **Format** | Use | +|---|---| +| **OpenAI** | `donkey.llm.client()` or any framework adapter. Default DDK proxies are `Format=OpenAI`. | +| **Anthropic** | `donkey.anthropic.client()` (native `AsyncAnthropic`). The proxy serves the native Messages route at `POST //v1/messages`; OpenAI-shape `/chat/completions` returns 404. | +| **Gemini** | No SDK adapter. Point a native `google-genai` client at `…/models/:generateContent` yourself, or reach Gemini as an upstream provider behind an OpenAI-format proxy (see below). | + + + **Ingress Format is not the same as the upstream provider.** The ingress + Format is the wire protocol *your request* speaks to the proxy. The upstream + provider is the model the proxy routes *to* after accepting it. A + model-based-routing proxy with OpenAI ingress already fans out to OpenAI, + Gemini, Azure OpenAI, Bedrock Anthropic, and NVIDIA upstreams, selected by the + `model` value in your request body. + + So to use Gemini or Claude models you don't need a Gemini- or Anthropic-format + proxy: send an OpenAI-format request naming that model to an OpenAI-format + proxy. + ## Injection depth differs by framework -How completely the SDK can inject its governed HTTP transport depends on what -each framework exposes: +How much of the SDK's HTTP layer reaches the request depends on what each +framework's constructor accepts. With **header injection**, the proxy auth and +attribution headers are sent. With **transport injection**, the SDK's shared +HTTP client is also used, which adds per-run correlation IDs and +`donkey.last_call`. | Framework | Header injection | Transport injection | Notes | |---|---|---|---| -| LangGraph | ✅ | ✅ | Best case — `default_headers` + custom async client. | +| LangGraph | ✅ | ✅ | `default_headers` plus a custom async client. | | Strands | ✅ | ✅ | Via `client_args`. | -| LlamaIndex | ✅ | ❌ | Receives a static `default_headers` snapshot, so per-run correlation and `donkey.last_call` are unavailable. `is_chat_model=True` is forced. | -| OpenAI Agents SDK | ✅ | ✅ | We build the `AsyncOpenAI` client ourselves. | -| Anthropic SDK | ✅ | ✅ | Returns a bare `client()`, not a model-bound object — see the [divergence note](/frameworks/anthropic). | -| MS Agent Framework | ✅ | ❌ | Receives a static `default_headers` snapshot, so per-run correlation and `donkey.last_call` are unavailable. Class name/kwarg **unverified**. | -| Google ADK | ✅ (`extra_headers`) | ❌ | LiteLLM owns the transport; correlation is per-client and the adapter cannot populate `donkey.last_call`. | -| CrewAI | ✅ (`extra_headers`) | ❌ | LiteLLM owns the transport; same correlation and `donkey.last_call` exemptions as Google ADK. | - - - The proxy contract is verified; the exact **constructor signatures** are still - being confirmed per framework — see [Verification policy](/concepts/verification). - +| OpenAI Agents SDK | ✅ | ✅ | The adapter builds the `AsyncOpenAI` client itself. | +| Anthropic SDK | ✅ | ✅ | Returns a bare `client()`, not a model-bound object — see the [Anthropic page](/frameworks/anthropic). | +| LlamaIndex | ✅ | ❌ | Static `default_headers` snapshot: no per-run correlation or `donkey.last_call`. `is_chat_model=True` is forced. | +| MS Agent Framework | ✅ | ❌ | Static `default_headers` snapshot: no per-run correlation or `donkey.last_call`. | +| Google ADK | ✅ (`extra_headers`) | ❌ | Calls go through ADK's `LiteLlm` model: correlation is per client and `donkey.last_call` is not populated. | +| CrewAI | ✅ (`extra_headers`) | ❌ | Calls go through CrewAI's LiteLLM layer: same behaviour as Google ADK. | + +See the [verification ledger](https://github.com/Donkey-Development-Kit/donkey-development-kit/blob/develop/docs/verified-apis.md) for how each constructor +signature the adapters depend on is checked. diff --git a/website/content/frameworks/langgraph.mdx b/website/content/frameworks/langgraph.mdx index 69b8bc59..55c29a6f 100644 --- a/website/content/frameworks/langgraph.mdx +++ b/website/content/frameworks/langgraph.mdx @@ -1,14 +1,23 @@ +--- +description: Use a governed LangChain ChatOpenAI in LangGraph, pointed at your Omni Gateway LLM proxy with full header and transport injection, typed refusals, and conformance testing. +--- + import { Tabs } from 'nextra/components' import OpenAiProxyTs from '../../components/openai-proxy-ts.mdx' -# LangGraph — governed model access +# LangGraph LangGraph (and LangChain more broadly) gets a governed `ChatOpenAI` pointed at -the Agent Fabric LLM proxy, with full header and transport injection. +your Agent Fabric LLM proxy. LangGraph is the **deep adapter**: every proxy +header and the SDK's shared async transport reach the native client, and the +adapter runs the full conformance suite in CI. + +**What you get** -> **Deep adapter — conformance-gated in blocking CI.** This is the best-case adapter: every -> proxy header and the shared async transport are injected into the native -> client. +- A native `langchain_openai.ChatOpenAI` — nothing LangGraph-specific wraps it. +- Per-run correlation IDs that reach every graph node. +- Typed gateway refusals (`PIIDetected`, `TokenBudgetExceeded`, …) inside nodes. +- A conformance suite you can run against your own graph. ## Install @@ -27,17 +36,16 @@ from donkey_kit.integrations.langgraph import chat_model llm = chat_model("gpt-4o") ``` -`llm` is a real, native **`langchain_openai.ChatOpenAI`** instance — nothing -LangGraph-specific wraps it. Drop it straight into your graph nodes or chains. +`llm` is a real `langchain_openai.ChatOpenAI` instance. Drop it straight into +your graph nodes or chains. -There's no first-party Agent Fabric TypeScript SDK yet (it's on the -[roadmap](/concepts/verification)). The proxy is OpenAI-compatible, so point the -official `openai` npm client at it — the same base URL and -`client_id`/`client_secret` headers also drop straight into **LangChain.js** -(`ChatOpenAI`, via its `configuration.baseURL` + `defaultHeaders`). +Call the proxy's OpenAI-compatible API with the official `openai` npm client. +The same base URL and `client_id`/`client_secret` headers also work with +**LangChain.js** (`ChatOpenAI`, via `configuration.baseURL` + +`defaultHeaders`). @@ -46,8 +54,8 @@ official `openai` npm client at it — the same base URL and ## Three ways to construct -**1. Off a shared `Donkey` instance** (reuses one HTTP client + lifecycle -across every adapter you touch in a run): +**1. Off a shared `Donkey` instance** (reuses one HTTP client and lifecycle +across every adapter you use in a run): ```python from donkey_kit import Donkey @@ -56,8 +64,11 @@ async with Donkey.from_env() as donkey: llm = donkey.langgraph.chat_model("gpt-4o") ``` +The adapter is also callable: `donkey.langgraph("gpt-4o")` is the same as +`donkey.langgraph.chat_model("gpt-4o")`. + **2. Module-level factory** (shortest — uses a cached, env-configured default -`Donkey` under the hood): +`Donkey`): ```python from donkey_kit.integrations.langgraph import chat_model @@ -75,13 +86,9 @@ async with Donkey.from_env() as donkey: llm = ChatOpenAI(model="gpt-4o", **donkey.langgraph.connection_kwargs()) ``` -The adapter is also **callable** as the shortest form of option 1 — -`donkey.langgraph("gpt-4o")` is exactly `donkey.langgraph.chat_model("gpt-4o")`, -returning the same native `ChatOpenAI`. +## Manual equivalent -## The manual equivalent (eject at any time) - -Everything above is sugar over this native constructor call: +The factories make this native constructor call for you: ```python from langchain_openai import ChatOpenAI @@ -93,28 +100,23 @@ llm = ChatOpenAI( default_headers=..., # client_id / client_secret header pair, not bearer http_async_client=..., # the SDK's shared httpx async client max_retries=0, # the SDK retries in its own transport layer - use_responses_api=True, # the proxy's live-verified data plane is /responses + use_responses_api=True, # the proxy's data plane is /responses ) ``` -Nothing here is hidden — `connection_kwargs()` returns exactly these keys, so -you can always drop the factory and construct `ChatOpenAI` by hand. Pass -`use_responses_api=False` to `chat_model(...)` (or omit it from your own -constructor) only if a deployment exposes chat-completions instead — the -Responses API (`/responses`) is the endpoint verified against a live proxy. - -## What "deep adapter" buys you +`connection_kwargs()` returns exactly these keys, so you can drop the factory +and construct `ChatOpenAI` by hand at any time. Pass `use_responses_api=False` +to `chat_model(...)` only if your deployment exposes chat-completions instead +of the Responses API (`/responses`). -LangGraph is the one adapter held to the conformance bar, so three graph-level -guarantees are tested end to end against a real compiled `StateGraph`, not just -a bare model call. +## Graph-level features -### Correlation IDs reach every node — for free +### Correlation IDs reach every node -Bind a run id once with `donkey.run(id=…)` and every node sees it via -`current_correlation_id()`, with nothing threaded through graph state. LangGraph -runs nodes on `asyncio` tasks that copy the current context, so the id -propagates on its own: +Bind a run ID once with `donkey.run(id=…)` and every node sees it via +`current_correlation_id()`, with nothing threaded through graph state. +LangGraph runs nodes on `asyncio` tasks that copy the current context, so the +ID propagates on its own: ```python from donkey_kit.core.telemetry import current_correlation_id @@ -129,10 +131,10 @@ async with donkey.run(id=ticket.id): ### Typed refusals inside a node -A node that calls the model directly gets whatever LangChain raises on a proxy -refusal — a framework-wrapped `OpenAIPermissionDeniedError`, not the SDK's typed -exception. Wrap the call in `typed_refusals()` and a gateway refusal comes back -through the [error taxonomy](/errors) instead: +On a proxy refusal, LangChain raises its own wrapped +`OpenAIPermissionDeniedError`, not the SDK's typed exception. Wrap the model +call in `typed_refusals()` and a gateway refusal comes back through the +[error taxonomy](/errors) instead: ```python from donkey_kit.integrations.langgraph import typed_refusals @@ -143,46 +145,42 @@ async def call_model(state): return {"messages": [reply]} ``` -A PII block now propagates out of `graph.ainvoke(...)` as `PIIDetected`, a budget -block as `TokenBudgetExceeded`, etc. — each carrying the correlation/call ids the -client sent. This is a helper plus a documented pattern, not a transport change: -transport-level errors with no HTTP response (connection/timeout) pass through -untouched. +A PII block now propagates out of `graph.ainvoke(...)` as `PIIDetected`, a +budget block as `TokenBudgetExceeded`, and so on — each carrying the +correlation and call IDs the client sent. Transport-level errors with no HTTP +response (connection failures, timeouts) pass through unchanged. ### `interrupt()` composes with typed refusals -A human-in-the-loop `interrupt()` and a typed refusal don't swallow each other: -the graph pauses cleanly at the interrupt (no refusal fires mid-pause), and on -resume a refusal in a downstream model node still surfaces as its typed -exception. +A human-in-the-loop `interrupt()` and a typed refusal don't interfere: the +graph pauses cleanly at the interrupt, and on resume a refusal in a downstream +model node still surfaces as its typed exception. ### Run the conformance suite against your own graph -The same suite that gates this adapter is a customer-facing pytest plugin you -can point at your own agent: +The suite that tests this adapter is also a pytest plugin you can point at +your own agent: ```bash pytest --donkey-conformance --agent=my_app:build ``` -`build` returns an object with an awaitable `run(text)`; the suite checks it -retries no budget refusal, surfaces `PIIDetected` typed, carries the correlation -id into its logs, and tolerates a response with no budget headers. The shipped -[`examples/langgraph`](https://github.com/Donkey-Development-Kit/donkey-development-kit/tree/main/python/examples/langgraph) -factory is exactly this shape. - -## Notes & limitations - -> LangGraph is the reference adapter: `base_url`, `api_key`, `default_headers`, -> and a custom `http_async_client` are all forwarded, so proxy auth headers and -> the SDK's transport (retries, correlation IDs) both reach every request. - -> `max_retries=0` is intentional — retry logic lives in the SDK's transport -> layer, not in the OpenAI client, so the two don't double-retry. - -The proxy is OpenAI-compatible but is **not** a full OpenAI API surface: there -is no `/v1` prefix on the base URL and no `/models` endpoint, and auth is a -`client_id`/`client_secret` header pair rather than a bearer token. See the -[error taxonomy](/errors) for how proxy rejections surface as typed -exceptions, and the [verification policy](/concepts/verification) page for -the current status of every constructor signature this adapter depends on. +`build` returns an object with an awaitable `run(text)`. The suite checks that +it doesn't retry a budget refusal, surfaces `PIIDetected` typed, carries the +correlation ID into its logs, and tolerates a response with no budget headers. +The [`examples/langgraph`](https://github.com/Donkey-Development-Kit/donkey-development-kit/tree/main/python/examples/langgraph) +factory has exactly this shape. See [Testing](/testing). + +## Notes + +- `base_url`, `api_key`, `default_headers`, and a custom `http_async_client` + are all forwarded, so proxy auth headers and the SDK's transport (retries, + correlation IDs) reach every request. +- `max_retries=0` is intentional: retries live in the SDK's transport layer, + so the SDK and the OpenAI client don't both retry. +- The proxy is OpenAI-compatible but not the full OpenAI API: the base URL has + no `/v1` prefix, there is no `/models` endpoint, and auth is a + `client_id`/`client_secret` header pair rather than a bearer token. + +See the [error taxonomy](/errors) for how proxy rejections surface as typed +exceptions. diff --git a/website/content/frameworks/llamaindex.mdx b/website/content/frameworks/llamaindex.mdx index f07e9582..25bf8175 100644 --- a/website/content/frameworks/llamaindex.mdx +++ b/website/content/frameworks/llamaindex.mdx @@ -1,18 +1,23 @@ +--- +description: Use a governed OpenAILike LLM in LlamaIndex, pointed at your Omni Gateway LLM proxy with is_chat_model forced on. +--- + import { Tabs } from 'nextra/components' import OpenAiProxyTs from '../../components/openai-proxy-ts.mdx' import AdapterErrorTaxonomyNote from '../../components/adapter-error-taxonomy-note.mdx' -# LlamaIndex — governed model access +# LlamaIndex + +LlamaIndex gets a governed `OpenAILike` LLM pointed at the Agent Fabric LLM +proxy, with the chat-model flag a chat-only gateway requires already set. -LlamaIndex gets a governed `OpenAILike` LLM pointed at the Agent Fabric LLM proxy, -with the one flag that a chat-only gateway absolutely requires forced on for -you. +**What you get** -> **Supported at `connection_kwargs()` — not conformance-tested (`BG §1.8`).** -> LlamaIndex receives a static `default_headers` snapshot, not the SDK's shared -> httpx client, so per-run correlation and `donkey.last_call` response -> observation are documented exemptions (see below). Watch the `is_chat_model` -> gotcha if you build the client by hand. +- A native `llama_index.llms.openai_like.OpenAILike`. +- `is_chat_model=True` and `is_function_calling_model=True` set for you. +- Supported at `connection_kwargs()`. The client receives a static + `default_headers` snapshot, so per-run correlation and `donkey.last_call` are + not available (see [Notes](#notes)). ## Install @@ -31,16 +36,15 @@ from donkey_kit.integrations.llamaindex import llm model = llm("gpt-4o") ``` -`model` is a real, native **`llama_index.llms.openai_like.OpenAILike`** -instance, ready to hand to any LlamaIndex query engine, chat engine, or agent. +`model` is a real `llama_index.llms.openai_like.OpenAILike` instance, ready to +hand to any LlamaIndex query engine, chat engine, or agent. -There's no first-party Agent Fabric TypeScript SDK yet (it's on the -[roadmap](/concepts/verification)). The proxy is OpenAI-compatible, so point the -official `openai` npm client at it — the same base URL and -`client_id`/`client_secret` headers also drop into **LlamaIndex.TS**. +Call the proxy's OpenAI-compatible API with the official `openai` npm client. +The same base URL and `client_id`/`client_secret` headers also work with +**LlamaIndex.TS**. @@ -76,7 +80,7 @@ async with Donkey.from_env() as donkey: model = OpenAILike(model="gpt-4o", **donkey.llamaindex.connection_kwargs()) ``` -## The manual equivalent (eject at any time) +## Manual equivalent ```python from llama_index.llms.openai_like import OpenAILike @@ -86,34 +90,29 @@ model = OpenAILike( api_base=..., # from DONKEY_LLM_PROXY_URL, no /v1 suffix api_key=..., default_headers=..., # client_id / client_secret header pair - is_chat_model=True, # see gotcha below — never omit this + is_chat_model=True, # required — see below is_function_calling_model=True, ) ``` -## Notes & limitations - -> **Gotcha: `OpenAILike` defaults `is_chat_model=False`.** Left at its -> default, `OpenAILike` silently routes requests to the completions endpoint -> instead of the chat endpoint — which fails against a chat-only proxy like -> the Agent Fabric LLM proxy. This is the single most common LlamaIndex-with-a- -> gateway bug. The adapter's `connection_kwargs()` always forces -> `is_chat_model=True` (and `is_function_calling_model=True`) so you don't -> have to remember to set it — but if you ever construct `OpenAILike` -> yourself outside the adapter, set it explicitly. - -> LlamaIndex uses `api_base` rather than `base_url` for its endpoint kwarg; -> `connection_kwargs()` already translates for you. - -> **This adapter cannot propagate per-run correlation or populate -> `donkey.last_call`.** LlamaIndex receives a static `default_headers` snapshot, -> which deliberately excludes the correlation ID bound later by -> `donkey.run(id=...)`, and not the SDK's httpx client. No response reaches -> `_on_response`, so gateway identity, routing, and usage fields cannot be -> observed either. When every adapter resolved on a `Donkey` is non-observing, -> the record reports `status == LastCallStatus.UNAVAILABLE`, `available == False`, -> and names the resolved adapters in `surface`. These are documented, asserted -> `correlation_id_propagated` and `gateway_identity_observed` conformance -> exemptions. +LlamaIndex uses `api_base` rather than `base_url`; `connection_kwargs()` +already translates for you. + +## Notes + +- **Always set `is_chat_model=True`.** `OpenAILike` defaults to + `is_chat_model=False`, which routes requests to the completions endpoint + instead of chat — and that fails against a chat-only proxy like the Omni + Gateway LLM proxy. `connection_kwargs()` always sets it (and + `is_function_calling_model=True`); set it yourself if you construct + `OpenAILike` outside the adapter. +- **No per-run correlation or `donkey.last_call`.** The client receives a + static `default_headers` snapshot, which excludes the correlation ID bound + later by `donkey.run(id=...)`, and the SDK's httpx client is not used. No + response reaches the SDK, so gateway identity, routing, and usage fields + can't be observed. When every adapter resolved on a `Donkey` is like this + one, `donkey.last_call` reports `status == LastCallStatus.UNAVAILABLE` and + `available == False`, and names the resolved adapters in `surface`. The + conformance suite asserts both as documented exemptions. diff --git a/website/content/frameworks/openai.mdx b/website/content/frameworks/openai.mdx index 8bc75732..02e4ef5c 100644 --- a/website/content/frameworks/openai.mdx +++ b/website/content/frameworks/openai.mdx @@ -1,17 +1,24 @@ +--- +description: Use a governed OpenAIChatCompletionsModel in the OpenAI Agents SDK, backed by a pre-built AsyncOpenAI client pointed at your Omni Gateway LLM proxy. +--- + import { Tabs } from 'nextra/components' import OpenAiProxyTs from '../../components/openai-proxy-ts.mdx' import AdapterErrorTaxonomyNote from '../../components/adapter-error-taxonomy-note.mdx' -# OpenAI Agents SDK — governed model access +# OpenAI Agents SDK The OpenAI Agents SDK (pip package `openai-agents`) gets a governed -`OpenAIChatCompletionsModel` backed by a pre-built `AsyncOpenAI` client — the -SDK constructs the OpenAI client itself, wired with its shared http client and -verified headers, and hands it to the Agents SDK ready-made. +`OpenAIChatCompletionsModel` backed by a pre-built `AsyncOpenAI` client. The +adapter builds that client itself, with the SDK's shared HTTP client and proxy +headers, and hands it to the Agents SDK ready-made. + +**What you get** -> **Supported at `connection_kwargs()` — not conformance-tested (`BG §1.8`).** Full header AND transport injection — -> the adapter builds the `AsyncOpenAI` client itself, so both travel together -> as one object. +- A native `agents.OpenAIChatCompletionsModel`. +- Full header **and** transport injection — both travel together in one + `AsyncOpenAI` object. +- Supported at `connection_kwargs()`. ## Install @@ -30,17 +37,15 @@ from donkey_kit.integrations.openai_agents import model llm = model("gpt-4o") ``` -`llm` is a real, native **`agents.OpenAIChatCompletionsModel`** instance — -pass it to your `Agent(model=...)` as you would any other Agents SDK model. +`llm` is a real `agents.OpenAIChatCompletionsModel` instance — pass it to +`Agent(model=...)` as you would any other Agents SDK model. -There's no first-party Agent Fabric TypeScript SDK yet (it's on the -[roadmap](/concepts/verification)). The proxy is OpenAI-compatible, so point the -official `openai` npm client at it — the same base URL and -`client_id`/`client_secret` headers also drop into the **OpenAI Agents SDK for -JS** (`@openai/agents`). +Call the proxy's OpenAI-compatible API with the official `openai` npm client. +The same base URL and `client_id`/`client_secret` headers also work with the +**OpenAI Agents SDK for JS** (`@openai/agents`). @@ -79,7 +84,7 @@ async with Donkey.from_env() as donkey: ) ``` -## The manual equivalent (eject at any time) +## Manual equivalent ```python from openai import AsyncOpenAI @@ -101,20 +106,16 @@ llm = OpenAIChatCompletionsModel( `connection_kwargs()` returns exactly one key, `openai_client`, holding this pre-built `AsyncOpenAI` instance. -## Notes & limitations - -> **Preferred pattern: hand the framework a pre-built OpenAI client.** -> Wherever a framework accepts a ready-made `AsyncOpenAI` instance instead of -> a set of loose kwargs, that's the adapter's preferred integration point — -> it guarantees the shared transport and every governed header travel -> together as one object, with no risk of a kwarg being dropped along the -> way. This is why header injection here is full even though the model -> object itself never sees `base_url` or `default_headers` directly. - -> The `openai-agents` package is distinct from the plain `openai` client — -> `agents.OpenAIChatCompletionsModel` lives in the Agents SDK, not in the base -> OpenAI Python SDK. Installing `donkey-kit[openai-agents]` pulls in -> `openai-agents` for you. For the raw governed client with no framework, use -> `donkey.openai()` (from `donkey-kit[llm]`) instead. +## Notes + +- **A pre-built client is the preferred integration point.** When a framework + accepts a ready-made `AsyncOpenAI` instead of loose kwargs, the shared + transport and every proxy header travel together as one object, with no + risk of a kwarg being dropped. That's why injection is full here even though + the model object never sees `base_url` or `default_headers` directly. +- `openai-agents` is distinct from the plain `openai` package: + `agents.OpenAIChatCompletionsModel` lives in the Agents SDK. Installing + `donkey-kit[openai-agents]` pulls it in for you. For the raw governed client + with no framework, use `donkey.openai()` (from `donkey-kit[llm]`) instead. diff --git a/website/content/frameworks/strands.mdx b/website/content/frameworks/strands.mdx index 6cbe9671..9fa9bf13 100644 --- a/website/content/frameworks/strands.mdx +++ b/website/content/frameworks/strands.mdx @@ -1,15 +1,23 @@ +--- +description: Use a governed OpenAIModel in Strands Agents, pointed at your Omni Gateway LLM proxy with full header and transport injection. +--- + import { Tabs } from 'nextra/components' import OpenAiProxyTs from '../../components/openai-proxy-ts.mdx' import AdapterErrorTaxonomyNote from '../../components/adapter-error-taxonomy-note.mdx' -# Strands Agents — governed model access +# Strands Agents + +Strands Agents gets a governed `OpenAIModel` pointed at the Agent Fabric LLM +proxy. The connection details travel in Strands' `client_args`, which Strands +passes straight through to its underlying OpenAI client. -Strands Agents gets a governed `OpenAIModel`, with proxy connection details -forwarded through Strands' `client_args`, which Strands passes straight -through to the underlying OpenAI client. +**What you get** -> **Supported at `connection_kwargs()` — not conformance-tested (`BG §1.8`).** `client_args` gives full header AND -> transport injection, on par with the LangGraph adapter. +- A native `strands.models.openai.OpenAIModel`. +- Full header **and** transport injection through `client_args` — per-run + correlation IDs and `donkey.last_call` work, as with LangGraph. +- Supported at `connection_kwargs()`. ## Install @@ -28,17 +36,15 @@ from donkey_kit.integrations.strands import model llm = model("gpt-4o") ``` -`llm` is a real, native **`strands.models.openai.OpenAIModel`** instance — -pass it to your `Agent` as you would any other Strands model. +`llm` is a real `strands.models.openai.OpenAIModel` instance — pass it to your +`Agent` as you would any other Strands model. -There's no first-party Agent Fabric TypeScript SDK yet (it's on the -[roadmap](/concepts/verification)). The proxy is OpenAI-compatible, so point the -official `openai` npm client at it — the same base URL and -`client_id`/`client_secret` headers also drop into the **Strands TypeScript -SDK** (`@strands-agents/sdk`). +Call the proxy's OpenAI-compatible API with the official `openai` npm client. +The same base URL and `client_id`/`client_secret` headers also work with the +**Strands TypeScript SDK** (`@strands-agents/sdk`). @@ -74,7 +80,7 @@ async with Donkey.from_env() as donkey: llm = OpenAIModel(model_id="gpt-4o", **donkey.strands.connection_kwargs()) ``` -## The manual equivalent (eject at any time) +## Manual equivalent ```python from strands.models.openai import OpenAIModel @@ -91,18 +97,16 @@ llm = OpenAIModel( ``` Everything the SDK injects lives inside the single `client_args` dict that -Strands forwards to its internal OpenAI client — nothing is bolted on -separately. - -## Notes & limitations +Strands forwards to its internal OpenAI client. -> Because Strands forwards `client_args` verbatim to the underlying OpenAI -> client, both header injection (`default_headers`) AND transport injection -> (`http_client`) are available — full conformance, same tier as LangGraph. +## Notes -> Strands also exposes lifecycle hooks (`BeforeToolCallEvent` and friends), -> which the SDK uses elsewhere for the policy-termination pattern — see the -> error taxonomy for how a `PolicyViolation` should end a run cleanly rather -> than triggering a retry loop. +- Strands forwards `client_args` verbatim to the underlying OpenAI client, so + both header injection (`default_headers`) and transport injection + (`http_client`) are available. +- Strands also exposes lifecycle hooks (`BeforeToolCallEvent` and friends). + The SDK uses them for the policy-termination pattern — see the error taxonomy + for how a `PolicyViolation` should end a run cleanly rather than trigger a + retry loop. diff --git a/website/content/hitl.mdx b/website/content/hitl.mdx index 3e3b4052..c4ed5cdc 100644 --- a/website/content/hitl.mdx +++ b/website/content/hitl.mdx @@ -3,26 +3,26 @@ description: One vocabulary for pause-and-ask-a-human, mapped onto each framewor --- import { Callout } from 'nextra/components' +import { Badge } from '../components' # Human-in-the-loop - - **Phase 2 — designed, not yet shipped.** The API shape below is a proposal; - the gateway step-up path is **unverified**. See [Roadmap](/roadmap) and - [Verification policy](/concepts/verification). - +Roadmap + +This capability is on the [Roadmap](/roadmap); the API shown here is the planned +design. Refunds over €100 need a human. Today that is bespoke code, per team, per -framework. +framework. DDK will let you declare the approval requirement on the tool itself: ```python @donkey.tool(approval="required", risk="financial") async def issue_refund(ticket_id: str, amount: float): ... ``` -When the agent calls `issue_refund`, the SDK raises `ApprovalRequired` (or -triggers the framework's own interrupt), records the pending approval against -the correlation ID, and resumes when you resolve it: +When the agent calls `issue_refund`, DDK raises `ApprovalRequired` (or triggers +the framework's own interrupt), records the pending approval against the +correlation ID, and resumes when you resolve it: ```python await donkey.approvals.resolve(approval_id, approved_by=reviewer.id) @@ -30,8 +30,8 @@ await donkey.approvals.resolve(approval_id, approved_by=reviewer.id) ## Mapped onto what your framework already has -The SDK does not introduce a new pause mechanism. It maps one vocabulary onto -the primitive each framework already ships: +DDK does not introduce a new pause mechanism. It maps one vocabulary onto the +primitive each framework already ships: | Framework / protocol | Native primitive | |---|---| @@ -40,16 +40,12 @@ the primitive each framework already ships: | Google ADK | before/after tool callbacks | | Strands | hooks | | MCP | elicitation | -| Omni Gateway | Trusted Agent Identity step-up (MFA) — unverified | +| Omni Gateway | Trusted Agent Identity step-up (MFA) | -## What the value actually is - - - **Every framework already has human-in-the-loop.** So be clear about what is - being added, or this looks like reinvention. - +## What DDK adds -Three things, none of which is a new mechanism: +Every framework already has human-in-the-loop. DDK adds three things on top, +none of which is a new mechanism: 1. **Normalisation** — one vocabulary across frameworks, so approval policy is not rewritten when a team switches from ADK to LangGraph. @@ -60,14 +56,13 @@ Three things, none of which is a new mechanism: gateway's identity layer for step-up MFA, rather than trusting a click in your own UI. -## What it will not become +## Out of scope -No approval queue and no approval UI — both are on the -[will-not-build list](/roadmap). This integrates with whatever you already -run: Slack, ServiceNow, or LangGraph's own checkpointer. Building a queue would -mean owning a durable store, an escalation model, and a notification system, -which is a product rather than a feature. - ---- + + DDK will not ship an approval queue or an approval UI — both are on the + [will-not-build list](/roadmap). It integrates with whatever you already run: + Slack, ServiceNow, or LangGraph's own checkpointer. + -**Status: Phase 2 — not yet shipped.** +A queue would mean owning a durable store, an escalation model, and a +notification system — a product rather than a feature. diff --git a/website/content/identity.mdx b/website/content/identity.mdx index f470a128..f30b4406 100644 --- a/website/content/identity.mdx +++ b/website/content/identity.mdx @@ -3,43 +3,44 @@ description: On-behalf-of token exchange, so the gateway can enforce per-user po --- import { Callout } from 'nextra/components' +import { Badge } from '../components' # Identity - - **Phase 2 — designed, not yet shipped.** The API shape below is a proposal. - The exact token-exchange endpoint and header the gateway expects are - **unverified**, and will not be guessed. See [Roadmap](/roadmap) and - [Verification policy](/concepts/verification). - +Roadmap + +This capability is on the [Roadmap](/roadmap); the API shown here is the planned +design. By default the gateway sees your **service**, not the person the agent is -acting for. That is fine until policy depends on who is asking. +acting for. That is fine until policy depends on who is asking. DDK will let you +run a block of agent code on behalf of a user: ```python async with donkey.as_user(id_token=slack_user_oidc_token): await hr_bot.answer(question) # the gateway sees the user, not just the service ``` -Inside that block, a user-scoped token is acquired via RFC 8693 token exchange -and attached to every governed call, so the gateway's Trusted Agent Identity -layer can apply per-user policy. +Inside that block, DDK acquires a user-scoped token via RFC 8693 token exchange +and attaches it to every governed call, so the gateway's Trusted Agent Identity +layer can apply per-user policy. The exact token-exchange endpoint and header +the gateway expects will be pinned to the gateway's documented contract. -## The problem this actually solves +## The problem this solves An HR bot must not answer a manager's salary question about a *different* manager's report. That decision belongs to the gateway — it is the component that holds the -identity policy, the org chart relationships, and the audit obligation. The -SDK's entire job is to get the user's token onto the request correctly so the -gateway can make the call it is already able to make. +identity policy, the org-chart relationships, and the audit obligation. DDK's +job is to get the user's token onto the request correctly so the gateway can +make the call it is already able to make. -## The boundary, stated plainly +## Where the boundary sits - **This does not implement authorisation logic.** Trusted Agent Identity is a - shipped MuleSoft feature. The SDK does the token-exchange plumbing and header + **DDK does not implement authorisation logic.** Trusted Agent Identity is a + MuleSoft gateway feature. DDK does the token-exchange plumbing and header placement — nothing more. "Check the user's role in the SDK" is client-side enforcement, and it is on the [will-not-build list](/roadmap): code in your process can be bypassed by code in your process. @@ -47,14 +48,9 @@ gateway can make the call it is already able to make. ## No silent fallback -The failure mode worth designing against is a token exchange that quietly -fails and lets the call proceed under the **service** identity. That would -mean per-user policy silently stops applying while everything still appears to -work — the worst possible outcome, because nothing looks broken. +The failure mode to design against is a token exchange that quietly fails and +lets the call proceed under the **service** identity. Per-user policy would stop +applying while everything still appears to work. -So entering `as_user()` either attaches a user-scoped token or raises. It never +So entering `as_user()` will either attach a user-scoped token or raise. It never degrades to the service identity without telling you. - ---- - -**Status: Phase 2 — not yet shipped.** diff --git a/website/content/index.mdx b/website/content/index.mdx index 3e535a6e..266e62bd 100644 --- a/website/content/index.mdx +++ b/website/content/index.mdx @@ -1,66 +1,205 @@ --- -description: An SDK for Agent Fabric — governed model access from your own agent framework, with budget, refusals, telemetry and simulation on one skeleton. +description: Governed by the gateway, understood by your code. DDK brings Agent Fabric and Omni Gateway awareness into your agent framework — typed refusals, budgets, telemetry and simulation, with native framework objects. --- import { Callout, Tabs } from 'nextra/components' -import { Badge, Card, CardGrid, Figure, Hero } from '../components' +import { Card, CardGrid, Figure, Hero } from '../components' - Consume Agent Fabric capabilities — governed model - access, and the budget, refusal, telemetry and simulation machinery that - hangs off it — from your own agent framework, in your own IDE,{' '} - without adopting Mule. + Governed by the gateway. Understood by your code. DDK + brings Omni Gateway awareness into the agents you already write — in + LangGraph, Google ADK, Strands, CrewAI, LlamaIndex, the OpenAI Agents SDK, + the Anthropic SDK or Microsoft Agent Framework — so governance is not + just enforced on your agents, it is respected by them. } media={
} actions={[ { label: 'Quickstart', href: '/quickstart', primary: true }, - { label: 'Pick your framework', href: '/frameworks' }, + { label: 'Feature overview', href: '/feature-overview' }, + { label: 'Examples', href: '/examples' }, ]} /> -You keep writing LangGraph, Google ADK, Strands, LlamaIndex, CrewAI, the OpenAI -Agents SDK, the Anthropic SDK, or Microsoft Agent Framework code exactly as you -do today. The SDK wires those frameworks to your organisation's governed **Omni -Gateway LLM proxy** so every model call is authenticated, attributed, and -policy-checked — and it hands you back your framework's **own native objects**, -never a wrapper you have to code around. +## Why gateway awareness + +Enterprises put a gateway in front of their AI traffic for good reasons: one +place to authenticate every caller, meter every token, block personal data, +stop prompt injection and attribute cost to the team that spent it. MuleSoft +**Omni Gateway**, managed through **Agent Fabric**, does exactly that — for +LLMs, MCP servers, APIs and agents alike. + +But a gateway that governs alone only solves half the problem. The agent on the +other side of the wire sees an opaque `403` or `429` and does what code does +with errors it does not understand: it retries a PII block as if it were a +network blip, burns the remaining budget on calls that will be refused, crashes +a nightly batch at 2am, and leaves no trace that connects its own run to the +gateway's audit log. Control is one-sided, and the cost of that shows up as +wasted tokens, broken runs and incident tickets. + +**DDK makes governance a collaboration.** It brings the gateway's view of the +world into your agent code: + +- a PII refusal arrives as a typed `PIIDetected`, not a generic HTTP error, so + the agent can redact and continue instead of retrying; +- the remaining token budget is an object the agent can **pace** against, + instead of a limit it discovers by failing; +- every call carries a correlation ID and cost tags, and emits an + OpenTelemetry span that lines up with the gateway's own audit trail; +- and you can rehearse all of it on your laptop, against a local simulator, + before an agent ever meets production policy. + +The gateway stays the enforcement point — it always has the final word. What +changes is that your agents become **good citizens** of the platform: aware of +the rules, efficient within them, and observable end to end. That is the +enterprise vision behind DDK — control at the proxy, efficiency in the agent — +built for AI engineers, developers and the AI teams who have to run their +agents in production. + + + + Authentication, cost attribution, PII blocking and budget limits are set + once at the gateway and inherited by every app that uses DDK. + + + The same MuleSoft API management plane that already governs thousands of + enterprise APIs, now covering model traffic. No second gateway to buy, + staff or audit. + + + One import, your framework, your IDE, your code. DDK returns native + framework objects, not wrappers — eight frameworks, three lines to eject. + + + Governed model access today, with governed tool access, agent-to-agent + calls and publishing to the Agent Fabric registry on the same foundation. + + - **"Agent Fabric" is a MuleSoft (Salesforce) product name**, not a generic - term. This project is an SDK *for* that product; it is descriptive, not a - first-party release. See [the trademark/support boundary in the README](https://github.com/Donkey-Development-Kit/donkey-development-kit). + **DDK is an open-source, community-driven project** (Apache-2.0). It is **not + an official Salesforce or MuleSoft product** and is not supported by + Salesforce. "Agent Fabric", "Anypoint", "MuleSoft" and "Omni Gateway" are + Salesforce trademarks; DDK uses them only to describe the platform it + connects to. Meet the people behind it on the [Team](/community/team) page. -## The one design rule +## Architecture -> **Adapters return the framework's native object — never a wrapper.** +DDK sits inside your agent process and speaks to the platform on three +fronts: governed calls through the gateway, assets published to the control +plane, and telemetry to your observability stack. -`donkey.langgraph.chat_model("gpt-4o")` returns a real `langchain_openai.ChatOpenAI`. -`donkey.llamaindex.llm("gpt-4o")` returns a real `OpenAILike`. You can hand these -straight to `create_agent`, a LlamaIndex query engine, a Strands `Agent`, -and so on. If you ever want to drop the SDK, you eject to three lines of native -constructor code — every framework page shows you exactly which three. +
+ +**The AI control plane** is where the platform team manages the AI estate: +the agent registry, cost control, gateway federation, and governance and +observability across every runtime. + +**Omni Gateway** is the single data-plane entry point for APIs, MCP servers, +LLMs and agents. Its policies do the enforcing — authentication and identity, +token budgets and rate limits, PII detection, prompt-injection and +content-safety guardrails, model routing and fallback, audit trails, contract +drift and tool-poisoning detection — before traffic reaches the managed +upstreams: enterprise APIs and integrations, MCP servers, LLM providers, and +other AI apps and agents. + +**DDK** is the developer-side half of that picture. It is wired into the +framework client your agent already uses and adds: + +- **Governed calls and typed refusals** — every model request goes through the + gateway with consumer credentials, correlation and attribution headers + injected; every policy rejection comes back as a typed exception such as + `PIIDetected` or `TokenBudgetExceeded`, never confused with an auth error. +- **Token budget awareness** — the gateway's rate-limit headers become a + `Budget` object with `remaining`, `pace()` and `wait_for_reset()`. +- **Local testing** — `donkey mock` and `donkey.simulate()` replay real + gateway rejection shapes on your laptop, and a pytest conformance suite + proves your agent handles each one before it ships. +- **OpenTelemetry GenAI spans** — each call emits a span carrying the policy + decision, policy type, budget and correlation ID, exported to whatever + observability stack you run (Grafana, Datadog, Jaeger, and others) and + joined to the gateway's audit record through the correlation ID. +- **Registry and agent-to-agent** — scanning your code to publish tools and + agent cards to the control plane, and serving or exposing your agent to + other agents over A2A. + +The division of labour is deliberate. The gateway enforces; DDK makes the +enforcement legible and actionable inside the agent. Nothing in DDK +re-implements a policy client-side, and nothing in your process can override +the gateway. + +## Before and after + +Take the most ordinary piece of agent code there is: one model call. + +
+ +**Without DDK**, a stock client talks straight to the model provider. It works +— and it is invisible. There are no centralised controls or policy +enforcement, no usage tracking, no record of which team or agent spent which +tokens, and nothing an auditor can follow. + +```python +import openai + +client = openai.OpenAI() +completion = client.chat.completions.create( + model="gpt-4o", + messages=[{"role": "user", "content": "Capital of Switzerland?"}], +) +print(completion.choices[0].message.content) +``` + +**With DDK**, the call is the same shape and the client is the same native +`openai.OpenAI` — but it now goes through your organisation's Omni Gateway. +The request is authenticated and attributed, policy is applied, the agent +knows how much budget it has left, and the platform team sees the usage per +model and per consumer in Agent Fabric. + +```python +from donkey_kit import Donkey + +with Donkey.from_env() as donkey: + client = donkey.llm.client(sync=True) # a real openai.OpenAI, governed + reply = client.chat.completions.create( + model="gpt-4o", + messages=[{"role": "user", "content": "Capital of Switzerland?"}], + ) + print(reply.choices[0].message.content) + print(donkey.budget.remaining, "tokens left") +``` + +Three lines changed. Governed, observable and attributed — without leaving +your framework. ## Make a governed call -The Python SDK returns native framework objects. From any other language you hit -the *same* governed proxy over its OpenAI-compatible HTTP API — the verified base -URL plus the `client_id` / `client_secret` headers. (No first-party TypeScript -SDK yet.) +From Python, DDK hands you your framework's own objects. From any other +language, call the same governed proxy over its OpenAI-compatible HTTP API +with the `client_id` / `client_secret` header pair. @@ -127,88 +266,42 @@ curl "${DONKEY_LLM_PROXY_URL}chat/completions" \ -## Why not just set `base_url`? +## Native objects, never wrappers -Fair question, and the honest answer shapes everything else here: - -> A stock OpenAI client with a `base_url` and two headers **can** reach the -> governed proxy. This SDK is not selling you that. - -What it sells is the **skeleton** — one place in your process where every -request enters and every response leaves. That is the only place where budget -headers, error classification, correlation IDs, cost tags, OTel spans, -simulation, and refusal handlers can all attach *without you wiring each one*. - - - - Point your framework at the governed proxy. Verified auth, attribution, - retries, and a typed error taxonomy — injected for you. - - - Typed refusals, budget pacing, a local simulator, a conformance plugin, - OTel GenAI spans, and correlation IDs — one release, because the skeleton - is worth the sum of what hangs on it. - - - Discover and bind governed MCP tools across frameworks, with governed-only - filtering. The page covers what's blocked and why. - - - -## What is live today +> **Adapters return the framework's native object — never a wrapper.** - - **Governed model access is live-verified** against a real Anypoint sandbox: - the proxy base-URL shape (`https:////`, no `/v1`), the - `client_id` + `client_secret` consumer-auth header pair, the attribution - headers, and six policy rejection shapes (auth `401`, PII `403`, - token-budget `429`, upstream passthrough, plus the regex prompt-guard and - Azure content-safety `403` blocks). The framework-free client and the - adapters are wired to that verified contract. - +`donkey.langgraph.chat_model("gpt-4o")` returns a real `langchain_openai.ChatOpenAI`. +`donkey.llamaindex.llm("gpt-4o")` returns a real `OpenAILike`. Hand them +straight to `create_agent`, a LlamaIndex query engine or a Strands `Agent`. +And if you ever want to drop DDK, you eject to three lines of native +constructor code — every [framework page](/frameworks) shows exactly which +three. -Beyond the live-verified contract above, shipped SDK capability pages say what -is implemented and call out any remaining platform-verification caveats. -Planned Phase 2 and Phase 3 pages carry roadmap badges and describe proposal -API shapes. Where a platform fact is not confirmed, the SDK raises -`NotImplementedError("blocked on verification: …")` rather than fabricate an -endpoint, header, or class name — see [Verification policy](/concepts/verification) -for the current ledger, and the [Roadmap](/roadmap) for what lands when. +A stock client with a `base_url` and two headers can reach the gateway. What +DDK adds is the **single place in your process** where every request enters +and every response leaves — which is where typed refusals, budget pacing, +correlation IDs, cost tags, spans and simulation all attach without you +wiring each one. ## Next steps - Make your first governed model call in under five minutes. + Your first governed call in minutes — no gateway or credentials needed. - A tour of everything governed model access gives you today. + Every capability, its purpose, and what it saves you. - - Per-framework install, quickstart, and the manual equivalent. + + Install and quickstart for each of the eight supported frameworks. - - Branch on governance outcomes instead of parsing error bodies. + + Runnable demos, from a first governed call to a full LangGraph agent. - What ships when — and what this SDK deliberately will not build. + What is available now and what is coming next, phase by phase. + + + DDK is open source — issues, docs, examples and adapters welcome. diff --git a/website/content/policies.mdx b/website/content/policies.mdx index 3ceb68a8..6052264a 100644 --- a/website/content/policies.mdx +++ b/website/content/policies.mdx @@ -3,19 +3,18 @@ description: Read the in-force policy set to avoid calls that will be refused --- import { Callout } from 'nextra/components' +import { Badge } from '../components' # Policy handshake - - **Phase 3 — blocked upstream, not merely unbuilt.** This needs a - policy-discovery endpoint that **does not exist on the gateway today**. It is - filed as an upstream gap; until that ships, this cannot be built without - inventing an endpoint. See [Roadmap](/roadmap) and - [Verification policy](/concepts/verification). - +Roadmap + +This capability is on the [Roadmap](/roadmap); the API shown here is the planned +design. It depends on a policy-discovery endpoint on the gateway, which does not +exist today. -On first connection, the SDK fetches the policy set in force for your -credentials and exposes it: +On first connection, DDK will fetch the policy set in force for your +credentials and expose it: ```python donkey.policies.models_allowed # ["gpt-4o", "claude-sonnet"] @@ -26,39 +25,34 @@ donkey.policies.content_safety.on # True donkey.policies.observed_at # when this view was fetched ``` -## Why it is worth an upstream ask +## What it enables -**Wasted calls.** An agent requests a model that is not in its allow-list. -Today that is one wasted round-trip and one refusal, every time. With the -handshake, the adapter picks from `models_allowed` at construction time and the -refusal never happens. Multiply by a few thousand tickets a day. +**Fewer wasted calls.** An agent that requests a model outside its allow-list +costs one round-trip and one refusal, every time. With the handshake, the +adapter picks from `models_allowed` at construction time and the refusal never +happens. Multiply by a few thousand tickets a day. -**Honest UX.** If `pii.mode == "mask"`, the gateway will redact rather than +**Better UX.** If `pii.mode == "mask"`, the gateway will redact rather than reject — so your bot can say *"some details were redacted"* instead of -*"request failed."* Same gateway behaviour, completely different experience, -and today the client has no way to know which mode is in force. +*"request failed."* Same gateway behaviour, a very different experience, and +today the client has no way to know which mode is in force. -**Cold start.** It is also what makes [budget](/budget) live rather than +**Warm start.** It also makes [budget](/budget) live rather than last-known-good: a fresh process currently knows nothing about its budget until its first response comes back. -## The hard rule +## The handshake is advisory - **The handshake is advisory.** It exists to avoid *wasted* calls and to - improve UX — never to make an access decision. The gateway still evaluates - every single request. + **The handshake never makes an access decision.** It exists to avoid *wasted* + calls and to improve UX. The gateway still evaluates every request. If the client's cached view and the gateway disagree, **the gateway wins** and -the client learns from the refusal. Any proposal to skip a gateway call -*"because the handshake said it's fine"* is client-side enforcement, which is -on the [will-not-build list](/roadmap). - -This is also why `observed_at` is part of the surface, exactly as it is on the -budget object: a policy view is a snapshot, and code that cannot tell a -snapshot from live state will eventually make a decision it should not have. - ---- +the client learns from the refusal. Skipping a gateway call *"because the +handshake said it's fine"* is client-side enforcement, which is on the +[will-not-build list](/roadmap). -**Status: Phase 3 — blocked on an upstream endpoint.** +This is also why `observed_at` is part of the surface, as it is on the budget +object: a policy view is a snapshot, and code that cannot tell a snapshot from +live state will eventually make a decision it should not have. diff --git a/website/content/publishing.mdx b/website/content/publishing.mdx index 056b8fc7..65d18508 100644 --- a/website/content/publishing.mdx +++ b/website/content/publishing.mdx @@ -1,53 +1,44 @@ --- -description: Derive a manifest and agent card from your code, then register them from CI — the Agent Scanner for your git repo. +description: Derive a manifest and agent card from your code, then register them with Exchange / Agent Registry from CI — the Agent Scanner for your git repo. --- import { Callout } from 'nextra/components' +import { Badge } from '../components' # Scan & publish - - **Phase 2 — designed, not yet shipped.** Command and API shapes below are a - proposal, and the Exchange publish API is currently - `blocked on verification`. See [Roadmap](/roadmap) and - [Verification policy](/concepts/verification). - - -`donkey scan` walks your repository, finds everything marked -[`@donkey.tool`](/cli), plus MCP server definitions and agent entry points, and -produces a manifest and A2A agent card. `donkey publish` registers it with -Exchange / Agent Registry. A GitHub Action runs both on every merge to `main`. +Roadmap -The point: **the registry becomes a consequence of the code, not a chore.** +This capability is on the [Roadmap](/roadmap); the API shown here is the planned design. -A support agent with six tools is registered by hand today, if at all, and the -tool list is stale within a week. With the Action, every merge updates it. +`donkey scan` walks your repository, finds everything marked +[`@donkey.tool`](/cli#donkeytool), plus MCP server definitions and agent entry +points, and produces a manifest and A2A agent card. `donkey publish` registers +them with Exchange / Agent Registry. A GitHub Action runs both on every merge to +`main`. -## Positioning: this complements Agent Scanners, it does not compete +**The registry becomes a consequence of the code, not a chore.** A support agent +with six tools registered by hand has a stale tool list within a week; with the +Action, every merge updates it. - - **MuleSoft's Agent Scanners already discover agents** from Agentforce, - Bedrock, Vertex AI, and Copilot Studio — at **runtime**. This feature is the - **design-time / CI complement**: it registers agents built in plain Python - that no cloud scanner can see. + + **This complements MuleSoft's Agent Scanners.** Agent Scanners discover agents + from Agentforce, Bedrock, Vertex AI, and Copilot Studio at **runtime**. Scan & + publish is the **design-time / CI** complement: it registers agents built in + plain Python that no cloud scanner can see. -The one-line version: *the Agent Scanner for your git repo.* Re-implementing -Agent Scanners is on the [will-not-build list](/roadmap). +## Scope: code-first assets -## Scope: code-first assets only - -Publication exists for assets that **originate in the developer's code**: +Publication is for assets that **originate in your code**: - an MCP server written in Python or TypeScript, - an agent exposed over [A2A](/a2a), - an agent exposed as a tool without an A2A surface. -It does **not** exist for assets the platform already owns. If an MCP server -was created by MCP Bridge from an existing API, that capability is already in -Exchange — publishing a second, code-derived descriptor for it creates two -catalog entries for one capability, which is exactly the failure a registry -exists to prevent. +It is not for assets the platform already owns. An MCP server created by MCP +Bridge from an existing API is already in Exchange; publishing a second, +code-derived descriptor would create two catalog entries for one capability. ### Collision check @@ -88,7 +79,7 @@ pub = Publication( ) ``` -`asset_type` drives which descriptor is required and how it is generated: +`asset_type` determines which descriptor is required and how it is generated: | `asset_type` | Descriptor | Generated from | |---|---|---| @@ -97,34 +88,19 @@ pub = Publication( | `AGENT` | agent descriptor (no A2A surface) | framework introspection, best-effort | | `API` | OpenAPI / AsyncAPI | user-supplied file; no generation | - - **Verification gate.** Exchange exposes first-class MCP-server and AI-agent - asset categories, but the exact token vocabulary used by publication and - discovery filters is unverified. `PublicationAssetType` currently shares the - SDK's `mcp` / `a2a-agent` discovery values; no blocked Exchange call sends - those assumed values until the direct contract is captured. The two features - share this dependency, so it is verified once and recorded once. - - ## `descriptor="auto"` — deriving the spec from code -Hand-maintained catalog descriptors go stale within one sprint. Generation is -the entire value of this object. +Hand-maintained catalog descriptors go stale within a sprint, so generation is +the core of this feature. -The key insight: **don't write a type-hint-to-JSON-Schema converter.** Every -supported framework already derives JSON Schema from function signatures, type -hints, and docstrings, because that is how tool calling works at all — `@tool` -in LangChain and Strands, `FunctionTool` in ADK and LlamaIndex, `@mcp.tool()` -in the MCP Python SDK, and each of CrewAI, the OpenAI Agents SDK, and the -Anthropic SDK's own conventions. Each already builds a schema and attaches it -to a tool object. +Every supported framework already derives JSON Schema from function signatures, +type hints, and docstrings — `@tool` in LangChain and Strands, `FunctionTool` in +ADK and LlamaIndex, `@mcp.tool()` in the MCP Python SDK, and the equivalent +conventions in CrewAI, the OpenAI Agents SDK, and the Anthropic SDK. The SDK +**asks the framework for the schema it already computed** rather than +re-deriving it, so the catalog documents exactly the schema the model sees. -So the SDK's job is to **ask the framework for the schema it already -computed**. Re-deriving it would produce a second, subtly different schema from -the one the model actually sees — worse than useless, because the catalog would -then document something other than the running behaviour. - -### Three derivation modes, ranked by fidelity +### Derivation modes ```python descriptor="auto" # object introspection — the default @@ -134,14 +110,13 @@ descriptor="auto:check" # generate, diff against committed file, fail on mism ``` - **`auto:live`** starts the server, performs the MCP initialize handshake, and - calls `tools/list` (plus `resources/list` and `prompts/list`). Highest - fidelity — it is exactly what a client sees — but the server must actually - run, with whatever credentials and network that needs. + calls `tools/list` (plus `resources/list` and `prompts/list`). It is exactly + what a client sees, but the server must actually run, with whatever + credentials and network that needs. - **`auto`** (the default) imports your module, locates the tool and agent objects, and reads their already-computed schemas. No server, no network, works in CI. Importing user code executes it, so tool definitions must be - import-safe — mitigated with an explicit entrypoint rather than package-wide - scanning: + import-safe; declare explicit entrypoints rather than scanning whole packages: ```toml [publication.entrypoints] @@ -150,14 +125,16 @@ descriptor="auto:check" # generate, diff against committed file, fail on mism ``` - **`auto:static`** parses decorators, signatures, and docstrings via AST - without executing anything. Useful only where importing user code is + without executing anything — for environments where importing user code is unacceptable. It cannot see tools registered in a loop, from config or a database, behind a feature flag, attached dynamically at startup, or built - from imported/generated pydantic models — so it emits a completeness warning - whenever it hits a pattern it cannot resolve, and it is never the default. + from imported/generated pydantic models, so it emits a completeness warning + whenever it hits a pattern it cannot resolve. It is never the default. +- **`auto:check`** generates the descriptor, diffs it against the committed + file, and fails on mismatch — useful as a CI gate. -`donkey publish --cross-check` runs `auto` and `auto:live` and diffs them. -Disagreement means either dynamic registration the object graph does not +`donkey publish --cross-check` runs `auto` and `auto:live` and diffs them. A +disagreement means either dynamic registration the object graph does not reflect, or a broken framework adapter. @@ -165,9 +142,11 @@ reflect, or a broken framework adapter. `{"type": "string"}` and says nothing about which departments are valid, or when to use this tool over a similar one. `auto` fails publication on a description that is tautological or missing, and `preview()` reports - description quality rather than a verdict. + description quality. ---- +## Related -**Status: Phase 2 — not yet shipped.** +- [CLI & decorators](/cli) — mark tools with `@donkey.tool` today. +- [A2A agents](/a2a) — serve and expose the agent whose card you publish. +- [Tool access](/tool-access) — discover and bind published tools from Exchange. diff --git a/website/content/quickstart.mdx b/website/content/quickstart.mdx index 9c52d0de..2d1deacf 100644 --- a/website/content/quickstart.mdx +++ b/website/content/quickstart.mdx @@ -1,109 +1,113 @@ +--- +description: From pip install to a governed model call, a typed refusal, a budget and an OpenTelemetry span — locally, with no gateway and no credentials. +--- + import { Callout, Steps } from 'nextra/components' +import { Badge, Output } from '../components' # Quickstart -Make your first **governed** model call and see the three things a bare -`base_url` + headers can't give you — a **typed refusal**, a **budget** object, -and a **span** — in under 15 minutes, with **no Anypoint credentials** and **no -real gateway**. - -You'll run the SDK against the bundled **local gateway simulator**: a -pure-Python stand-in that replays the *same* captured rejection fixtures the -SDK's typed errors are tested against, so refusals light up locally exactly as -they would against a live proxy. - - - The simulator is a **fixture replay, never a real gateway** (BG §1.4). It - enforces no policy and forwards no traffic, and every response it serves - carries `x-donkey-simulator: true` so it can't be mistaken for one. It also - **ignores auth** — that's why the credentials below are throwaway - placeholders. See [Verification policy](/concepts/verification). - +Make your first governed model call, then see what a plain `base_url` can't +give you — a **typed refusal**, a **budget** and a **span** — in a few +minutes, on your laptop. You start against DDK's **local gateway simulator**, +so you need no Anypoint account and no credentials. Switching to your real +Omni Gateway later is an environment change, not a code change. + +## What you can use + +| Capability | Status | Learn more | +|---|---|---| +| Governed model access for 8 frameworks | Live | [Frameworks](/frameworks) | +| Typed refusals | Live | [Typed refusals](/errors) | +| Budget & pacing | Live | [Budget & pacing](/budget) | +| OpenTelemetry spans & cost attribution | Live | [Telemetry & cost](/telemetry) | +| Local simulator & `simulate()` | Live | [Local simulator](/simulator) | +| Conformance testing with pytest | Live | [Testing & conformance](/testing) | +| CLI (`init`, `doctor`, `mock`, `test`) & decorators | Live | [CLI & decorators](/cli) | +| Governed tool access (MCP) | Roadmap | [Tool access](/tool-access) | +| A2A agents (`serve`, `expose`, `dev`) | Roadmap | [A2A agents](/a2a) | +| On-behalf-of identity | Roadmap | [Identity](/identity) | +| Human-in-the-loop | Roadmap | [Human-in-the-loop](/hitl) | +| Scan & publish to the registry | Roadmap | [Scan & publish](/publishing) | +| Policy handshake | Roadmap | [Policy handshake](/policies) | + +## Your first governed call ### Install -Install the base SDK, the raw LLM client, the simulator, and OpenTelemetry (so -you can see a span): - ```bash pip install "donkey-kit[llm,local,otel]" ``` -- `llm` — the OpenAI client `donkey.llm.client()` hands back. -- `local` — the local gateway simulator (`starlette` + `uvicorn`). -- `otel` — the OpenTelemetry SDK, so the `donkey.llm.chat` span is emitted. +`llm` adds the OpenAI client, `local` the gateway simulator, `otel` the +OpenTelemetry SDK. Quote the extras — zsh treats unquoted brackets as a glob. ### Start the local gateway -In one terminal, boot the simulator. It binds `127.0.0.1:8080` and stays in the -foreground: +In one terminal: ```bash donkey mock ``` -It serves the captured fixtures over a real TCP port — the exact surface a stock -client points `DONKEY_LLM_PROXY_URL` at. +The simulator listens on `127.0.0.1:8080` and replays real gateway responses, +including every refusal shape. It enforces no policy and ignores +credentials, and every response carries `x-donkey-simulator: true`. -### Point the SDK at it +### Point DDK at it -In a second terminal, set the three governed-access values. `Donkey.from_env()` -requires the `client_id` / `client_secret` header pair to *exist*, but the -simulator never checks it — so these are placeholders, not real credentials: +In a second terminal: ```bash export DONKEY_LLM_PROXY_URL="http://127.0.0.1:8080" -export DONKEY_LLM_PROXY_CLIENT_ID="local" # placeholder — simulator ignores auth -export DONKEY_LLM_PROXY_CLIENT_SECRET="local" # placeholder — simulator ignores auth +export DONKEY_LLM_PROXY_CLIENT_ID="local" # placeholder — the simulator ignores auth +export DONKEY_LLM_PROXY_CLIENT_SECRET="local" ``` - - **No Anypoint credentials at any step.** This is the whole point of the - gateway-free path: you see the SDK's governance surfaces work before you ever - obtain a single credential. - - -### See a span and a budget - -Wire a `ConsoleSpanExporter` so spans print to your terminal, then make one -governed call. `donkey.llm.client(sync=True)` returns a real `openai.OpenAI`, -but routed through the Donkey transport — which is what makes the call -auto-emit a span and update the budget: - -```python -from opentelemetry import trace -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor +### Make the call +```python filename="hello.py" from donkey_kit import Donkey -# Send spans to the console. core.telemetry reads the global provider when it -# opens each span, so set this before the first call. -provider = TracerProvider() -provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter())) -trace.set_tracer_provider(provider) - with Donkey.from_env() as donkey: client = donkey.llm.client(sync=True) # a real openai.OpenAI, governed - reply = client.responses.create(model="gpt-5.1", input="Say hi in three words.") + reply = client.responses.create( + model="gpt-5.1", + input="Tell me a one-sentence bedtime story about a unicorn.", + ) print(reply.output_text) print("budget remaining:", donkey.budget.remaining, "tokens") ``` -You'll see a `donkey.llm.chat` span printed with `gen_ai.usage.*` token counts, -`donkey.policy.decision` (`allow`), `donkey.budget.remaining`, and the -correlation id. And `donkey.budget` is a real -[`Budget`](/feature-overview) object updated from the response's rate-limit -header — not a header you parse yourself. + +```text +A sleepy unicorn named Luma painted soft silver stars across the night sky with her glowing horn, then curled up on a moonbeam so all the children below could fall asleep beneath her gentle, sparkling light. +budget remaining: 99500 tokens +``` + + +`donkey.llm.client()` returns the OpenAI SDK's own client, routed through +DDK — so credentials, correlation and attribution headers are injected, and +`donkey.budget` is updated from the gateway's response. The simulator replays +a response captured from a real gateway, so you get this story whatever you +ask; against your Omni Gateway the model answers your actual prompt. + + + + + Prefer one command? Run `python -m examples.quickstart.main` from the + `python/` directory of the + [SDK repository](https://github.com/Donkey-Development-Kit/donkey-development-kit/tree/develop/python/examples/quickstart): + it boots the simulator for you and runs the steps above end to end. + -### Catch a typed refusal +## Catch a typed refusal Live -The simulator serves any of its captured rejection shapes on demand: a request -whose model id is `donkey-sim/` is served that exact rejection body. -Ask for the PII shape, and bridge the raw response into the taxonomy with -`donkey_kit.core.errors.classify()`: +Ask the simulator for a specific rejection by using the model id +`donkey-sim/`, then bridge the raw response into DDK's taxonomy with +`classify()`: ```python import openai @@ -113,53 +117,139 @@ from donkey_kit.core.errors import classify try: client.responses.create( model="donkey-sim/pii-detected", - input="My SSN is 123-45-6789, please store it.", + input="Email the report to jane.doe@example.com.", ) except openai.APIStatusError as exc: - governed = classify(exc.response) # bridge into the taxonomy + governed = classify(exc.response) if isinstance(governed, PIIDetected): print("blocked, entities:", governed.entities) ``` - - `donkey.llm.client()` gives you the framework's own client, so a refusal - arrives as `openai.APIStatusError`, **not** a `DonkeyError` — until you bridge - it with `classify()`. The framework adapters (`donkey.langgraph`, …) do this - for you; see the [Error taxonomy](/errors). - + +```text +blocked, entities: ['Email'] +``` + - +A PII block is not an auth error, and a policy `429` must never be retried — +typed refusals let the agent react correctly. The framework adapters +(`donkey.langgraph`, …) classify for you. See [Typed refusals](/errors). - - **Prefer to run it all at once?** The whole flow above is a single runnable - script — `examples/quickstart/main.py` — that boots the simulator for you and - needs no environment setup. It's executed and timed on a clean machine in CI, - so this quickstart can never silently rot. - +## Pace against the budget Live -## When you're ready: point at your real gateway +The gateway's token window is a first-class object. Keep a reserve in hand +and wait for the window to reset instead of failing: -The *same code* runs against your Omni Gateway proxy — you only change the -environment. Stop the simulator and set the three real governed-access values. -The proxy authenticates on a `client_id` / `client_secret` **header pair** -(consumer auth) — **not** a bearer token, and separate from any Anypoint -control-plane credential. +```python +from donkey_kit import BudgetReserveReached + +async with Donkey.from_env() as donkey: + while True: + try: + async with donkey.budget.pace(reserve=0.05): + await enrich(batch) + break + except BudgetReserveReached: + await donkey.budget.wait_for_reset() +``` - - **Don't have these values yet?** They come from the platform, not the SDK — - create the proxy, then request access to mint the consumer credentials. The - official docs own each step (linking rather than restating so this page can't - drift from them): - - - **[Create the model proxy](https://docs.mulesoft.com/general/model-proxy-create-model-proxy)** - on the Omni Gateway — its consumer endpoint is your `DONKEY_LLM_PROXY_URL` - (with **no `/v1`**). Choose **Format=OpenAI**: that's the verified - data-plane path the SDK targets ([why](/frameworks#which-wire-format-does-your-proxy-speak)). - - **[Request access](https://docs.mulesoft.com/exchange/to-request-access)** in - Exchange — the proxy is protected by a client-ID enforcement policy, so - requesting access registers a client application and mints the pair that - becomes `DONKEY_LLM_PROXY_CLIENT_ID` / `DONKEY_LLM_PROXY_CLIENT_SECRET`. - +See [Budget & pacing](/budget). + +## See a span Live + +With `otel` installed, every governed call emits an OpenTelemetry GenAI span. +Print them to the console: + +```python +from opentelemetry import trace +from opentelemetry.sdk.trace import TracerProvider +from opentelemetry.sdk.trace.export import ConsoleSpanExporter, SimpleSpanProcessor + +provider = TracerProvider() +provider.add_span_processor(SimpleSpanProcessor(ConsoleSpanExporter())) +trace.set_tracer_provider(provider) # set before the first call +``` + +Put these lines at the top of `hello.py` and run it again. Alongside the +story, the console prints the span: + + +```text +{ + "name": "donkey.llm.chat", + "context": { + "trace_id": "0x604e2f805df7f12cfdc402f83a66c8f4", + "span_id": "0x212735c846ec8f18", + "trace_state": "[]" + }, + "kind": "SpanKind.INTERNAL", + "parent_id": null, + "status": { + "status_code": "UNSET" + }, + "attributes": { + "gen_ai.request.model": "gpt-5.1", + "gen_ai.system": "openai", + "gen_ai.response.model": "gpt-5.1", + "donkey.routing.type": "ModelBased", + "donkey.routing.fallback": false, + "gen_ai.usage.input_tokens": 17, + "gen_ai.usage.output_tokens": 51, + "donkey.usage.cached_tokens": 0, + "donkey.usage.cache_write_tokens": 0, + "donkey.usage.reasoning_tokens": 0, + "donkey.policy.decision": "allow", + "donkey.budget.remaining": 99000, + "donkey.correlation_id": "e2fec0b686694055846da9446fa24c96" + }, + ... +} +``` + + +Each `donkey.llm.chat` span carries `gen_ai.usage.*` token counts, +`donkey.policy.decision`, `donkey.budget.remaining` and the correlation ID. +Set `OTEL_EXPORTER_OTLP_ENDPOINT` and DDK exports over OTLP with no code at +all. See [Telemetry & cost](/telemetry). + +## Attribute cost to a run Live + +Group one logical task under a single correlation ID and cost tags, so the +gateway record, your logs and your spans all join up: + +```python +async with donkey.run(id=ticket.id, team="support", project="triage"): + await triage_agent.ainvoke(ticket) +``` + +Or fold it into one line with a decorator: + +```python +@donkey.governed(team="support") +async def handle_ticket(ticket): ... +``` + +See [CLI & decorators](/cli). + +## Test the refusal branch Live + +Inject a real gateway refusal in-process — no server, no network: + +```python +from donkey_kit import PIIDetected + +with donkey.simulate(PIIDetected): + await agent.ainvoke(...) # the next call fails as a real PIIDetected +``` + +Then grade your own agent against every refusal shape with the pytest +conformance suite. See [Testing & conformance](/testing). + +## Connect to your Omni Gateway + +The same code runs against your real gateway — stop the simulator and set +three values. The proxy authenticates on a `client_id` / `client_secret` +header pair (consumer auth), not a bearer token. ```bash export DONKEY_LLM_PROXY_URL="https:////" # note: no /v1 @@ -167,28 +257,39 @@ export DONKEY_LLM_PROXY_CLIENT_ID="" export DONKEY_LLM_PROXY_CLIENT_SECRET="" ``` -Nothing in the Python changes — same `Donkey.from_env()`, same client, same -typed refusals, same span and budget. From here: - -- **[Pick your framework](/frameworks)** — get your framework's own native model - object (LangGraph, ADK, Strands, LlamaIndex, CrewAI, the OpenAI Agents SDK, - the Anthropic SDK, Agent Framework) instead of the raw client, in three lines. - The proxy is OpenAI-compatible, so any OpenAI-compatible TypeScript SDK works - the same way. - -## Where to go next - -- **[LangGraph support-triage demo](/frameworks/langgraph)** — this quickstart - scaled to a real agent: `examples/langgraph/main.py` triages five support - tickets through a two-node LangGraph graph against the simulator, blocks the - PII-laden one as a typed `PIIDetected`, and exports every call's span to a - bundled local OTLP collector — end to end, still no gateway. Run it with - `python -m examples.langgraph.main`. It's the Phase-1 acceptance demo, timed in - CI. -- **[Feature overview](/feature-overview)** — everything governed model access - gives you, including `donkey.budget` pacing and the telemetry span contract. -- **[Error taxonomy](/errors)** — every rejection shape, its discriminator, and - the typed exception `classify()` returns for it. -- **[Verification policy](/concepts/verification)** — what's confirmed against a - live proxy vs. what raises a clear error, and why the simulator only ever - replays captured shapes. +Where the values come from: + +- **[Create the model proxy](https://docs.mulesoft.com/general/model-proxy-create-model-proxy)** + on Omni Gateway with **Format=OpenAI**. Its consumer endpoint is your + `DONKEY_LLM_PROXY_URL`. +- **[Request access](https://docs.mulesoft.com/exchange/to-request-access)** + to the proxy in Exchange. That registers a client application and issues the + `client_id` / `client_secret` pair. + +Then check the setup: + +```bash +donkey doctor +``` + + +```text +[ok] config env (3 fields) +[ok] gateway reachable, responded +[ok] credentials client_id accepted +[ok] model accepted by the proxy +[i] budget 99,000 / 100,000 remaining, resets in 59s, observed 0s ago +``` + + +`donkey doctor` tells a wrong URL from wrong credentials from a model that is +not on the proxy's allow-list, instead of one opaque failure. + +## Next steps + +- **[Pick your framework](/frameworks)** — get a native LangGraph, ADK, + Strands, LlamaIndex, CrewAI, OpenAI Agents SDK, Anthropic SDK or Agent + Framework object in three lines. +- **[Examples](/examples)** — runnable demos for every capability on this page. +- **[Scenarios](/scenarios)** — support triage, a nightly batch and an internal + copilot, built end to end. diff --git a/website/content/reference/configuration.mdx b/website/content/reference/configuration.mdx index 1bf47d0f..96968649 100644 --- a/website/content/reference/configuration.mdx +++ b/website/content/reference/configuration.mdx @@ -1,3 +1,7 @@ +--- +description: Every DDK configuration value — LLM proxy URL and credentials, JWT model-wallet auth, attribution, cost tags, telemetry, and Anypoint control-plane settings — as env vars, config file, or code. +--- + import { Callout } from 'nextra/components' # Configuration @@ -12,7 +16,7 @@ The three required values for the LLM proxy: | Env var | Meaning | |---|---| | `DONKEY_LLM_PROXY_URL` | Proxy base URL: `https:////` — **no `/v1`**. | -| `DONKEY_LLM_PROXY_CLIENT_ID` | Consumer client id (the per-agent identity). | +| `DONKEY_LLM_PROXY_CLIENT_ID` | Consumer client ID (the per-agent identity). | | `DONKEY_LLM_PROXY_CLIENT_SECRET` | Consumer client secret. | @@ -23,33 +27,29 @@ The three required values for the LLM proxy: The stock gateway also accepts a **single** colon-joined header — `authorization: Bearer :` or `apikey: :` — which its - `dataweave-headers-transformation` policy splits back into the pair (see - [Governance](/concepts/governance)). That is a convenience for thin clients - with one auth field; **DDK does not use it and will not add an apikey/bearer - emit mode**. DDK always sends the two-header pair because `client_id` is the - per-agent [attribution](/concepts/attribution) unit, and a colon-joined value - is **not** an alternate auth: the policy ignores it once a `client_id` header - is present, and `validated(need="llm")` hard-requires the pair in client-id - mode. + `dataweave-headers-transformation` policy splits back into the pair. DDK doesn't use that form: it always + sends the two-header pair, because `client_id` is the per-agent + [attribution](/telemetry#cost-attribution-tags) unit. The colon-joined value is not an + alternative once a `client_id` header is present — the policy ignores it. -Missing required fields are reported **all at once** with their env-var names, so -you fix configuration in a single pass rather than one error per run. +Missing required fields are reported **all at once** with their env-var names, +so you can fix configuration in a single pass. ## JWT / model-wallet auth mode A **model-wallet** proxy identifies the caller from an IdP-issued **JWT** plus a -durable wallet-selector client id, with Client ID Enforcement disabled and **no +durable wallet-selector client ID, with Client ID Enforcement disabled and **no `client_secret`**. Select it with `DONKEY_LLM_PROXY_AUTH=jwt`: | Env var | `DonkeyConfig` field | Meaning | |---|---|---| | `DONKEY_LLM_PROXY_AUTH` | `llm_proxy_auth` | Data-plane auth mode: `client-id` (default) or `jwt`. | -| `DONKEY_LLM_PROXY_WALLET_CLIENT_ID` | `llm_proxy_wallet_client_id` | The wallet's system-generated client id, sent as the `X-Client-Id` header. Required in `jwt` mode. | +| `DONKEY_LLM_PROXY_WALLET_CLIENT_ID` | `llm_proxy_wallet_client_id` | The wallet's system-generated client ID, sent as the `X-Client-Id` header. Required in `jwt` mode. | In `jwt` mode the required fields are `llm_proxy_url` **and** `llm_proxy_wallet_client_id` — **not** `client_id` / `client_secret`. The -rotating JWT is **not** a config value: it is supplied through an +rotating JWT is **not** a config value: supply it through an [`AuthProvider`](/identity) passed as `Donkey(llm_auth=…)`, so the SDK can re-fetch it as it rotates and refresh it once on a `401`: @@ -71,10 +71,10 @@ donkey = Donkey( `jwt` mode is **async-only** — the rotating credential is fetched from an async `AuthProvider`, so the blocking client (`sync=True`) is refused with an actionable error. Use client-id auth for a synchronous caller. It also works - only for adapters routed through the SDK's shared HTTP client (the raw client - and LangGraph): frameworks handed a one-time `default_headers` snapshot (ADK, + only for adapters that use the SDK's shared HTTP client (the raw client and + LangGraph). Frameworks given a one-time `default_headers` snapshot (ADK, CrewAI, LlamaIndex, MS Agent Framework) pin the token at construction and - cannot refresh it — an [asserted conformance exemption](/testing). + can't refresh it — see [Testing](/testing). ## Optional attribution @@ -87,9 +87,9 @@ donkey = Donkey( ## Cost-attribution tags A fixed set of dimensions set once and emitted on every call (both as request -headers and as `donkey.cost.*` span attributes). Overridable per run via +headers and as `donkey.cost.*` span attributes). Override them per run with `donkey.run(team=…, project=…, env=…, enduser_id=…)`. The key set is fixed — an -unknown dimension is a configuration error, not a silently-dropped tag. See +unknown dimension is a configuration error, not a silently dropped tag. See [Telemetry & cost](/telemetry). | Env var | `Donkey.from_env` kwarg | Meaning | @@ -97,7 +97,7 @@ unknown dimension is a configuration error, not a silently-dropped tag. See | `DONKEY_COST_TEAM` | `team` | Owning team. | | `DONKEY_COST_PROJECT` | `project` | Project / workload. | | `DONKEY_COST_ENV` | `env` | Deployment environment (e.g. `prod`). | -| `DONKEY_COST_ENDUSER_ID` | `enduser_id` | End-user id (the `enduser.id` tag). | +| `DONKEY_COST_ENDUSER_ID` | `enduser_id` | End-user ID (the `enduser.id` tag). | In `.donkey-kit.toml` these live under a `[donkey.cost]` table (the end-user dimension keeps its dotted key): @@ -110,12 +110,12 @@ env = "prod" "enduser.id" = "user-42" ``` -The gateway-side request-header **names** are unverified, so they are overridable -placeholders — point them at your real names with `DONKEY_COST_TEAM_HEADER`, -`DONKEY_COST_PROJECT_HEADER`, `DONKEY_COST_ENV_HEADER`, -`DONKEY_COST_ENDUSER_HEADER` (or the matching `cost_*_header` config keys). The -`donkey.cost.*` span attributes carry the full value regardless -([Verification policy](/concepts/verification)). +The request-header **names** the gateway reads for these tags aren't +published, so the SDK uses placeholder names you can override to match your +gateway: `DONKEY_COST_TEAM_HEADER`, `DONKEY_COST_PROJECT_HEADER`, +`DONKEY_COST_ENV_HEADER`, `DONKEY_COST_ENDUSER_HEADER` (or the matching +`cost_*_header` config keys). The `donkey.cost.*` span attributes carry the +full value regardless. ## Telemetry @@ -126,21 +126,39 @@ placeholders — point them at your real names with `DONKEY_COST_TEAM_HEADER`, `telemetry_capture_content` defaults to `false` on purpose: spans are emitted inside your process, **upstream of the gateway's PII masking**, so capturing -content re-exports the very text the platform masks. Opting in is you assuming -that obligation — enable it only for a trusted collector. See -[Telemetry & cost](/telemetry). +content re-exports the very text the platform masks. Enable it only for a +trusted collector. See [Telemetry & cost](/telemetry). + +## Anypoint control plane -## Anypoint control plane (used by later phases) +A separate credential from the LLM proxy, used by features that call the +Anypoint control plane (for example, the connected-app token and the +[CLI](/cli)). You don't need these for governed model access. -These match the Anypoint CLI's own variable names (CLI-verified): -`ANYPOINT_CLIENT_ID`, `ANYPOINT_CLIENT_SECRET`, `ANYPOINT_ORG`, `ANYPOINT_ENV`, -`ANYPOINT_BEARER`, and `ANYPOINT_HOST` (default `anypoint.mulesoft.com`). +| Env var | Meaning | +|---|---| +| `ANYPOINT_CLIENT_ID` | Connected-app client ID. | +| `ANYPOINT_CLIENT_SECRET` | Connected-app client secret. | +| `ANYPOINT_ORG_ID` | Anypoint organization ID. | +| `ANYPOINT_ENV` | Anypoint environment (default `Sandbox`). | +| `ANYPOINT_REGION` | Control-plane region: `us` (default), `eu`, `ca`, or `jp`. | +| `ANYPOINT_BASE_URL` | Explicit control-plane base URL; overrides the region. | ## Config file -Instead of env vars you can use `.donkey-kit.toml` in your project. Secrets and -local dev credentials belong in `.donkey-kit.local.toml` (git-ignored), never -in the committed file. +Instead of env vars you can put non-secret values in a `[donkey]` table in +`.donkey-kit.toml`, read from the working directory (or `$XDG_CONFIG_HOME`). +Keys are the `DonkeyConfig` field names: + +```toml +[donkey] +llm_proxy_url = "https:////" +llm_proxy_client_id = "…" +``` + +`donkey init` generates this file from your current environment. Keep secrets +(`llm_proxy_client_secret`, `client_secret`) out of the committed file and +supply them as environment variables. ## Programmatic diff --git a/website/content/reference/unsupported-boundary.mdx b/website/content/reference/unsupported-boundary.mdx index f79cc988..ab8a7b4c 100644 --- a/website/content/reference/unsupported-boundary.mdx +++ b/website/content/reference/unsupported-boundary.mdx @@ -1,18 +1,22 @@ +--- +description: Which MuleSoft platform APIs DDK calls, whether each is documented for third-party use, and the project's support statement. +--- + import { Callout } from 'nextra/components' +import { Badge } from '../../components' # Unsupported boundary -Enterprise buyers ask: *which platform APIs does this SDK call, and are they -supported for third-party use?* Having the answer pre-written turns a procurement -stall into a short conversation. +Which platform APIs does the SDK call, and are they supported for third-party +use? This page answers that question for security and procurement reviews. -The authoritative, maintained list lives in the repository at +The maintained list lives in the repository at [`docs/unsupported-boundary.md`](https://github.com/Donkey-Development-Kit/donkey-development-kit/blob/main/docs/unsupported-boundary.md). -It is separate from the [verification ledger](/concepts/verification): verification -records how a fact was established, while this page records whether MuleSoft -publishes the contract for third-party use. +It is separate from the [verification ledger](https://github.com/Donkey-Development-Kit/donkey-development-kit/blob/develop/docs/verified-apis.md): the +ledger records how a fact was established, while this page records whether +MuleSoft publishes the contract for third-party use. -Every platform API the shipping SDK calls is classified: +Every platform API the SDK calls is classified: | Classification | Meaning | |---|---| @@ -20,35 +24,37 @@ Every platform API the shipping SDK calls is classified: | **Documented, no SLA for third-party use** | May break; we'll fix. | | **Undocumented** | Should be empty. Anything here needs a written justification and an owner. | -## Current shipping boundary +## Current boundary -The SDK currently reaches two platform destinations. The rows below classify -the distinct contracts it consumes at those destinations. Everything else that -needs an unverified endpoint is blocked before network I/O. +The SDK reaches two platform destinations: the Anypoint connected-app token +endpoint and the Model Proxy. The rows below classify each contract it +consumes there. Any feature that would need an unconfirmed endpoint stops +before making a network request. | Destination / contract | Classification | SDK use | |---|---|---| | Anypoint connected-app token endpoint | **Documented and public** | Retrieves an OAuth bearer token with client credentials. | -| Model Proxy OpenAI-format `/responses` endpoint | **Documented and public** | Sends buffered or streaming model requests with the documented `client_id` / `client_secret` headers and reads OpenAI-format usage. The raw client can also call documented OpenAI-native routes such as `/chat/completions`, but this repository has live-probed only `/responses`. | -| Live-captured Model Proxy policy refusals | **Documented and public** | Classifies Client ID Enforcement, token-rate-limit, and PII responses observed against a deployed proxy. | -| Documentation-derived Model Proxy policy refusals | **Documented and public** | Classifies the **Injection Protection** contract (`x-injection-protection: blocked`) from its official policy page; its body is still pending direct live capture (no proxy deployed). The Regex Prompt Guard, Azure Content Safety, and Amazon Bedrock Guardrails shapes are now live-captured — see the verification page. | -| Upstream provider error pass-through | **Documented, no SLA for third-party use** | Classifies the live-captured nested non-`429` `4xx` provider envelope as `UpstreamRequestError`; generic `5xx` responses become `UpstreamModelError` by status only. The envelope schema belongs to the upstream provider, and MuleSoft's public Model Proxy page states no pass-through compatibility contract. | +| Model Proxy OpenAI-format `/responses` endpoint | **Documented and public** | Sends buffered or streaming model requests with the documented `client_id` / `client_secret` headers and reads OpenAI-format usage. The raw client can also call documented OpenAI-native routes such as `/chat/completions`; `/responses` is the route tested against a deployed proxy. | +| Model Proxy policy refusals (observed) | **Documented and public** | Classifies Client ID Enforcement, token-rate-limit, PII, Regex Prompt Guard, Azure Content Safety, and Amazon Bedrock Guardrails responses captured from a deployed proxy. | +| Model Proxy policy refusals (from documentation) | **Documented and public** | Classifies Injection Protection responses from its official policy page. | +| Upstream provider error pass-through | **Documented, no SLA for third-party use** | Classifies the nested non-`429` `4xx` provider envelope as `UpstreamRequestError`; generic `5xx` responses become `UpstreamModelError` by status only. The envelope schema belongs to the upstream provider, and MuleSoft's public Model Proxy page states no pass-through compatibility contract. | | `x-llm-proxy-ratelimit` success-budget sentence | **Documented, no SLA for third-party use** | Updates `donkey.budget`; an absent or changed value is ignored. | | Gateway identity and routing extension headers | **Documented, no SLA for third-party use** | Populates `donkey.last_call`; missing or unrecognised values become `None`. | - Exchange search and resolution, API Manager governed-state reads, MCP discovery - and binding, and provisioning/publication are not hidden dependencies. They - raise `NotImplementedError("blocked on verification: ...")` before making a - network request. The SDK also does not call a Model Proxy `/models` endpoint, - because live verification established that no such catalog endpoint exists. + Exchange search and resolution, API Manager governed-state reads, MCP + discovery and binding, and provisioning/publication are + Roadmap — not hidden dependencies. Today they + raise `NotImplementedError` before making any network request. The SDK also + never calls a Model Proxy `/models` endpoint, because the proxy has no model + catalog endpoint. -The full ledger links each contract to its official documentation, SDK consumer, -verification evidence, and maintenance owner. Its **Undocumented surfaces** +The full ledger links each contract to its official documentation, SDK +consumer, evidence, and maintenance owner. Its **Undocumented surfaces** section is empty. -## Support statement (the trademark/support boundary) +## Support statement Donkey Development Kit is an independent, community-maintained project with @@ -57,8 +63,8 @@ section is empty. "Omni Gateway" are Salesforce trademarks. -## Why this matters (verification discipline) +## Why the boundary stays small -The SDK's rule against inventing endpoints exists to keep this boundary honest: a -call the SDK makes is either against a classified, known API or it doesn't happen -at all. See [Verification policy](/concepts/verification). +The SDK doesn't invent endpoints: every call it makes is against a classified, +known API, or it doesn't happen at all. See the +[verification ledger](https://github.com/Donkey-Development-Kit/donkey-development-kit/blob/develop/docs/verified-apis.md). diff --git a/website/content/roadmap.mdx b/website/content/roadmap.mdx index 3bbc836a..7839b3da 100644 --- a/website/content/roadmap.mdx +++ b/website/content/roadmap.mdx @@ -1,160 +1,114 @@ --- -description: What ships when — the five phases, the six-piece minimum, and what this SDK deliberately will not build. +description: Where DDK is going — five phases in order, each with its goal, live progress and the issues in its milestone. --- import { Callout } from 'nextra/components' -import { Badge, Card, CardGrid } from '../components' +import { Badge, RoadmapPhase, RoadmapUpdated } from '../components' # Roadmap -This page separates SDK behavior that is **shipped today**, platform contracts -that are **verified live**, and capabilities that are **designed but not yet -shipped**. +DDK is delivered in five phases. Each phase is a GitHub milestone with a clear +goal; the progress bars and issue lists below come straight from those +milestones. - **How to read the badges.** Live means the - platform-facing behavior is verified against a real Anypoint sandbox and - usable now. Shipped means the SDK capability is - implemented and available now; it does not imply that every platform-facing - contract it uses is live-verified. Those caveats remain explicit on the - capability page and in the [Verification policy](/concepts/verification). - Phase 2 and friends mean *designed, with an - agreed acceptance bar, but not yet shipped* — the API shapes on those pages - are proposals and will move. Nothing here is described as working before it - does. + Complete every issue in the milestone is closed ·{' '} + In progress work has landed and more is open ·{' '} + Planned designed, not started. Want to help + move a phase forward? See [Contribute](/community/contribute). -## Why the SDK exists at all - -The honest framing, because it determines everything below: - -> A stock OpenAI client with a `base_url` and two headers can reach the -> governed proxy. The SDK is not selling you that. - -What it sells is the **skeleton**: one place in your process where every -request enters and every response leaves. That is the only place where budget -headers, error classification, correlation IDs, cost tags, OTel spans, -simulation, and refusal handlers can all attach *without you wiring each one*. - -The consequence, and the reason the roadmap is shaped the way it is: **the -skeleton is worth exactly the sum of what hangs on it.** Which is why the six -pieces below are one release and not spread across a year. - -## Phase 1 — the six-piece minimum - -The bar for Phase 1 is deliberately concrete: a developer who tries the SDK -for 15 minutes should find three things they cannot get from `base_url` plus -headers, and one of them should save them from a production incident. - -Four core rejection shapes are live-verified; the remaining typed refusal paths -are shipped with explicit verification caveats. Budget and pacing, the local -simulator, testing and conformance, telemetry and cost, and the CLI and -decorators are shipped SDK capabilities; their pages call out any remaining -platform-verification caveats. - - - - A `403` from a PII policy is not an auth error, and a policy `429` must - never be retried. Branch on governance outcomes instead of parsing bodies. - - - Remaining token budget as a first-class object, so an overnight batch - paces itself instead of dying at 2am. - - - A local server that replays real rejection fixtures, so you can test your - PII branch without a gateway. - - - `simulate()` for in-process refusal injection, plus a pytest plugin you - run against *your own* agent. - - - OpenTelemetry GenAI spans, per-run correlation IDs, and validated - cost-attribution tags. - - - `donkey doctor` tells wrong credentials from wrong URL from - model-not-allowed, instead of one opaque failure. - - - -Alongside those: [Model access](/frameworks) is live today, LangGraph becomes -the one deep, conformance-gated adapter, and the other seven frameworks are -supported at the `connection_kwargs()` level. - -## Phase 2 — differentiate - - - - Discover governed MCP tools and bind them as native framework tools, with - allow/deny filtering. - - - `serve`, `expose`, and `dev` — make your agent callable by other agents, - wrapping the official `a2a-sdk`. - - - On-behalf-of token exchange, so per-user policy reaches the gateway and - never silently falls back to the service identity. - - - One vocabulary for pause-and-ask-a-human, mapped onto each framework's - native interrupt. - - - Derive a manifest and agent card from your code, then register them — - from CI, not by hand. - - - -Also in Phase 2: declarative refusal-reaction handlers, a classification -registry so custom gateway policies become typed exceptions, and kill-switch -awareness. - -## Phase 3 — platform capabilities - - - - Read the in-force policy set and stop wasting calls that will be refused — - advisory only, the gateway always wins. - - - -Most of this phase is gated on the platform, not on effort: it needs a -policy-discovery endpoint that does not exist yet. Structured output on the -`.parse()` path and evaluation hooks are the parts that are not blocked. - -## Phases 4 and 5 - -**Phase 4 — enterprise readiness.** Independent security review and -supply-chain hardening, a latency and overhead budget enforced in CI, a full -pass over every remediation string, the public API contract and deprecation -policy, and compliance evidence mapping. - -**Phase 5 — complete rollout.** A TypeScript port of the six pieces, remaining -framework adapters brought back one at a time by demand, and 1.0 with -stability guarantees. The TypeScript gate is deliberate: Python -product-market fit first, because starting earlier means fixing every gateway -change twice. - -## What this SDK will not build - -Refusals, not backlog. At each of these boundaries the job is to make the -platform's own capability reachable and typed, not to reproduce it: +## Phase 1 — Build the MVP + + + **Version 0.1.0 · Goal:** a developer who tries DDK for fifteen minutes + finds three things they cannot get from a `base_url` and two headers — and + one of them saves them from a production incident. + + The shared governed transport and the capabilities that hang off it: typed + refusals, budget & pacing, the local simulator, `simulate()` and the + conformance suite, OpenTelemetry GenAI spans, correlation IDs and cost tags. + Plus a deep LangGraph adapter, `connection_kwargs()` for seven more + frameworks, the decorators and CLI, the documentation site, and the PyPI + release. + + +## Phase 2 — Differentiate, go beyond + + + **Version 0.2.0 · Goal:** capabilities no generic LLM client offers, + because they depend on the platform behind the gateway. + + Governed tool access (MCP discovery and binding), A2A `serve` / `expose` / + `dev`, on-behalf-of identity, human-in-the-loop, scan & publish to the + registry with a GitHub Action, declarative refusal handlers and a + classification registry for custom policies, kill-switch awareness, and a + second deep framework adapter chosen by demand. + + +## Phase 3 — Platform capabilities + + + **Version 0.3.0 · Goal:** agents that know the rules before they call, not + only after they are refused. + + The policy handshake (read the in-force policy set, advisory only), + policy updates pushed to the code at run boundaries, a governed + structured-output path, evaluation hooks on the run span, monetary spend and + wallet warnings, and run-level cost rollups. Much of this phase depends on + new gateway endpoints tracked under Upstream gaps. + + +## Phase 4 — Enterprise readiness + + + **Version 0.4.0 · Goal:** everything a security, compliance and platform + team asks for before rolling DDK out broadly. + + Independent security review and supply-chain hardening, a latency and + overhead budget enforced in CI, a full pass over every error message, the + public API contract and deprecation policy, compliance evidence mapping (EU + AI Act Art. 12, ISO 42001, OWASP LLM Top 10), log shipping, data residency, + workload identity, air-gapped operation and gateway failover. + + +## Phase 5 — Complete rollout + + + **Version 1.0.0 · Goal:** a stable, multi-language SDK with guarantees. + + A TypeScript port of the core capabilities with conformance scenarios shared + across Python and TypeScript, the remaining framework adapters brought to the + full bar by demand, framework-docs partnerships and launch channels, and the + 1.0 release with stability guarantees. + + +## Cross-cutting tracks + +These milestones run alongside every phase. + + + Every endpoint, header, class name and constructor argument DDK relies on is + checked against the real platform and the installed framework packages + before features are built on it. + + + + Requests to the Omni Gateway team for platform capabilities DDK needs — such + as a budget-query endpoint, policy discovery, a dry-run mode and richer + guardrail verdicts. They land whenever the gateway ships them. + + +## What DDK will not build + +At each of these boundaries the job is to make the platform's own capability +reachable and typed, not to reproduce it: - Client-side policy enforcement — the gateway is the enforcement point. - Client-side semantic caching. - A provisioning control plane competing with API Manager or Terraform. -- Re-implementations of Agent Scanners, Kill Switch, or Trusted Agent Identity. +- Re-implementations of Agent Scanners, Kill Switch or Trusted Agent Identity. - An approval UI or queue. - An evaluation framework. - The gateway inside your agent process. -- A home-grown A2A protocol implementation — the official `a2a-sdk` is wrapped. - - - **API shapes on planned Phase 2 and Phase 3 pages are proposals.** They exist - so the design can be argued about concretely, and they will change before - they ship. What is committed is the *behaviour* and the acceptance bar, not - the signature. - +- A home-grown A2A protocol implementation — the official `a2a-sdk` is used. diff --git a/website/content/scenarios/index.mdx b/website/content/scenarios/index.mdx index 3ac148d6..ac742de3 100644 --- a/website/content/scenarios/index.mdx +++ b/website/content/scenarios/index.mdx @@ -33,7 +33,7 @@ Support triage runs **completely** today — it is the Phase-1 acceptance artefact (`BG §1.8`), timed in CI so its first-run experience can't rot. The nightly-batch and internal-copilot walkthroughs run their **governed-call and refusal-handling** paths against the simulator now; where a step depends on a -surface that is still [blocked on verification](/concepts/verification) — MCP +surface that is still blocked on verification — MCP tool access, agent identity, the kill switch — the page says so in the reader's terms and shows the shape without inventing an endpoint (verification discipline). A "blocked" note here means *known, deliberate, and tracked*, never *guessed*. diff --git a/website/content/scenarios/internal-copilot.mdx b/website/content/scenarios/internal-copilot.mdx index 206fbeee..c0ee9260 100644 --- a/website/content/scenarios/internal-copilot.mdx +++ b/website/content/scenarios/internal-copilot.mdx @@ -12,7 +12,7 @@ and when the guardrail fires you need the refusal to (a) surface as something you can branch on and (b) carry an id that joins the block back to the run in your own logs. This page runs that content-safety branch against the [local simulator](/simulator), and is honest about the parts of a full internal -copilot that are still [blocked on verification](/concepts/verification). +copilot that are still blocked on verification. ## What it demonstrates @@ -111,7 +111,7 @@ an endpoint: The content-safety **discriminator** (the vendor `…-action: reject` header) is typed by `donkey_kit.core.errors.classify()`, but its exact body is **documented-but-not-live-captured** — pinned from the policy pages and pending -a live sandbox round-trip ([Verification policy](/concepts/verification)). The +a live sandbox round-trip. The simulator replays the captured fixture so the branch runs today; no verification row flips to `verified` until a live capture confirms the shape. The correlation mechanism (`donkey.run()` → `X-Correlation-Id` → `.correlation_id`) is shipped diff --git a/website/content/scenarios/nightly-batch.mdx b/website/content/scenarios/nightly-batch.mdx index 51cec0b4..d25d4a8d 100644 --- a/website/content/scenarios/nightly-batch.mdx +++ b/website/content/scenarios/nightly-batch.mdx @@ -124,8 +124,7 @@ The budget object, `pace()`, and `wait_for_reset()` are **shipped** (Phase 1). The windowed-counter behaviour you're pacing against here is the [simulator's](/simulator) — a faithful replay of the observed live contract (prose window on the `200`, numeric `x-token-*` trio on the `429`), never a -header shape the gateway does not emit ([Verification -policy](/concepts/verification)). The end-to-end assertion against the simulator +header shape the gateway does not emit. The end-to-end assertion against the simulator is exactly what this scenario runs. ## Where to go next diff --git a/website/content/scenarios/support-triage.mdx b/website/content/scenarios/support-triage.mdx index b7e95e5d..6515e3ac 100644 --- a/website/content/scenarios/support-triage.mdx +++ b/website/content/scenarios/support-triage.mdx @@ -107,7 +107,7 @@ async with donkey.run(id=f"ticket-{i}"): The proxy *contract* the demo depends on — the base URL shape (no `/v1`), the `client_id`/`client_secret` header pair, the attribution headers, and the four live-verified rejection shapes including PII — is **live-verified** -([Verification policy](/concepts/verification)). `ChatOpenAI` / `StateGraph` are +(see the [verification ledger](https://github.com/Donkey-Development-Kit/donkey-development-kit/blob/develop/docs/verified-apis.md)). `ChatOpenAI` / `StateGraph` are the frameworks' own classes and `.ainvoke` is their documented API: construction via the SDK factory is the verified surface, and everything after is the framework's own runtime. diff --git a/website/content/simulator.mdx b/website/content/simulator.mdx index ae750471..92b75357 100644 --- a/website/content/simulator.mdx +++ b/website/content/simulator.mdx @@ -1,39 +1,33 @@ --- -description: A local server that replays real gateway rejections, so you can develop against a governed gateway without having one. +description: Run a local server that replays real gateway rejections, so you can build and test failure handling without a governed gateway. --- import { Callout } from 'nextra/components' +import { Badge } from '../components' # Local simulator - - **Phase 1 — the replay server ships, with scenario scripting.** `donkey mock` - boots a local server that replays the captured rejection fixtures, - honesty-stamps every response, and synthesises a budget window on the happy - path. The repeatable `--scenario` flag scripts a specific failure on demand — - frequency-based PII blocks, a shrinking budget window, injection matching. - See [Roadmap](/roadmap) and [Verification policy](/concepts/verification). - +Live -Nobody can make the production gateway emit a PII block on cue. Which means -the branch of your agent that handles `PIIDetected` has **never executed** -before the day it executes on a real customer's data. +Nobody can make the production gateway emit a PII block on cue, so the branch +of your agent that handles `PIIDetected` usually runs for the first time on a +real customer's data. -`donkey mock` is a local HTTP server that behaves like the governed proxy -*for the failure paths*. It replays captured rejection fixtures on demand and -replays a captured happy-path completion for everything else. +`donkey mock` fixes that. It is a local HTTP server that behaves like the +governed LLM proxy *for the failure paths*: it replays captured rejection +fixtures on demand and a captured happy-path completion for everything else. ```bash -pip install "donkey-kit[local]" +pip install "donkey-kit[cli,local]" donkey mock --port 8080 # --host defaults to 127.0.0.1 ``` -Then point **any** client at `http://localhost:8080` — the SDK, a stock OpenAI +Point **any** client at `http://localhost:8080` — the SDK, a stock OpenAI client, or cURL. To trigger a specific rejection, set the request's **`model`** to the sentinel `donkey-sim/`: ```bash -# note: no /v1 segment — the governed proxy (and the simulator) has none +# no /v1 segment — the governed proxy (and the simulator) has none curl -s http://localhost:8080/responses \ -H 'content-type: application/json' \ -d '{"model": "donkey-sim/pii-detected"}' -i | head -1 @@ -44,17 +38,27 @@ The selectable shapes are `token-rate-limit`, `pii-detected`, `injection-protection`, `regex-prompt-guard`, `content-safety`, `content-moderation`, `model-not-found`, `upstream-5xx`, and `client-id-missing` — the eight documented rejections plus the consumer-auth `401`. Any other -`model` value gets the happy path. The `donkey-sim/` prefix is -a simulator-only control surface; the real gateway never interprets it. +`model` value gets the happy path. The `donkey-sim/` prefix is a simulator-only +control surface; the real gateway never interprets it. -## Scenario scripting +Because each body is the *same fixture* the SDK's error classifier is tested +against, a sentinel request surfaces in your agent as the typed exception — a +[`PIIDetected`](/errors), not a raw `403`. -The sentinel makes *you* decide which call fails. A **scenario** makes the -simulator decide, on a rule you script once at boot — so a failure fires on its -own, deterministically, across a whole run. Pass `--scenario` once per rule -(repeatable); the grammar is `:`. +## Budget windows on the happy path -Three scenarios ship, each mapping to a worked example below: +Happy-path `200` responses carry a synthesised `x-llm-proxy-ratelimit` budget +window that decrements on every call, so the pacing logic from +[Budget & pacing](/budget) runs end-to-end against the simulator. For a real +windowed counter that resets and emits the `429` on exhaustion, use the +[`budget` scenario](#scenario-scripting). + +## Scenario scripting + +The sentinel lets *you* decide which call fails. A **scenario** lets the +simulator decide, on a rule you set once at boot, so failures fire on their own +and deterministically across a whole run. Pass `--scenario` once per rule; the +grammar is `:`. ```bash donkey mock --port 8080 \ @@ -64,169 +68,92 @@ donkey mock --port 8080 \ ``` When more than one is set, they are evaluated `injection` → `pii_block` → -`budget`, and the first that fires wins. The `donkey-sim/` sentinel still -takes precedence over all of them — it's an explicit "force this exact shape" +`budget`, and the first that fires wins. The `donkey-sim/` sentinel +takes precedence over all of them — it is an explicit "force this exact shape" override. -### Scenario A — `pii_block:every=N` + + The sentinel is for one-off failures; a scenario runs across a whole session. + Use both together to cover specific calls and background failure rates. + + +### `pii_block:every=N` -Nobody can make the production gateway emit a PII block on cue, so the branch of -your agent that handles `PIIDetected` may never have run. `pii_block:every=5` -serves the captured `pii-detected` **403** on every fifth `POST /responses`, -deterministically — the other four get the happy path. +Serves the captured `pii-detected` **403** on every Nth `POST /responses`; the +other calls get the happy path. ```bash donkey mock --scenario pii_block:every=5 ``` -Because the body is the *same fixture* `donkey_kit.core.errors.classify()` is -tested against, call five lights up as a typed -[`PIIDetected`](/errors), not a raw `403`, and your masking logic runs in your -own terminal before the PR is opened. +Every fifth call raises a typed [`PIIDetected`](/errors) in your agent, so your +masking logic runs in your own terminal before the pull request is opened. -### Scenario B — `budget:limit=,window=` +### `budget:limit=,window=` -`budget:limit=20000,window=60s` shrinks an hour-long budget window to one -minute, so the pacing and resume logic from [Budget & pacing](/budget) is tested -end-to-end in ninety seconds instead of "we'll find out tonight." +Runs a **real, wall-clock-windowed token counter**. Shrink an hour-long window +to a minute and your [pacing and resume logic](/budget) is exercised in ninety +seconds. ```bash donkey mock --scenario budget:limit=20000,window=60s ``` -It runs a **real, wall-clock-windowed token counter**. Each call deducts the -happy-path completion's own reported token usage (override with `cost=`). -While budget remains, the happy-path `200` carries the live -`x-llm-proxy-ratelimit` prose header the real gateway emits, decreasing as you -spend. Once the window's budget is spent, calls are served the captured -`token-rate-limit` **429** — with `x-token-remaining` and `x-token-reset` -recomputed from the counter and the real milliseconds left in the window — until -the window rolls over and the budget resets. `duration` accepts `ms`, `s`, `m` -(a bare number is seconds). +Each call deducts the happy-path completion's reported token usage (override +with `cost=`). While budget remains, the `200` carries the +`x-llm-proxy-ratelimit` prose header, decreasing as you spend. Once the window +is spent, calls receive the captured `token-rate-limit` **429**, with +`x-token-remaining` and `x-token-reset` recomputed from the counter and the +milliseconds left in the window, until the window rolls over. `duration` +accepts `ms`, `s`, or `m` (a bare number is seconds). - The numeric `x-token-*` trio appears on the `429`, and the prose - `x-llm-proxy-ratelimit` header on the `200` — matching the observed live - contract, where the happy-path `200` carries the window as prose and the - numeric headers appear only on the `429`. The simulator never emits a header - shape the gateway does not - ([Verification policy](/concepts/verification)). + The numeric `x-token-*` headers appear only on the `429`, and the prose + `x-llm-proxy-ratelimit` header only on the `200` — matching the real gateway. + The simulator never emits a header shape the gateway does not. -### Scenario C — `injection:on-pattern=` +### `injection:on-pattern=` -`injection:on-pattern="ignore previous"` serves the `injection-protection` -**400** on any request whose text contains the substring (case-insensitive), -matched against the request's `input` / `messages` / `instructions` content. +Serves the `injection-protection` **400** on any request whose `input`, +`messages`, or `instructions` text contains the substring (case-insensitive). ```bash donkey mock --scenario injection:on-pattern="ignore previous" ``` -A request carrying a prompt-injection attempt lights up as a typed -[`PromptInjectionBlocked`](/errors) (via the `x-injection-protection: blocked` -discriminator), so the "I can't help with that" branch of a guardrailed bot is -exercised before it faces a real attack. - -## Why this is the one feature `base_url` cannot give you - -Everything else the SDK does happens on the client side, so a determined -developer could hand-roll it. This cannot be hand-rolled, because **the value -is on the server side.** "Develop against a governed gateway without having a -gateway" is not an ergonomics improvement; it is a capability you otherwise do -not have. +The request surfaces as a typed [`PromptInjectionBlocked`](/errors) (via the +`x-injection-protection: blocked` discriminator), so the refusal branch of a +guardrailed bot is exercised before it faces a real attack. -## Testing the branch that has never run - -Point your agent at the simulator and send `model: "donkey-sim/pii-detected"` -on the calls you want to fail. Your masking logic runs in your own terminal, -before the pull request is opened, instead of in production — and because the -body is the *same fixture* `classify()` is tested against, it lights up as a -typed `PIIDetected`, not a raw `403`. - -The happy path carries a synthesised `x-llm-proxy-ratelimit` budget window that -decrements on every call, so the pacing and resume logic from -[Budget & pacing](/budget) runs end-to-end against the simulator instead of -"we'll find out tonight." For a *real* windowed counter that resets and emits -the `429` on exhaustion, use the [`budget` scenario](#scenario-scripting). - - - Deciding *which* calls fail by hand (send the `donkey-sim/` sentinel) - and scripting a rule so failures fire on their own - ([`--scenario`](#scenario-scripting)) are complementary — the sentinel is a - one-off, a scenario runs across a whole session. - - -## The honesty guarantee - -A simulator that could be mistaken for a real gateway would be worse than no -simulator, so three rules are non-negotiable: +## How the simulator works - **Every simulator response carries `x-donkey-simulator: true`.** It can - never be confused for a real gateway in a log, a trace, or a screenshot — - and the same holds for the framework-generated `405`/`500` responses, not - just the ones the simulator hand-builds. + **Every simulator response carries `x-donkey-simulator: true`** — including + framework-generated `405`/`500` responses — so a simulated response is never + confused with a real gateway in a log, a trace, or a screenshot. -And the fixtures are the **same files** used by the `classify()` tests. One -source of truth: if the gateway contract drifts, the simulator and the error -taxonomy fail together, rather than the simulator quietly teaching you a -contract that no longer exists. - -Because "same files, both fail together" only catches a fixture edit that -flips something a test asserts on, a committed **integrity manifest** pins the -sha256 of every fixture the simulator serves. Its source copy lives at -`tests/fixtures/fixtures.lock`; wheel builds copy it beside the packaged -fixtures, so an installed wheel can verify that its replayed bytes still match -the committed manifest. A *benign* byte edit — reformatted JSON, an added field -— fails the lock loudly instead of silently drifting the replayed bytes. A -genuine re-capture is accepted by regenerating the source lock -(`python -m donkey_kit.simulator.fixtures --relock`) and committing it. - -### It replays; it does not evaluate - - - **The simulator replays captured shapes — it does not evaluate policy.** It - tests how your agent handles a refusal, never *which* prompts get refused. - *You* choose the refusal (the `donkey-sim/` sentinel, or a - `--scenario` rule); the simulator never inspects a prompt and decides it - violates a policy, because the real gateway's policy logic does not live on - your laptop. Testing against a real policy *configuration* — "would this - exact prompt be blocked by the rules we deployed?" — needs gateway-side - dry-run mode, tracked in - [#250](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/250). - - -## Acceptance bar - -- A **stock non-SDK client** receives byte-identical rejection bodies and the - exact discriminator headers — proof the simulator is honest rather than - merely convenient. Asserted against both plain `httpx` and a stock - `openai.OpenAI(base_url="http://localhost:8080", …)` client, comparing the - status, body bytes and headers the client actually received (not the SDK's - parsed view) - ([#189](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/189)). ✅ -- The fixtures are the *same files* the `classify()` contract tests load, so - the two fail together if the gateway contract ever drifts — and a committed - integrity manifest fails loudly on any byte edit that the "both fail together" - rule would otherwise miss. ✅ -- The simulator replays captured shapes; it does **not** evaluate policy - (real policy-configuration testing needs gateway - [dry-run mode, #250](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/250)). ✅ -- Every response carries `x-donkey-simulator: true`, including the - framework-generated `405`/`500`. ✅ -- Ships as `pip install "donkey-kit[local]"`, pure-Python, no Docker. ✅ -- Happy-path responses emit a synthesised `x-llm-proxy-ratelimit` prose window - so budget pacing works against it — matching the observed live contract, where - the `200` carries the window as prose and the numeric `x-token-*` trio appears - only on the `429` - (recorded in [`docs/verified-apis.md`](https://github.com/Donkey-Development-Kit/donkey-development-kit/blob/main/docs/verified-apis.md)). ✅ -- Declarative scenario scripting with three worked examples (Scenarios A, B, C). - `pii_block:every=N` fails every Nth call, `budget:limit=,window=` runs a real - windowed counter, `injection:on-pattern=` matches request text. ✅ - ---- - -**Status: Phase 1 — the fixture-replay server (`donkey mock`) and `--scenario` -scripting are shipped.** +- **It replays; it does not evaluate policy.** The simulator tests how your + agent handles a refusal, never *which* prompts get refused. You choose the + refusal (the `donkey-sim/` sentinel or a `--scenario` rule); the + simulator never inspects a prompt and decides it violates a policy, and it + ignores authentication. Testing whether a prompt would be blocked by your + deployed policy configuration requires the real gateway. +- **It works with any client.** A stock, non-SDK client — plain `httpx` or + `openai.OpenAI(base_url="http://localhost:8080", …)` — receives the + byte-identical rejection bodies and exact discriminator headers. +- **Its fixtures are the SDK's test fixtures.** The simulator serves the same + files the error classifier is tested against, so the simulator and the + [error taxonomy](/errors) cannot drift apart. The wheel ships a sha256 + integrity manifest of every fixture, so a changed byte fails loudly rather + than silently altering what the simulator replays. +- **It is pure Python.** `pip install "donkey-kit[local]"` adds Starlette and + Uvicorn; no Docker required. + +## Related + +- [Testing & conformance](/testing) — `simulate()` for in-process unit tests, + and the `gateway` pytest fixture that runs this simulator on an ephemeral port. +- [Error taxonomy](/errors) — the typed exceptions each shape maps to. +- [CLI](/cli) — the full `donkey` command reference. diff --git a/website/content/telemetry.mdx b/website/content/telemetry.mdx index 9e52108b..016627d3 100644 --- a/website/content/telemetry.mdx +++ b/website/content/telemetry.mdx @@ -2,20 +2,16 @@ description: OpenTelemetry GenAI spans, per-run correlation IDs, and cost-attribution tags — in the dashboard you already run. --- -import { Callout } from 'nextra/components' +import { Callout, Tabs } from 'nextra/components' +import { Badge } from '../components' # Telemetry & cost - - **Shipped in Phase 1.** OpenTelemetry GenAI span export (#194), correlation - IDs (#195), cost-attribution tags (#196), and routing and resilience signals - (#309) are available. Platform-facing verification caveats are called out - below; see [Roadmap](/roadmap) and - [Verification policy](/concepts/verification). - +Live -Two pieces of the six-piece minimum land here, because they answer the same -two questions: *what happened?* and *who pays for it?* +DDK answers two questions about every governed call: *what happened?* and *who +pays for it?* It does so with OpenTelemetry GenAI spans, per-run correlation +IDs, cost-attribution tags, and routing and resilience signals. ## OpenTelemetry GenAI spans @@ -42,25 +38,26 @@ donkey.usage.reasoning_tokens = 96 ``` The three `donkey.usage.*` counts carry the cost-relevant detail tokens the -semconv has no pinned key for — cached / cache-write prompt tokens and -reasoning-model thinking tokens. They are read from the response `usage` block's -detail sub-objects and are **omitted, never `0`,** when the provider reports no -detail counts. When the response passes through the SDK's shared HTTP client, -the same counts are exposed per-call on `donkey.last_call`. +semantic conventions have no pinned key for — cached / cache-write prompt tokens +and reasoning-model thinking tokens. They are read from the response `usage` +block's detail sub-objects and are **omitted, never `0`,** when the provider +reports no detail counts. When the response passes through the SDK's shared HTTP +client, the same counts are exposed per-call on `donkey.last_call`. -Export goes over OTLP to wherever you already send spans. **Nothing in the -emit path is Anypoint-specific.** +A **refused** request still produces a span, with `donkey.policy.decision=refuse` +and `otel.status_code=ERROR`. A streaming response produces **exactly one** span, +with token counts filled in at stream end. -That is the point: if your team already runs Langfuse, Datadog, or Phoenix, -then "policy refusals per hour by type" and "tokens per ticket" show up in the -dashboard you already have, with no new tooling to adopt. +Export goes over OTLP to wherever you already send spans. **Nothing in the +emit path is Anypoint-specific**, so if your team already runs Grafana, Elastic, +Honeycomb, New Relic, Datadog, Dynatrace, Langfuse or Phoenix, "policy refusals +per hour by type" and "tokens per ticket" show up in the dashboard you already +have. See [Send spans to your observability backend](#send-spans-to-your-observability-backend). ### Zero-config export - - **Shipped in Phase 1** (#194). Set the standard OpenTelemetry endpoint env - var and spans flow — **no SDK-specific variable**. - +Set the standard OpenTelemetry endpoint env var and spans flow — there is **no +SDK-specific variable**. ```bash pip install "donkey-kit[otel]" @@ -72,16 +69,16 @@ python -m my_app # spans flow, refused calls includ `Donkey.from_env()` reads `OTEL_EXPORTER_OTLP_ENDPOINT` (or the traces-specific `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT`) and, when one is set, installs an OTLP exporter behind a batch processor. The network flush runs on that background -thread, off your request path — which is how instrumentation stays under the -1 ms/call bar. The `[otel]` extra ships the **http/protobuf** exporter; +thread, off your request path, keeping instrumentation overhead under +1 ms per call. The `[otel]` extra ships the **http/protobuf** exporter; `OTEL_EXPORTER_OTLP_PROTOCOL=grpc` is honoured only if you also install `opentelemetry-exporter-otlp-proto-grpc`. **With no endpoint set, the export path is inert and silent** — no exporter is -built, nothing connects, nothing is printed. And if your process already -configures its own OpenTelemetry provider (say via `opentelemetry-instrument`), -Donkey rides it rather than replacing it, so your spans flow through the -pipeline you already set up. +built, nothing connects, nothing is printed. If your process already configures +its own OpenTelemetry provider (say via `opentelemetry-instrument`), DDK rides it +rather than replacing it, so your spans flow through the pipeline you already +set up. Opt out of telemetry entirely with a single flag: @@ -89,31 +86,235 @@ Opt out of telemetry entirely with a single flag: export DONKEY_TELEMETRY=false # or telemetry = false in .donkey-kit.toml ``` -### Two honest caveats +### Send spans to your observability backend + +Any backend that accepts OTLP works with the same three standard variables: +`OTEL_EXPORTER_OTLP_ENDPOINT`, `OTEL_EXPORTER_OTLP_HEADERS` for the vendor's +credentials, and `OTEL_SERVICE_NAME`. There is no DDK-specific setting and no +code change between backends. Pick yours: + + + + +Grafana Cloud stores traces in **Tempo**. In the Grafana Cloud portal, open your +stack and choose **Configure** on the **OpenTelemetry** tile. It generates the +endpoint for your region and a token, already base64-encoded as +`instanceID:token`. + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="https://otlp-gateway-prod-.grafana.net/otlp" +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic%20" +export OTEL_SERVICE_NAME="support-triage" +``` + +Grafana's own note for Python: write the space after `Basic` as `%20`. Find the +spans in **Explore → Tempo** or in **Application Observability**. For +production, Grafana recommends sending through **Grafana Alloy** (its +OpenTelemetry Collector) rather than straight from the app. + +Docs: [Send data to the Grafana Cloud OTLP endpoint](https://grafana.com/docs/grafana-cloud/send-data/otlp/send-data-otlp/) + + + + +On Elastic Cloud Hosted or Serverless, copy the **Managed OTLP endpoint** from +Kibana (**Add data → Applications → OpenTelemetry**), which also generates an +API key. Prefix the key with `ApiKey` yourself; Kibana gives you the bare key. + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="https://" +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=ApiKey%20" +export OTEL_SERVICE_NAME="support-triage" +``` + +Spans appear in Kibana under **Observability → Applications**, grouped by +service. Self-managed Elasticsearch exposes the same protocol at +`/_otlp`; its API key needs `create_doc` and `auto_configure` +privileges. + +Docs: [Send OTLP data to Elastic](https://www.elastic.co/docs/solutions/observability/get-started/quickstart-elastic-cloud-otel-endpoint) · +[Elasticsearch OTLP/HTTP endpoint](https://www.elastic.co/docs/manage-data/ingest/otlp-endpoint) + + + + +Create an ingest API key in your Honeycomb environment settings. + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="https://api.honeycomb.io" # EU: https://api.eu1.honeycomb.io +export OTEL_EXPORTER_OTLP_HEADERS="x-honeycomb-team=" +export OTEL_SERVICE_NAME="support-triage" +``` + +Honeycomb creates a dataset named after `OTEL_SERVICE_NAME`. Query +`donkey.policy.decision`, `donkey.cost.team` or `gen_ai.usage.output_tokens` like +any other field; for example, a `COUNT` where `donkey.policy.decision = refuse`, +grouped by `donkey.policy.type`, gives you refusals by type. + +Docs: [Using the Honeycomb OpenTelemetry endpoint](https://docs.honeycomb.io/send-data/opentelemetry/) + + + + +Authenticate with your account's **license key** in an `api-key` header. + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="https://otlp.nr-data.net" # EU: https://otlp.eu01.nr-data.net +export OTEL_EXPORTER_OTLP_HEADERS="api-key=" +export OTEL_SERVICE_NAME="support-triage" +``` + +New Relic recommends `http/protobuf`, which is DDK's default. The service shows +up under **APM & Services**, and span attributes are queryable with NRQL, for +example `SELECT count(*) FROM Span WHERE donkey.policy.decision = 'refuse' FACET donkey.policy.type`. + +Docs: [New Relic OTLP endpoint](https://docs.newrelic.com/docs/opentelemetry/best-practices/opentelemetry-otlp/) + + + + +The simplest route is the **Datadog Agent**, which accepts OTLP once its receiver +is enabled. Start the Agent with +`DD_OTLP_CONFIG_RECEIVER_PROTOCOLS_HTTP_ENDPOINT=0.0.0.0:4318` and expose port +`4318`, then point DDK at it. No credentials are needed in the app; the Agent +holds the API key. + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318" # the Agent's host +export OTEL_SERVICE_NAME="support-triage" +``` + +Spans appear in **APM → Traces**. Datadog also offers an agentless OTLP traces +intake that takes a `dd-api-key` header; its URL depends on your Datadog site. + +Docs: [OTLP ingestion by the Datadog Agent](https://docs.datadoghq.com/opentelemetry/setup/otlp_ingest_in_the_agent/) · +[OTLP traces intake endpoint](https://docs.datadoghq.com/opentelemetry/setup/otlp_ingest/traces/) + + + + +Create an access token with the `openTelemetryTrace.ingest` scope. + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="https://.live.dynatrace.com/api/v2/otlp" +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Api-Token%20" +export OTEL_SERVICE_NAME="support-triage" +``` + +Use `live.dynatrace.com`, not `live.apps.dynatrace.com`. Through an ActiveGate the +base URL is `https://:9999/e//api/v2/otlp`. Spans +appear in **Distributed Tracing**. + +Docs: [Dynatrace OTLP API endpoints](https://docs.dynatrace.com/docs/ingest-from/opentelemetry/otlp-api) + + + + +Langfuse is built for LLM traces, so the `gen_ai.*` model and token attributes +on DDK spans are what it reads. Authenticate with your project's public and +secret key, base64-encoded: + +```bash +AUTH_STRING=$(echo -n "pk-lf-...:sk-lf-..." | base64) +export OTEL_EXPORTER_OTLP_ENDPOINT="https://cloud.langfuse.com/api/public/otel" # US: https://us.cloud.langfuse.com/api/public/otel +export OTEL_EXPORTER_OTLP_HEADERS="Authorization=Basic%20${AUTH_STRING},x-langfuse-ingestion-version=4" +export OTEL_SERVICE_NAME="support-triage" +``` + +The `x-langfuse-ingestion-version=4` header makes spans appear in real time +instead of after a delay of up to ten minutes. Self-hosted Langfuse (v3.22 or +later) takes the same path on your own host. + +Docs: [Langfuse OpenTelemetry](https://langfuse.com/docs/opentelemetry/get-started) + + + + +Phoenix is open source and runs locally with no account, which makes it a +quick way to look at LLM spans during development: + +```bash +docker run -p 6006:6006 -p 4317:4317 arizephoenix/phoenix:latest +``` + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:6006" +export OTEL_SERVICE_NAME="support-triage" +``` + +Open `http://localhost:6006` to browse the spans. A Phoenix deployment with +authentication enabled also needs its API key as a bearer token: +`OTEL_EXPORTER_OTLP_HEADERS="Authorization=Bearer%20"`. + +Docs: [Phoenix configuration](https://arize.com/docs/phoenix/self-hosting/configuration) · +[Get started with tracing](https://arize.com/docs/phoenix/get-started/get-started-tracing) + + + + +Jaeger's all-in-one container is the fastest way to see spans on your laptop, +with no account and no credentials: + +```bash +docker run --rm --name jaeger -p 16686:16686 -p 4318:4318 \ + cr.jaegertracing.io/jaegertracing/jaeger:latest +``` + +```bash +export OTEL_EXPORTER_OTLP_ENDPOINT="http://localhost:4318" +export OTEL_SERVICE_NAME="support-triage" +``` + +Open `http://localhost:16686`, pick the `support-triage` service and search. +Storage is in memory, so traces are gone when the container stops. + +Docs: [Jaeger getting started](https://www.jaegertracing.io/docs/latest/getting-started/) + + + + +A few rules hold for every backend: + +- **Base endpoint vs traces endpoint.** `OTEL_EXPORTER_OTLP_ENDPOINT` is a base + URL: the exporter appends `/v1/traces`. If you set + `OTEL_EXPORTER_OTLP_TRACES_ENDPOINT` instead, give the full URL including + `/v1/traces`, or the backend answers `404`. +- **Encode spaces in headers.** Header values such as `Basic ` or + `ApiKey ` should be written with `%20` in place of the space; separate + several headers with commas. +- **Protocol.** DDK exports `http/protobuf`, which every backend above accepts. + For gRPC, set `OTEL_EXPORTER_OTLP_PROTOCOL=grpc` and install + `opentelemetry-exporter-otlp-proto-grpc`. +- **In production, consider a Collector.** Sending to an + [OpenTelemetry Collector](https://opentelemetry.io/docs/collector/) (or a + vendor distribution such as Grafana Alloy) lets you batch, retry, redact and + fan out to several backends without touching the app. +- **Prompts stay private by default.** None of these backends receives prompt or + completion text unless you opt in; see + [Message content stays off spans by default](#message-content-stays-off-spans-by-default). + +### Semantic-convention stability and sinks - **The GenAI conventions are still `Development` status upstream.** Attribute - names can change. So the semconv version is **pinned**, and spans are - **dual-emitted**: `gen_ai.*` at the pinned version, plus a stable `donkey.*` + **The GenAI conventions are still `Development` status upstream**, so + attribute names can change. DDK **pins** the semconv version and + **dual-emits**: `gen_ai.*` at the pinned version, plus a stable `donkey.*` namespace under this project's control. Your dashboards do not break when upstream renames something. -Second: whether Anypoint Monitoring or Agent Visualizer **ingests** OTLP GenAI -spans is not publicly documented. So this page promises *"exports OTLP"* and -lets the sink be your choice. It does not promise your spans appear in Agent -Visualizer, because that has not been confirmed — see -[Verification policy](/concepts/verification). +DDK exports standard OTLP and leaves the sink to you. Whether Anypoint +Monitoring or Agent Visualizer ingests OTLP GenAI spans is not publicly +documented, so don't rely on spans appearing there. ### Message content stays off spans by default Spans carry **metadata only** — model, token counts, policy decision, cost tags, -correlation id. They do **not** carry prompt or completion text. This is a -deliberate boundary, not an omission: +correlation id. They do **not** carry prompt or completion text. **Spans are emitted upstream of the gateway's PII masking.** The Omni Gateway - masks sensitive content in *its* logs; a Donkey span is created inside your + masks sensitive content in *its* logs; a DDK span is created inside your process, before the request reaches the gateway. Putting message text on the span would re-export the very content the platform masks — straight to whatever OTLP collector you have wired up. So capturing content is **opt-in, @@ -132,22 +333,17 @@ donkey = Donkey.from_env(telemetry_capture_content=True) `telemetry_capture_content` resolves along the standard precedence (kwarg → env → `.donkey-kit.toml` → default) and **defaults to `False`**. When enabled, content is emitted under the pinned semconv attribute names — -`gen_ai.prompt` and `gen_ai.completion` — and no others. When off (the default), -those attributes never reach a span, and the allowlist that builds every span -drops any content-shaped attribute a call site hands it, so there is no accidental +`gen_ai.prompt` and `gen_ai.completion` — and no others. When off, those +attributes never reach a span, and the allowlist that builds every span drops +any content-shaped attribute a call site hands it, so there is no accidental path for message text to leak. ## Routing & resilience - - **Shipped in Phase 1** (#309). Routing, fallback, and served-model signals - are available through the shared transport and on OpenTelemetry spans. - - The gateway can fail over between providers when one degrades ("Enhanced Resilience for Intelligent Routing"). It reports what it *did* with each request on the response — which provider and model served it, how it routed, and whether -that was a **fallback**. Donkey reads those signals off the shared transport, so +that was a **fallback**. DDK reads those signals off the shared transport, so you get them with **no framework required** — the raw `donkey.llm.client()` path benefits just as the deep adapters do. @@ -168,19 +364,19 @@ r.substituted # False — served_model != requested_model ``` They also land on the span (`gen_ai.response.model`, `donkey.routing.type`, -`donkey.routing.fallback`) — the single most useful thing to have on hand when -latency spikes: it tells an operator whether a slow call was routed normally or +`donkey.routing.fallback`) — the most useful thing to have on hand when latency +spikes: it tells an operator whether a slow call was routed normally or recovered from a degraded provider. ### When `last_call` is unavailable `donkey.last_call` is populated only when the governed response passes through the SDK's shared httpx client. Four `connection_kwargs()`-only adapters route -outside that response path: ADK and CrewAI use LiteLLM's transport, while +outside that response path: ADK and CrewAI send requests through LiteLLM, while LlamaIndex and Microsoft Agent Framework receive only `default_headers`. -That static snapshot deliberately excludes the correlation ID bound later by -`donkey.run(id=...)`, so those two adapters also carry the asserted -`correlation_id_propagated` exemption. +That static snapshot excludes the correlation ID bound later by +`donkey.run(id=...)`, so those two adapters also do not propagate the run's +correlation ID. When every adapter resolved on a `Donkey` is one of those four, a cold read reports the limitation explicitly. For a `Donkey` that resolved only ADK: @@ -196,12 +392,11 @@ This is different from `UNOBSERVED`, which means the current context has not yet received a governed response. On an unavailable surface the SDK cannot observe any response-derived `last_call` field, including gateway identity, routing, fallback, and usage. If multiple non-observing adapters were resolved, -`surface` lists their names. Each limitation is asserted as the -`gateway_identity_observed` conformance exemption rather than silently skipped. +`surface` lists their names. ### Two behaviours worth knowing -**The SDK never double-retries a fallback.** Donkey retries `502/503/504` with +**The SDK never double-retries a fallback.** DDK retries `502/503/504` with backoff, but if the gateway already failed over internally, a `503` it marked as a fallback is **not** retried again — a second recovery layer stacked on a working first one just multiplies latency against an outage the gateway already @@ -221,21 +416,16 @@ donkey = Donkey.from_env(on_model_substitution="raise") `DONKEY_ON_MODEL_SUBSTITUTION` → `.donkey-kit.toml` → default) and **defaults to `"off"`**. - - **TypeScript parity is planned (Phase 5).** The same signals will surface as - `donkey.lastRouting.servedModel` / `.fallback` once the TypeScript SDK ships. - +### TypeScript parity Roadmap -## Correlation IDs +The same signals will surface as `donkey.lastRouting.servedModel` / `.fallback` +in the TypeScript SDK. - - **Shipped in Phase 1** (#195). See - [Observability](/concepts/observability) for the full detail. - +## Correlation IDs Set a per-**run** id once, and every call inside the block carries it — on the wire, on every span, and on every exception — with nothing threaded through your -framework state: +framework state. ```python async with donkey.run(id=ticket.id): @@ -251,6 +441,12 @@ async with donkey.run(id=ticket.id): `DonkeyError.call_id`. Unique per logical request, stable across that request's retries, so one call is pinpointable within a run. +The gateway reads the inbound `X-Correlation-Id` and echoes it verbatim on the +response, so the request and response `x-correlation-id` are the same value. +`X-Donkey-Request-Id` is a client-owned per-call id the gateway does not consume. +Both header names are overridable (`correlation_header` / `call_id_header`) for a +gateway that expects different ones. + Propagation is contextvar-based, so it reaches through framework nodes (every LangGraph node, for instance) without threading an argument through every function, and concurrent runs never leak into each other. It works with or @@ -268,31 +464,16 @@ async def handle_ticket(ticket): await triage_agent.run(ticket) ``` -Each call opens its own run — a fresh run/correlation id (a "run of one") — and -binds the optional per-run cost tags, the OTel span, and typed refusals, exactly -the scope `donkey.run()` establishes. It wraps **both sync and async** callables -and is usable bare (`@donkey.governed`) or parametrised. There is deliberately +Each call opens its own run — a fresh run/correlation id — and binds the +optional per-run cost tags, the OTel span, and typed refusals, exactly the scope +`donkey.run()` establishes. It wraps **both sync and async** callables and is +usable bare (`@donkey.governed`) or parametrised. There is deliberately **no `id=`**: pinning one id across every call would collapse unrelated runs into -a single correlation, so when you need a specific id, reach for `donkey.run(id=…)` +a single correlation, so when you need a specific id, use `donkey.run(id=…)` directly. - - **Verified (#522).** The gateway **reads** the inbound `X-Correlation-Id` and - echoes it verbatim on the response, so the request and response - `x-correlation-id` are the same value — the client→gateway join key is real. - `X-Donkey-Request-Id` is a **client-owned** per-call id the gateway does not - consume; it lives on `DonkeyError.call_id`. Both names stay overridable - (`correlation_header` / `call_id_header`) for a gateway that expects different - ones ([Verification policy](/concepts/verification)). - - ## Cost-attribution tags - - **Shipped in Phase 1** (#196). Completes the six-piece minimum alongside the - correlation IDs above. - - A small, fixed set of tags — `team`, `project`, `env`, `enduser.id` — set once and emitted on every call, both as request headers and as `donkey.cost.*` span attributes: @@ -309,49 +490,26 @@ kwargs, then `DONKEY_COST_*` env vars, then a `[donkey.cost]` table in `.donkey-kit.toml`. Per-run overrides layer on top: `donkey.run(team=…, project=…, env=…, enduser_id=…)` wins **per field** for its block and the rest fall back to the configured tags. The key set is **fixed** — an unknown -dimension is a configuration error, never a silently-dropped header. - -### The question this answers - -Finance asks what the support agent cost last month versus the HR bot. - -Without tags, both agents share one `client_id`, and the honest answer is -*"we don't know."* With tags it is a group-by. - -And for compliance: *"prove the HR bot's answer to user X on date Y went -through the content-safety policy."* The correlation ID on the log line joins -to the gateway record, and the span carries `enduser.id` and -`donkey.policy.type`. That is what an EU AI Act Article 12 log request looks -like in practice — one query, not an investigation. +dimension is a configuration error, never a silently-dropped header. Values are +**validated** — fixed keys, bounded length — so nobody stuffs a JSON blob into a +header. - **Verified negative (#522).** The Anypoint LLM Gateway has no inbound cost-tag - ingestion — it meters cost from token usage per API instance and consuming - client application, not from a client header. So the `X-Anypoint-Cost-*` - request-header names are a forward-looking convention (harmless — nothing reads - them); the **authoritative** carrier is the `donkey.cost.*` OTel span attribute, - which this SDK controls end to end. The names stay overridable (`cost_*_header`) - for a gateway that does read one. + The Anypoint LLM Gateway does not ingest cost tags from request headers — it + meters cost from token usage per API instance and consuming client + application. The **authoritative** carrier is the `donkey.cost.*` OTel span + attribute. The `X-Anypoint-Cost-*` request headers are a convention nothing + currently reads; their names are overridable (`cost_*_header`) for a gateway + that does read one. -The tags are **validated** — fixed keys, bounded length — so nobody stuffs a -JSON blob into a header. - -## Acceptance bar - -- Zero-config: `Donkey.from_env()` plus `OTEL_EXPORTER_OTLP_ENDPOINT` produces - spans, with no SDK-specific environment variable. -- A **refused** request still produces a span, with - `donkey.policy.decision=refuse` and `otel.status_code=ERROR`. -- A streaming response produces **exactly one** span, with token counts filled - in at stream end. -- Opt-out behind a single flag, and under 1 ms of overhead — benchmarked in CI, - not asserted in prose. -- Every `DonkeyError` exposes `.correlation_id` (the run id) and `.call_id` (the - per-call id), each matching the header sent. +### The question this answers ---- +Finance asks what the support agent cost last month versus the HR bot. Without +tags, both agents share one `client_id` and there is no way to split the bill. +With tags it is a group-by. -**Status: Phase 1 — OpenTelemetry GenAI spans and OTLP export, per-run -correlation IDs, cost-attribution tags, and routing and resilience signals are -shipped. Platform-facing verification caveats are called out above.** +For compliance — *"prove the HR bot's answer to user X on date Y went through +the content-safety policy"* — the correlation ID on the log line joins to the +gateway record, and the span carries `enduser.id` and `donkey.policy.type`. An +EU AI Act Article 12 log request becomes one query, not an investigation. diff --git a/website/content/testing.mdx b/website/content/testing.mdx index a1709a12..619a4771 100644 --- a/website/content/testing.mdx +++ b/website/content/testing.mdx @@ -1,31 +1,34 @@ --- -description: In-process refusal injection, a pytest plugin you run against your own agent, and an out-of-process gateway fixture. +description: Test your agent's failure handling — in-process refusal injection, a pytest conformance suite you run against your own agent, and an out-of-process gateway fixture. --- import { Callout } from 'nextra/components' +import { Badge } from '../components' # Testing & conformance - - **Phase 1 — all three ship.** In-process refusal injection - (`donkey.simulate()`), the pytest conformance plugin - ([#191](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/191)), - and an out-of-process `gateway` fixture - ([#278](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/278)) - are all shipped. The conformance plugin runs your agent **in process** through - its own `Donkey`; the `gateway` fixture serves the simulator on a **real port** - for a subject that does not import the SDK. See [Roadmap](/roadmap) and - [Verification policy](/concepts/verification). - +Live -Three test-time tools, and the second one is the most important of the set. +Three test-time tools, all serving the same captured gateway rejections: + +- **`simulate()`** — make the next N calls through a `Donkey` return a chosen + refusal, in process, with no server. +- **The conformance plugin** — a pytest suite you point at *your own* agent to + find governance bugs you did not know you had. +- **The `gateway` fixture** — a real simulator on an ephemeral port, for + subjects that do not import the SDK, with a spy on what it received. + +```bash +pip install "donkey-kit[test]" # simulate() + conformance plugin +pip install "donkey-kit[test,local]" # add the gateway fixture +``` ## `simulate()` — in-process, no server A context manager that makes the next N calls through a `Donkey` return a -chosen refusal — the **same captured rejection fixture** `classify()` and the -[local simulator](/simulator) are tested against, so it lights up as exactly -that typed refusal, not a hand-rolled stand-in. No network, no server, fast +chosen refusal. It injects the **same captured rejection fixture** the error +classifier and the [local simulator](/simulator) use, so your agent sees exactly +that typed refusal — not a hand-rolled stand-in. No network, no server, fast enough for unit tests: ```python @@ -38,39 +41,37 @@ async def test_agent_masks_pii(): assert "****" in result.draft_reply ``` -`times=N` (default `1`) sets how many calls are refused before requests proceed -normally; retries of a single logical call count once, and the previous -transport is restored on exit even if the block raises (nesting composes). -Every injected response carries `x-donkey-simulator: true`, so a simulated -refusal is never mistaken for a real gateway response in a log or trace. - -The selectable refusals are the ones `classify()` produces from a captured -fixture: `TokenBudgetExceeded`, `PIIDetected`, `PromptInjectionBlocked`, -`ContentSafetyBlocked`, `UpstreamRequestError`, `UpstreamModelError`, `AuthError`, -and the generic `PolicyViolation` (the still-under-documented content-moderation -shape, [#253](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/253)). -Asking for a refusal with no captured fixture — e.g. a client-side -`ConfigError`, which no gateway response produces — is a clear `ValueError`, -never a silent miss. +- `times=N` (default `1`) sets how many calls are refused before requests + proceed normally. Retries of a single logical call count once. +- The previous transport is restored on exit, even if the block raises, and + nested `simulate()` blocks compose. +- Every injected response carries `x-donkey-simulator: true`, so a simulated + refusal is never mistaken for a real gateway response in a log or trace. + +The selectable refusals are the ones produced from a captured fixture: +`TokenBudgetExceeded`, `PIIDetected`, `PromptInjectionBlocked`, +`ContentSafetyBlocked`, `UpstreamRequestError`, `UpstreamModelError`, +`AuthError`, and the generic `PolicyViolation` (the content-moderation shape). +Asking for a refusal no gateway response produces — for example a client-side +`ConfigError` — raises a `ValueError`. - `simulate()` injects the fixture **verbatim**. Shaping the body — specific - PII entity types, a custom message, a shrinking budget window — is the - follow-up [#188](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/188), - shared with the simulator's scenario scripting. + `simulate()` injects the fixture verbatim. For scripted behaviour such as + every-Nth-call PII blocks or a shrinking budget window, use the + [`gateway` fixture](#the-gateway-fixture) with + `set_scenarios(...)`, or [`donkey mock --scenario`](/simulator#scenario-scripting). -## The conformance plugin — run it against *your* agent +## The conformance plugin — run it against your agent -This is the part that matters. `pip install "donkey-kit[test]"` publishes a -pytest plugin exposing a `donkey` fixture and a conformance suite you point at -your own agent: +Installing `donkey-kit[test]` registers a pytest plugin with a conformance suite +you point at your own agent factory: ```bash pytest --donkey-conformance --agent=my_app.agent:build ``` -It asks questions you probably cannot currently answer about your own code: +It answers questions you probably cannot currently answer about your own code: - Does your agent **retry** a `TokenBudgetExceeded`? *It must not* — a policy refusal is terminal, and retrying it burns budget to earn the same refusal. @@ -78,20 +79,40 @@ It asks questions you probably cannot currently answer about your own code: - Does it propagate the correlation ID into its own logs? - Does it still work when budget headers are absent entirely? -The output is a table of scenario → pass / fail / **exempt**. +Each scenario becomes one pytest item, and the run prints a table of +scenario → pass / fail / **exempt**. A failure reads as a finding about your +agent — for example, *"your agent retried a budget refusal 3 times"* — found in +CI, before production. + +The factory is called once per scenario and receives the `donkey` fixture if it +declares one. With `--donkey-conformance`, the plugin runs the suite +exclusively in place of normal test collection; without the flag it is inert. +You can also run it through the CLI with `donkey test --agent my_app.agent:build` +(see [CLI](/cli#donkey-test)). + +### Exemptions + + + **Exemptions are asserted, never silently skipped.** If your agent + legitimately cannot satisfy a scenario, record it in a `KNOWN_LIMITATIONS` + mapping of `{scenario: reason}` as an explicit, reviewable claim. + -## The `gateway` fixture — an out-of-process server tests assert against +By default the plugin reads a `KNOWN_LIMITATIONS` attribute from the `--agent` +module; point elsewhere with `--donkey-known-limitations=module:NAME`. The +mapping is validated at collection time, so an unknown scenario key or an empty +reason fails the run before any scenario executes. -`simulate()` and the conformance plugin both run your subject **in process**. -Some subjects cannot be: a containerised agent, a Node service, an A2A client, a -manual `curl` — none of which import `donkey_kit`. Those need a real listener on a -real port, and they need the other half of the question `simulate()` cannot -answer: *what did the gateway actually receive?* "Did my agent stop after the +## The `gateway` fixture + +`simulate()` and the conformance plugin run your subject **in process**. Some +subjects cannot be: a containerised agent, a Node service, an A2A client, a +manual `curl`. Those need a real listener on a real port — and often you need +to know *what the gateway actually received*. "Did my agent stop after the refusal, or retry four more times?" is answerable only from the gateway's side. -The `gateway` fixture (shipped in `donkey-kit[test]`, beside `donkey`) boots the -[local simulator](/simulator) on an **ephemeral port**, hands your test its -`.url`, and records every request in a spy you can assert on: +The `gateway` fixture boots the [local simulator](/simulator) on an ephemeral +port, hands your test its `.url`, and records every request: ```python async def test_agent_stops_after_a_refusal(gateway): @@ -103,43 +124,19 @@ async def test_agent_stops_after_a_refusal(gateway): ``` - `gateway.url` is a real `http://127.0.0.1:` any process can point - `DONKEY_LLM_PROXY_URL` at — it does not import the SDK. + `DONKEY_LLM_PROXY_URL` at. - The port is bound to `0`, so parallel `pytest -n` runs never collide. - `gateway.requests_received` counts requests; `gateway.requests` exposes each - one's method, path and headers, with `client_secret` **redacted**. -- `gateway.set_scenarios(...)` arms the same - [scenario scripting](/simulator#scenario-scripting) (`pii_block`, `injection`, - `budget`) **per test**, not just per process. -- Every response carries `x-donkey-simulator: true`, so a fixture reply is never - mistaken for a real gateway. + one's method, path, and headers, with `client_secret` **redacted**. +- `gateway.set_scenarios(...)` arms [scenario scripting](/simulator#scenario-scripting) + (`pii_block`, `injection`, `budget`) **per test**. +- Every response carries `x-donkey-simulator: true`. - The server is torn down when the test exits, including on failure. -It serves the **same captured rejection fixtures** as `simulate()` and the -conformance plugin, so the three cannot drift apart. Because it starts a real -server, it needs the `[local]` extra (`starlette` + `uvicorn`); taking the -fixture without it raises an `ImportError` naming the exact `pip install`. - -## Why this is the moment the SDK earns trust - -A team installs the SDK and on day one gets a failing test that says *"your -agent retried a budget refusal 3 times."* - -That is a real bug they did not know they had, found in CI, phrased in a way -nobody argues with. It is worth more than any feature list, because it is the -SDK telling the team something true about their own code rather than asking -them to take a claim on faith. - - - **Exemptions are asserted, never silently skipped.** If your agent - legitimately cannot satisfy a scenario, it is recorded in - `KNOWN_LIMITATIONS` as an explicit, reviewable claim. A quiet - `pytest.skip` would turn the suite into decoration — the same rule this - repo already applies to its own adapters. - - -## How this differs from the simulator +The fixture needs the `[local]` extra (Starlette + Uvicorn); requesting it +without that extra raises an `ImportError` naming the exact `pip install`. -They all exist, they share fixtures, and they are not the same tool: +## Choosing a tool | | [`donkey mock`](/simulator) | `gateway` fixture | `simulate()` | |---|---|---|---| @@ -150,30 +147,4 @@ They all exist, they share fixtures, and they are not the same tool: | Works with a stock OpenAI client | Yes | Yes | No — it hooks the `Donkey` transport | | Asserts on what the gateway received | No | Yes (`requests_received`) | No | -They read the **same rejection fixtures**, so they cannot drift apart. - -## Acceptance bar - -- `simulate()` works with no network and no server running. ✅ -- `times=N` is honoured exactly; call N+1 proceeds normally, and nesting or an - exception restores the previous transport. ✅ -- It replays the *same fixtures* `classify()` and the simulator are tested - against, so the three fail together if the gateway contract ever drifts. ✅ -- The plugin registers as a pytest entry point, and - `pytest --donkey-conformance --agent=my_app.agent:build` turns each scenario - into one pytest item and prints a scenario table. ✅ -- Exemptions must be asserted in code — a `KNOWN_LIMITATIONS` mapping next to - your factory (or `--donkey-known-limitations=module:NAME`), validated at - collection time so a bad key or empty reason fails the run loudly; a silent - skip is never possible. ✅ -- The `gateway` fixture binds an ephemeral port, exposes `.url` to a process that - does not import `donkey_kit`, records requests with `client_secret` redacted, - takes scenarios per test, and tears down on exit. ✅ - ---- - -**Status: Phase 1 — `simulate()`, the conformance plugin -([#191](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/191)), -and the `gateway` fixture -([#278](https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/278)) -are all shipped.** +All three read the **same rejection fixtures**, so they cannot drift apart. diff --git a/website/content/tool-access/a2a.mdx b/website/content/tool-access/a2a.mdx index f079d2a6..f99e3a24 100644 --- a/website/content/tool-access/a2a.mdx +++ b/website/content/tool-access/a2a.mdx @@ -1,11 +1,19 @@ -import { Callout } from 'nextra/components' +--- +description: Bind a governed A2A agent from the registry as a native tool, so your agent can delegate to it without learning the A2A protocol. +--- + +import { Badge } from '../../components' # A2A agent tools +Roadmap + +This capability is on the [Roadmap](/roadmap); the API shown here is the planned design. + Agent Broker is an A2A server, and A2A-compliant agents in the registry can be -consumed directly from Python. `AgentHandle.as_tool()` wraps a remote A2A -agent as a callable tool in each framework — the same "resolve, then bind" -shape as MCP tools, just for a remote agent instead of a remote tool server. +consumed directly from Python. `AgentHandle.as_tool()` wraps a remote A2A agent +as a callable tool in your framework — the same "resolve, then bind" shape as +MCP tools, for a remote agent instead of a remote tool server. ```python handle = await donkey.registry.resolve_agent("com.acme/claims-triage-agent/1.0.0") @@ -14,15 +22,14 @@ tool = handle.as_tool() # callable in your framework's native tool shap agent = create_react_agent(donkey.langgraph.chat_model("gpt-4o"), [tool]) ``` -This is what lets a Python agent **delegate** to an Agent Broker agent without -the developer learning the A2A protocol. Under the hood, `as_tool()` uses the -community `a2a-sdk` for the protocol handling — the SDK does not implement -JSON-RPC itself. +This lets a Python agent **delegate** to an Agent Broker agent without you +learning the A2A protocol. Under the hood, `as_tool()` uses the official +`a2a-sdk` for protocol handling. ## Install -A2A support is secondary but cheap, and ships behind its own extra so it -doesn't add a dependency to installs that only need MCP tools: +A2A support ships behind its own extra, so installs that only need MCP tools +carry no extra dependency: ```bash pip install "donkey-kit[a2a]" @@ -34,11 +41,5 @@ pip install "donkey-kit[a2a]" registry surface that resolves MCP servers into `McpServerHandle`s. See [Discovery, search & filter](/tool-access/discovery). - `as_tool()` returns a framework-native callable, following the same - per-framework binding conventions as [Framework binding](/tool-access/binding). - - - A2A agent tools are in scope alongside the rest of Phase 2, but - are lower priority than MCP discovery and binding — see - [Verification policy](/concepts/verification) for what's confirmed versus - designed. - + conventions as [Framework binding](/tool-access/binding). +- To make *your* agent callable by others over A2A, see [A2A agents](/a2a). diff --git a/website/content/tool-access/binding.mdx b/website/content/tool-access/binding.mdx index 505ea690..4dcd5d94 100644 --- a/website/content/tool-access/binding.mdx +++ b/website/content/tool-access/binding.mdx @@ -1,30 +1,39 @@ +--- +description: Turn discovered governed MCP servers into each framework's native tool objects, with auth, lifecycle, name collisions, and filtering handled for you. +--- + import { Callout } from 'nextra/components' +import { Badge } from '../../components' # Framework binding -Once [discovery](/tool-access/discovery) hands you a `ToolSet`, binding is the -step that turns governed MCP servers into a framework's own tool objects — -nothing wrapped, nothing re-implemented. +Roadmap + +This capability is on the [Roadmap](/roadmap); the API shown here is the planned design. + +Once [discovery](/tool-access/discovery) hands you a `ToolSet`, binding turns +governed MCP servers into a framework's own tool objects — nothing wrapped, +nothing re-implemented. ## MCP session management MCP servers created by MCP Bridge are gateway endpoints speaking streamable HTTP, protected by gateway policies. The SDK's session layer handles four -things so you don't have to: +things for you: - **Auth.** Client-credentials OAuth is the machine-to-machine case. Strands' `MCPClient` already builds streamable HTTP with a `client_credentials` grant internally; every other framework needs headers supplied explicitly. - `McpServerHandle.auth_headers()` returns a ready-to-use header dict, and it - is refreshed automatically on a `401`. + `McpServerHandle.auth_headers()` returns a ready-to-use header dict, refreshed + automatically on a `401`. - **Connection lifecycle.** MCP clients are stateful, and several frameworks connect lazily. `donkey.tools.discover()` never opens a connection — it - returns handles. The underlying connection opens on first tool use. + returns handles, and the connection opens on first tool use. - **Multi-server aggregation.** `ToolSet` wraps N `McpServerHandle`s. When two servers expose a tool with the same name, the collision is resolved by - prefixing with the server's short name — for example `hr__get_employee` — - and the mapping is surfaced on `ToolSet.name_map` so you can see exactly why - the model called that name. + prefixing the server's short name — for example `hr__get_employee` — and the + mapping is available on `ToolSet.name_map`, so you can see exactly why the + model called that name. - **Filtering.** Enterprise MCP servers can expose dozens of tools. Handing 60 tool descriptors to a model degrades it and inflates token cost, so filter before you bind: @@ -45,20 +54,8 @@ filtered = tools.filter(predicate=lambda t: t.name.startswith("get_")) ## Per-framework binding -| Framework | Binding | -|---|---| -| LangGraph | `langchain_mcp_adapters.client.MultiServerMCPClient({...}).get_tools()` — the SDK builds the connection dict from your handles, transport `"streamable_http"`, headers injected. | -| Google ADK | `McpToolset(connection_params=StreamableHTTPConnectionParams(url=..., headers=...), tool_filter=[...])`, passed straight into `LlmAgent(tools=[...])`. | -| MS Agent Framework | its MCP client/tool class for streamable HTTP (name pending live verification); the framework's own docs reference hosted MCP tools and MCP clients for tool integration. | -| OpenAI Agents SDK | `agents.mcp.MCPServerStreamableHttp(params={"url": ..., "headers": ...})`, passed into `Agent(mcp_servers=[...])` (pending live verification). | -| Anthropic SDK | no native client-side MCP binding in the `anthropic` SDK itself; bind via the MCP Python SDK's streamable-HTTP client and pass the resulting tool schemas to `messages.create(tools=...)` (pending live verification). | -| CrewAI | its MCP adapter for streamable-HTTP servers (name pending live verification), yielding native `crewai` tool objects for a `Crew`/`Agent`. | -| LlamaIndex | `llama_index.tools.mcp.BasicMCPClient` + `McpToolSpec(...).to_tool_list_async()`. | -| Strands | `MCPClient(lambda: streamablehttp_client(url, headers=...))` — implements `ToolProvider`, so it can be passed directly into `Agent(tools=[...])` with automatic lifecycle management. | - -Each binding returns the framework's **native** tool type — a LangGraph -`BaseTool`, an ADK `McpToolset`, a Strands `MCPClient`, and so on. `ToolSet` -exposes one method per installed integration: +`ToolSet` exposes one method per installed integration, each returning the +framework's **native** tool type: ```python ts = await donkey.tools.discover(domain="hr") @@ -69,16 +66,27 @@ ts.llamaindex() # -> list[FunctionTool] # etc. ``` -Note the shape difference: ADK and Strands want a toolset/provider object, -LangGraph and LlamaIndex want a flat tool list. The SDK does not force a -uniform return type — each method matches its framework's own idiom, and the -difference is called out in that method's docstring. - - - The proxy contract discovery and session management depend on is designed - against MCP Bridge, but the binding classes marked "pending live - verification" above have not yet been confirmed against an installed - framework. See [Verification policy](/concepts/verification) for how these - get unblocked, and the [error taxonomy](/errors) for how a binding failure - or a `401` surfaces as a typed exception. - +Note the shape difference: ADK and Strands take a toolset/provider object, +while LangGraph and LlamaIndex take a flat tool list. Each method matches its +framework's own idiom rather than forcing a uniform return type, and its +docstring calls out the difference. + +| Framework | Binding | +|---|---| +| LangGraph | `langchain_mcp_adapters.client.MultiServerMCPClient({...}).get_tools()` — the SDK builds the connection dict from your handles, transport `"streamable_http"`, headers injected. | +| Google ADK | `McpToolset(connection_params=StreamableHTTPConnectionParams(url=..., headers=...), tool_filter=[...])`, passed straight into `LlmAgent(tools=[...])`. | +| MS Agent Framework | The framework's MCP client/tool class for streamable HTTP. | +| OpenAI Agents SDK | `agents.mcp.MCPServerStreamableHttp(params={"url": ..., "headers": ...})`, passed into `Agent(mcp_servers=[...])`. | +| Anthropic SDK | The `anthropic` SDK has no native client-side MCP binding; the SDK binds via the MCP Python SDK's streamable-HTTP client and passes the resulting tool schemas to `messages.create(tools=...)`. | +| CrewAI | The framework's MCP adapter for streamable-HTTP servers, yielding native `crewai` tool objects for a `Crew`/`Agent`. | +| LlamaIndex | `llama_index.tools.mcp.BasicMCPClient` + `McpToolSpec(...).to_tool_list_async()`. | +| Strands | `MCPClient(lambda: streamablehttp_client(url, headers=...))` — implements `ToolProvider`, so it can be passed directly into `Agent(tools=[...])` with automatic lifecycle management. | + +A binding failure or a `401` surfaces as a typed exception from the +[error taxonomy](/errors). + +## Related + +- [Discovery, search & filter](/tool-access/discovery) — produce the `ToolSet`. +- [A2A agent tools](/tool-access/a2a) — bind a remote agent the same way. +- [Frameworks](/frameworks) — governed model access per framework. diff --git a/website/content/tool-access/discovery.mdx b/website/content/tool-access/discovery.mdx index 9c4cdec2..0b9252f8 100644 --- a/website/content/tool-access/discovery.mdx +++ b/website/content/tool-access/discovery.mdx @@ -1,10 +1,19 @@ +--- +description: Find governed MCP tools by name, governance criteria, domain, tags, asset type, and environment — and see why any tool was included or excluded. +--- + import { Callout } from 'nextra/components' +import { Badge } from '../../components' # Discovery, search & filter +Roadmap + +This capability is on the [Roadmap](/roadmap); the API shown here is the planned design. + `donkey.tools.discover(...)` is the one entry point for finding governed tools. It narrows the catalog by **name/description (search)**, **governance**, domain, -tags, asset type, and environment — so an agent binds only the tools it needs +tags, asset type, and environment, so an agent binds only the tools it needs rather than the entire catalog. It returns a [`ToolSet`](/tool-access/binding) whose per-framework methods hand back native tool objects. @@ -19,7 +28,7 @@ tools = await donkey.tools.discover(governed_only=True, search="*accounts*") tools = await donkey.tools.discover(domain="hr", governed_only=True) ``` -## The full filter surface +## Filter reference Every argument is optional and combines with `AND` semantics: @@ -41,7 +50,7 @@ tools = await donkey.tools.discover( | `governed_only` | `bool \| GovernanceCriteria \| None` | `True` applies the default criteria; pass a `GovernanceCriteria` (e.g. `STRICT`) for explicit rules; `None` = unfiltered. | | `domain` | `str \| None` | Catalog domain. | | `tags` | `list[str] \| None` | All listed tags must be present. | -| `asset_types` | `list[AssetType] \| None` | e.g. `["mcp"]`; exact Exchange filter tokens remain unverified. | +| `asset_types` | `list[AssetType] \| None` | Restrict by asset type, e.g. `["mcp"]`. | | `environment` | `str \| None` | Which environment governance is computed against. | | `limit` | `int` | Max results (default 50). | @@ -49,20 +58,17 @@ tools = await donkey.tools.discover( `discover(...)` is the high-level facade over `ExchangeRegistry.search()`. On the facade, `search` is the text/glob filter (the registry's `query`) and `governed_only` is the governance predicate (the registry's `governed`). The - glob is applied server-side where Exchange supports a wildcard/substring query, - and client-side otherwise — the behaviour you see is identical either way. - Which path Exchange supports is a live-verification item; see - [Verification policy](/concepts/verification). + glob runs server-side where Exchange supports it and client-side otherwise — + the results are identical either way. -## "Governed" is a computed predicate, not a flag +## "Governed" is a computed predicate Publication to Exchange says nothing about whether an asset is fronted by a gateway, has policies applied, or passes the org's rulesets — there is no single -boolean to query. "Governed" is **computed** by joining state across systems, and -it is **environment-scoped**: an asset governed in Production may be ungoverned in -Sandbox. `GovernanceCriteria` makes every condition explicit and independently -checkable: +boolean to query. "Governed" is **computed** by joining state across systems, +and it is **environment-scoped**: an asset governed in Production may be +ungoverned in Sandbox. `GovernanceCriteria` makes every condition explicit: ```python from donkey_kit.registry.governance import GovernanceCriteria, STRICT @@ -88,23 +94,23 @@ STRICT = GovernanceCriteria( ) ``` -`governed_only=STRICT` passes it straight through: +Pass it straight through: ```python tools = await donkey.tools.discover(domain="hr", governed_only=STRICT) ``` - `allow_unknown` matters more than it looks. If the platform doesn't expose, say, - ruleset results, then `require_governance_pass=True` with `allow_unknown=False` - silently filters the whole catalog to zero — so every filtered-out asset carries - a **reason**, surfaced by `explain()`. + `allow_unknown` matters more than it looks. If the platform doesn't expose, + say, ruleset results, then `require_governance_pass=True` with + `allow_unknown=False` filters the whole catalog to zero — so every + filtered-out asset carries a **reason**, surfaced by `explain()`. ## `explain()` — why a tool was included or excluded -`governed_only=True` returning an empty list is otherwise indistinguishable from a -broken credential. `explain()` is a first-class, always-available method: +Without it, `governed_only=True` returning an empty list is indistinguishable +from a broken credential. `explain()` reports every check: ```python report = await donkey.registry.explain(ref, criteria=STRICT) @@ -118,20 +124,18 @@ report = await donkey.registry.explain(ref, criteria=STRICT) Each check is `True` (passed), `False` (failed, with the reason), or `None` (couldn't be evaluated — resolved via `allow_unknown`). The empty-result warning -message references `explain()` directly. +message points you to `explain()`. ## Defaults and performance -- **Unfiltered by default.** `governed_only` defaults to `None` in v1, with a - startup log line stating discovery is unfiltered. Flipping the default to `True` - is a breaking change reserved for a later major version. +- **Unfiltered by default.** `governed_only` defaults to `None`, and a startup + log line states that discovery is unfiltered. Changing the default to `True` + is reserved for a future major version. - **Warm the index.** The governance join is a bulk operation, not one API call - per asset. A long-running agent process can build the index at startup with - `donkey.registry.warm(environment=...)` so first discovery is fast. + per asset. A long-running agent can build the index at startup with + `donkey.registry.warm(environment=...)` so the first discovery is fast. ## Related - [Framework binding](/tool-access/binding) — turn a `ToolSet` into native tools. -- [Pinning & lockfile](/tool-access/lockfile) — pin resolved versions for prod. -- [Governance & environments](/concepts/environments) — the `Governance` object - you can pass as `governance=` to scope discovery to a target. +- [Pinning & lockfile](/tool-access/lockfile) — pin resolved versions for production. diff --git a/website/content/tool-access/index.mdx b/website/content/tool-access/index.mdx index bada8e37..a9bd556e 100644 --- a/website/content/tool-access/index.mdx +++ b/website/content/tool-access/index.mdx @@ -1,26 +1,21 @@ --- -description: Phase 2 — discover governed MCP tools and bind them into any framework as native tool objects. +description: Discover the governed MCP tools your organisation publishes in Exchange and bind them into any framework as native tool objects. --- -import { Callout } from 'nextra/components' -import { Card, CardGrid } from '../../components' +import { Badge, Card, CardGrid } from '../../components' # Tool access - - **Phase 2 — designed, not yet shipped.** This documents governed tool access as designed in the build guide. The platform contracts it depends on are being live-verified; until then, unverified paths raise `NotImplementedError("blocked on verification: …")` rather than guess. See [Verification policy](/concepts/verification). - +Roadmap -Governed tool access is the differentiating feature: discover the MCP tools your -organisation has published and governed, filter them down to what an agent -actually needs, and bind them into any of the eight frameworks as that -framework's **native tool objects** — the same "no wrapper" rule as -[model access](/frameworks). +This capability is on the [Roadmap](/roadmap); the API shown here is the planned design. -## The target developer experience +Governed tool access lets an agent discover the MCP tools your organisation has +published and governed, filter them down to what it actually needs, and bind +them into any of the eight supported frameworks as that framework's **native +tool objects** — the same "no wrapper" approach as [model access](/frameworks). -Two lines take you from "our enterprise has a governed tool catalog" to "my -LangGraph agent can use it": +## Two lines from catalog to agent ```python tools = await donkey.tools.discover(domain="hr", tags=["approved"]) @@ -28,15 +23,14 @@ agent = create_react_agent(donkey.langgraph.chat_model("gpt-4o"), tools.langgrap ``` Everything else in this section — search filters, session management, -per-framework binding, pinning, and A2A tool handles — exists to make those -two lines hold up in production. +per-framework binding, pinning, and A2A tool handles — makes those two lines +hold up in production. -## What discovery returns +## Discover and filter -`donkey.tools.discover(...)` is also the filter and search entry point. The -same call narrows the catalog by name/description glob, governance, domain, -tags, and asset type, so an agent binds only the tools it needs instead of the -whole catalog: +`donkey.tools.discover(...)` is also the search and filter entry point. One call +narrows the catalog by name/description glob, governance, domain, tags, and +asset type, so an agent binds only the tools it needs: ```python tools = await donkey.tools.discover( @@ -50,11 +44,8 @@ tools = await donkey.tools.discover( ) ``` -`governed_only` and the `GovernanceCriteria` vocabulary are covered on the -[Discovery, search & filter](/tool-access/discovery) page — "governed" is a -computed predicate, not a flag Exchange exposes directly. See also the -[environments and governance object](/concepts/environments) page for how -governance is scoped per environment. +"Governed" is a computed, environment-scoped predicate rather than a flag in +Exchange — see [Discovery, search & filter](/tool-access/discovery). ## In this section @@ -72,5 +63,3 @@ governance is scoped per environment. Treat a governed agent-to-agent endpoint as another bindable tool. - -**Status: planned — not yet shipped.** diff --git a/website/content/tool-access/lockfile.mdx b/website/content/tool-access/lockfile.mdx index 7ee904da..b0ec696b 100644 --- a/website/content/tool-access/lockfile.mdx +++ b/website/content/tool-access/lockfile.mdx @@ -1,18 +1,27 @@ +--- +description: Pin governed tool versions and content digests in a donkey.lock so catalog changes become reviewable diffs, not runtime surprises. +--- + import { Callout } from 'nextra/components' +import { Badge } from '../../components' # Pinning & lockfile -Governed tool catalogs change under you. A platform team edits an MCP -server's tool schema, bumps a policy, or republishes an asset — and if your -agent resolves `version="latest"` at startup, that change silently alters -agent behaviour in production. Nobody notices until the agent starts calling -a tool with a different signature. +Roadmap + +This capability is on the [Roadmap](/roadmap); the API shown here is the planned design. + +Governed tool catalogs change under you. A platform team edits an MCP server's +tool schema, bumps a policy, or republishes an asset — and if your agent +resolves `version="latest"` at startup, that change silently alters agent +behaviour in production. Pinning and the lockfile keep what your agent binds +under version control. ## Pin by default -`donkey.tools.discover()` defaults to requiring an explicit version on every -asset reference. `version="latest"` is still allowed, but only with a logged -warning — it is an opt-in escape hatch, not the default path: +`donkey.tools.discover()` requires an explicit version on every asset reference +by default. `version="latest"` is allowed, but logs a warning — it is an opt-in +escape hatch, not the default path: ```python # explicit — no warning @@ -25,8 +34,8 @@ tools = await donkey.tools.discover(domain="hr", version="latest") ## The lockfile `donkey.tools.lock()` resolves the current discovery call and writes a -`donkey.lock` file recording the resolved versions and content digests of -every asset it touched: +`donkey.lock` file recording the resolved versions and content digests of every +asset it touched: ```bash python -c "import asyncio; from donkey_kit import Donkey; asyncio.run(Donkey.from_env().tools.lock())" @@ -42,36 +51,33 @@ assets: digest: sha256:... ``` -Once a `donkey.lock` exists, pass `locked=True` to discovery and it refuses to -resolve anything not already in the lockfile: +Once a `donkey.lock` exists, pass `locked=True` and discovery refuses to resolve +anything not already in the lockfile: ```python tools = await donkey.tools.discover(domain="hr", locked=True) # raises if discovery would resolve an asset/version not in donkey.lock ``` -Commit `donkey.lock` alongside your agent code. A version bump becomes a -reviewable diff in a pull request instead of a runtime surprise. - - Teams shipping agents to production need this, and it's the kind of thing - nobody else provides for a governed tool catalog — treat it as a required - step before a deploy, not an optional extra. + Commit `donkey.lock` alongside your agent code and treat it as a required step + before deploying. A version bump then becomes a reviewable diff in a pull + request instead of a runtime surprise. ## Registry caching Registry lookups (`ExchangeRegistry.search()`, `resolve_mcp()`, `resolve_agent()` — see [Discovery, search & filter](/tool-access/discovery)) -are cached with a configurable TTL: +are cached with a configurable TTL (default 300 seconds): ```python donkey = Donkey.from_env(registry_cache_ttl_s=300) ``` -- `donkey.registry.refresh()` invalidates the cache and re-fetches on next - use — call it after a platform team publishes a change you need to see - immediately. +- `donkey.registry.refresh()` invalidates the cache and re-fetches on next use — + call it after a platform team publishes a change you need to see immediately. +- Set `DONKEY_REGISTRY_CACHE_TTL_S` to change the TTL from the environment. - Set `DONKEY_NO_CACHE=1` to bypass the cache entirely, for debugging a discovery result that looks stale. diff --git a/website/content/use-with-your-agent.mdx b/website/content/use-with-your-agent.mdx index bb9fd016..4fc2a126 100644 --- a/website/content/use-with-your-agent.mdx +++ b/website/content/use-with-your-agent.mdx @@ -3,13 +3,16 @@ description: Point your coding assistant at these docs — llms.txt, llms-full.t --- import { Callout, Tabs } from 'nextra/components' +import { Badge } from '../components' # Use these docs with your agent +Live + You probably ask your coding assistant before you open a docs site. These docs are published in the [llms.txt convention](https://llmstxt.org) so an assistant -can read them directly and write **correct, governed** code — the verified base -URL with no `/v1`, the `client_id` / `client_secret` header pair, the typed +can read them directly and write **correct, governed** code — the base URL with +no `/v1`, the `client_id` / `client_secret` header pair, the typed refusal taxonomy — instead of guessing. ## What's published @@ -81,9 +84,8 @@ https://donkey-development-kit.github.io/donkey-development-kit/llms.txt - These docs describe both live-verified surfaces and design-stage ones. The - artifacts reproduce the pages verbatim, including each page's verification - status — so an assistant sees the same "blocked on verification" notes you do. - Sanity-check generated code against the [Verification policy](/concepts/verification) - before running it against a real gateway. + These docs cover both live capabilities and ones on the [Roadmap](/roadmap). + Roadmap pages show the planned API, which is not callable yet. When you ask an + assistant to write code, tell it to use only capabilities marked **Live**, and + review generated code before running it against a real gateway. diff --git a/website/data/roadmap.json b/website/data/roadmap.json new file mode 100644 index 00000000..780ba335 --- /dev/null +++ b/website/data/roadmap.json @@ -0,0 +1,1686 @@ +{ + "repo": "Donkey-Development-Kit/donkey-development-kit", + "generatedAt": "2026-09-24T09:35:41.948Z", + "milestones": [ + { + "number": 11, + "title": "Phase 1 — Build the MVP (0.1.0)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/milestone/11", + "open": 15, + "closed": 113, + "issues": [ + { + "number": 9, + "title": "Epic: Testing — the adapter conformance kit, contract fixtures, nightly matrix", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/9", + "epic": true + }, + { + "number": 10, + "title": "core/config: DonkeyConfig resolution order and all-at-once validation", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/10", + "epic": false + }, + { + "number": 11, + "title": "core/transport: the single shared HTTP client that injects every governed header", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/11", + "epic": false + }, + { + "number": 12, + "title": "core/auth: AuthProvider protocol, AnypointConnectedApp, StaticToken, ChainedAuth", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/12", + "epic": false + }, + { + "number": 13, + "title": "core/errors: the governed error taxonomy and classify()", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/13", + "epic": false + }, + { + "number": 14, + "title": "core/telemetry: correlation IDs, run_context, and optional OpenTelemetry spans", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/14", + "epic": false + }, + { + "number": 16, + "title": "Donkey facade: lazy per-framework adapters, curated ImportError, async and sync lifecycle", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/16", + "epic": false + }, + { + "number": 17, + "title": "core: keep core framework-free — import-linter contracts as a blocking CI gate", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/17", + "epic": false + }, + { + "number": 18, + "title": "llm: donkey.llm.client() — the framework-free governed OpenAI client (async + blocking overloads)", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/18", + "epic": false + }, + { + "number": 24, + "title": "adapter/langgraph: chat_model() returning a native ChatOpenAI (Tier 1)", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/24", + "epic": false + }, + { + "number": 33, + "title": "adapters: connection_kwargs() and module-level factories across all eight frameworks", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/33", + "epic": false + }, + { + "number": 179, + "title": "transport: add the four lifecycle hooks every Phase 1 feature attaches to", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/179", + "epic": false + }, + { + "number": 180, + "title": "donkey: Donkey.openai() returns a native openai.AsyncOpenAI", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/180", + "epic": false + }, + { + "number": 181, + "title": "errors: re-align classify() to the documented v1.11-v1.13 rejection contracts", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/181", + "epic": false + }, + { + "number": 182, + "title": "errors: require a non-empty remediation on every PolicyViolation", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/182", + "epic": false + }, + { + "number": 183, + "title": "errors: prove policy refusals are terminal and openai's built-in 429 retry never fires", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/183", + "epic": false + }, + { + "number": 184, + "title": "errors: honest fall-through for guardrail verdicts whose shape is unconfirmed", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/184", + "epic": false + }, + { + "number": 185, + "title": "budget: Budget as a first-class object parsed from x-token-* headers", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/185", + "epic": false + }, + { + "number": 186, + "title": "budget: wait_for_reset() and pace(reserve=) raising BudgetReserveReached", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/186", + "epic": false + }, + { + "number": 187, + "title": "simulator: donkey mock — a local HTTP server that replays real rejection fixtures", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/187", + "epic": false + }, + { + "number": 188, + "title": "simulator: scenario scripting for PII, budget and injection", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/188", + "epic": false + }, + { + "number": 189, + "title": "simulator: the honesty guarantee — x-donkey-simulator header and a stock-client test", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/189", + "epic": false + }, + { + "number": 190, + "title": "testing: donkey.simulate() — in-process refusal injection with no server", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/190", + "epic": false + }, + { + "number": 191, + "title": "testing: the pytest plugin and the conformance suite customers run against their own agent", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/191", + "epic": false + }, + { + "number": 192, + "title": "telemetry: OTel GenAI spans at a pinned semconv version, dual-emitted with a stable donkey.* namespace", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/192", + "epic": false + }, + { + "number": 193, + "title": "telemetry: a refused request still produces a span, and streaming produces exactly one", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/193", + "epic": false + }, + { + "number": 194, + "title": "telemetry: zero-config OTLP export, one-flag opt-out, <1ms overhead benchmarked in CI", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/194", + "epic": false + }, + { + "number": 195, + "title": "telemetry: per-call and per-run correlation IDs via donkey.run()", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/195", + "epic": false + }, + { + "number": 196, + "title": "telemetry: validated cost-attribution tags (team, project, env, enduser.id)", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/196", + "epic": false + }, + { + "number": 197, + "title": "adapters: demote seven adapters to connection_kwargs-only and cut the conformance roster to LangGraph", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/197", + "epic": false + }, + { + "number": 198, + "title": "adapter/langgraph: deepen to the one conformance-gated adapter", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/198", + "epic": false + }, + { + "number": 199, + "title": "demos: langgraph-support-triage implements Scenario A end-to-end against the simulator", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/199", + "epic": false + }, + { + "number": 200, + "title": "decorators: @donkey.governed and @donkey.tool", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/200", + "epic": false + }, + { + "number": 201, + "title": "cli: the donkey command surface — init, plus consistent flags and exit codes across all four", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/201", + "epic": false + }, + { + "number": 202, + "title": "cli: donkey doctor — tell wrong credentials from wrong URL from model-not-allowed", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/202", + "epic": false + }, + { + "number": 203, + "title": "docs: a 15-minute quickstart that needs no gateway", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/203", + "epic": false + }, + { + "number": 204, + "title": "docs: scenario pages, the refusal cookbook, and per-framework connection_kwargs pages", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/204", + "epic": false + }, + { + "number": 205, + "title": "docs: publish llms.txt so coding assistants can read the SDK docs", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/205", + "epic": false + }, + { + "number": 206, + "title": "release: publish to PyPI with semver and a changelog", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/206", + "epic": false + }, + { + "number": 207, + "title": "meta: replace the build plan with the phase-based plan and realign CLAUDE.md and the skills", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/207", + "epic": false + }, + { + "number": 259, + "title": "epic: Phase 1 — Build the MVP", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/259", + "epic": true + }, + { + "number": 266, + "title": "docs: migrate the ~497 legacy §N.N citations to BG §N.N and retire spec/archive", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/266", + "epic": false + }, + { + "number": 267, + "title": "docs: realign the docs site to the phase model and cut the retired surfaces", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/267", + "epic": false + }, + { + "number": 270, + "title": "docs: drop the Omni Gateway architecture image from the feature overview", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/270", + "epic": false + }, + { + "number": 273, + "title": "Remove demos/ after moving it to a dedicated companion repo", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/273", + "epic": false + }, + { + "number": 277, + "title": "adapters: free donkey.openai by renaming the Agents SDK adapter", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/277", + "epic": false + }, + { + "number": 278, + "title": "testing: an out-of-process gateway fixture tests can assert against", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/278", + "epic": false + }, + { + "number": 281, + "title": "Git-ignore the .playwright-mcp/ local MCP artifact directory", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/281", + "epic": false + }, + { + "number": 285, + "title": "mypy --strict fails on core/telemetry.py get_tracer under opentelemetry-api 1.44", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/285", + "epic": false + }, + { + "number": 286, + "title": "Negative Retry-After header makes the retry sleep raise ValueError", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/286", + "epic": false + }, + { + "number": 289, + "title": "errors: type the documented Regex-Prompt-Guard and Content-Safety refusals; reconcile PIIDetected.entities", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/289", + "epic": false + }, + { + "number": 292, + "title": "Realign ARCHITECTURE.md with the code and add an implement-stage skill", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/292", + "epic": false + }, + { + "number": 295, + "title": "testing: end-to-end pace→resume test against the local gateway simulator", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/295", + "epic": false + }, + { + "number": 296, + "title": "Codify \"deferred scope → dedicated follow-up issue\" as a stated rule in the skills", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/296", + "epic": false + }, + { + "number": 306, + "title": "telemetry: keep prompt and completion content out of spans by default", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/306", + "epic": false + }, + { + "number": 307, + "title": "budget: surface cached and reasoning token counts from the live usage payload", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/307", + "epic": false + }, + { + "number": 339, + "title": "Publish donkey-kit to PyPI and wire project URLs", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/339", + "epic": false + }, + { + "number": 349, + "title": "Wire the DDK stacked-black logo into README and the docs site", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/349", + "epic": false + }, + { + "number": 352, + "title": "budget: Budget never observes on a live 200 — the gateway emits x-llm-proxy-ratelimit, not x-token-*", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/352", + "epic": false + }, + { + "number": 353, + "title": "simulator: happy-path budget overlay emits x-token-*, but the live 200 emits x-llm-proxy-ratelimit", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/353", + "epic": false + }, + { + "number": 362, + "title": "skeleton: define the last-call record once — #307 and #309 each assume a different container, and gateway identity is surfaced nowhere", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/362", + "epic": false + }, + { + "number": 363, + "title": "refusals: overriding correlation_header silently breaks DonkeyError.correlation_id read-back through classify()", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/363", + "epic": false + }, + { + "number": 366, + "title": "Quickstart Configure step omits how to get client_id/client_secret", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/366", + "epic": false + }, + { + "number": 379, + "title": "errors: a typed GatewayUnavailable for transport-level failures", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/379", + "epic": false + }, + { + "number": 383, + "title": "Move Claude config & skills to a private companion repo", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/383", + "epic": false + }, + { + "number": 397, + "title": "docs: document @donkey.governed / @donkey.tool decorators (follow-up to #396)", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/397", + "epic": false + }, + { + "number": 400, + "title": "Sandbox test suite against the provisioned LLM Gateway proxies", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/400", + "epic": false + }, + { + "number": 410, + "title": "release: add workflow_dispatch → TestPyPI dry-run path (dev snapshots aren't GitHub Releases)", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/410", + "epic": false + }, + { + "number": 413, + "title": "docs: README logo uses a relative path → broken on the PyPI project page", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/413", + "epic": false + }, + { + "number": 415, + "title": "docs: document the versioning & naming convention publicly (it lives only in a private skill)", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/415", + "epic": false + }, + { + "number": 418, + "title": "chore(release): bump version to 0.1.0.dev1 for a fresh TestPyPI dry-run", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/418", + "epic": false + }, + { + "number": 423, + "title": "Upgrade website to Nextra 4 + Next 16 (supersedes #422)", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/423", + "epic": false + }, + { + "number": 425, + "title": "Distinguish internal and fork contributors in CONTRIBUTING.md", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/425", + "epic": false + }, + { + "number": 434, + "title": "docs: build plan still says standing-invariant § anchors resolve into the deleted archive", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/434", + "epic": false + }, + { + "number": 438, + "title": "chore(website): pin @xmldom/xmldom to 0.9.12 to clear 13 Dependabot alerts", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/438", + "epic": false + }, + { + "number": 440, + "title": "Set least-privilege GITHUB_TOKEN permissions in ci, nightly-matrix, rebrand-completeness workflows", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/440", + "epic": false + }, + { + "number": 445, + "title": "Gitignore the .claude/settings.json symlink in the public repo", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/445", + "epic": false + }, + { + "number": 448, + "title": "bug(skeleton): Donkey.aclose()/close() leaks the control-plane auth's httpx client", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/448", + "epic": false + }, + { + "number": 449, + "title": "test(skeleton): no unit coverage for the AnypointConnectedApp OAuth2 token path", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/449", + "epic": false + }, + { + "number": 450, + "title": "docs(skeleton): core/auth.py narrates the (verified) OAuth2 token path as UNVERIFIED", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/450", + "epic": false + }, + { + "number": 451, + "title": "bug(refusals): classify() AuthError cites control-plane §1 for a data-plane §2 rejection", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/451", + "epic": false + }, + { + "number": 452, + "title": "bug(budget): pace()+wait_for_reset() recovery loop can livelock after window exhaustion", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/452", + "epic": false + }, + { + "number": 453, + "title": "test(simulator): packaged (pip install) fixture-resolution path is never exercised", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/453", + "epic": false + }, + { + "number": 454, + "title": "docs(testing): gateway-identity conformance exemptions asserted in code but not published as credibility", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/454", + "epic": false + }, + { + "number": 455, + "title": "docs(telemetry): telemetry.mdx banners the shipped OTel GenAI feature as \"not yet shipped\"", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/455", + "epic": false + }, + { + "number": 456, + "title": "docs(adapters): adapter import comments falsely claim framework class names are \"verified\"", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/456", + "epic": false + }, + { + "number": 457, + "title": "test(adapters): AgentFrameworkAdapter.policy_middleware() and chat_client() blocked-path are uncovered", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/457", + "epic": false + }, + { + "number": 458, + "title": "chore(cli): refused provisioning control-plane is marketed in --help and exported in top-level __all__", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/458", + "epic": false + }, + { + "number": 459, + "title": "docs: README headlines \"provisioning-as-code\" as a flagship capability (a \"Do not build\" surface)", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/459", + "epic": false + }, + { + "number": 460, + "title": "docs(cli): stale citations in provisioning/cli.py docstrings (deleted-v1 milestones; wrong BG §1.8)", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/460", + "epic": false + }, + { + "number": 461, + "title": "test(cli): provisioning spec/planner/applier/lint and the hidden CLI commands have zero coverage", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/461", + "epic": false + }, + { + "number": 462, + "title": "bug(tools): ToolSet.filter() mutates the receiver in place instead of returning an independent view", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/462", + "epic": false + }, + { + "number": 463, + "title": "bug(tools): duplicate AssetType symbols — package exports the Enum, AssetRef.type expects the Literal", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/463", + "epic": false + }, + { + "number": 464, + "title": "test(tools): no test exercises the _verify.blocked() guards in registry/ and tools/session.py", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/464", + "epic": false + }, + { + "number": 465, + "title": "docs: docs/unsupported-boundary.md is an empty template but README promises it classifies platform APIs", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/465", + "epic": false + }, + { + "number": 471, + "title": "Distinguish shipped Phase 1 cards from roadmap proposals", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/471", + "epic": false + }, + { + "number": 484, + "title": "AuthError.remediation gives data-plane advice for control-plane token rejections", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/484", + "epic": false + }, + { + "number": 486, + "title": "Replace remaining M0/M4 milestone citations outside provisioning", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/486", + "epic": false + }, + { + "number": 487, + "title": "pace() retry loop spins at zero delay when reset_at is None", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/487", + "epic": false + }, + { + "number": 488, + "title": "Correct the \"one retry\" claim for pace() after the window resets", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/488", + "epic": false + }, + { + "number": 489, + "title": "Inject the clock into pace() and route the e2e retry through pace()", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/489", + "epic": false + }, + { + "number": 491, + "title": "Expose fixture-lock helpers that work outside a source checkout", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/491", + "epic": false + }, + { + "number": 496, + "title": "Retire M0/M1/M4 milestone vocabulary outside python/src", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/496", + "epic": false + }, + { + "number": 497, + "title": "Clarify paired export verification messages", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/497", + "epic": false + }, + { + "number": 501, + "title": "Cover remaining verification-blocked surfaces", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/501", + "epic": false + }, + { + "number": 504, + "title": "Reconcile header-only adapters with correlation conformance", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/504", + "epic": false + }, + { + "number": 509, + "title": "Support JWT/model-wallet auth mode for the LLM proxy data plane", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/509", + "epic": false + }, + { + "number": 510, + "title": "Document apikey/bearer auth convenience and DDK's two-header stance", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/510", + "epic": false + }, + { + "number": 515, + "title": "Archive daily PyPI adoption metrics; TestPyPI publishes none", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/515", + "epic": false + }, + { + "number": 517, + "title": "Drop :8081 from the sandbox proxies.toml base_url guidance", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/517", + "epic": false + }, + { + "number": 522, + "title": "Verify inbound correlation and cost-tag request header names against the live gateway", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/522", + "epic": false + }, + { + "number": 533, + "title": "Group Typed refusals under the six-piece-minimum sidebar separator", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/533", + "epic": false + }, + { + "number": 535, + "title": "Docs site typography: tighten sidebar items and hero tagline size", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/535", + "epic": false + }, + { + "number": 541, + "title": "Document the proxy ingress Format (OpenAI/Anthropic/Gemini) to adapter mapping", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/541", + "epic": false + }, + { + "number": 542, + "title": "request_id is None on Bedrock routes: x-request-id is the upstream provider's header, not the gateway's", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/542", + "epic": false + }, + { + "number": 548, + "title": "classify() types a Gemini upstream 400 as a gateway PolicyViolation: list-shaped error envelope is discarded", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/548", + "epic": false + }, + { + "number": 556, + "title": "epic(docs): restructure the docs site into end-user documentation organised by capability pillars", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/556", + "epic": true + }, + { + "number": 557, + "title": "docs(intro): rewrite the Introduction around gateway awareness, with architecture and before/after sections", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/557", + "epic": false + }, + { + "number": 558, + "title": "docs(quickstart): simplify the Quickstart and mark each capability Live or Roadmap", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/558", + "epic": false + }, + { + "number": 559, + "title": "docs(feature-overview): high-level capability tour with the gateway-aware image", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/559", + "epic": false + }, + { + "number": 560, + "title": "docs(nav): restructure the sidebar from phase separators into capability pillars", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/560", + "epic": false + }, + { + "number": 561, + "title": "docs(pages): remove issue refs, phase labels and verification narration from capability pages", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/561", + "epic": false + }, + { + "number": 562, + "title": "docs(examples): add an Examples pillar built on the donkey-development-kit-demos repo", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/562", + "epic": false + }, + { + "number": 563, + "title": "docs(roadmap): phases in order with live milestone progress and the issues in each", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/563", + "epic": false + }, + { + "number": 564, + "title": "docs(community): add a Team (About us) page with Creator/Contributor pills", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/564", + "epic": false + }, + { + "number": 565, + "title": "docs(community): add a Contribute page condensed from CONTRIBUTING.md", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/565", + "epic": false + }, + { + "number": 566, + "title": "docs(site): open-source disclaimer footer, metadata and optimised images", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/566", + "epic": false + }, + { + "number": 568, + "title": "errors: flip Amazon Bedrock Guardrails refusal from documented-only to VERIFIED (LIVE) after the acceptance capture", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/568", + "epic": false + } + ] + }, + { + "number": 12, + "title": "Phase 2 — Differentiate, go beyond (0.2.0)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/milestone/12", + "open": 33, + "closed": 1, + "issues": [ + { + "number": 15, + "title": "core/cache: TTLCache for tokens and registry lookups, with DONKEY_NO_CACHE escape hatch", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/15", + "epic": false + }, + { + "number": 42, + "title": "tools: ToolSet.filter() — allow/deny/predicate filtering and descriptor token accounting", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/42", + "epic": false + }, + { + "number": 208, + "title": "refusals: declarative reaction handlers registered once, applied at the transport", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/208", + "epic": false + }, + { + "number": 209, + "title": "refusals: enforce that a handler can react but never un-refuse", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/209", + "epic": false + }, + { + "number": 210, + "title": "classify: a registry so custom gateway policies become typed exceptions", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/210", + "epic": false + }, + { + "number": 211, + "title": "hitl: one vocabulary for pause-and-ask-a-human", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/211", + "epic": false + }, + { + "number": 212, + "title": "hitl: map the vocabulary onto LangGraph interrupt() and Command(resume=)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/212", + "epic": false + }, + { + "number": 213, + "title": "hitl: mappings for OpenAI Agents, ADK, Strands and MCP elicitation", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/213", + "epic": false + }, + { + "number": 214, + "title": "hitl: route escalation through gateway step-up (Trusted Agent Identity)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/214", + "epic": false + }, + { + "number": 215, + "title": "identity: donkey.as_user() — RFC 8693 token exchange and header placement", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/215", + "epic": false + }, + { + "number": 216, + "title": "identity: a missing or expired user token fails typed, never falls back to service identity", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/216", + "epic": false + }, + { + "number": 217, + "title": "scanner: donkey scan walks the repo and writes donkey.yaml + agent-card.json", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/217", + "epic": false + }, + { + "number": 218, + "title": "scanner: donkey publish registers the manifest with Exchange / Agent Registry", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/218", + "epic": false + }, + { + "number": 219, + "title": "scanner: the scan-and-publish GitHub Action", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/219", + "epic": false + }, + { + "number": 220, + "title": "killswitch: recognise the kill-switch rejection and stop cleanly", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/220", + "epic": false + }, + { + "number": 221, + "title": "tools: donkey.tools.discover() returns governed MCP tools as native framework tools", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/221", + "epic": false + }, + { + "number": 222, + "title": "tools: allow/deny filtering and descriptor token accounting", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/222", + "epic": false + }, + { + "number": 223, + "title": "adapters: a second deep adapter, chosen by Phase 1 demand", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/223", + "epic": false + }, + { + "number": 224, + "title": "a2a: donkey.serve() — an A2A listener in one line, wrapping the official a2a-sdk", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/224", + "epic": false + }, + { + "number": 225, + "title": "a2a: governance-aware inbound — correlation, tags, spans and typed refusals on served tasks", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/225", + "epic": false + }, + { + "number": 226, + "title": "a2a: donkey expose — provision the A2A proxy, policies and registry entry in one command", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/226", + "epic": false + }, + { + "number": 227, + "title": "a2a: donkey dev — a gateway in front of the laptop, real or simulated", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/227", + "epic": false + }, + { + "number": 260, + "title": "epic: Phase 2 — Differentiate, go beyond", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/260", + "epic": true + }, + { + "number": 308, + "title": "telemetry: adopt an inbound correlation ID and propagate it across MCP and A2A hops", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/308", + "epic": false + }, + { + "number": 309, + "title": "skeleton: surface gateway-side routing and fallback, and never double-retry it", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/309", + "epic": false + }, + { + "number": 310, + "title": "skeleton: per-request business group for shared, multi-tenant gateway clients", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/310", + "epic": false + }, + { + "number": 311, + "title": "budget + telemetry: state and test the concurrency contract under parallel fan-out", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/311", + "epic": false + }, + { + "number": 312, + "title": "tools: a typed refusal for a tool outside the consumer's MCP access profile", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/312", + "epic": false + }, + { + "number": 313, + "title": "refusals: type federated guardrail verdicts (Akamai, CrowdStrike, Google Armor)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/313", + "epic": false + }, + { + "number": 314, + "title": "killswitch: carry the kill reason — manual, anomaly, or eval drift", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/314", + "epic": false + }, + { + "number": 322, + "title": "testing: --donkey-mode server path for the conformance plugin", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/322", + "epic": false + }, + { + "number": 367, + "title": "skeleton: bring-your-own-key via a configurable proxy key header (Dynamic key)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/367", + "epic": false + }, + { + "number": 444, + "title": "conformance: wire the Phase 2–2.8 scenario bodies as their features land", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/444", + "epic": true + }, + { + "number": 503, + "title": "scanner: define how a @donkey.tool is attributed to an MCP server / Exchange asset", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/503", + "epic": false + } + ] + }, + { + "number": 13, + "title": "Phase 3 — Platform capabilities (0.3.0)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/milestone/13", + "open": 10, + "closed": 0, + "issues": [ + { + "number": 19, + "title": "llm/catalog: ModelHandle, resolve(), and an honest list_models() with no /models endpoint", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/19", + "epic": false + }, + { + "number": 228, + "title": "policies: the policy handshake client — read the in-force policy set", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/228", + "epic": false + }, + { + "number": 229, + "title": "policies: advisory only — the gateway always wins, and the client learns from the refusal", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/229", + "epic": false + }, + { + "number": 230, + "title": "policies: to-the-code push — refresh at run boundaries, never mid-run", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/230", + "epic": false + }, + { + "number": 231, + "title": "llm: keep the .parse() structured-output path governed, and support it in the simulator", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/231", + "epic": false + }, + { + "number": 232, + "title": "telemetry: evaluation hooks that attach a score to the run span", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/232", + "epic": false + }, + { + "number": 261, + "title": "epic: Phase 3 — Platform capabilities", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/261", + "epic": true + }, + { + "number": 315, + "title": "budget: monetary spend and the wallet threshold warning", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/315", + "epic": false + }, + { + "number": 316, + "title": "telemetry: arbitrary validated cost dimensions beyond the fixed four", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/316", + "epic": false + }, + { + "number": 317, + "title": "telemetry: a run-scoped rollup answering \"what did this task cost\"", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/317", + "epic": false + } + ] + }, + { + "number": 14, + "title": "Phase 4 — Enterprise readiness (0.4.0)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/milestone/14", + "open": 14, + "closed": 0, + "issues": [ + { + "number": 233, + "title": "security: independent security review and supply-chain hardening", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/233", + "epic": false + }, + { + "number": 234, + "title": "perf: a latency and overhead budget enforced in CI", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/234", + "epic": false + }, + { + "number": 235, + "title": "errors: a full pass over every remediation string and error message", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/235", + "epic": false + }, + { + "number": 236, + "title": "docs: migration guide, deprecation policy, and the public API contract", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/236", + "epic": false + }, + { + "number": 237, + "title": "compliance: map correlation and span data to EU AI Act Art. 12 and ISO 42001 evidence", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/237", + "epic": false + }, + { + "number": 238, + "title": "telemetry: structured log shipping to the control plane", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/238", + "epic": false + }, + { + "number": 239, + "title": "config: data-residency and region routing hints", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/239", + "epic": false + }, + { + "number": 240, + "title": "identity: evaluate workload identity (SPIFFE / Entra Agent ID) for agent-to-gateway auth", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/240", + "epic": false + }, + { + "number": 241, + "title": "ops: support model, issue templates and release cadence", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/241", + "epic": false + }, + { + "number": 262, + "title": "epic: Phase 4 — Enterprise readiness", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/262", + "epic": true + }, + { + "number": 318, + "title": "refusals: map the typed refusal taxonomy to the OWASP LLM Top 10", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/318", + "epic": false + }, + { + "number": 319, + "title": "provisioning: state the boundary against the platform's LLM Gateway IaC", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/319", + "epic": false + }, + { + "number": 320, + "title": "ops: air-gapped and private-cloud operation, proved by an egress-denied CI job", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/320", + "epic": false + }, + { + "number": 380, + "title": "skeleton: opt-in failover to a second Connected-Mode gateway endpoint", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/380", + "epic": false + } + ] + }, + { + "number": 15, + "title": "Phase 5 — Complete rollout (1.0.0)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/milestone/15", + "open": 6, + "closed": 0, + "issues": [ + { + "number": 242, + "title": "typescript: port the six-piece minimum", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/242", + "epic": false + }, + { + "number": 243, + "title": "typescript: shared conformance scenarios across Python and TypeScript", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/243", + "epic": false + }, + { + "number": 244, + "title": "adapters: bring back remaining frameworks by demand, each to the full bar", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/244", + "epic": false + }, + { + "number": 245, + "title": "gtm: framework-docs partner PRs and launch channels", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/245", + "epic": false + }, + { + "number": 246, + "title": "release: 1.0.0 with stability guarantees", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/246", + "epic": false + }, + { + "number": 263, + "title": "epic: Phase 5 — Complete rollout", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/263", + "epic": true + } + ] + }, + { + "number": 16, + "title": "Verification", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/milestone/16", + "open": 26, + "closed": 5, + "issues": [ + { + "number": 1, + "title": "Epic: M0 — verify every endpoint, header and class name before building on it", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/1", + "epic": true + }, + { + "number": 21, + "title": "Verify the business-group attribution header name (highest-priority M0 unknown)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/21", + "epic": false + }, + { + "number": 22, + "title": "Verify Anypoint control-plane region hosts for EU, CA and JP Hyperforce", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/22", + "epic": false + }, + { + "number": 23, + "title": "Verify connected-app scopes and admin-context requirements per operation", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/23", + "epic": false + }, + { + "number": 34, + "title": "Verify the eight framework constructor signatures against installed packages (§8)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/34", + "epic": false + }, + { + "number": 53, + "title": "Verify the Exchange asset search API and whether wildcard query is server-side", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/53", + "epic": false + }, + { + "number": 54, + "title": "Verify whether applied policies can be fetched in bulk per environment", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/54", + "epic": false + }, + { + "number": 65, + "title": "Verify Local Mode capabilities: can it run the LLM Proxy and MCP Bridge, and does it need a licence artifact", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/65", + "epic": false + }, + { + "number": 79, + "title": "Verify Exchange publication mechanics: metadata/tags for the digest, lifecycle states, docs pages", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/79", + "epic": false + }, + { + "number": 88, + "title": "Verify the MCP Bridge provisioning API, or confirm the Terraform/CLI fallback", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/88", + "epic": false + }, + { + "number": 101, + "title": "meta: verified-apis.md as the living source of truth, and the _verify.blocked discipline", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/101", + "epic": false + }, + { + "number": 114, + "title": "Verify Exchange draft/unlisted publication states and provenance metadata for scanner-derived assets", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/114", + "epic": false + }, + { + "number": 123, + "title": "Verify Flex Gateway Connected Mode registration from a laptop or CI runner, and the per-developer instance cost", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/123", + "epic": false + }, + { + "number": 135, + "title": "Verify PDK capabilities before any custom policy is designed (§6.9.7)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/135", + "epic": false + }, + { + "number": 253, + "title": "verify: re-confirm the v1.11-v1.13 rejection contracts against current docs and a sandbox", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/253", + "epic": false + }, + { + "number": 254, + "title": "verify: the A2A proxy provisioning API, or confirm there is no supported path", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/254", + "epic": false + }, + { + "number": 255, + "title": "verify: the Trusted Agent Identity token-exchange endpoint and header", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/255", + "epic": false + }, + { + "number": 256, + "title": "verify: the Agent Kill Switch rejection shape", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/256", + "epic": false + }, + { + "number": 257, + "title": "verify: whether Anypoint Monitoring or Agent Visualizer ingests OTLP GenAI spans", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/257", + "epic": false + }, + { + "number": 258, + "title": "verify: self-managed gateway image availability and licensing for local development", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/258", + "epic": false + }, + { + "number": 279, + "title": "verify: whether the local simulator can be hosted on a2d-ai.com", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/279", + "epic": false + }, + { + "number": 300, + "title": "verify: whether the gateway echoes the client x-correlation-id or mints its own", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/300", + "epic": false + }, + { + "number": 301, + "title": "verify: model-wallet spend headers and the threshold-warning signal", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/301", + "epic": false + }, + { + "number": 302, + "title": "verify: upstream error envelopes for Bedrock, Gemini and Microsoft Foundry", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/302", + "epic": false + }, + { + "number": 303, + "title": "verify: the MCP access-profile denial shape", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/303", + "epic": false + }, + { + "number": 304, + "title": "verify: the Anthropic-native Messages route on the LLM proxy", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/304", + "epic": false + }, + { + "number": 305, + "title": "verify: federated guardrail verdict shapes for Akamai, CrowdStrike and Google Armor", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/305", + "epic": false + }, + { + "number": 354, + "title": "docs/verify: answer #253 — the live 200 does carry budget state, as prose in x-llm-proxy-ratelimit", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/354", + "epic": false + }, + { + "number": 372, + "title": "verify: model-wallet JWT ingress auth (X-Client-Id + client_id claim, IdP/JWKS)", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/372", + "epic": false + }, + { + "number": 520, + "title": "agent_framework.chat_client() passes an unsupported model_id kwarg", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/520", + "epic": false + }, + { + "number": 540, + "title": "verify: whether the LLM proxy exposes a Gemini-native ingress route", + "state": "closed", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/540", + "epic": false + } + ] + }, + { + "number": 17, + "title": "Upstream gaps", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/milestone/17", + "open": 7, + "closed": 0, + "issues": [ + { + "number": 247, + "title": "upstream: 429 carries no Retry-After, only x-token-reset", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/247", + "epic": false + }, + { + "number": 248, + "title": "upstream: no budget-query endpoint — remaining budget is only visible in-band", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/248", + "epic": false + }, + { + "number": 249, + "title": "upstream: no policy-discovery endpoint — clients cannot ask what is in force", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/249", + "epic": false + }, + { + "number": 250, + "title": "upstream: no dry-run mode — policy cannot be evaluated without blocking", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/250", + "epic": false + }, + { + "number": 251, + "title": "upstream: guardrail-verdict error contract is under-documented", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/251", + "epic": false + }, + { + "number": 252, + "title": "upstream: no outbound-connect or tunnel path for A2A ingress", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/252", + "epic": false + }, + { + "number": 381, + "title": "upstream: no highly available or multi-region LLM proxy endpoint", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/381", + "epic": false + } + ] + } + ] +} diff --git a/website/next.config.mjs b/website/next.config.mjs index f709fa5e..40ba3911 100644 --- a/website/next.config.mjs +++ b/website/next.config.mjs @@ -3,8 +3,15 @@ import nextra from 'nextra' // Nextra 4 is App-Router only: `theme` / `themeConfig` are gone — the docs theme // is composed from //