From 368bc7b0b9cf481a91e271df29f05c1a61518fd0 Mon Sep 17 00:00:00 2001 From: Tommaso Bolis Date: Thu, 24 Sep 2026 14:25:52 +0200 Subject: [PATCH 1/3] chore: bump version to 0.1.0.dev10 (#591) --- python/pyproject.toml | 2 +- python/src/donkey_kit/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/python/pyproject.toml b/python/pyproject.toml index 1f280faf..b70018fc 100644 --- a/python/pyproject.toml +++ b/python/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "donkey-kit" -version = "0.1.0.dev9" +version = "0.1.0.dev10" description = "An SDK for consuming Agent Fabric capabilities (governed model and tool access) from your own agent framework." readme = "README.md" requires-python = ">=3.10" diff --git a/python/src/donkey_kit/__init__.py b/python/src/donkey_kit/__init__.py index 764f7d3e..52807090 100644 --- a/python/src/donkey_kit/__init__.py +++ b/python/src/donkey_kit/__init__.py @@ -52,7 +52,7 @@ PublicationAssetType, ) -__version__ = "0.1.0.dev9" +__version__ = "0.1.0.dev10" __all__ = [ "STRICT", From 9a3fe5ec78947cf8061eb4526f237a13bac71641 Mon Sep 17 00:00:00 2001 From: Tommaso Bolis Date: Thu, 24 Sep 2026 14:57:32 +0200 Subject: [PATCH 2/3] =?UTF-8?q?docs(verify):=20live-verify=20the=20semanti?= =?UTF-8?q?c-routing=20success=20header=20(=C2=A70.3,=20#589)=20(#596)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #589 --- docs/verified-apis.md | 1 + .../anypoint/semantic_routing/README.md | 62 +++++++++++++++++++ .../semantic_routing/request.success.http | 18 ++++++ .../semantic_routing/responses.code.body.json | 1 + .../responses.code.headers.txt | 24 +++++++ .../responses.finance.body.json | 35 +++++++++++ .../responses.finance.headers.txt | 36 +++++++++++ .../responses.offtopic.body.json | 35 +++++++++++ .../responses.offtopic.headers.txt | 36 +++++++++++ 9 files changed, 248 insertions(+) create mode 100644 python/tests/fixtures/anypoint/semantic_routing/README.md create mode 100644 python/tests/fixtures/anypoint/semantic_routing/request.success.http create mode 100644 python/tests/fixtures/anypoint/semantic_routing/responses.code.body.json create mode 100644 python/tests/fixtures/anypoint/semantic_routing/responses.code.headers.txt create mode 100644 python/tests/fixtures/anypoint/semantic_routing/responses.finance.body.json create mode 100644 python/tests/fixtures/anypoint/semantic_routing/responses.finance.headers.txt create mode 100644 python/tests/fixtures/anypoint/semantic_routing/responses.offtopic.body.json create mode 100644 python/tests/fixtures/anypoint/semantic_routing/responses.offtopic.headers.txt diff --git a/docs/verified-apis.md b/docs/verified-apis.md index 39eab37a..37975323 100644 --- a/docs/verified-apis.md +++ b/docs/verified-apis.md @@ -99,6 +99,7 @@ request against the deployed gateway. | Auth: header name / model | `core/transport.py`, `core/config.py` | VERIFIED (LIVE) | **`client_id` + `client_secret` request headers** (NOT bearer) — enforced by `client-id-enforcement` 1.3.3. A consumer **credential pair**, mapped to an Anypoint client application | 2026-08-28 | live probe + `policy:list` | | Auth: model-wallet ingress (alt to `client_id`/`client_secret`, #372) | `core/transport.py`, `core/config.py` — contract verified; SDK wiring landed in #509 (`llm_proxy_auth="jwt"` + `llm_proxy_wallet_client_id` + `Donkey(llm_auth=…)`) | VERIFIED (LIVE) | Wallet-backed model proxies auth via an **IdP-issued JWT + a client ID**, with **no `client_secret`**: the client ID travels as the **`X-Client-Id`** request header (exact casing confirmed; value = the wallet's system-generated `clientId`, `ddk-model-wallet` here — read it from the omni API response, don't assume) which selects the wallet (echoed back as response header **`x-model-wallet-selected`**), *and* as a `client_id` **JWT claim** read by the LLM Proxy Core Policy at `#[authentication.properties.claims.client_id]` after the **JWT Validation** policy validates + publishes claims. **The JWT rides as `Authorization: Bearer `** — this was an *assumption* from the doc read (Bearer is nowhere quoted on the source page); now **confirmed** (policy `jwtOrigin: httpBearerAuthenticationHeader`; missing header → `400 {"error":"JWT Token is required."}`, `www-authenticate: Bearer`; invalid/expired token → `401 {"error":"Invalid token."}`). No `client_id`/`client_secret` request headers are sent and the call still succeeds → the default **DataWeave Headers Transformation** + **Client ID Enforcement** policies are confirmed **disabled**. Requires an org **IdP** (JWKS URL / signing key — "orgs without an IdP can't use model wallets"). A **parallel** ingress model, NOT a replacement of the LIVE `client_id`/`client_secret` pair above. | 2026-09-21 | live probe — `tests/fixtures/anypoint/model_wallet/` (`ddk-model-wallet`, instance `21186246`, Sandbox); shape corroborates `docs.mulesoft.com/general/exp-model-wallets-manage` | | Model routing | `llm/client.py`; consumed by `core/lastcall.py` + `core/transport.py` (#309) | VERIFIED (LIVE) | `model-based-routing` 1.0.3 reads `model` from body → provider. Response headers `x-llm-proxy-routing-type: ModelBased`, `x-llm-proxy-routing-fallback: false`, `x-llm-proxy-llm-provider: openai`, `x-llm-proxy-llm-model: gpt-5.1`, `x-llm-proxy-model-based-routing-success: Request successfully matched. …`. As of #309 the SDK surfaces these at `donkey.last_call` (`served_provider`/`served_model`/`routing_type`/`fallback`, `substituted` when served≠requested), emits them as `gen_ai.response.model` / `donkey.routing.type` / `donkey.routing.fallback` span attrs, and **does not retry** a `503` a fallback header already marked failed-over (#309, must not fight #183) | 2026-08-28 | live probe (`responses.success.headers.txt`) | +| Semantic routing (#589) | `llm/client.py`; the four shared routing headers already consumed by `core/lastcall.py` + `core/transport.py` (#309); matched-topic/score surfacing is #590 | VERIFIED (LIVE) | A **semantic-routing** proxy emits the **same four routing headers as model-based** — `x-llm-proxy-routing-type` (value **`Semantic`** vs `ModelBased`), `-routing-fallback`, `-llm-provider`, `-llm-model` — so #309's `LastCall` parsing already works unchanged for the semantic case. Additionally, a **semantic-only** header carries the match detail: **`x-llm-proxy-semantic-routing-success`**, exact format `Request successfully matched '{topic}' topic (Provider: {provider}, Model: {model}). Score: {score}.` (observed: Finance → `openai`/`gpt-5-mini` score `0.62`; Code → `gemini`/`gemini-2.5-flash` score `0.62`). Note the format is `(Provider: …, Model: …)` — **not** a `provider/model` slug — and the score is a bare `0.xx`. Also present alongside: `x-llm-proxy-request-success` (`Request completed successfully. Provider: …, Model: ….`) and, on OpenAI-served routes, cost hints `x-llm-proxy-input-tokens-cost-per-1m` / `-output-tokens-cost-per-1m` (out of scope for #590 — see its Out of scope). **Fallback (`routing-fallback: true`) was NOT reproducible on this SSC** (§0.3 — not fabricated): with a 10-vector `text-embedding-3-small` store, cosine similarity floors ~`0.51`–`0.53` (above the `0.5` threshold, comparison strictly-greater-than) even for gibberish/off-topic prompts, so every request matched a primary topic with `routing-fallback: false`. The fallback-branch header shape therefore remains uncaptured. | 2026-09-24 | live probe against `ddk-semantic-advanced-routing` (instance `21194930`, DDK/Sandbox, private-space) — captured in `python/tests/fixtures/anypoint/semantic_routing/` | | Token accounting (cost attribution) | `llm/*`, telemetry | VERIFIED (LIVE) | response `usage: {input_tokens, output_tokens, total_tokens, input_tokens_details, output_tokens_details}` — the detail sub-objects carry `cached_tokens` / `cache_write_tokens` (under `input_tokens_details`) and `reasoning_tokens` (under `output_tokens_details`), surfaced per-call at `donkey.last_call` and on the span (#307, see §3); also upstream `x-ratelimit-*` headers passed through (Go-style **duration-string** resets, e.g. `0s`/`12ms` — distinct from the gateway's own `x-llm-proxy-ratelimit`/`x-token-*` window, see §4) | 2026-08-28 | `responses.success.body.json` | | Request fields passed through | `llm/client.py` | VERIFIED (LIVE) | OpenAI body passed through verbatim; response returned verbatim (model `gpt-5.1`→`gpt-5.1-2025-11-13`, full Responses object) | 2026-08-28 | live probe | | Streaming support | `llm/client.py` | VERIFIED (LIVE) | `"stream": true` → `200`, `content-type: text/event-stream`, chunked SSE (`event: response.created` / `response.in_progress` / …); same `x-llm-proxy-*` headers | 2026-08-28 | live probe (`responses.stream.*`) | diff --git a/python/tests/fixtures/anypoint/semantic_routing/README.md b/python/tests/fixtures/anypoint/semantic_routing/README.md new file mode 100644 index 00000000..7fc77fb8 --- /dev/null +++ b/python/tests/fixtures/anypoint/semantic_routing/README.md @@ -0,0 +1,62 @@ +# Semantic routing — LIVE capture (docs/verified-apis.md §3, #589) + +Captured **2026-09-24** from a real deployed Agent Network LLM proxy provisioned +with **semantic-based routing** (advanced Semantic Service Configuration, an +external Azure AI Search vector store) in the DDK sandbox, env **Sandbox** +(svc id `14d3b31e-4e3b-4d90-b77a-63c9d6b7ea6a`, from the +`x-envoy-decorator-operation` header). Proxy asset `ddk-semantic-advanced-routing`, +API Manager instance **`21194930`**, deployed to the `private-space-omni-gateway` +Flex Gateway. Two topics — **Finance** → `openai`/`gpt-5-mini`, **Code** → +`gemini`/`gemini-2.5-flash` — with `openai` `text-embedding-3-small` embeddings +and a `0.5` fallback threshold. + +This is the **first live confirmation of the semantic-routing header contract**. +It resolves issue **#589** and unblocks **#590** (surfacing the matched topic + +similarity score on `donkey.last_call`). It is the semantic sibling of the +model-based routing captured in `../llm_proxy/responses.success.policy-applied.headers.txt`. + +## What is verified (all witnessed live) + +- **The four shared routing headers are emitted identically to model-based**, + with `x-llm-proxy-routing-type: Semantic` (vs `ModelBased`) — so the #309 + `LastCall` / `transport` parsing of `routing-type` / `routing-fallback` / + `llm-provider` / `llm-model` already works unchanged for the semantic case. +- **A semantic-only header carries the match detail:** + `x-llm-proxy-semantic-routing-success`, with the exact format + + ``` + Request successfully matched '{topic}' topic (Provider: {provider}, Model: {model}). Score: {score}. + ``` + + Note the `(Provider: …, Model: …)` shape — **not** a `provider/model` slug — + and the bare `0.xx` score. This is the string #590 must parse for + `matched_topic` + `routing_score`. See `responses.{finance,code,offtopic}.headers.txt`. +- **`x-llm-proxy-request-success`** (`Request completed successfully. Provider: + …, Model: ….`) and, on the OpenAI-served route, **cost hints** + `x-llm-proxy-input-tokens-cost-per-1m` / `-output-tokens-cost-per-1m` are also + present. Those cost headers are recorded but **out of scope for #590**. +- **Client ID Enforcement is on** — same `client_id`/`client_secret` pair as the + other CIE proxies (no `client_id` → 401 `{"error":"Client ID is not present"}`, + `www-authenticate: Client-ID-Enforcement`; verified during the probe, not + re-captured here since it is identical to `../llm_proxy/reject.client-id-missing.*`). + +## What could NOT be captured — the fallback branch (§0.3, not fabricated) + +`x-llm-proxy-routing-fallback: true` was **not reproducible** on this SSC. With a +10-vector `text-embedding-3-small` store the cosine similarity floors around +`0.51`–`0.53` (above the `0.5` threshold, and the comparison is +strictly-greater-than) even for gibberish and deliberately off-topic prompts — +so every request matched a *primary* topic with `routing-fallback: false`. The +`responses.offtopic.*` capture is such a low-but-above-threshold match +(`Finance`, score `0.51`), **not** an actual fallback. The fallback-branch header +shape therefore remains uncaptured; it would need a proxy whose threshold is +raised (or whose topic coverage is thinner) to observe, which is a +provisioning-side change out of scope for #589. + +## How this was captured + +The data-plane requests were issued directly against instance `21194930` with +the shared `ddk-test-client` consumer's already-approved contract. Consumer +`client_id`/`client_secret` are **redacted** from `request.success.http` and were +never persisted here; the Cloudflare `set-cookie` line was stripped from the +captured header dumps as noise. diff --git a/python/tests/fixtures/anypoint/semantic_routing/request.success.http b/python/tests/fixtures/anypoint/semantic_routing/request.success.http new file mode 100644 index 00000000..1b1e9575 --- /dev/null +++ b/python/tests/fixtures/anypoint/semantic_routing/request.success.http @@ -0,0 +1,18 @@ +# Live semantic-routing request (consumer creds redacted). +# Same OpenAI /chat/completions wire shape as ../llm_proxy/ — the routing +# decision is made by prompt SEMANTICS, not the body `model` field (which is a +# placeholder here). The topic each prompt matched is echoed in the +# `x-llm-proxy-semantic-routing-success` response header (see responses.*.headers.txt). + +POST /ddk-semantic-advanced-routing/chat/completions HTTP/1.1 +Host: private-space-omni-gateway-qrpud8.wxdmnv.usa-e2.cloudhub.io +Content-Type: application/json +client_id: +client_secret: + +{"model": "any-placeholder", "messages": [{"role": "user", "content": "How do I calculate compound interest?"}]} + +# Prompt → matched topic (captured 2026-09-24): +# "How do I calculate compound interest?" → Finance (openai/gpt-5-mini, score 0.62) → responses.finance.* +# "Write a Python function to sort a list." → Code (gemini/gemini-2.5-flash, score 0.62) → responses.code.* +# "What is the airspeed velocity of an unladen swallow?" → Finance (openai/gpt-5-mini, score 0.51) → responses.offtopic.* diff --git a/python/tests/fixtures/anypoint/semantic_routing/responses.code.body.json b/python/tests/fixtures/anypoint/semantic_routing/responses.code.body.json new file mode 100644 index 00000000..4dfeef05 --- /dev/null +++ b/python/tests/fixtures/anypoint/semantic_routing/responses.code.body.json @@ -0,0 +1 @@ +{"id":"xhq1asymC_a2sOIPiOqpkQ8","object":"chat.completion","created":0,"model":"gemini-2.5-flash","choices":[{"index":0,"finish_reason":"stop","message":{"role":"assistant","refusal":"","content":"Python provides excellent built-in ways to sort lists, which are highly optimized (using the Timsort algorithm). You rarely need to implement a sorting algorithm from scratch in Python for practical purposes.\n\nHere are a few ways to write a function to sort a list, demonstrating the built-in methods:\n\n---\n\n### Method 1: Using `sorted()` (Recommended for most cases)\n\nThe `sorted()` function returns a **new sorted list** without modifying the original list.\n\n```python\ndef sort_list_with_sorted(input_list, reverse=False):\n \"\"\"\n Sorts a list using the built-in sorted() function.\n Returns a new sorted list, leaving the original list unchanged.\n\n Args:\n input_list (list): The list to be sorted.\n reverse (bool): If True, the list is sorted in descending order.\n Defaults to False (ascending order).\n\n Returns:\n list: A new sorted list.\n \"\"\"\n return sorted(input_list, reverse=reverse)\n\n# --- Example Usage ---\nmy_list = [3, 1, 4, 1, 5, 9, 2, 6]\nprint(f\"Original list: {my_list}\")\n\n# Ascending sort\nsorted_ascending = sort_list_with_sorted(my_list)\nprint(f\"Sorted ascending (new list): {sorted_ascending}\")\nprint(f\"Original list after ascending sort: {my_list}\") # Original is unchanged\n\n# Descending sort\nsorted_descending = sort_list_with_sorted(my_list, reverse=True)\nprint(f\"Sorted descending (new list): {sorted_descending}\")\nprint(f\"Original list after descending sort: {my_list}\") # Original is still unchanged\n\n# Sorting a list of strings\nstring_list = [\"banana\", \"apple\", \"cherry\", \"date\"]\nprint(f\"\\nOriginal string list: {string_list}\")\nprint(f\"Sorted strings: {sort_list_with_sorted(string_list)}\")\n\n# Sorting with a custom key (e.g., by length of strings)\ndef sort_list_by_length(input_list):\n return sorted(input_list, key=len)\n\nprint(f\"\\nSorted strings by length: {sort_list_by_length(string_list)}\")\n```\n\n---\n\n### Method 2: Using `list.sort()` (In-place sort)\n\nThe `list.sort()` method sorts the list **in-place**, meaning it modifies the original list and returns `None`.\n\n```python\ndef sort_list_in_place(input_list, reverse=False):\n \"\"\"\n Sorts a list using the built-in list.sort() method (in-place).\n Modifies the original list and returns None.\n\n Args:\n input_list (list): The list to be sorted (will be modified).\n reverse (bool): If True, the list is sorted in descending order.\n Defaults to False (ascending order).\n \"\"\"\n input_list.sort(reverse=reverse)\n # This function doesn't explicitly return the list,\n # as its primary effect is the modification of `input_list`.\n # It implicitly returns None.\n\n# --- Example Usage ---\nmy_list_2 = [7, 2, 8, 0, 1, 5]\nprint(f\"\\nOriginal list_2: {my_list_2}\")\n\n# Ascending sort (in-place)\nsort_list_in_place(my_list_2)\nprint(f\"List_2 after in-place ascending sort: {my_list_2}\") # my_list_2 is modified\n\n# Descending sort (in-place)\nmy_list_3 = [7, 2, 8, 0, 1, 5]\nprint(f\"Original list_3: {my_list_3}\")\nsort_list_in_place(my_list_3, reverse=True)\nprint(f\"List_3 after in-place descending sort: {my_list_3}\") # my_list_3 is modified\n```\n\n---\n\n### Key Differences and When to Use Which:\n\n* **`sorted(iterable)`**:\n * Returns a **new sorted list**.\n * The original iterable remains unchanged.\n * Can be used on any iterable (lists, tuples, sets, strings, dictionaries, etc.).\n * Generally preferred when you need a sorted version but also want to preserve the original data.\n\n* **`list.sort()`**:\n * Sorts the list **in-place**.\n * Modifies the original list directly.\n * Returns `None`.\n * Can **only** be used on lists.\n * More memory-efficient if you no longer need the unsorted version, as it doesn't create a new list.\n\n### Advanced Sorting with `key` Parameter\n\nBoth `sorted()` and `list.sort()` accept a `key` parameter, which is a function that will be called on each element prior to making comparisons.\n\n```python\ndef sort_list_with_key(input_list, key_func, reverse=False):\n \"\"\"\n Sorts a list using the built-in sorted() function with a custom key function.\n\n Args:\n input_list (list): The list to be sorted.\n key_func (function): A function to be called on each list element\n prior to making comparisons.\n reverse (bool): If True, the list is sorted in descending order.\n\n Returns:\n list: A new sorted list.\n \"\"\"\n return sorted(input_list, key=key_func, reverse=reverse)\n\n# --- Example Usage with Key ---\nstudents = [\n {'name': 'Alice', 'age': 30, 'grade': 'A'},\n {'name': 'Bob', 'age': 25, 'grade': 'B'},\n {'name': 'Charlie', 'age': 30, 'grade': 'C'},\n {'name': 'David', 'age': 22, 'grade': 'A'}\n]\n\nprint(f\"\\nOriginal students list: {students}\")\n\n# Sort by age (ascending)\nsorted_by_age = sort_list_with_key(students, key_func=lambda s: s['age'])\nprint(f\"Sorted by age: {sorted_by_age}\")\n\n# Sort by age (descending), then by name (ascending) for ties\n# This requires a slightly different approach for multiple keys,\n# you can use tuples in the key function\nsorted_by_age_name = sorted(students, key=lambda s: (s['age'], s['name']))\nprint(f\"Sorted by age then name: {sorted_by_age_name}\")\n\n# Sort by grade (descending)\nsorted_by_grade_desc = sort_list_with_key(students, key_func=lambda s: s['grade'], reverse=True)\nprint(f\"Sorted by grade (descending): {sorted_by_grade_desc}\")\n\n# Sort by absolute value\nnumbers_with_negatives = [3, -1, -4, 2, 0]\nsorted_by_abs = sort_list_with_key(numbers_with_negatives, key_func=abs)\nprint(f\"\\nOriginal numbers: {numbers_with_negatives}\")\nprint(f\"Sorted by absolute value: {sorted_by_abs}\")\n```\n\n---\n\n### Implementing a Basic Sorting Algorithm from Scratch (For educational purposes only)\n\nWhile not recommended for production code, here's an example of a simple sorting algorithm like **Bubble Sort** implemented in Python to show how it works conceptually.\n\n```python\ndef bubble_sort(input_list):\n \"\"\"\n Implements the Bubble Sort algorithm to sort a list in ascending order.\n Modifies the list in-place.\n\n Args:\n input_list (list): The list to be sorted (will be modified).\n \"\"\"\n n = len(input_list)\n # Traverse through all list elements\n for i in range(n - 1):\n # Last i elements are already in place, so we don't need to check them\n swapped = False\n for j in range(n - 1 - i):\n # Traverse the list from 0 to n-i-1\n # Swap if the element found is greater than the next element\n if input_list[j] > input_list[j + 1]:\n input_list[j], input_list[j + 1] = input_list[j + 1], input_list[j]\n swapped = True\n # If no two elements were swapped by inner loop, then the list is sorted\n if not swapped:\n break\n\n# --- Example Usage of Bubble Sort ---\nmy_list_bubble = [64, 34, 25, 12, 22, 11, 90]\nprint(f\"\\nOriginal list for Bubble Sort: {my_list_bubble}\")\nbubble_sort(my_list_bubble)\nprint(f\"List after Bubble Sort: {my_list_bubble}\")\n\nmy_list_bubble_2 = [5, 1, 4, 2, 8]\nprint(f\"Original list for Bubble Sort: {my_list_bubble_2}\")\nbubble_sort(my_list_bubble_2)\nprint(f\"List after Bubble Sort: {my_list_bubble_2}\")\n```\n\n**Conclusion:**\n\nFor almost all real-world applications in Python, you should use `sorted()` or `list.sort()` because they are highly efficient, well-tested, and easy to use. Implementing your own sorting algorithm is primarily for learning purposes.","tool_calls":null},"logprobs":null}],"usage":{"prompt_tokens":10,"completion_tokens":2219,"total_tokens":3409,"completion_tokens_details":{"accepted_prediction_tokens":2219,"audio_tokens":0,"reasoning_tokens":1180,"rejected_prediction_tokens":0},"prompt_tokens_details":{"audio_tokens":0,"cached_tokens":0}},"service_tier":"default"} \ No newline at end of file diff --git a/python/tests/fixtures/anypoint/semantic_routing/responses.code.headers.txt b/python/tests/fixtures/anypoint/semantic_routing/responses.code.headers.txt new file mode 100644 index 00000000..218a9286 --- /dev/null +++ b/python/tests/fixtures/anypoint/semantic_routing/responses.code.headers.txt @@ -0,0 +1,24 @@ +HTTP/1.1 200 OK +x-gemini-service-tier: standard +content-type: application/json; charset=UTF-8 +vary: X-Origin,Referer,Origin,Accept-Encoding +date: Thu, 24 Sep 2026 12:43:02 GMT +server: Anypoint Flex Gateway +x-xss-protection: 0 +x-frame-options: SAMEORIGIN +x-content-type-options: nosniff +server-timing: gfet4t7; dur=16193 +alt-svc: h3=":443"; ma=2592000,h3-29=":443"; ma=2592000 +accept-ranges: none +x-llm-proxy-semantic-routing-success: Request successfully matched 'Code' topic (Provider: gemini, Model: gemini-2.5-flash). Score: 0.62. +x-llm-proxy-routing-type: Semantic +x-llm-proxy-routing-fallback: false +x-llm-proxy-llm-provider: gemini +x-llm-proxy-llm-model: gemini-2.5-flash +x-llm-proxy-request-success: Request completed successfully. Provider: gemini, Model: gemini-2.5-flash. +x-envoy-decorator-operation: api-instance-21194930.14d3b31e-4e3b-4d90-b77a-63c9d6b7ea6a.svc +transfer-encoding: chunked +x-correlation-id: d72f3ffa-2019-458d-b44c-ceb8867d2105 +Strict-Transport-Security: max-age=31536000; includeSubdomains +Connection: Keep-Alive + diff --git a/python/tests/fixtures/anypoint/semantic_routing/responses.finance.body.json b/python/tests/fixtures/anypoint/semantic_routing/responses.finance.body.json new file mode 100644 index 00000000..dc33b324 --- /dev/null +++ b/python/tests/fixtures/anypoint/semantic_routing/responses.finance.body.json @@ -0,0 +1,35 @@ +{ + "id": "chatcmpl-ERd4VU9Og5T4W8ZwvOHbI79ucgKvr", + "object": "chat.completion", + "created": 1790253755, + "model": "gpt-5-mini-2025-08-07", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Compound interest means you earn interest on the principal and on previously earned interest. The basic formula and how to use it:\n\n1) Standard (discrete) compounding\n- Formula: A = P (1 + r/n)^(n t)\n - A = amount after t years\n - P = principal (initial amount)\n - r = annual nominal interest rate (as a decimal, e.g., 5% = 0.05)\n - n = number of compounding periods per year (1 for annual, 12 for monthly, 365 for daily, etc.)\n - t = time in years\n\n- Interest earned = A − P\n\nExample:\n- P = $1,000, r = 5% = 0.05, n = 12, t = 3 years\n- A = 1000*(1 + 0.05/12)^(12*3) ≈ 1000*(1.0041667)^36 ≈ $1,161.62\n- Interest earned ≈ $161.62\n\n2) Continuous compounding\n- Formula: A = P e^(r t)\n - e ≈ 2.71828\n- Example with same numbers: A = 1000 * e^(0.05*3) ≈ 1000 * 1.16183 ≈ $1,161.83\n\n3) Effective annual rate (EAR)\n- EAR = (1 + r/n)^n − 1\n- If r = 5% and n = 12, EAR ≈ (1 + 0.05/12)^12 − 1 ≈ 0.05116 = 5.116%\n\n4) Solving for time or rate\n- To find t given A: t = ln(A/P) / (n ln(1 + r/n))\n- For continuous compounding: t = ln(A/P) / r\n- To find r given A and t (discrete): r = n[(A/P)^(1/(n t)) − 1]\n\nTips\n- Always convert percentage rates to decimals before using formulas.\n- Make sure t and n use the same time units (t in years if n is per year).\n- Use a calculator or spreadsheet; most have functions for power and e^x.\n\nIf you want, tell me P, r, n and t and I’ll compute A and interest for you.", + "refusal": null, + "annotations": [] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 13, + "completion_tokens": 962, + "total_tokens": 975, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 448, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": null +} diff --git a/python/tests/fixtures/anypoint/semantic_routing/responses.finance.headers.txt b/python/tests/fixtures/anypoint/semantic_routing/responses.finance.headers.txt new file mode 100644 index 00000000..d033c784 --- /dev/null +++ b/python/tests/fixtures/anypoint/semantic_routing/responses.finance.headers.txt @@ -0,0 +1,36 @@ +HTTP/1.1 200 OK +date: Thu, 24 Sep 2026 12:42:44 GMT +content-type: application/json +content-length: 2236 +cf-ray: a401deaf697ee88e-CMH +cf-cache-status: DYNAMIC +server: Anypoint Flex Gateway +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +access-control-expose-headers: X-Request-ID,CF-Ray,CF-Ray +openai-organization: user-9c8mvuhcimhkzwlcpxtztz7a +openai-processing-ms: 8835 +openai-project: proj_UpRmBxcPjQR5y8Tap22ULwcw +openai-version: 2020-10-01 +x-openai-proxy-wasm: v0.1 +x-ratelimit-limit-requests: 500 +x-ratelimit-limit-tokens: 500000 +x-ratelimit-remaining-requests: 499 +x-ratelimit-remaining-tokens: 499988 +x-ratelimit-reset-requests: 120ms +x-ratelimit-reset-tokens: 1ms +x-request-id: req_3dbaaccb9a634f669e7245070f0160d3 +alt-svc: h3=":443"; ma=86400 +x-llm-proxy-semantic-routing-success: Request successfully matched 'Finance' topic (Provider: openai, Model: gpt-5-mini). Score: 0.62. +x-llm-proxy-routing-type: Semantic +x-llm-proxy-routing-fallback: false +x-llm-proxy-llm-provider: openai +x-llm-proxy-llm-model: gpt-5-mini +x-llm-proxy-input-tokens-cost-per-1m: 0.25 +x-llm-proxy-output-tokens-cost-per-1m: 2 +x-llm-proxy-request-success: Request completed successfully. Provider: openai, Model: gpt-5-mini. +x-envoy-decorator-operation: api-instance-21194930.14d3b31e-4e3b-4d90-b77a-63c9d6b7ea6a.svc +x-correlation-id: 63810e66-86e1-4a0f-b958-2204b646a8f2 +Strict-Transport-Security: max-age=31536000; includeSubdomains +Connection: Keep-Alive + diff --git a/python/tests/fixtures/anypoint/semantic_routing/responses.offtopic.body.json b/python/tests/fixtures/anypoint/semantic_routing/responses.offtopic.body.json new file mode 100644 index 00000000..bf52b8c6 --- /dev/null +++ b/python/tests/fixtures/anypoint/semantic_routing/responses.offtopic.body.json @@ -0,0 +1,35 @@ +{ + "id": "chatcmpl-ERd4x9soCUGxQuMDFcKmhDISaFFIB", + "object": "chat.completion", + "created": 1790253783, + "model": "gpt-5-mini-2025-08-07", + "choices": [ + { + "index": 0, + "message": { + "role": "assistant", + "content": "Ah — the classic. Which swallow do you mean: African or European?\n\n- European swallow (barn swallow, Hirundo rustica): roughly 11 m/s ≈ 40 km/h ≈ 25 mph (typical cruising speed cited by ornithological estimates).\n- \"African swallow\" covers several species with different sizes and speeds, so it’s species-dependent; many small swallows have similar cruising speeds.\n\nNote: \"airspeed velocity\" means speed through the air (not groundspeed). And yes — \"unladen\" means not carrying a coconut.", + "refusal": null, + "annotations": [] + }, + "finish_reason": "stop" + } + ], + "usage": { + "prompt_tokens": 18, + "completion_tokens": 763, + "total_tokens": 781, + "prompt_tokens_details": { + "cached_tokens": 0, + "audio_tokens": 0 + }, + "completion_tokens_details": { + "reasoning_tokens": 640, + "audio_tokens": 0, + "accepted_prediction_tokens": 0, + "rejected_prediction_tokens": 0 + } + }, + "service_tier": "default", + "system_fingerprint": null +} diff --git a/python/tests/fixtures/anypoint/semantic_routing/responses.offtopic.headers.txt b/python/tests/fixtures/anypoint/semantic_routing/responses.offtopic.headers.txt new file mode 100644 index 00000000..3b574c8f --- /dev/null +++ b/python/tests/fixtures/anypoint/semantic_routing/responses.offtopic.headers.txt @@ -0,0 +1,36 @@ +HTTP/1.1 200 OK +date: Thu, 24 Sep 2026 12:43:11 GMT +content-type: application/json +content-length: 1285 +cf-ray: a401df633d797929-CMH +cf-cache-status: DYNAMIC +server: Anypoint Flex Gateway +strict-transport-security: max-age=31536000; includeSubDomains; preload +x-content-type-options: nosniff +access-control-expose-headers: X-Request-ID,CF-Ray,CF-Ray +openai-organization: user-9c8mvuhcimhkzwlcpxtztz7a +openai-processing-ms: 8138 +openai-project: proj_UpRmBxcPjQR5y8Tap22ULwcw +openai-version: 2020-10-01 +x-openai-proxy-wasm: v0.1 +x-ratelimit-limit-requests: 500 +x-ratelimit-limit-tokens: 500000 +x-ratelimit-remaining-requests: 499 +x-ratelimit-remaining-tokens: 499985 +x-ratelimit-reset-requests: 120ms +x-ratelimit-reset-tokens: 1ms +x-request-id: req_c5beead2e7ef4819bee2ad0a1c9d15c8 +alt-svc: h3=":443"; ma=86400 +x-llm-proxy-semantic-routing-success: Request successfully matched 'Finance' topic (Provider: openai, Model: gpt-5-mini). Score: 0.51. +x-llm-proxy-routing-type: Semantic +x-llm-proxy-routing-fallback: false +x-llm-proxy-llm-provider: openai +x-llm-proxy-llm-model: gpt-5-mini +x-llm-proxy-input-tokens-cost-per-1m: 0.25 +x-llm-proxy-output-tokens-cost-per-1m: 2 +x-llm-proxy-request-success: Request completed successfully. Provider: openai, Model: gpt-5-mini. +x-envoy-decorator-operation: api-instance-21194930.14d3b31e-4e3b-4d90-b77a-63c9d6b7ea6a.svc +x-correlation-id: 5eb8eb28-6d1c-4137-9ab6-b172b2beceac +Strict-Transport-Security: max-age=31536000; includeSubdomains +Connection: Keep-Alive + From 1fcf053df2d6d2a3d328f68853b2e7febedb7bb9 Mon Sep 17 00:00:00 2001 From: Amir Khan <86777111+amirkhan-ak-sf@users.noreply.github.com> Date: Thu, 24 Sep 2026 15:08:51 +0200 Subject: [PATCH 3/3] docs(site): live Roadmap from GitHub milestones, and sync pages with recent issues (#594) * docs(roadmap): load milestone status live from GitHub, keep the snapshot as fallback (#592) The Roadmap page rendered a build-time snapshot that only refreshed on pushes to main. RoadmapPhase/RoadmapUpdated are now a client component that refetches milestones and issues from the public GitHub REST API after hydration (four requests, no token, cached 30 min in localStorage) and fall back to the snapshot on any failure. docs.yml rebuilds every 6 hours from main so the snapshot stays fresh for no-JS visitors and search. * docs(site): sync pages with issues filed since the restructure (#593) - Roadmap: phase prose covers every open milestone topic, including the TypeSafe Jev epic and gateway semantic caching/routing; the non-goal now separates gateway semantic caching from client-side caching. - Telemetry & gateway-identity: known limitation for provider/model names reading as substituted (#586); semantic caching & routing as Roadmap. - Frameworks: TypeSafe Jev as a Roadmap decision model, not OpenAI-compatible. - Gateway identity: request_id is the upstream provider's id (#542). * docs(site): correct budget headers, config coverage and error tree against the code (#593) - Budget: success/403 carry the prose x-llm-proxy-ratelimit header, only the 429 carries x-token-*; Budget parses both and stays None until observed. - Configuration: auth providers section (the AuthProvider link pointed at the unrelated Identity page), DONKEY_LLM_PROXY_KEY, correlation header overrides, and a Behaviour table (timeout, retries, model substitution, registry cache, DONKEY_NO_CACHE). - Errors: the tree lists BudgetReserveReached, ModelSubstituted and the tool access / registry / publishing errors. - Feature overview: the framework-free client card links to the Quickstart, not the OpenAI Agents SDK page. - Telemetry: last_call also carries identity and usage fields. --- .github/workflows/docs.yml | 7 + website/components/index.tsx | 89 +------ website/components/roadmap.tsx | 230 ++++++++++++++++++ website/content/budget.mdx | 26 +- website/content/errors.mdx | 6 +- website/content/examples/gateway-identity.mdx | 11 +- website/content/feature-overview.mdx | 7 +- website/content/frameworks/index.mdx | 11 + website/content/reference/configuration.mdx | 45 +++- website/content/roadmap.mdx | 33 ++- website/content/telemetry.mdx | 32 +++ website/data/roadmap.json | 174 +++++++++++-- website/public/budget.md | 26 +- website/public/errors.md | 6 +- website/public/examples/gateway-identity.md | 11 +- website/public/feature-overview.md | 5 +- website/public/frameworks.md | 11 + website/public/llms-full.txt | 167 +++++++++++-- website/public/reference/configuration.md | 45 +++- website/public/roadmap.md | 33 ++- website/public/telemetry.md | 30 +++ website/scripts/generate-roadmap.mjs | 9 +- 22 files changed, 836 insertions(+), 178 deletions(-) create mode 100644 website/components/roadmap.tsx diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index e41ef0cf..e046942a 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -13,6 +13,10 @@ on: paths: - 'website/**' - '.github/workflows/docs.yml' + # A rebuild every 6 hours refreshes the Roadmap snapshot (data/roadmap.json) that + # visitors without JavaScript, and search indexing, see. + schedule: + - cron: '17 */6 * * *' workflow_dispatch: # Least privilege + the OIDC token deploy-pages needs to publish. @@ -34,7 +38,10 @@ jobs: build: runs-on: ubuntu-latest steps: + # Scheduled runs start from the default branch (develop); always publish main. - uses: actions/checkout@v7 + with: + ref: ${{ github.event_name == 'schedule' && 'main' || github.ref }} - uses: actions/setup-node@v7 with: node-version: 20 diff --git a/website/components/index.tsx b/website/components/index.tsx index 3f7f948b..447d6d54 100644 --- a/website/components/index.tsx +++ b/website/components/index.tsx @@ -1,7 +1,6 @@ import React from 'react' import Link from 'next/link' import NextImage from 'next/image' -import roadmap from '../data/roadmap.json' /* Presentation components for the docs pages. * @@ -168,93 +167,7 @@ export function Figure({ /* ---------------------------------------------------------------- roadmap */ -type RoadmapIssue = { - number: number - title: string - state: string - url: string - epic: boolean -} - -function IssueList({ issues }: { issues: RoadmapIssue[] }) { - return ( -
    - {issues.map(issue => ( -
  • - - {issue.state === 'open' ? '○' : '●'} - - - #{issue.number} - {' '} - {issue.title} -
  • - ))} -
- ) -} - -// Renders one milestone from data/roadmap.json (see scripts/generate-roadmap.mjs): -// a status pill, a progress bar, and its issues split into open and done. -export function RoadmapPhase({ - milestone, - children, -}: { - milestone: string - children?: React.ReactNode -}) { - const m = roadmap.milestones.find(x => x.title.startsWith(milestone)) - if (!m) return null - const total = m.open + m.closed - const pct = total ? Math.round((m.closed / total) * 100) : 0 - const status = - m.open === 0 && total > 0 - ? { label: 'Complete', tone: 'live' as Tone } - : m.closed > 0 - ? { label: 'In progress', tone: 'accent' as Tone } - : { label: 'Planned', tone: 'roadmap' as Tone } - const open = m.issues.filter(i => i.state === 'open') - const done = m.issues.filter(i => i.state !== 'open') - return ( -
-
- - {m.title} - - {status.label} -
- {children ?
{children}
: null} -
- -
-

- {pct}% complete · {m.closed} done · {m.open} open -

- {open.length ? ( -
- Open issues ({open.length}) - -
- ) : null} - {done.length ? ( -
- Done ({done.length}) - -
- ) : null} -
- ) -} - -export function RoadmapUpdated() { - const date = new Date(roadmap.generatedAt) - return ( - - Status synced from GitHub milestones on{' '} - {date.toLocaleDateString('en-GB', { day: 'numeric', month: 'long', year: 'numeric' })}. - - ) -} +export { RoadmapPhase, RoadmapUpdated } from './roadmap' /* ------------------------------------------------------------------- team */ diff --git a/website/components/roadmap.tsx b/website/components/roadmap.tsx new file mode 100644 index 00000000..6751018d --- /dev/null +++ b/website/components/roadmap.tsx @@ -0,0 +1,230 @@ +'use client' + +import React, { useEffect, useState } from 'react' +import snapshot from '../data/roadmap.json' + +/* Roadmap milestones, live from GitHub. + * + * The server render (and any visitor without JavaScript) shows the build-time + * snapshot in data/roadmap.json, written by scripts/generate-roadmap.mjs. After + * hydration the page refetches the milestones and their issues from the public + * GitHub REST API and re-renders. No token is ever used here: the site is + * public, so anything shipped to the browser is readable by every visitor. + * The unauthenticated limit is 60 requests/hour per IP, and visitors behind a + * corporate NAT share one IP; one load costs four requests, so the result is + * cached in localStorage across tabs. Any failure, including a rate-limit 403, + * keeps the snapshot. */ + +type RoadmapIssue = { + number: number + title: string + state: string + url: string + epic: boolean +} + +type Milestone = { + number: number + title: string + state: string + url: string + open: number + closed: number + issues: RoadmapIssue[] +} + +type Roadmap = { generatedAt: string; milestones: Milestone[]; live?: boolean } + +const API = `https://api.github.com/repos/${snapshot.repo}` +const CACHE_KEY = 'ddk-roadmap-live-v1' +const CACHE_TTL_MS = 30 * 60 * 1000 + +type GhLabel = { name: string } +type GhIssue = { + number: number + title: string + state: string + html_url: string + labels: GhLabel[] + pull_request?: unknown + milestone: { number: number } | null +} +type GhMilestone = { number: number; title: string; state: string; html_url: string } + +async function gh(path: string): Promise { + const res = await fetch(`${API}${path}`, { headers: { accept: 'application/vnd.github+json' } }) + if (!res.ok) throw new Error(`GET ${path} → ${res.status}`) + return res.json() as Promise +} + +async function fetchLive(): Promise { + const milestones = await gh('/milestones?state=all&per_page=100') + const issues: GhIssue[] = [] + for (let page = 1; page <= 20; page++) { + const batch = await gh(`/issues?milestone=*&state=all&per_page=100&page=${page}`) + issues.push(...batch.filter(i => !i.pull_request)) + if (batch.length < 100) break + } + return { + generatedAt: new Date().toISOString(), + live: true, + milestones: milestones.map(m => { + const own = issues + .filter(i => i.milestone?.number === m.number) + .map(i => ({ + number: i.number, + title: i.title, + state: i.state, + url: i.html_url, + epic: i.labels.some(l => l.name === 'epic'), + })) + .sort((a, b) => a.number - b.number) + return { + number: m.number, + title: m.title, + state: m.state, + url: m.html_url, + open: own.filter(i => i.state === 'open').length, + closed: own.filter(i => i.state !== 'open').length, + issues: own, + } + }), + } +} + +function readCache(): Roadmap | null { + try { + const raw = localStorage.getItem(CACHE_KEY) + if (!raw) return null + const { at, data } = JSON.parse(raw) as { at: number; data: Roadmap } + return Date.now() - at < CACHE_TTL_MS ? data : null + } catch { + return null + } +} + +function writeCache(data: Roadmap) { + try { + localStorage.setItem(CACHE_KEY, JSON.stringify({ at: Date.now(), data })) + } catch { + // Storage full or disabled: the live data still renders, just uncached. + } +} + +// One request per page load, shared by every RoadmapPhase on the page. +let inflight: Promise | null = null + +function loadLive(): Promise { + if (!inflight) { + const cached = readCache() + inflight = cached + ? Promise.resolve(cached) + : fetchLive() + .then(data => { + writeCache(data) + return data + }) + .catch(() => null) + } + return inflight +} + +function useRoadmap(): Roadmap { + const [data, setData] = useState(snapshot as Roadmap) + useEffect(() => { + let active = true + loadLive().then(live => { + if (active && live) setData(live) + }) + return () => { + active = false + } + }, []) + return data +} + +type Tone = 'live' | 'roadmap' | 'accent' + +function IssueList({ issues }: { issues: RoadmapIssue[] }) { + return ( +
    + {issues.map(issue => ( +
  • + + {issue.state === 'open' ? '○' : '●'} + + + #{issue.number} + {' '} + {issue.title} +
  • + ))} +
+ ) +} + +// One milestone: a status pill, a progress bar, and its issues split into open and done. +export function RoadmapPhase({ + milestone, + children, +}: { + milestone: string + children?: React.ReactNode +}) { + const roadmap = useRoadmap() + const m = roadmap.milestones.find(x => x.title.startsWith(milestone)) + if (!m) return null + const total = m.open + m.closed + const pct = total ? Math.round((m.closed / total) * 100) : 0 + const status: { label: string; tone: Tone } = + m.open === 0 && total > 0 + ? { label: 'Complete', tone: 'live' } + : m.closed > 0 + ? { label: 'In progress', tone: 'accent' } + : { label: 'Planned', tone: 'roadmap' } + const open = m.issues.filter(i => i.state === 'open') + const done = m.issues.filter(i => i.state !== 'open') + return ( +
+
+ + {m.title} + + {status.label} +
+ {children ?
{children}
: null} +
+ +
+

+ {pct}% complete · {m.closed} done · {m.open} open +

+ {open.length ? ( +
+ Open issues ({open.length}) + +
+ ) : null} + {done.length ? ( +
+ Done ({done.length}) + +
+ ) : null} +
+ ) +} + +export function RoadmapUpdated() { + const roadmap = useRoadmap() + if (roadmap.live) { + return Status is live from the GitHub milestones. + } + const date = new Date(roadmap.generatedAt) + return ( + + Status synced from GitHub milestones on{' '} + {date.toLocaleDateString('en-GB', { day: 'numeric', month: 'long', year: 'numeric', timeZone: 'UTC' })}. + + ) +} diff --git a/website/content/budget.mdx b/website/content/budget.mdx index cd706b81..9e66b267 100644 --- a/website/content/budget.mdx +++ b/website/content/budget.mdx @@ -9,19 +9,28 @@ import { Badge } from '../components' Live -The governed proxy reports your token budget on **every response**, in -`x-token-*` headers. Parsing those yourself in every call site is tedious, and -most code skips it — so the first time budget matters is the moment it runs out. +Once a token-rate-limit policy is applied, the governed proxy reports your +token budget on its responses, in **two shapes** depending on the response: -With DDK **you never parse a header.** Every response updates a `Budget` object -on the `Donkey` instance. +| Response | Header | Example | +|---|---|---| +| Success `200` (and a `403` refusal) | `x-llm-proxy-ratelimit`, as prose | `Token rate limit: 10000 tokens remaining of 10000 limit. Reset in 56711ms.` | +| Budget refusal `429` | `x-token-limit`, `x-token-remaining`, `x-token-reset` | numeric values | + +Parsing those yourself in every call site is tedious, and most code skips +it — so the first time budget matters is the moment it runs out. + +With DDK **you never parse a header.** Every response that carries either shape +updates a `Budget` object on the `Donkey` instance. Where both are present the +numeric values win, and the prose header fills any field they leave unset. +Until the first such response, every field is `None`, never a misleading zero. ## The object ```python donkey.budget.limit # int, tokens per window donkey.budget.remaining # int, from the last response -donkey.budget.reset_at # datetime, converted from x-token-reset (ms) — not raw +donkey.budget.reset_at # datetime, converted from the ms-to-reset value — not raw donkey.budget.observed_at # when headers were last seen (staleness) donkey.budget.fraction_used # 0.0–1.0 ``` @@ -97,8 +106,9 @@ last-known-good — see [Roadmap](/roadmap). ## Behaviour guarantees -- `x-token-reset` (milliseconds) is converted to a `datetime`, accurate to the - second. +- The reset value (milliseconds *until* reset, in both `x-token-reset` and the + prose `Reset in …ms`) is converted to a `datetime` anchored to + `observed_at`, accurate to the second. - `pace()` raises before the request that would cross the reserve, never after a `429`. - After `reset_at`, `pace()` no longer refuses, so a wait-and-retry loop can diff --git a/website/content/errors.mdx b/website/content/errors.mdx index 0837284c..c276212a 100644 --- a/website/content/errors.mdx +++ b/website/content/errors.mdx @@ -61,7 +61,11 @@ DonkeyError # base of the whole tree │ └─ ContentSafetyBlocked # Azure Content Safety / Bedrock Guardrails vendor reject header; .categories ├─ GatewayUnavailable # transport failure — gateway unreachable, NO response; .base_url/.cause (ungoverned) ├─ UpstreamRequestError # upstream 4xx; .code/.error_type/.param -└─ UpstreamModelError # upstream 5xx — provider error, retryable +├─ UpstreamModelError # upstream 5xx — provider error, retryable +├─ BudgetReserveReached # client-side, from budget.pace(); .fraction_used/.reserve/.reset_at +├─ ModelSubstituted # client-side, opt-in; .requested_model/.served_model/.served_provider +└─ ToolInvocationError, RegistryError, PublicationDrift + # tool access, registry and publishing (Roadmap surfaces) ``` `GatewayUnavailable` is deliberately **not** under `PolicyViolation`: it is the one diff --git a/website/content/examples/gateway-identity.mdx b/website/content/examples/gateway-identity.mdx index 9428fc6b..f9f955db 100644 --- a/website/content/examples/gateway-identity.mdx +++ b/website/content/examples/gateway-identity.mdx @@ -204,13 +204,20 @@ except Exception as err: `OBSERVED` means the SDK saw a response (fields may still be `None` if the gateway said nothing); `UNAVAILABLE` marks adapter surfaces that never route through the SDK's transport. -- **`request_id` is the gateway's own id** (`x-request-id`) — the same field - `classify()` puts on a refusal, now present on a 200 too. +- **`request_id` is the upstream provider's id, passed through by the + gateway** — `x-request-id` for OpenAI, `x-amzn-requestid` for Amazon Bedrock, + `apim-request-id` for Azure OpenAI. Quote it to the provider's support team. + It is the same field `classify()` puts on a refusal, now present on a 200 too, + and `None` on a route whose provider forwards no id. - **Absent counts are `None`, never `0`.** `cached_tokens` are billed at the cached rate; `reasoning_tokens` are output you never see. - **`ModelSubstituted` is not a `PolicyViolation`.** The request succeeded, against a model you did not choose. `last_call` is still populated before the raise. +- **`provider/model` names read as substituted.** On a model-based routing + proxy the gateway reports the served model without the `provider/` prefix, + so `substituted` is currently `True` on every such call. Use `fallback` to + detect a real failover there (see [Telemetry & cost](/telemetry#two-behaviours-worth-knowing)). The simulator's captured success response was recorded against `gpt-5.1`, so diff --git a/website/content/feature-overview.mdx b/website/content/feature-overview.mdx index 7663a4ea..9df2060f 100644 --- a/website/content/feature-overview.mdx +++ b/website/content/feature-overview.mdx @@ -32,7 +32,7 @@ attaches there once, so you never wire it call by call. correlation, attribution and retry policy injected. No wrapper to code around, three lines to eject. - + **Purpose:** use the OpenAI SDK directly. **Advantage:** `donkey.llm.client()` returns a native `AsyncOpenAI` (or `OpenAI` with `sync=True`) — Chat Completions and Responses, streaming included — governed @@ -162,5 +162,6 @@ shared `Donkey` (`donkey.langgraph.chat_model(...)`), a module-level factory DDK makes the platform's capabilities reachable and typed; it does not reproduce them. Policy enforcement, semantic caching, provisioning, agent scanners, kill switch, trusted agent identity, approval UIs and evaluation all -stay with Agent Fabric and Omni Gateway. See the [Roadmap](/roadmap#what-ddk-will-not-build) -for the full list. +stay with Agent Fabric and Omni Gateway. Where the platform exposes a signal, +such as a semantic-cache hit or the routing decision, DDK surfaces it to your +code. See the [Roadmap](/roadmap#what-ddk-will-not-build) for the full list. diff --git a/website/content/frameworks/index.mdx b/website/content/frameworks/index.mdx index 08469995..cd8b674c 100644 --- a/website/content/frameworks/index.mdx +++ b/website/content/frameworks/index.mdx @@ -104,6 +104,17 @@ for it: pick the adapter that matches your proxy. proxy. +### Decision models: TypeSafe Jev Roadmap + +[TypeSafe Jev](https://typesafe.ai/blog/introducing-system-one-models-and-jev) +is a *System One* model: instead of generating text it answers typed questions +(yes/no probability, a choice among options, or a score on a scale) with +calibrated confidence. Its API is **not OpenAI-compatible**, so none of the +adapters above, and no OpenAI-format proxy, can call it. Planned support +returns TypeSafe's own client pointed at Jev behind Omni Gateway, with the same +auth, correlation, typed refusals, spans, budget and simulator support as LLM +calls. + ## Injection depth differs by framework How much of the SDK's HTTP layer reaches the request depends on what each diff --git a/website/content/reference/configuration.mdx b/website/content/reference/configuration.mdx index 96968649..0e3dbf15 100644 --- a/website/content/reference/configuration.mdx +++ b/website/content/reference/configuration.mdx @@ -36,6 +36,10 @@ The three required values for the LLM proxy: Missing required fields are reported **all at once** with their env-var names, so you can fix configuration in a single pass. +| Optional env var | `DonkeyConfig` field | Meaning | +|---|---|---| +| `DONKEY_LLM_PROXY_KEY` | `llm_proxy_key` | Fills the OpenAI SDK's mandatory `api_key` slot. The proxy ignores it, so leave it unset unless a tool insists on a real-looking value. | + ## JWT / model-wallet auth mode A **model-wallet** proxy identifies the caller from an IdP-issued **JWT** plus a @@ -50,7 +54,7 @@ durable wallet-selector client ID, with Client ID Enforcement disabled and **no In `jwt` mode the required fields are `llm_proxy_url` **and** `llm_proxy_wallet_client_id` — **not** `client_id` / `client_secret`. The rotating JWT is **not** a config value: supply it through an -[`AuthProvider`](/identity) passed as `Donkey(llm_auth=…)`, so the SDK can +[`AuthProvider`](#auth-providers) passed as `Donkey(llm_auth=…)`, so the SDK can re-fetch it as it rotates and refresh it once on a `401`: ```python @@ -77,6 +81,22 @@ donkey = Donkey( can't refresh it — see [Testing](/testing). +### Auth providers + +An `AuthProvider` (in `donkey_kit.core.auth`) is any object with two async +methods: `token()` returns the current credential and `invalidate()` drops a +cached one. The transport calls `invalidate()` and retries exactly once when a +downstream call returns `401`. + +| Provider | Use it for | +|---|---| +| `StaticToken(token)` | A token injected out-of-band, for example from CI. Never refreshes. | +| `AnypointConnectedApp(client_id=…, client_secret=…, control_plane_url=…, http_client=…)` | OAuth2 client credentials against the Anypoint token endpoint. Caches the token in memory and refreshes it 60 seconds before expiry. | +| `ChainedAuth(*providers)` | Tries providers in order; the first that yields a token wins. | + +For a rotating JWT from your IdP, implement the two methods yourself and pass +the object as `llm_auth`. + ## Optional attribution | Env var | Meaning | @@ -84,6 +104,19 @@ donkey = Donkey( | `DONKEY_APP_NAME` | Human-readable app name, surfaced on telemetry. | | `DONKEY_BUSINESS_GROUP` | Business group for attribution. | +## Correlation headers + +Per-call and per-run correlation IDs ride on request headers. The gateway's +inbound header names aren't published, so the SDK uses placeholder names you +can override to match your gateway. The IDs also appear on spans and +exceptions regardless of the header names. See +[Telemetry & cost](/telemetry#correlation-ids). + +| Env var | `DonkeyConfig` field | Meaning | +|---|---|---| +| `DONKEY_CORRELATION_HEADER` | `correlation_header` | Request header that carries the per-run correlation ID. | +| `DONKEY_CALL_ID_HEADER` | `call_id_header` | Request header that carries the per-call ID. | + ## Cost-attribution tags A fixed set of dimensions set once and emitted on every call (both as request @@ -129,6 +162,16 @@ inside your process, **upstream of the gateway's PII masking**, so capturing content re-exports the very text the platform masks. Enable it only for a trusted collector. See [Telemetry & cost](/telemetry). +## Behaviour + +| Env var | `DonkeyConfig` field | Default | Meaning | +|---|---|---|---| +| `DONKEY_TIMEOUT_S` | `timeout_s` | `60.0` | HTTP timeout for governed calls, in seconds. | +| `DONKEY_MAX_RETRIES` | `max_retries` | `3` | Retries for transient upstream failures (`502` / `503` / `504`) with backoff. Policy refusals are never retried, and a gateway fallback is never retried twice. | +| `DONKEY_ON_MODEL_SUBSTITUTION` | `on_model_substitution` | `off` | `off` surfaces a model substitution on `donkey.last_call`; `raise` turns it into `ModelSubstituted`. See [Telemetry & cost](/telemetry#two-behaviours-worth-knowing). | +| `DONKEY_REGISTRY_CACHE_TTL_S` | `registry_cache_ttl_s` | `300` | How long registry lookups (used by [tool access](/tool-access)) are cached in memory, in seconds. | +| `DONKEY_NO_CACHE` | — | unset | Set to `1`, `true` or `yes` to bypass that in-memory registry cache. | + ## Anypoint control plane A separate credential from the LLM proxy, used by features that call the diff --git a/website/content/roadmap.mdx b/website/content/roadmap.mdx index 7839b3da..f106763c 100644 --- a/website/content/roadmap.mdx +++ b/website/content/roadmap.mdx @@ -8,8 +8,8 @@ import { Badge, RoadmapPhase, RoadmapUpdated } from '../components' # Roadmap DDK is delivered in five phases. Each phase is a GitHub milestone with a clear -goal; the progress bars and issue lists below come straight from those -milestones. +goal; the progress bars and issue lists below are loaded from those milestones +when you open the page. Complete every issue in the milestone is closed ·{' '} @@ -30,7 +30,8 @@ milestones. conformance suite, OpenTelemetry GenAI spans, correlation IDs and cost tags. Plus a deep LangGraph adapter, `connection_kwargs()` for seven more frameworks, the decorators and CLI, the documentation site, and the PyPI - release. + release. Closing out the phase: steering the gateway's semantic cache and + surfacing cache hits and semantic-routing matches on `last_call`. ## Phase 2 — Differentiate, go beyond @@ -42,8 +43,18 @@ milestones. Governed tool access (MCP discovery and binding), A2A `serve` / `expose` / `dev`, on-behalf-of identity, human-in-the-loop, scan & publish to the registry with a GitHub Action, declarative refusal handlers and a - classification registry for custom policies, kill-switch awareness, and a - second deep framework adapter chosen by demand. + classification registry for custom policies, typed federated guardrail + verdicts, kill-switch awareness with the kill reason, and a second deep + framework adapter chosen by demand. + + Around them: an inbound correlation ID carried across MCP and A2A hops, a + stated concurrency contract for budget and telemetry under parallel fan-out, + a per-request business group for shared multi-tenant clients, bring-your-own + provider keys, and a token and registry cache. Also in this phase: governed + access to [TypeSafe Jev](https://typesafe.ai/blog/introducing-system-one-models-and-jev), + a System One decision model, with the same typed refusals, spans, budget and + simulator support as LLM calls, plus worked patterns for confidence-gated + review, tool ranking and budget-aware routing. ## Phase 3 — Platform capabilities @@ -54,8 +65,9 @@ milestones. The policy handshake (read the in-force policy set, advisory only), policy updates pushed to the code at run boundaries, a governed - structured-output path, evaluation hooks on the run span, monetary spend and - wallet warnings, and run-level cost rollups. Much of this phase depends on + structured-output path, a model catalog with honest model resolution, + evaluation hooks on the run span, monetary spend and wallet warnings, custom + cost dimensions, and run-level cost rollups. Much of this phase depends on new gateway endpoints tracked under Upstream gaps. @@ -69,7 +81,9 @@ milestones. overhead budget enforced in CI, a full pass over every error message, the public API contract and deprecation policy, compliance evidence mapping (EU AI Act Art. 12, ISO 42001, OWASP LLM Top 10), log shipping, data residency, - workload identity, air-gapped operation and gateway failover. + workload identity, air-gapped operation, opt-in gateway failover, a clear + boundary against the platform's infrastructure-as-code, and a support model + with a release cadence. ## Phase 5 — Complete rollout @@ -105,7 +119,8 @@ At each of these boundaries the job is to make the platform's own capability reachable and typed, not to reproduce it: - Client-side policy enforcement — the gateway is the enforcement point. -- Client-side semantic caching. +- Client-side semantic caching — DDK steers the gateway's semantic cache and + reports hits, but never caches responses itself. - A provisioning control plane competing with API Manager or Terraform. - Re-implementations of Agent Scanners, Kill Switch or Trusted Agent Identity. - An approval UI or queue. diff --git a/website/content/telemetry.mdx b/website/content/telemetry.mdx index 016627d3..875259b9 100644 --- a/website/content/telemetry.mdx +++ b/website/content/telemetry.mdx @@ -368,6 +368,12 @@ They also land on the span (`gen_ai.response.model`, `donkey.routing.type`, spikes: it tells an operator whether a slow call was routed normally or recovered from a degraded provider. +The same record also carries the call's identity and usage: `request_id` (the +upstream provider's id), `api_instance_id` and `environment_id` (which gateway +instance served it), and the token counts including cached and reasoning +tokens. The [gateway identity example](/examples/gateway-identity) walks +through every field. + ### When `last_call` is unavailable `donkey.last_call` is populated only when the governed response passes through @@ -416,6 +422,32 @@ donkey = Donkey.from_env(on_model_substitution="raise") `DONKEY_ON_MODEL_SUBSTITUTION` → `.donkey-kit.toml` → default) and **defaults to `"off"`**. + + **Known limitation with `provider/model` names.** On a model-based routing + proxy you address models as `provider/model` (for example + `openai/gpt-5-mini`), but the gateway reports the served model without the + provider prefix (`gpt-5-mini`). DDK currently compares the two verbatim, so + `last_call.substituted` reads `True` on every such call, and + `on_model_substitution="raise"` raises `ModelSubstituted` even when + `last_call.fallback` is `False`. Until this is fixed, check + `last_call.fallback` to detect a real failover, and leave + `on_model_substitution` at `"off"` on model-based proxies. + + +### Semantic caching & semantic routing Roadmap + +Omni Gateway can answer a request from its **semantic cache** (a similar +prompt was answered before, so no provider round-trip and no token cost) and +can **route semantically** (pick the model by matching the prompt to a +topic). Both happen at the gateway. DDK will not cache anything or compute +embeddings itself; it will let you steer the gateway's cache per request, for +example to skip it, not store a response, or override the TTL or similarity +threshold, and report what happened: + +- the cache outcome (`hit`, `miss`, `bypass`, `no-store`) and similarity score + on `donkey.last_call` and the span; +- the matched routing topic and its score, beside the routing fields above. + ### TypeScript parity Roadmap The same signals will surface as `donkey.lastRouting.servedModel` / `.fallback` diff --git a/website/data/roadmap.json b/website/data/roadmap.json index 780ba335..6efa46da 100644 --- a/website/data/roadmap.json +++ b/website/data/roadmap.json @@ -1,14 +1,14 @@ { "repo": "Donkey-Development-Kit/donkey-development-kit", - "generatedAt": "2026-09-24T09:35:41.948Z", + "generatedAt": "2026-09-24T12:48:52.545Z", "milestones": [ { "number": 11, "title": "Phase 1 — Build the MVP (0.1.0)", "state": "open", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/milestone/11", - "open": 15, - "closed": 113, + "open": 7, + "closed": 125, "issues": [ { "number": 9, @@ -825,86 +825,114 @@ { "number": 556, "title": "epic(docs): restructure the docs site into end-user documentation organised by capability pillars", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/556", "epic": true }, { "number": 557, "title": "docs(intro): rewrite the Introduction around gateway awareness, with architecture and before/after sections", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/557", "epic": false }, { "number": 558, "title": "docs(quickstart): simplify the Quickstart and mark each capability Live or Roadmap", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/558", "epic": false }, { "number": 559, "title": "docs(feature-overview): high-level capability tour with the gateway-aware image", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/559", "epic": false }, { "number": 560, "title": "docs(nav): restructure the sidebar from phase separators into capability pillars", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/560", "epic": false }, { "number": 561, "title": "docs(pages): remove issue refs, phase labels and verification narration from capability pages", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/561", "epic": false }, { "number": 562, "title": "docs(examples): add an Examples pillar built on the donkey-development-kit-demos repo", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/562", "epic": false }, { "number": 563, "title": "docs(roadmap): phases in order with live milestone progress and the issues in each", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/563", "epic": false }, { "number": 564, "title": "docs(community): add a Team (About us) page with Creator/Contributor pills", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/564", "epic": false }, { "number": 565, "title": "docs(community): add a Contribute page condensed from CONTRIBUTING.md", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/565", "epic": false }, { "number": 566, "title": "docs(site): open-source disclaimer footer, metadata and optimised images", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/566", "epic": false }, { "number": 568, "title": "errors: flip Amazon Bedrock Guardrails refusal from documented-only to VERIFIED (LIVE) after the acceptance capture", - "state": "open", + "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/568", "epic": false + }, + { + "number": 587, + "title": "Support gateway semantic caching: steer requests and surface hits", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/587", + "epic": false + }, + { + "number": 590, + "title": "Surface the semantic-routing match topic and score on last_call", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/590", + "epic": false + }, + { + "number": 592, + "title": "docs(roadmap): fetch milestone status live from GitHub, keep the snapshot as fallback", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/592", + "epic": false + }, + { + "number": 593, + "title": "docs(site): sync pages with the current code and the issues filed since the restructure", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/593", + "epic": false } ] }, @@ -913,7 +941,7 @@ "title": "Phase 2 — Differentiate, go beyond (0.2.0)", "state": "open", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/milestone/12", - "open": 33, + "open": 47, "closed": 1, "issues": [ { @@ -1153,6 +1181,104 @@ "state": "open", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/503", "epic": false + }, + { + "number": 572, + "title": "epic(jev): support TypeSafe Jev (System One models) as a governed decision surface", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/572", + "epic": true + }, + { + "number": 573, + "title": "verify: govern TypeSafe's System One endpoint through Omni Gateway and capture its shapes", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/573", + "epic": false + }, + { + "number": 574, + "title": "adapters: donkey.typesafe — a governed native TypeSafeClient / AsyncTypeSafeClient", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/574", + "epic": false + }, + { + "number": 575, + "title": "retry: tell a gateway budget 429 apart from TypeSafe's own 429/529 on the Jev route", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/575", + "epic": false + }, + { + "number": 576, + "title": "refusals: classify System One failures into the DDK error taxonomy", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/576", + "epic": false + }, + { + "number": 577, + "title": "telemetry: a correlated span per System One call, with usage and confidence", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/577", + "epic": false + }, + { + "number": 578, + "title": "budget: expose the Jev route's rate-limit window through donkey.budget", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/578", + "epic": false + }, + { + "number": 579, + "title": "simulator: serve /v1/systemone in donkey mock and simulate()", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/579", + "epic": false + }, + { + "number": 580, + "title": "spike: run system-one-adapter over a governed DDK proxy (the OpenAI-compatible route)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/580", + "epic": false + }, + { + "number": 581, + "title": "example(jev): confidence-gated human review through the HITL vocabulary", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/581", + "epic": false + }, + { + "number": 582, + "title": "example(jev): rank governed MCP tools and A2A agents with a Choice question", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/582", + "epic": false + }, + { + "number": 583, + "title": "example(jev): advisory pre-screen to avoid gateway refusals (gateway stays the enforcement point)", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/583", + "epic": false + }, + { + "number": 584, + "title": "example(jev): budget-aware cascade across code, small and large governed models", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/584", + "epic": false + }, + { + "number": 585, + "title": "docs(site): document TypeSafe Jev support, examples and System One sections", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/585", + "epic": false } ] }, @@ -1401,7 +1527,7 @@ "title": "Verification", "state": "open", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/milestone/16", - "open": 26, + "open": 28, "closed": 5, "issues": [ { @@ -1620,6 +1746,20 @@ "state": "closed", "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/540", "epic": false + }, + { + "number": 588, + "title": "verify: the semantic-caching policy header contract on the LLM proxy", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/588", + "epic": false + }, + { + "number": 589, + "title": "verify: the semantic-routing success header (topic + score) on the LLM proxy", + "state": "open", + "url": "https://github.com/Donkey-Development-Kit/donkey-development-kit/issues/589", + "epic": false } ] }, diff --git a/website/public/budget.md b/website/public/budget.md index f59f96e3..f3b96edf 100644 --- a/website/public/budget.md +++ b/website/public/budget.md @@ -2,19 +2,28 @@ Live -The governed proxy reports your token budget on **every response**, in -`x-token-*` headers. Parsing those yourself in every call site is tedious, and -most code skips it — so the first time budget matters is the moment it runs out. +Once a token-rate-limit policy is applied, the governed proxy reports your +token budget on its responses, in **two shapes** depending on the response: -With DDK **you never parse a header.** Every response updates a `Budget` object -on the `Donkey` instance. +| Response | Header | Example | +|---|---|---| +| Success `200` (and a `403` refusal) | `x-llm-proxy-ratelimit`, as prose | `Token rate limit: 10000 tokens remaining of 10000 limit. Reset in 56711ms.` | +| Budget refusal `429` | `x-token-limit`, `x-token-remaining`, `x-token-reset` | numeric values | + +Parsing those yourself in every call site is tedious, and most code skips +it — so the first time budget matters is the moment it runs out. + +With DDK **you never parse a header.** Every response that carries either shape +updates a `Budget` object on the `Donkey` instance. Where both are present the +numeric values win, and the prose header fills any field they leave unset. +Until the first such response, every field is `None`, never a misleading zero. ## The object ```python donkey.budget.limit # int, tokens per window donkey.budget.remaining # int, from the last response -donkey.budget.reset_at # datetime, converted from x-token-reset (ms) — not raw +donkey.budget.reset_at # datetime, converted from the ms-to-reset value — not raw donkey.budget.observed_at # when headers were last seen (staleness) donkey.budget.fraction_used # 0.0–1.0 ``` @@ -88,8 +97,9 @@ last-known-good — see [Roadmap](https://donkey-development-kit.github.io/donke ## Behaviour guarantees -- `x-token-reset` (milliseconds) is converted to a `datetime`, accurate to the - second. +- The reset value (milliseconds *until* reset, in both `x-token-reset` and the + prose `Reset in …ms`) is converted to a `datetime` anchored to + `observed_at`, accurate to the second. - `pace()` raises before the request that would cross the reserve, never after a `429`. - After `reset_at`, `pace()` no longer refuses, so a wait-and-retry loop can diff --git a/website/public/errors.md b/website/public/errors.md index 0fb3e4da..3aaa807b 100644 --- a/website/public/errors.md +++ b/website/public/errors.md @@ -54,7 +54,11 @@ DonkeyError # base of the whole tree │ └─ ContentSafetyBlocked # Azure Content Safety / Bedrock Guardrails vendor reject header; .categories ├─ GatewayUnavailable # transport failure — gateway unreachable, NO response; .base_url/.cause (ungoverned) ├─ UpstreamRequestError # upstream 4xx; .code/.error_type/.param -└─ UpstreamModelError # upstream 5xx — provider error, retryable +├─ UpstreamModelError # upstream 5xx — provider error, retryable +├─ BudgetReserveReached # client-side, from budget.pace(); .fraction_used/.reserve/.reset_at +├─ ModelSubstituted # client-side, opt-in; .requested_model/.served_model/.served_provider +└─ ToolInvocationError, RegistryError, PublicationDrift + # tool access, registry and publishing (Roadmap surfaces) ``` `GatewayUnavailable` is deliberately **not** under `PolicyViolation`: it is the one diff --git a/website/public/examples/gateway-identity.md b/website/public/examples/gateway-identity.md index eba80c57..3c4bc7f9 100644 --- a/website/public/examples/gateway-identity.md +++ b/website/public/examples/gateway-identity.md @@ -192,13 +192,20 @@ except Exception as err: `OBSERVED` means the SDK saw a response (fields may still be `None` if the gateway said nothing); `UNAVAILABLE` marks adapter surfaces that never route through the SDK's transport. -- **`request_id` is the gateway's own id** (`x-request-id`) — the same field - `classify()` puts on a refusal, now present on a 200 too. +- **`request_id` is the upstream provider's id, passed through by the + gateway** — `x-request-id` for OpenAI, `x-amzn-requestid` for Amazon Bedrock, + `apim-request-id` for Azure OpenAI. Quote it to the provider's support team. + It is the same field `classify()` puts on a refusal, now present on a 200 too, + and `None` on a route whose provider forwards no id. - **Absent counts are `None`, never `0`.** `cached_tokens` are billed at the cached rate; `reasoning_tokens` are output you never see. - **`ModelSubstituted` is not a `PolicyViolation`.** The request succeeded, against a model you did not choose. `last_call` is still populated before the raise. +- **`provider/model` names read as substituted.** On a model-based routing + proxy the gateway reports the served model without the `provider/` prefix, + so `substituted` is currently `True` on every such call. Use `fallback` to + detect a real failover there (see [Telemetry & cost](https://donkey-development-kit.github.io/donkey-development-kit/telemetry.md#two-behaviours-worth-knowing)). The simulator's captured success response was recorded against `gpt-5.1`, so asking for any other model id shows up as a substitution in narrative demo diff --git a/website/public/feature-overview.md b/website/public/feature-overview.md index f6fddd29..e917ba7e 100644 --- a/website/public/feature-overview.md +++ b/website/public/feature-overview.md @@ -139,5 +139,6 @@ shared `Donkey` (`donkey.langgraph.chat_model(...)`), a module-level factory DDK makes the platform's capabilities reachable and typed; it does not reproduce them. Policy enforcement, semantic caching, provisioning, agent scanners, kill switch, trusted agent identity, approval UIs and evaluation all -stay with Agent Fabric and Omni Gateway. See the [Roadmap](https://donkey-development-kit.github.io/donkey-development-kit/roadmap.md#what-ddk-will-not-build) -for the full list. +stay with Agent Fabric and Omni Gateway. Where the platform exposes a signal, +such as a semantic-cache hit or the routing decision, DDK surfaces it to your +code. See the [Roadmap](https://donkey-development-kit.github.io/donkey-development-kit/roadmap.md#what-ddk-will-not-build) for the full list. diff --git a/website/public/frameworks.md b/website/public/frameworks.md index 783f21bc..3b64d39c 100644 --- a/website/public/frameworks.md +++ b/website/public/frameworks.md @@ -91,6 +91,17 @@ for it: pick the adapter that matches your proxy. proxy: send an OpenAI-format request naming that model to an OpenAI-format proxy. +### Decision models: TypeSafe Jev Roadmap + +[TypeSafe Jev](https://typesafe.ai/blog/introducing-system-one-models-and-jev) +is a *System One* model: instead of generating text it answers typed questions +(yes/no probability, a choice among options, or a score on a scale) with +calibrated confidence. Its API is **not OpenAI-compatible**, so none of the +adapters above, and no OpenAI-format proxy, can call it. Planned support +returns TypeSafe's own client pointed at Jev behind Omni Gateway, with the same +auth, correlation, typed refusals, spans, budget and simulator support as LLM +calls. + ## Injection depth differs by framework How much of the SDK's HTTP layer reaches the request depends on what each diff --git a/website/public/llms-full.txt b/website/public/llms-full.txt index cf578a14..1be42a36 100644 --- a/website/public/llms-full.txt +++ b/website/public/llms-full.txt @@ -686,8 +686,9 @@ shared `Donkey` (`donkey.langgraph.chat_model(...)`), a module-level factory DDK makes the platform's capabilities reachable and typed; it does not reproduce them. Policy enforcement, semantic caching, provisioning, agent scanners, kill switch, trusted agent identity, approval UIs and evaluation all -stay with Agent Fabric and Omni Gateway. See the [Roadmap](https://donkey-development-kit.github.io/donkey-development-kit/roadmap.md#what-ddk-will-not-build) -for the full list. +stay with Agent Fabric and Omni Gateway. Where the platform exposes a signal, +such as a semantic-cache hit or the routing decision, DDK surfaces it to your +code. See the [Roadmap](https://donkey-development-kit.github.io/donkey-development-kit/roadmap.md#what-ddk-will-not-build) for the full list. --- @@ -786,6 +787,17 @@ for it: pick the adapter that matches your proxy. proxy: send an OpenAI-format request naming that model to an OpenAI-format proxy. +### Decision models: TypeSafe Jev Roadmap + +[TypeSafe Jev](https://typesafe.ai/blog/introducing-system-one-models-and-jev) +is a *System One* model: instead of generating text it answers typed questions +(yes/no probability, a choice among options, or a score on a scale) with +calibrated confidence. Its API is **not OpenAI-compatible**, so none of the +adapters above, and no OpenAI-format proxy, can call it. Planned support +returns TypeSafe's own client pointed at Jev behind Omni Gateway, with the same +auth, correlation, typed refusals, spans, budget and simulator support as LLM +calls. + ## Injection depth differs by framework How much of the SDK's HTTP layer reaches the request depends on what each @@ -1945,7 +1957,11 @@ DonkeyError # base of the whole tree │ └─ ContentSafetyBlocked # Azure Content Safety / Bedrock Guardrails vendor reject header; .categories ├─ GatewayUnavailable # transport failure — gateway unreachable, NO response; .base_url/.cause (ungoverned) ├─ UpstreamRequestError # upstream 4xx; .code/.error_type/.param -└─ UpstreamModelError # upstream 5xx — provider error, retryable +├─ UpstreamModelError # upstream 5xx — provider error, retryable +├─ BudgetReserveReached # client-side, from budget.pace(); .fraction_used/.reserve/.reset_at +├─ ModelSubstituted # client-side, opt-in; .requested_model/.served_model/.served_provider +└─ ToolInvocationError, RegistryError, PublicationDrift + # tool access, registry and publishing (Roadmap surfaces) ``` `GatewayUnavailable` is deliberately **not** under `PolicyViolation`: it is the one @@ -2111,19 +2127,28 @@ Source: https://donkey-development-kit.github.io/donkey-development-kit/budget.m Live -The governed proxy reports your token budget on **every response**, in -`x-token-*` headers. Parsing those yourself in every call site is tedious, and -most code skips it — so the first time budget matters is the moment it runs out. +Once a token-rate-limit policy is applied, the governed proxy reports your +token budget on its responses, in **two shapes** depending on the response: + +| Response | Header | Example | +|---|---|---| +| Success `200` (and a `403` refusal) | `x-llm-proxy-ratelimit`, as prose | `Token rate limit: 10000 tokens remaining of 10000 limit. Reset in 56711ms.` | +| Budget refusal `429` | `x-token-limit`, `x-token-remaining`, `x-token-reset` | numeric values | + +Parsing those yourself in every call site is tedious, and most code skips +it — so the first time budget matters is the moment it runs out. -With DDK **you never parse a header.** Every response updates a `Budget` object -on the `Donkey` instance. +With DDK **you never parse a header.** Every response that carries either shape +updates a `Budget` object on the `Donkey` instance. Where both are present the +numeric values win, and the prose header fills any field they leave unset. +Until the first such response, every field is `None`, never a misleading zero. ## The object ```python donkey.budget.limit # int, tokens per window donkey.budget.remaining # int, from the last response -donkey.budget.reset_at # datetime, converted from x-token-reset (ms) — not raw +donkey.budget.reset_at # datetime, converted from the ms-to-reset value — not raw donkey.budget.observed_at # when headers were last seen (staleness) donkey.budget.fraction_used # 0.0–1.0 ``` @@ -2197,8 +2222,9 @@ last-known-good — see [Roadmap](https://donkey-development-kit.github.io/donke ## Behaviour guarantees -- `x-token-reset` (milliseconds) is converted to a `datetime`, accurate to the - second. +- The reset value (milliseconds *until* reset, in both `x-token-reset` and the + prose `Reset in …ms`) is converted to a `datetime` anchored to + `observed_at`, accurate to the second. - `pace()` raises before the request that would cross the reserve, never after a `429`. - After `reset_at`, `pace()` no longer refuses, so a wait-and-retry loop can @@ -2710,6 +2736,12 @@ They also land on the span (`gen_ai.response.model`, `donkey.routing.type`, spikes: it tells an operator whether a slow call was routed normally or recovered from a degraded provider. +The same record also carries the call's identity and usage: `request_id` (the +upstream provider's id), `api_instance_id` and `environment_id` (which gateway +instance served it), and the token counts including cached and reasoning +tokens. The [gateway identity example](https://donkey-development-kit.github.io/donkey-development-kit/examples/gateway-identity.md) walks +through every field. + ### When `last_call` is unavailable `donkey.last_call` is populated only when the governed response passes through @@ -2758,6 +2790,30 @@ donkey = Donkey.from_env(on_model_substitution="raise") `DONKEY_ON_MODEL_SUBSTITUTION` → `.donkey-kit.toml` → default) and **defaults to `"off"`**. + **Known limitation with `provider/model` names.** On a model-based routing + proxy you address models as `provider/model` (for example + `openai/gpt-5-mini`), but the gateway reports the served model without the + provider prefix (`gpt-5-mini`). DDK currently compares the two verbatim, so + `last_call.substituted` reads `True` on every such call, and + `on_model_substitution="raise"` raises `ModelSubstituted` even when + `last_call.fallback` is `False`. Until this is fixed, check + `last_call.fallback` to detect a real failover, and leave + `on_model_substitution` at `"off"` on model-based proxies. + +### Semantic caching & semantic routing Roadmap + +Omni Gateway can answer a request from its **semantic cache** (a similar +prompt was answered before, so no provider round-trip and no token cost) and +can **route semantically** (pick the model by matching the prompt to a +topic). Both happen at the gateway. DDK will not cache anything or compute +embeddings itself; it will let you steer the gateway's cache per request, for +example to skip it, not store a response, or override the TTL or similarity +threshold, and report what happened: + +- the cache outcome (`hit`, `miss`, `bypass`, `no-store`) and similarity score + on `donkey.last_call` and the span; +- the matched routing topic and its score, beside the routing fields above. + ### TypeScript parity Roadmap The same signals will surface as `donkey.lastRouting.servedModel` / `.fallback` @@ -6504,13 +6560,20 @@ except Exception as err: `OBSERVED` means the SDK saw a response (fields may still be `None` if the gateway said nothing); `UNAVAILABLE` marks adapter surfaces that never route through the SDK's transport. -- **`request_id` is the gateway's own id** (`x-request-id`) — the same field - `classify()` puts on a refusal, now present on a 200 too. +- **`request_id` is the upstream provider's id, passed through by the + gateway** — `x-request-id` for OpenAI, `x-amzn-requestid` for Amazon Bedrock, + `apim-request-id` for Azure OpenAI. Quote it to the provider's support team. + It is the same field `classify()` puts on a refusal, now present on a 200 too, + and `None` on a route whose provider forwards no id. - **Absent counts are `None`, never `0`.** `cached_tokens` are billed at the cached rate; `reasoning_tokens` are output you never see. - **`ModelSubstituted` is not a `PolicyViolation`.** The request succeeded, against a model you did not choose. `last_call` is still populated before the raise. +- **`provider/model` names read as substituted.** On a model-based routing + proxy the gateway reports the served model without the `provider/` prefix, + so `substituted` is currently `True` on every such call. Use `fallback` to + detect a real failover there (see [Telemetry & cost](https://donkey-development-kit.github.io/donkey-development-kit/telemetry.md#two-behaviours-worth-knowing)). The simulator's captured success response was recorded against `gpt-5.1`, so asking for any other model id shows up as a substitution in narrative demo @@ -6530,8 +6593,8 @@ Source: https://donkey-development-kit.github.io/donkey-development-kit/roadmap. # Roadmap DDK is delivered in five phases. Each phase is a GitHub milestone with a clear -goal; the progress bars and issue lists below come straight from those -milestones. +goal; the progress bars and issue lists below are loaded from those milestones +when you open the page. Complete every issue in the milestone is closed ·{' '} In progress work has landed and more is open ·{' '} @@ -6549,7 +6612,8 @@ milestones. conformance suite, OpenTelemetry GenAI spans, correlation IDs and cost tags. Plus a deep LangGraph adapter, `connection_kwargs()` for seven more frameworks, the decorators and CLI, the documentation site, and the PyPI - release. + release. Closing out the phase: steering the gateway's semantic cache and + surfacing cache hits and semantic-routing matches on `last_call`. ## Phase 2 — Differentiate, go beyond @@ -6559,8 +6623,18 @@ milestones. Governed tool access (MCP discovery and binding), A2A `serve` / `expose` / `dev`, on-behalf-of identity, human-in-the-loop, scan & publish to the registry with a GitHub Action, declarative refusal handlers and a - classification registry for custom policies, kill-switch awareness, and a - second deep framework adapter chosen by demand. + classification registry for custom policies, typed federated guardrail + verdicts, kill-switch awareness with the kill reason, and a second deep + framework adapter chosen by demand. + + Around them: an inbound correlation ID carried across MCP and A2A hops, a + stated concurrency contract for budget and telemetry under parallel fan-out, + a per-request business group for shared multi-tenant clients, bring-your-own + provider keys, and a token and registry cache. Also in this phase: governed + access to [TypeSafe Jev](https://typesafe.ai/blog/introducing-system-one-models-and-jev), + a System One decision model, with the same typed refusals, spans, budget and + simulator support as LLM calls, plus worked patterns for confidence-gated + review, tool ranking and budget-aware routing. ## Phase 3 — Platform capabilities @@ -6569,8 +6643,9 @@ milestones. The policy handshake (read the in-force policy set, advisory only), policy updates pushed to the code at run boundaries, a governed - structured-output path, evaluation hooks on the run span, monetary spend and - wallet warnings, and run-level cost rollups. Much of this phase depends on + structured-output path, a model catalog with honest model resolution, + evaluation hooks on the run span, monetary spend and wallet warnings, custom + cost dimensions, and run-level cost rollups. Much of this phase depends on new gateway endpoints tracked under Upstream gaps. ## Phase 4 — Enterprise readiness @@ -6582,7 +6657,9 @@ milestones. overhead budget enforced in CI, a full pass over every error message, the public API contract and deprecation policy, compliance evidence mapping (EU AI Act Art. 12, ISO 42001, OWASP LLM Top 10), log shipping, data residency, - workload identity, air-gapped operation and gateway failover. + workload identity, air-gapped operation, opt-in gateway failover, a clear + boundary against the platform's infrastructure-as-code, and a support model + with a release cadence. ## Phase 5 — Complete rollout @@ -6611,7 +6688,8 @@ At each of these boundaries the job is to make the platform's own capability reachable and typed, not to reproduce it: - Client-side policy enforcement — the gateway is the enforcement point. -- Client-side semantic caching. +- Client-side semantic caching — DDK steers the gateway's semantic cache and + reports hits, but never caches responses itself. - A provisioning control plane competing with API Manager or Terraform. - Re-implementations of Agent Scanners, Kill Switch or Trusted Agent Identity. - An approval UI or queue. @@ -6784,6 +6862,10 @@ The three required values for the LLM proxy: Missing required fields are reported **all at once** with their env-var names, so you can fix configuration in a single pass. +| Optional env var | `DonkeyConfig` field | Meaning | +|---|---|---| +| `DONKEY_LLM_PROXY_KEY` | `llm_proxy_key` | Fills the OpenAI SDK's mandatory `api_key` slot. The proxy ignores it, so leave it unset unless a tool insists on a real-looking value. | + ## JWT / model-wallet auth mode A **model-wallet** proxy identifies the caller from an IdP-issued **JWT** plus a @@ -6798,7 +6880,7 @@ durable wallet-selector client ID, with Client ID Enforcement disabled and **no In `jwt` mode the required fields are `llm_proxy_url` **and** `llm_proxy_wallet_client_id` — **not** `client_id` / `client_secret`. The rotating JWT is **not** a config value: supply it through an -[`AuthProvider`](https://donkey-development-kit.github.io/donkey-development-kit/identity.md) passed as `Donkey(llm_auth=…)`, so the SDK can +[`AuthProvider`](#auth-providers) passed as `Donkey(llm_auth=…)`, so the SDK can re-fetch it as it rotates and refresh it once on a `401`: ```python @@ -6823,6 +6905,22 @@ donkey = Donkey( CrewAI, LlamaIndex, MS Agent Framework) pin the token at construction and can't refresh it — see [Testing](https://donkey-development-kit.github.io/donkey-development-kit/testing.md). +### Auth providers + +An `AuthProvider` (in `donkey_kit.core.auth`) is any object with two async +methods: `token()` returns the current credential and `invalidate()` drops a +cached one. The transport calls `invalidate()` and retries exactly once when a +downstream call returns `401`. + +| Provider | Use it for | +|---|---| +| `StaticToken(token)` | A token injected out-of-band, for example from CI. Never refreshes. | +| `AnypointConnectedApp(client_id=…, client_secret=…, control_plane_url=…, http_client=…)` | OAuth2 client credentials against the Anypoint token endpoint. Caches the token in memory and refreshes it 60 seconds before expiry. | +| `ChainedAuth(*providers)` | Tries providers in order; the first that yields a token wins. | + +For a rotating JWT from your IdP, implement the two methods yourself and pass +the object as `llm_auth`. + ## Optional attribution | Env var | Meaning | @@ -6830,6 +6928,19 @@ donkey = Donkey( | `DONKEY_APP_NAME` | Human-readable app name, surfaced on telemetry. | | `DONKEY_BUSINESS_GROUP` | Business group for attribution. | +## Correlation headers + +Per-call and per-run correlation IDs ride on request headers. The gateway's +inbound header names aren't published, so the SDK uses placeholder names you +can override to match your gateway. The IDs also appear on spans and +exceptions regardless of the header names. See +[Telemetry & cost](https://donkey-development-kit.github.io/donkey-development-kit/telemetry.md#correlation-ids). + +| Env var | `DonkeyConfig` field | Meaning | +|---|---|---| +| `DONKEY_CORRELATION_HEADER` | `correlation_header` | Request header that carries the per-run correlation ID. | +| `DONKEY_CALL_ID_HEADER` | `call_id_header` | Request header that carries the per-call ID. | + ## Cost-attribution tags A fixed set of dimensions set once and emitted on every call (both as request @@ -6875,6 +6986,16 @@ inside your process, **upstream of the gateway's PII masking**, so capturing content re-exports the very text the platform masks. Enable it only for a trusted collector. See [Telemetry & cost](https://donkey-development-kit.github.io/donkey-development-kit/telemetry.md). +## Behaviour + +| Env var | `DonkeyConfig` field | Default | Meaning | +|---|---|---|---| +| `DONKEY_TIMEOUT_S` | `timeout_s` | `60.0` | HTTP timeout for governed calls, in seconds. | +| `DONKEY_MAX_RETRIES` | `max_retries` | `3` | Retries for transient upstream failures (`502` / `503` / `504`) with backoff. Policy refusals are never retried, and a gateway fallback is never retried twice. | +| `DONKEY_ON_MODEL_SUBSTITUTION` | `on_model_substitution` | `off` | `off` surfaces a model substitution on `donkey.last_call`; `raise` turns it into `ModelSubstituted`. See [Telemetry & cost](https://donkey-development-kit.github.io/donkey-development-kit/telemetry.md#two-behaviours-worth-knowing). | +| `DONKEY_REGISTRY_CACHE_TTL_S` | `registry_cache_ttl_s` | `300` | How long registry lookups (used by [tool access](https://donkey-development-kit.github.io/donkey-development-kit/tool-access.md)) are cached in memory, in seconds. | +| `DONKEY_NO_CACHE` | — | unset | Set to `1`, `true` or `yes` to bypass that in-memory registry cache. | + ## Anypoint control plane A separate credential from the LLM proxy, used by features that call the diff --git a/website/public/reference/configuration.md b/website/public/reference/configuration.md index e4cee7f4..021bc04e 100644 --- a/website/public/reference/configuration.md +++ b/website/public/reference/configuration.md @@ -28,6 +28,10 @@ The three required values for the LLM proxy: Missing required fields are reported **all at once** with their env-var names, so you can fix configuration in a single pass. +| Optional env var | `DonkeyConfig` field | Meaning | +|---|---|---| +| `DONKEY_LLM_PROXY_KEY` | `llm_proxy_key` | Fills the OpenAI SDK's mandatory `api_key` slot. The proxy ignores it, so leave it unset unless a tool insists on a real-looking value. | + ## JWT / model-wallet auth mode A **model-wallet** proxy identifies the caller from an IdP-issued **JWT** plus a @@ -42,7 +46,7 @@ durable wallet-selector client ID, with Client ID Enforcement disabled and **no In `jwt` mode the required fields are `llm_proxy_url` **and** `llm_proxy_wallet_client_id` — **not** `client_id` / `client_secret`. The rotating JWT is **not** a config value: supply it through an -[`AuthProvider`](https://donkey-development-kit.github.io/donkey-development-kit/identity.md) passed as `Donkey(llm_auth=…)`, so the SDK can +[`AuthProvider`](#auth-providers) passed as `Donkey(llm_auth=…)`, so the SDK can re-fetch it as it rotates and refresh it once on a `401`: ```python @@ -67,6 +71,22 @@ donkey = Donkey( CrewAI, LlamaIndex, MS Agent Framework) pin the token at construction and can't refresh it — see [Testing](https://donkey-development-kit.github.io/donkey-development-kit/testing.md). +### Auth providers + +An `AuthProvider` (in `donkey_kit.core.auth`) is any object with two async +methods: `token()` returns the current credential and `invalidate()` drops a +cached one. The transport calls `invalidate()` and retries exactly once when a +downstream call returns `401`. + +| Provider | Use it for | +|---|---| +| `StaticToken(token)` | A token injected out-of-band, for example from CI. Never refreshes. | +| `AnypointConnectedApp(client_id=…, client_secret=…, control_plane_url=…, http_client=…)` | OAuth2 client credentials against the Anypoint token endpoint. Caches the token in memory and refreshes it 60 seconds before expiry. | +| `ChainedAuth(*providers)` | Tries providers in order; the first that yields a token wins. | + +For a rotating JWT from your IdP, implement the two methods yourself and pass +the object as `llm_auth`. + ## Optional attribution | Env var | Meaning | @@ -74,6 +94,19 @@ donkey = Donkey( | `DONKEY_APP_NAME` | Human-readable app name, surfaced on telemetry. | | `DONKEY_BUSINESS_GROUP` | Business group for attribution. | +## Correlation headers + +Per-call and per-run correlation IDs ride on request headers. The gateway's +inbound header names aren't published, so the SDK uses placeholder names you +can override to match your gateway. The IDs also appear on spans and +exceptions regardless of the header names. See +[Telemetry & cost](https://donkey-development-kit.github.io/donkey-development-kit/telemetry.md#correlation-ids). + +| Env var | `DonkeyConfig` field | Meaning | +|---|---|---| +| `DONKEY_CORRELATION_HEADER` | `correlation_header` | Request header that carries the per-run correlation ID. | +| `DONKEY_CALL_ID_HEADER` | `call_id_header` | Request header that carries the per-call ID. | + ## Cost-attribution tags A fixed set of dimensions set once and emitted on every call (both as request @@ -119,6 +152,16 @@ inside your process, **upstream of the gateway's PII masking**, so capturing content re-exports the very text the platform masks. Enable it only for a trusted collector. See [Telemetry & cost](https://donkey-development-kit.github.io/donkey-development-kit/telemetry.md). +## Behaviour + +| Env var | `DonkeyConfig` field | Default | Meaning | +|---|---|---|---| +| `DONKEY_TIMEOUT_S` | `timeout_s` | `60.0` | HTTP timeout for governed calls, in seconds. | +| `DONKEY_MAX_RETRIES` | `max_retries` | `3` | Retries for transient upstream failures (`502` / `503` / `504`) with backoff. Policy refusals are never retried, and a gateway fallback is never retried twice. | +| `DONKEY_ON_MODEL_SUBSTITUTION` | `on_model_substitution` | `off` | `off` surfaces a model substitution on `donkey.last_call`; `raise` turns it into `ModelSubstituted`. See [Telemetry & cost](https://donkey-development-kit.github.io/donkey-development-kit/telemetry.md#two-behaviours-worth-knowing). | +| `DONKEY_REGISTRY_CACHE_TTL_S` | `registry_cache_ttl_s` | `300` | How long registry lookups (used by [tool access](https://donkey-development-kit.github.io/donkey-development-kit/tool-access.md)) are cached in memory, in seconds. | +| `DONKEY_NO_CACHE` | — | unset | Set to `1`, `true` or `yes` to bypass that in-memory registry cache. | + ## Anypoint control plane A separate credential from the LLM proxy, used by features that call the diff --git a/website/public/roadmap.md b/website/public/roadmap.md index ae058cf0..d35d4457 100644 --- a/website/public/roadmap.md +++ b/website/public/roadmap.md @@ -1,8 +1,8 @@ # Roadmap DDK is delivered in five phases. Each phase is a GitHub milestone with a clear -goal; the progress bars and issue lists below come straight from those -milestones. +goal; the progress bars and issue lists below are loaded from those milestones +when you open the page. Complete every issue in the milestone is closed ·{' '} In progress work has landed and more is open ·{' '} @@ -20,7 +20,8 @@ milestones. conformance suite, OpenTelemetry GenAI spans, correlation IDs and cost tags. Plus a deep LangGraph adapter, `connection_kwargs()` for seven more frameworks, the decorators and CLI, the documentation site, and the PyPI - release. + release. Closing out the phase: steering the gateway's semantic cache and + surfacing cache hits and semantic-routing matches on `last_call`. ## Phase 2 — Differentiate, go beyond @@ -30,8 +31,18 @@ milestones. Governed tool access (MCP discovery and binding), A2A `serve` / `expose` / `dev`, on-behalf-of identity, human-in-the-loop, scan & publish to the registry with a GitHub Action, declarative refusal handlers and a - classification registry for custom policies, kill-switch awareness, and a - second deep framework adapter chosen by demand. + classification registry for custom policies, typed federated guardrail + verdicts, kill-switch awareness with the kill reason, and a second deep + framework adapter chosen by demand. + + Around them: an inbound correlation ID carried across MCP and A2A hops, a + stated concurrency contract for budget and telemetry under parallel fan-out, + a per-request business group for shared multi-tenant clients, bring-your-own + provider keys, and a token and registry cache. Also in this phase: governed + access to [TypeSafe Jev](https://typesafe.ai/blog/introducing-system-one-models-and-jev), + a System One decision model, with the same typed refusals, spans, budget and + simulator support as LLM calls, plus worked patterns for confidence-gated + review, tool ranking and budget-aware routing. ## Phase 3 — Platform capabilities @@ -40,8 +51,9 @@ milestones. The policy handshake (read the in-force policy set, advisory only), policy updates pushed to the code at run boundaries, a governed - structured-output path, evaluation hooks on the run span, monetary spend and - wallet warnings, and run-level cost rollups. Much of this phase depends on + structured-output path, a model catalog with honest model resolution, + evaluation hooks on the run span, monetary spend and wallet warnings, custom + cost dimensions, and run-level cost rollups. Much of this phase depends on new gateway endpoints tracked under Upstream gaps. ## Phase 4 — Enterprise readiness @@ -53,7 +65,9 @@ milestones. overhead budget enforced in CI, a full pass over every error message, the public API contract and deprecation policy, compliance evidence mapping (EU AI Act Art. 12, ISO 42001, OWASP LLM Top 10), log shipping, data residency, - workload identity, air-gapped operation and gateway failover. + workload identity, air-gapped operation, opt-in gateway failover, a clear + boundary against the platform's infrastructure-as-code, and a support model + with a release cadence. ## Phase 5 — Complete rollout @@ -82,7 +96,8 @@ At each of these boundaries the job is to make the platform's own capability reachable and typed, not to reproduce it: - Client-side policy enforcement — the gateway is the enforcement point. -- Client-side semantic caching. +- Client-side semantic caching — DDK steers the gateway's semantic cache and + reports hits, but never caches responses itself. - A provisioning control plane competing with API Manager or Terraform. - Re-implementations of Agent Scanners, Kill Switch or Trusted Agent Identity. - An approval UI or queue. diff --git a/website/public/telemetry.md b/website/public/telemetry.md index ee370a2f..566b7096 100644 --- a/website/public/telemetry.md +++ b/website/public/telemetry.md @@ -327,6 +327,12 @@ They also land on the span (`gen_ai.response.model`, `donkey.routing.type`, spikes: it tells an operator whether a slow call was routed normally or recovered from a degraded provider. +The same record also carries the call's identity and usage: `request_id` (the +upstream provider's id), `api_instance_id` and `environment_id` (which gateway +instance served it), and the token counts including cached and reasoning +tokens. The [gateway identity example](https://donkey-development-kit.github.io/donkey-development-kit/examples/gateway-identity.md) walks +through every field. + ### When `last_call` is unavailable `donkey.last_call` is populated only when the governed response passes through @@ -375,6 +381,30 @@ donkey = Donkey.from_env(on_model_substitution="raise") `DONKEY_ON_MODEL_SUBSTITUTION` → `.donkey-kit.toml` → default) and **defaults to `"off"`**. + **Known limitation with `provider/model` names.** On a model-based routing + proxy you address models as `provider/model` (for example + `openai/gpt-5-mini`), but the gateway reports the served model without the + provider prefix (`gpt-5-mini`). DDK currently compares the two verbatim, so + `last_call.substituted` reads `True` on every such call, and + `on_model_substitution="raise"` raises `ModelSubstituted` even when + `last_call.fallback` is `False`. Until this is fixed, check + `last_call.fallback` to detect a real failover, and leave + `on_model_substitution` at `"off"` on model-based proxies. + +### Semantic caching & semantic routing Roadmap + +Omni Gateway can answer a request from its **semantic cache** (a similar +prompt was answered before, so no provider round-trip and no token cost) and +can **route semantically** (pick the model by matching the prompt to a +topic). Both happen at the gateway. DDK will not cache anything or compute +embeddings itself; it will let you steer the gateway's cache per request, for +example to skip it, not store a response, or override the TTL or similarity +threshold, and report what happened: + +- the cache outcome (`hit`, `miss`, `bypass`, `no-store`) and similarity score + on `donkey.last_call` and the span; +- the matched routing topic and its score, beside the routing fields above. + ### TypeScript parity Roadmap The same signals will surface as `donkey.lastRouting.servedModel` / `.fallback` diff --git a/website/scripts/generate-roadmap.mjs b/website/scripts/generate-roadmap.mjs index 50030072..f9a2ece6 100644 --- a/website/scripts/generate-roadmap.mjs +++ b/website/scripts/generate-roadmap.mjs @@ -1,10 +1,13 @@ // Snapshot the delivery milestones and their issues from the GitHub API into -// data/roadmap.json, which the Roadmap page renders. +// data/roadmap.json. The Roadmap page renders this snapshot on the server and +// for visitors without JavaScript; in the browser it then refetches the same +// data live from GitHub (components/roadmap.tsx) and falls back to this file +// when that fails. // // The snapshot is committed so that `next build` never depends on the network: // when the API is unreachable or rate-limited, the previous snapshot is kept -// and the build carries on. The Pages workflow runs this before every build -// (and on a daily schedule), so the published page tracks the milestones. +// and the build carries on. The Pages workflow runs this before every build, +// including its scheduled rebuild every 6 hours. // // Set GITHUB_TOKEN to lift the unauthenticated rate limit (60 requests/hour). //