diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ff0790..82f83e9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,47 @@ Format follows [Keep a Changelog](https://keepachangelog.com/). Versions use [Se ## [Unreleased] +## [1.23.0] — 2026-10-15 + +### Added +- Current vendor models, checked against primary docs on 2026-10-15 (#112, phase 2): + - Workers AI: `@cf/meta/llama-3.1-8b-instruct-fp8`, `@cf/ibm-granite/granite-4.0-h-micro`, `@cf/aisingapore/gemma-sea-lion-v4-27b-it`. + - OpenAI: `gpt-6-astra`, `gpt-5.6-sol`, `gpt-5.6-terra`, `gpt-5.6-luna`, `gpt-5.5`. + - Anthropic: `claude-fable-5-1`, `claude-opus-5-5`, `claude-sonnet-5-5`, `claude-haiku-5-5`. +- `ModelCatalogEntry.explicitOnly` (optional): the model is served when a request names it, but recommendations and automatic routing skip it. The new OpenAI and Anthropic models and SEA-LION are explicit-only, so OpenAI and Anthropic defaults do not change in this release. +- `MODELS` constants for the new models and for current Workers AI models (`CF_GLM_5_3`, `CF_GLM_5_3_FLASH`, `CF_KIMI_K2_6`, `CF_DEEPSEEK_V4_PRO`, `CF_DEEPSEEK_V4_FLASH`, `CF_GPT_OSS_120B`, `CF_GPT_OSS_20B`, `CF_GEMMA_4_26B`, `CF_QWEN_3_8_27B`, `CF_LLAMA_3_3_70B_FAST`, `CF_LLAMA_3_1_8B_FP8`, `CF_GRANITE_4_MICRO`). + +### Changed +- The package is documented as a Cloudflare-first router. A test requires a Workers AI model to rank first for every use case when all providers are configured and healthy (no health, ledger or request context). +- Workers AI models that Cloudflare deprecated are now `retired` in the catalog and are no longer auto-routed: Llama 3.1 8B / 70B, Llama 3 8B, Llama 2 7B int8, Phi-2 and Mistral 7B v0.1 (2026-05-30), OpenChat 3.5, TinyLlama and the four Qwen 1.5 models (2025-10-01). The adapter still accepts them by name. Their `MODELS` constants are marked `@deprecated`. +- `gpt-4`, `gpt-4-turbo` and `gpt-3.5-turbo` (OpenAI shutdown 2026-10-23) and the invalid `gpt-3.5-turbo-16k` are `retired` in the catalog and no longer auto-routed. The adapter still accepts them by name. +- `gpt-4o-mini` stays the OpenAI default. `getRoutingInfo()` reports its 2027-01-14 shutdown in `deprecationWarning`; response metadata does not, because the model is still `active`. +- Claude Opus 4.6, Sonnet 4.6 and Haiku 4.5 stay the Anthropic defaults; their descriptions note that Anthropic lists them as legacy. +- Routing changes from the corrected data: + - Cloudflare long-context requests (and `getProviderDefaultModel('cloudflare', ...)` for long requests) now pick `@cf/zai-org/glm-5.3-flash` instead of `@cf/deepseek-ai/deepseek-v4-flash-0731`, because DeepSeek V4 Flash's context window was corrected from 1.31M to 1.05M tokens and long-context scoring weighs context size. + - `@cf/meta/llama-3.1-8b-instruct-fp8` and `@cf/ibm-granite/granite-4.0-h-micro` join the recommendation lists and fallback pools; neither ranks first for any use case. + - OpenAI's fallback pool shrinks to `gpt-4o-mini` now that `gpt-4`, `gpt-4-turbo` and `gpt-3.5-turbo` are retired. +- OpenAI responses that name a dated snapshot are now priced by their alias. This includes the default `gpt-4o-mini`, whose responses name `gpt-4o-mini-2024-07-18` and were recorded at $0 cost in 1.22; ledgers, budgets and cost tracking will now show real spend for it. +- `@cf/qwen/qwen2.5-coder-32b-instruct` requests that include tools now throw a `ConfigurationError` before any call, because the catalog no longer claims function calling for it (see Fixed). +- OpenAI requests to reasoning models (GPT-5 and later, o-series) send `max_completion_tokens` instead of `max_tokens` and omit `temperature`. `max_completion_tokens` also covers reasoning tokens, so a small `maxTokens` can leave no room for the answer. Only requests that name one of these models are affected. +- Anthropic requests to Claude 5.x models omit `temperature` (non-default values return 400) and default to `max_tokens: 16000` when the request sets none, because these models think by default and thinking counts against `max_tokens`. Preflight cost estimates use the same default. +- Anthropic forced tool choice (`toolChoice: 'required'` or a named function) on Fable 5.1, Opus 5.5 and Sonnet 5.5 now throws a `ConfigurationError` before any API call; those models return 400 for it. Use `toolChoice: 'auto'`. Haiku 5.5 and earlier models still accept it. + +### Fixed +- Anthropic JSON mode no longer sends an assistant prefill to Claude Opus 4.6, Sonnet 4.6 or the 5.x models, which reject prefill with a 400. This affected the 1.22 default for JSON requests (`claude-sonnet-4-6`). These models rely on the JSON system instruction; Haiku 4.5 still uses the prefill. +- Streaming JSON mode on models that are prefilled (Haiku 4.5) now restores the opening `{` on the first text chunk, as non-streaming responses already did. +- OpenAI cost was $0 when a response named a dated snapshot (for example `gpt-4o-mini-2024-07-18` or `gpt-5.5-2026-04-23`); IDs with a date suffix are now priced by their alias. Snapshot IDs without a date suffix (for example `gpt-3.5-turbo-0125`) are still not priced. Prompts over 272K tokens on the new 1M-context models are billed at 2x input and 1.5x output. +- Claude Opus 4.8 through the AI Gateway unified catalog no longer receives `temperature` or `top_p`, which Claude 4.7 and later reject. +- Workers AI context windows: Llama 3.3 70B FP8 Fast 24,000 (was 131,072), Llama 3.2 1B 60,000, Llama 3.2 3B 80,000, Qwen3 30B 32,768, Kimi K2.6 and K2.7 Code 262,144, DeepSeek V4 Flash 1,048,576 (was 1,310,720). Kimi K2.7 Code supports vision. +- `@cf/qwen/qwen2.5-coder-32b-instruct` no longer claims function calling; Cloudflare's model page does not list it. +- `deepseek/deepseek-v4-pro` and `anthropic/claude-opus-4.8` are priced from the AI Gateway unified catalog ($1.74/$3.48 and $5/$25 per million tokens) instead of $0, and `deepseek/deepseek-v4-pro` has a 131,072-token context. +- Anthropic cache reads use per-model rates (Fable 5.1 2.5%, Opus 5.5 and Sonnet 5.5 5% of input), and Haiku 5.5 prompts over 100K tokens, cache included, are billed at the 5x long-prompt rate. + +### Known limitations +- Claude 5.x thinking blocks are not passed back with tool results, which Anthropic requires for multi-turn tool use. This is why the 5.x models are explicit-only. The adapter also sends `tool_result` blocks without `tool_use_id`, which likely breaks multi-turn tool use on every Claude model (#124). +- `gpt-6-luna` and `gpt-6.1-sol` are not supported yet (tool-calling restrictions on Chat Completions). +- The 16,000-token `max_tokens` default for Claude 5.x does not change the 30-second default `timeout`. It times the wait for response headers, which for a non-streaming request arrive when the response is complete. Long thinking responses from Fable 5.1 or Opus 5.5 can exceed it, and a timeout is retried (`maxRetries`, default 3), so one request can take about two minutes before fallback and each attempt may be billed. Set a larger `timeout` on the Anthropic provider config or use streaming. + ## [1.22.1] — 2026-10-15 ### Fixed diff --git a/README.md b/README.md index 66e80de..5a06c01 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ Development status and resume notes: [issue sweep checkpoint](docs/maintenance/issue-sweep.md). The checkpoint identifies APIs and fixes that are still unreleased. -A multi-provider LLM abstraction layer with automatic failover, graduated circuit breakers, cost tracking, and intelligent retry. Built for Cloudflare Workers but runs anywhere with a standard `fetch` API. Extracted from a production orchestration platform handling 80K+ LOC across multiple services. +A Cloudflare-first LLM router: Workers AI models are the primary route, and OpenAI, Anthropic, Cerebras, Groq and NVIDIA NIM serve as secondary providers behind the same interface, with automatic failover, graduated circuit breakers, cost tracking, and intelligent retry. When Workers AI is configured and healthy, catalog routing ranks a Workers AI model first for every use case; breaker state, ledger pressure and request requirements (for example vision) can still move a request to another provider. Built for Cloudflare Workers but runs anywhere with a standard `fetch` API. Extracted from a production orchestration platform handling 80K+ LOC across multiple services. ## Features @@ -156,9 +156,9 @@ const llm = LLMProviders.fromEnv(env, { | Provider | Models | Streaming | Tools | Notes | |----------|--------|-----------|-------|-------| -| **OpenAI** | GPT-4o Mini, GPT-4 Turbo, GPT-4, GPT-3.5 Turbo | Yes | Yes | Default: `gpt-4o-mini` | -| **Anthropic** | Claude Opus 4.6, Sonnet 4.6, Sonnet 4, Haiku 4.5, 3.7 Sonnet, 3.5 Sonnet/Haiku, 3 Opus/Sonnet | Yes | Yes | Default: `claude-haiku-4-5-20251001` | -| **Cloudflare** | Nemotron-3 120B, Claude Opus 4.8 (CF-managed), Kimi K2.6/K2.7, DeepSeek-R1-Distill, QwQ-32B, GLM-4.7-Flash, DeepSeek V4 Pro, Gemma 4 26B, Llama 4 Scout, GPT-OSS 120B/20B, LLaMA 3.x, Mistral Small 3.1, Qwen 2.5/3, and more | Yes | Nemotron, Kimi, GLM, GPT-OSS, Gemma 4, Llama 4 Scout, Mistral Small | Default is request-aware and catalog-driven; thinking models (DeepSeek-R1, QwQ, GLM) are RESEARCH-only | +| **Cloudflare** (primary) | GLM-5.3 / 5.3 Flash / 5.2 / 4.7 Flash, Kimi K2.6 / K2.7 Code, DeepSeek V4 Flash / Pro, GPT-OSS 120B / 20B, Gemma 4 26B, Qwen 3.8 27B / 3 30B / 2.5 Coder, Llama 4 Scout / 3.3 70B Fast / 3.2 / 3.1 8B FP8, Nemotron-3 120B, Granite 4.0 Micro, Mistral Small 3.1, SEA-LION v4 (explicit requests only), DeepSeek-R1-Distill, QwQ-32B; plus `deepseek/deepseek-v4-pro` and `anthropic/claude-opus-4.8` from the AI Gateway unified catalog | Yes | Most current models (see catalog `supportsTools`) | Default is request-aware and catalog-driven; thinking models (DeepSeek-R1, QwQ, GLM-4.7 Flash) are RESEARCH-only. Kimi, GLM-5.x and DeepSeek V4 need a Workers Paid plan or AI Gateway credits. Models Cloudflare deprecated (Llama 3.1/3 8B, Llama 3.1 70B, Llama 2 7B, Mistral 7B, Phi-2, Qwen 1.5, TinyLlama, OpenChat) are still accepted when named but are never auto-routed; Kimi K2.5 and Gemma 3 12B are no longer accepted | +| **OpenAI** | GPT-4o Mini (shutdown 2027-01-14); GPT-6 Astra, GPT-5.6 Sol / Terra / Luna, GPT-5.5 (explicit requests only); GPT-4, GPT-4 Turbo, GPT-3.5 Turbo (shutdown 2026-10-23, no longer auto-routed) | Yes | Yes | Default: `gpt-4o-mini`. GPT-5 and later get `max_completion_tokens` (which includes reasoning tokens) and no `temperature` | +| **Anthropic** | Opus 4.6, Sonnet 4.6, Haiku 4.5 (legacy at Anthropic); Fable 5.1, Opus 5.5, Sonnet 5.5, Haiku 5.5 (explicit requests only); retired 4 / 3.x IDs | Yes | Yes | Default: catalog-driven by request shape: `claude-haiku-4-5-20251001` for plain requests, `claude-sonnet-4-6` for tools, JSON mode (`response_format`), vision and long context. `claude-opus-4-6` is picked only when a caller asks for the high-performance or research workload (`metadata.useCase` on factory requests, or the workload helpers). Claude 5.x requests omit `temperature` and default `max_tokens` to 16,000; a non-streaming request can exceed the 30-second default `timeout`, which is then retried (`maxRetries`, default 3), so raise `timeout` or stream for long thinking responses; Fable 5.1, Opus 5.5 and Sonnet 5.5 reject forced tool choice. JSON mode prefills `{` only on models that accept it | | **Cerebras** | GPT-OSS 120B, ZAI-GLM 4.7, LLaMA 3.1 8B *(deprecated 2026-05-27)*, Qwen 3 235B *(deprecated 2026-05-27)* | Yes | GLM, Qwen, GPT-OSS | ~2,200 tok/s | | **Groq** | LLaMA 3.3 70B Versatile, LLaMA 3.1 8B Instant, GPT-OSS 120B, Compound, Compound Mini | Yes | LLaMA 3.3 70B, GPT-OSS 120B | Ultra-fast inference; Compound systems run built-in tools | | **NVIDIA NIM** | Llama 3.3/3.1 70B, Llama 4 Maverick, Nemotron 70B/49B/253B, Mistral Large 2, DeepSeek V4 Flash/Pro | Yes | Llama, Nemotron, Mistral Large 2 | Costs $0 placeholder — dev-tier credits; set real rates in `CreditLedger` | @@ -551,11 +551,15 @@ try { import { MODELS, getRecommendedModel } from '@stackbilt/llm-providers'; // Current-gen models -MODELS.CLAUDE_OPUS_4_6; // 'claude-opus-4-6' +MODELS.CF_GLM_5_3_FLASH; // '@cf/zai-org/glm-5.3-flash' +MODELS.CF_KIMI_K2_6; // '@cf/moonshotai/kimi-k2.6' +MODELS.CF_GPT_OSS_120B; // '@cf/openai/gpt-oss-120b' +MODELS.CF_GEMMA_4_26B; // '@cf/google/gemma-4-26b-a4b-it' MODELS.CLAUDE_SONNET_4_6; // 'claude-sonnet-4-6' MODELS.CLAUDE_HAIKU_4_5; // 'claude-haiku-4-5-20251001' -MODELS.GPT_4O; // 'gpt-4o' (not routed by this package) -MODELS.GPT_4O_MINI; // 'gpt-4o-mini' +MODELS.GPT_4O_MINI; // 'gpt-4o-mini' (shutdown 2027-01-14) +MODELS.CLAUDE_OPUS_5_5; // 'claude-opus-5-5' (explicit requests only) +MODELS.GPT_6_ASTRA; // 'gpt-6-astra' (explicit requests only) MODELS.CEREBRAS_ZAI_GLM_4_7; // 'zai-glm-4.7' MODELS.GROQ_COMPOUND; // 'groq/compound' (RESEARCH; built-in tools) MODELS.GROQ_COMPOUND_MINI; // 'groq/compound-mini' diff --git a/docs/maintenance/issue-sweep.md b/docs/maintenance/issue-sweep.md index f9d9d1a..242f447 100644 --- a/docs/maintenance/issue-sweep.md +++ b/docs/maintenance/issue-sweep.md @@ -7,7 +7,7 @@ Checkpoint: 2026-10-15. Repository: `Stackbilt-dev/llm-providers`. The sweep below is **merged to `main`** and is versioned as 1.22.0 by release PR [#121](https://github.com/Stackbilt-dev/llm-providers/pull/121) (npm publish is a separate operator step): [#119](https://github.com/Stackbilt-dev/llm-providers/pull/119) (sweep plus review fixes) and [#120](https://github.com/Stackbilt-dev/llm-providers/pull/120) (#116 hook isolation). Each went through adversarial review rounds; review-driven compatibility fixes are recorded in the 1.22.0 CHANGELOG entry. The issues stay open for the remaining scope in the table. No live or billable vendor requests were made. 1. Inspect `git status --short`. Preserve the pre-existing `src/__tests__/factory.test.ts` modification; its diff is line-ending-only when checked with `--ignore-space-at-eol`. -2. Next implementation targets: #112 vendor IDs/pricing, then #109 atomic spend reservations (needs a design pass first). +2. #112 phase 1 shipped in 1.22.1; phase 2 (current models, Workers AI deprecations, reasoning-model request parameters, `explicitOnly`) shipped in 1.23.0 via [#123](https://github.com/Stackbilt-dev/llm-providers/pull/123). Next: #124 (Anthropic `tool_use_id`, tool error text and Claude 5.x thinking blocks in tool loops; prerequisite for routing the 5.x models), then #109 atomic spend reservations (needs a design pass first). 3. Run the validation commands below after substantive code changes, then open reviewable PRs. `issue-corpus/Stackbilt-dev__llm-providers/` is a June 2026 historical export. `.agent-memory/status.json` belongs to the archive/upload machinery. Neither is the current task checkpoint; preserve their historical/managed contents. @@ -28,7 +28,7 @@ The sweep below is **merged to `main`** and is versioned as 1.22.0 by release PR | [#116](https://github.com/Stackbilt-dev/llm-providers/issues/116) | Quota observers and streaming terminal/schema observers are guarded; stream body deadlines, cancellation, fault/race tests and awaited stream settlement were added. | Hook replay fixed in #120: every factory/adapter hook is guarded. Remaining: non-streaming body deadlines, retry sleep aborts, Retry-After/vendor error fixtures, unproduced hook events, floating non-streaming cache/quota writes, request/tool-loop signals, workerd and coverage baselines | | [#118](https://github.com/Stackbilt-dev/llm-providers/issues/118) | Coordinating roadmap receives this checkpoint and priorities. | Track reviewed/merged implementation phases separately from local progress. | -Other open items are unchanged: #112 (vendor audit, next), #114 (failure demo/README restructuring/repo cleanup), #117 (optional AI SDK adapter), #77 (supply-chain workflow), #78/#79/#80 (new provider proposals). #90/#95 have existing code surfaces but were not re-audited for closure during this sweep. +Other open items are unchanged: #112 (phase 2 shipped in 1.23.0; remaining: Claude 5.x thinking-block round trip and `tool_use_id` in tool loops, tracked in #124 (then lift `explicitOnly` on the 5.x and GPT-5.6 models), Cerebras/Groq/NVIDIA prices and IDs, GPT-6 Luna / 6.1 Sol tool-calling restrictions, OpenAI cached-input pricing, custom-model escape hatch, unknown-cost semantics, provenance canary), #114 (failure demo/README restructuring/repo cleanup), #117 (optional AI SDK adapter), #77 (supply-chain workflow), #78/#79/#80 (new provider proposals). #90/#95 have existing code surfaces but were not re-audited for closure during this sweep. ## Public behavior to retain diff --git a/package-lock.json b/package-lock.json index ba47a9d..0b4332a 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "@stackbilt/llm-providers", - "version": "1.22.1", + "version": "1.23.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "@stackbilt/llm-providers", - "version": "1.22.1", + "version": "1.23.0", "license": "Apache-2.0", "devDependencies": { "@cloudflare/workers-types": "^4.0.0", diff --git a/package.json b/package.json index aac299f..b63b615 100755 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@stackbilt/llm-providers", - "version": "1.22.1", + "version": "1.23.0", "description": "Multi-LLM failover with circuit breakers, cost tracking, and intelligent retry. Cloudflare Workers native.", "author": "Stackbilt ", "license": "Apache-2.0", diff --git a/src/__tests__/model-catalog.test.ts b/src/__tests__/model-catalog.test.ts index 5caa262..4665012 100644 --- a/src/__tests__/model-catalog.test.ts +++ b/src/__tests__/model-catalog.test.ts @@ -168,8 +168,8 @@ describe('getRoutingInfo', () => { it('returns deprecation warning for a compatibility-lifecycle model', () => { const info = getRoutingInfo( - { messages: [{ role: 'user', content: 'hi' }], model: 'gpt-4' }, - ['openai'] + { messages: [{ role: 'user', content: 'hi' }], model: 'qwen-3-235b-a22b-instruct-2507' }, + ['cerebras'] ); expect(info.modelLifecycle).toBe('compatibility'); expect(info.deprecationWarning).toBeTruthy(); @@ -444,9 +444,11 @@ describe('new CF Workers AI models (v1.16.0)', () => { } }); - it('prefers the current DeepSeek V4 Flash lane for Cloudflare long-context routing', () => { + it('routes Cloudflare long-context requests to an active 1M-context model', () => { const ranked = rankModels('LONG_CONTEXT', ['cloudflare']); - expect(ranked[0]?.model).toBe('@cf/deepseek-ai/deepseek-v4-flash-0731'); + expect(ranked[0]?.lifecycle).toBe('active'); + expect(ranked[0]?.capabilities.maxContextLength).toBeGreaterThanOrEqual(1_048_576); + expect(ranked.slice(0, 3).map(entry => entry.model)).toContain('@cf/deepseek-ai/deepseek-v4-flash-0731'); }); }); @@ -472,7 +474,7 @@ describe('current CF Workers AI models (August 2026)', () => { }); it('records the documented context windows and vision capabilities', () => { - expect(getCatalogEntry('@cf/deepseek-ai/deepseek-v4-flash-0731')?.capabilities.maxContextLength).toBe(1_310_720); + expect(getCatalogEntry('@cf/deepseek-ai/deepseek-v4-flash-0731')?.capabilities.maxContextLength).toBe(1_048_576); expect(getCatalogEntry('@cf/deepseek-ai/deepseek-v4-pro-0813')?.capabilities.maxContextLength).toBe(1_048_576); expect(getCatalogEntry('@cf/zai-org/glm-5.3')?.capabilities.maxContextLength).toBe(1_048_576); expect(getCatalogEntry('@cf/qwen/qwen3.8-27b')?.capabilities.supportsVision).toBe(true); diff --git a/src/__tests__/model-refresh.test.ts b/src/__tests__/model-refresh.test.ts new file mode 100644 index 0000000..8b2fa23 --- /dev/null +++ b/src/__tests__/model-refresh.test.ts @@ -0,0 +1,497 @@ +/** + * Model list refresh (#112, phase 2). + * + * Expected values are taken from primary vendor documentation, checked 2026-10-15: + * - Workers AI: developers.cloudflare.com/workers-ai/models/ (each model page) and + * /workers-ai/platform/pricing/ (updated 2026-10-15); deprecations from the + * 2026-05-08 "planned model deprecations" changelog post and the 2025-09-18 entry. + * - OpenAI: developers.openai.com/api/docs/{models,pricing,deprecations} + * - Anthropic: platform.claude.com/docs/en/about-claude/{models/overview,pricing,model-deprecations} + */ +import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest'; +import { AnthropicProvider } from '../providers/anthropic'; +import { CloudflareProvider } from '../providers/cloudflare'; +import { OpenAIProvider, isOpenAIReasoningModel } from '../providers/openai'; +import { + getCatalogEntry, getRoutingInfo, MODEL_RECOMMENDATIONS, rankModels, type ModelRecommendationUseCase, +} from '../model-catalog'; +import { LLMProviders, MODELS } from '../index'; +import { defaultCircuitBreakerManager } from '../utils/circuit-breaker'; +import { CreditLedger } from '../utils/credit-ledger'; +import type { LLMRequest } from '../types'; + +const request: LLMRequest = { messages: [{ role: 'user', content: 'hello' }] }; +const JSON_HEADERS = new Headers({ 'content-type': 'application/json' }); +const USE_CASES: ModelRecommendationUseCase[] = [ + 'COST_EFFECTIVE', 'HIGH_PERFORMANCE', 'BALANCED', 'TOOL_CALLING', 'LONG_CONTEXT', 'VISION', 'RESEARCH']; + +function jsonReply(body: unknown) { + return { ok: true, status: 200, headers: JSON_HEADERS, json: async () => body }; +} +function openaiReply(model: string, promptTokens = 1, completionTokens = 1) { + return jsonReply({ + id: 'chatcmpl_1', object: 'chat.completion', created: 1, model, + choices: [{ index: 0, message: { role: 'assistant', content: 'ok' }, finish_reason: 'stop' }], + usage: { prompt_tokens: promptTokens, completion_tokens: completionTokens, total_tokens: promptTokens + completionTokens }, + }); +} +function anthropicReply(model: string, usage: Record = { input_tokens: 1, output_tokens: 1 }) { + return jsonReply({ + id: 'msg_1', type: 'message', role: 'assistant', content: [{ type: 'text', text: 'ok' }], + model, stop_reason: 'end_turn', stop_sequence: null, usage, + }); +} +function sentBody(fetchMock: ReturnType): Record { + return JSON.parse(fetchMock.mock.calls[0][1].body as string); +} + +beforeEach(() => { defaultCircuitBreakerManager.resetAll(); }); +afterEach(() => { vi.unstubAllGlobals(); vi.restoreAllMocks(); }); + +describe('Cloudflare-first routing', () => { + it.each(USE_CASES)('ranks a Workers AI model first for %s when every provider is configured', useCase => { + const ranked = rankModels(useCase, ['openai', 'anthropic', 'cloudflare', 'cerebras', 'groq', 'nvidia']); + expect(ranked[0]?.provider).toBe('cloudflare'); + }); +}); + +describe('Workers AI catalog', () => { + const removed = [ + '@cf/meta/llama-3.1-8b-instruct', '@cf/meta/llama-3.1-70b-instruct', '@cf/meta/llama-3-8b-instruct', + '@cf/meta/llama-2-7b-chat-int8', '@cf/microsoft/phi-2', '@cf/mistral/mistral-7b-instruct-v0.1', + '@cf/openchat/openchat-3.5-0106', '@cf/tinyllama/tinyllama-1.1b-chat-v1.0', '@cf/qwen/qwen1.5-0.5b-chat', + '@cf/qwen/qwen1.5-1.8b-chat', '@cf/qwen/qwen1.5-14b-chat-awq', '@cf/qwen/qwen1.5-7b-chat-awq', + ]; + + it.each(removed)('%s is retired and never recommended', model => { + expect(getCatalogEntry(model)?.lifecycle).toBe('retired'); + expect(Object.values(MODEL_RECOMMENDATIONS).flat()).not.toContain(model); + }); + + it('keeps removed IDs accepted by the adapter for explicit callers', () => { + const provider = new CloudflareProvider({ ai: { run: vi.fn() } as unknown as Ai }); + for (const model of removed) expect(provider.models).toContain(model); + }); + + it.each([ + ['@cf/meta/llama-3.3-70b-instruct-fp8-fast', 24_000], + ['@cf/meta/llama-3.2-1b-instruct', 60_000], + ['@cf/meta/llama-3.2-3b-instruct', 80_000], + ['@cf/qwen/qwen3-30b-a3b-fp8', 32_768], + ['@cf/moonshotai/kimi-k2.6', 262_144], + ['@cf/moonshotai/kimi-k2.7-code', 262_144], + ['@cf/deepseek-ai/deepseek-v4-flash-0731', 1_048_576], + ['deepseek/deepseek-v4-pro', 131_072], + ])('%s has the documented %i-token context window', (model, context) => { + expect(getCatalogEntry(model)?.capabilities.maxContextLength).toBe(context); + }); + + it('never auto-routes SEA-LION', () => { + const model = '@cf/aisingapore/gemma-sea-lion-v4-27b-it'; + expect(getCatalogEntry(model)?.explicitOnly).toBe(true); + expect(Object.values(MODEL_RECOMMENDATIONS).flat()).not.toContain(model); + }); + + it('does not forward temperature or top_p to Claude Opus 4.8 through the AI Gateway catalog', async () => { + const run = vi.fn().mockResolvedValue({ + id: 'msg_1', type: 'message', role: 'assistant', content: [{ type: 'text', text: 'ok' }], + stop_reason: 'end_turn', usage: { input_tokens: 1, output_tokens: 1 }, + }); + const provider = new CloudflareProvider({ ai: { run } as unknown as Ai, maxRetries: 0 }); + await provider.generateResponse({ ...request, model: 'anthropic/claude-opus-4.8', temperature: 0.2, topP: 0.5, maxTokens: 50 }); + const body = run.mock.calls[0][1] as Record; + expect(body).not.toHaveProperty('temperature'); + expect(body).not.toHaveProperty('top_p'); + expect(body.max_tokens).toBe(50); + }); + + it('does not claim function calling where the model page does not list it', () => { + const coder = getCatalogEntry('@cf/qwen/qwen2.5-coder-32b-instruct')!; + expect(coder.capabilities.supportsTools).toBe(false); + expect(coder.useCases).not.toContain('TOOL_CALLING'); + const toolRequest: LLMRequest = { + ...request, + tools: [{ type: 'function', function: { name: 'lookup', description: 'Lookup', parameters: { type: 'object' } } }], + }; + const ranked = rankModels('TOOL_CALLING', ['cloudflare'], { request: toolRequest }); + expect(ranked.map(entry => entry.model)).not.toContain(coder.model); + }); + + it.each([ + ['@cf/meta/llama-3.1-8b-instruct-fp8', 0.152, 0.287], + ['@cf/ibm-granite/granite-4.0-h-micro', 0.017, 0.112], + ['@cf/aisingapore/gemma-sea-lion-v4-27b-it', 0.351, 0.555], + ['deepseek/deepseek-v4-pro', 1.74, 3.48], + ['anthropic/claude-opus-4.8', 5, 25], + ])('%s costs $%s/$%s per million tokens in the adapter and the catalog', async (model, inPerM, outPerM) => { + const run = vi.fn().mockResolvedValue({ response: 'ok', usage: { prompt_tokens: 1_000_000, completion_tokens: 1_000_000 } }); + const provider = new CloudflareProvider({ ai: { run } as unknown as Ai, maxRetries: 0 }); + expect(provider.models).toContain(model); + const entry = getCatalogEntry(model)!; + expect(entry.provider).toBe('cloudflare'); + expect(entry.capabilities.inputTokenCost * 1000).toBeCloseTo(inPerM, 6); + expect(entry.capabilities.outputTokenCost * 1000).toBeCloseTo(outPerM, 6); + if (model.startsWith('@cf/')) { + const response = await provider.generateResponse({ ...request, model }); + expect(response.usage.cost).toBeCloseTo(inPerM + outPerM, 6); + } + }); +}); + +describe('OpenAI models', () => { + const current = ['gpt-6-astra', 'gpt-5.6-sol', 'gpt-5.6-terra', 'gpt-5.6-luna', 'gpt-5.5']; + + it.each(['gpt-4', 'gpt-4-turbo', 'gpt-3.5-turbo', 'gpt-3.5-turbo-16k'])('%s is out of routing', model => { + expect(getCatalogEntry(model)?.lifecycle).toBe('retired'); + expect(Object.values(MODEL_RECOMMENDATIONS).flat()).not.toContain(model); + }); + + it('keeps gpt-4o-mini as the OpenAI default and warns about its 2027-01-14 shutdown', () => { + const info = getRoutingInfo(request, ['openai']); + expect(info.model).toBe('gpt-4o-mini'); + expect(info.modelLifecycle).toBe('active'); + expect(info.deprecationWarning).toContain('2027-01-14'); + }); + + it.each(current)('%s is accepted by name but never auto-routed', model => { + expect(new OpenAIProvider({ apiKey: 'k' }).models).toContain(model); + expect(getCatalogEntry(model)).toMatchObject({ provider: 'openai', lifecycle: 'active', explicitOnly: true }); + expect(Object.values(MODEL_RECOMMENDATIONS).flat()).not.toContain(model); + expect(getRoutingInfo({ ...request, model }, ['openai']).model).toBe(model); + }); + + it('prices a dated snapshot in the response by its alias', async () => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(openaiReply('gpt-5.5-2026-04-23', 200_000, 1_000_000))); + const response = await new OpenAIProvider({ apiKey: 'k', maxRetries: 0 }).generateResponse({ ...request, model: 'gpt-5.5' }); + expect(response.usage.cost).toBeCloseTo(1 + 30, 6); + }); + + it('bills prompts over 272K tokens at 2x input and 1.5x output', async () => { + const provider = new OpenAIProvider({ apiKey: 'k', maxRetries: 0 }); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(openaiReply('gpt-5.6-luna', 272_000, 1_000_000))); + expect((await provider.generateResponse({ ...request, model: 'gpt-5.6-luna' })).usage.cost).toBeCloseTo(0.0544 + 1.2, 6); + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(openaiReply('gpt-5.6-luna', 300_000, 1_000_000))); + expect((await provider.generateResponse({ ...request, model: 'gpt-5.6-luna' })).usage.cost).toBeCloseTo(0.12 + 1.8, 6); + }); + + it.each(current)('sends max_completion_tokens and no temperature to %s', async model => { + expect(isOpenAIReasoningModel(model)).toBe(true); + const fetchMock = vi.fn().mockResolvedValue(openaiReply(model)); + vi.stubGlobal('fetch', fetchMock); + await new OpenAIProvider({ apiKey: 'k', maxRetries: 0 }) + .generateResponse({ ...request, model, temperature: 0.2, maxTokens: 500 }); + const body = sentBody(fetchMock); + expect(body.max_completion_tokens).toBe(500); + expect(body).not.toHaveProperty('max_tokens'); + expect(body).not.toHaveProperty('temperature'); + }); + + it('keeps max_tokens and temperature for non-reasoning models', async () => { + expect(isOpenAIReasoningModel('gpt-4o-mini')).toBe(false); + const fetchMock = vi.fn().mockResolvedValue(openaiReply('gpt-4o-mini')); + vi.stubGlobal('fetch', fetchMock); + await new OpenAIProvider({ apiKey: 'k', maxRetries: 0 }) + .generateResponse({ ...request, model: 'gpt-4o-mini', temperature: 0.2, maxTokens: 500 }); + const body = sentBody(fetchMock); + expect(body).toMatchObject({ max_tokens: 500, temperature: 0.2 }); + expect(body).not.toHaveProperty('max_completion_tokens'); + }); + + it('sends the same reasoning parameters on the streaming path', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true, body: new ReadableStream({ start: c => c.close() }) }); + vi.stubGlobal('fetch', fetchMock); + const stream = await new OpenAIProvider({ apiKey: 'k', maxRetries: 0 }) + .streamResponse({ ...request, model: 'gpt-5.6-sol', temperature: 0.2, maxTokens: 500 }); + await stream.getReader().read().catch(() => undefined); + const body = sentBody(fetchMock); + expect(body.max_completion_tokens).toBe(500); + expect(body).not.toHaveProperty('temperature'); + }); +}); + +describe('Anthropic models', () => { + const current: Array<[string, number, number]> = [ + ['claude-fable-5-1', 10, 50], + ['claude-opus-5-5', 4, 20], + ['claude-sonnet-5-5', 2, 10], + ['claude-haiku-5-5', 0.1, 0.5], + ]; + + it.each(current)('%s is accepted by name, never auto-routed, and priced at $%s/$%s per million tokens', (model, inPerM, outPerM) => { + expect(new AnthropicProvider({ apiKey: 'k' }).models).toContain(model); + const entry = getCatalogEntry(model)!; + expect(entry).toMatchObject({ provider: 'anthropic', lifecycle: 'active', explicitOnly: true }); + expect(Object.values(MODEL_RECOMMENDATIONS).flat()).not.toContain(model); + expect(entry.capabilities.maxContextLength).toBe(1_000_000); + expect(entry.capabilities.inputTokenCost * 1000).toBeCloseTo(inPerM, 6); + expect(entry.capabilities.outputTokenCost * 1000).toBeCloseTo(outPerM, 6); + }); + + it.each(['claude-opus-4-6', 'claude-sonnet-4-6', 'claude-haiku-4-5-20251001'])('%s stays routable with a legacy note', model => { + const entry = getCatalogEntry(model)!; + expect(entry.lifecycle).toBe('active'); + expect(entry.capabilities.description).toMatch(/legacy/); + }); + + it('keeps Anthropic routing on Claude 4.x', () => { + expect(getRoutingInfo(request, ['anthropic']).model).toBe('claude-haiku-4-5-20251001'); + expect(getRoutingInfo({ ...request, images: [{ data: 'x', mimeType: 'image/png' }] }, ['anthropic']).model) + .toBe('claude-sonnet-4-6'); + }); + + it.each(['claude-opus-4-6', 'claude-sonnet-4-6', 'claude-fable-5-1', 'claude-opus-5-5', 'claude-sonnet-5-5', 'claude-haiku-5-5'])( + 'JSON mode on %s ends with a user turn and returns the text unchanged', async model => { + const fetchMock = vi.fn().mockResolvedValue(jsonReply({ + id: 'msg_1', type: 'message', role: 'assistant', content: [{ type: 'text', text: 'Result: {"a":1}' }], + model, stop_reason: 'end_turn', stop_sequence: null, usage: { input_tokens: 1, output_tokens: 1 }, + })); + vi.stubGlobal('fetch', fetchMock); + const response = await new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }) + .generateResponse({ ...request, model, response_format: { type: 'json_object' } }); + const messages = sentBody(fetchMock).messages as Array<{ role: string }>; + expect(messages[messages.length - 1].role).toBe('user'); + expect(sentBody(fetchMock).system).toMatch(/valid JSON only/); + // Without a prefill there is no consumed '{' to restore. + expect(response.message).toBe('Result: {"a":1}'); + }); + + it('JSON mode on Claude Haiku 4.5 still prefills and restores the opening brace', async () => { + const fetchMock = vi.fn().mockResolvedValue(jsonReply({ + id: 'msg_1', type: 'message', role: 'assistant', content: [{ type: 'text', text: '"a":1}' }], + model: 'claude-haiku-4-5-20251001', stop_reason: 'end_turn', stop_sequence: null, usage: { input_tokens: 1, output_tokens: 1 }, + })); + vi.stubGlobal('fetch', fetchMock); + const response = await new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }) + .generateResponse({ ...request, model: 'claude-haiku-4-5-20251001', response_format: { type: 'json_object' } }); + const messages = sentBody(fetchMock).messages as Array<{ role: string; content: unknown }>; + expect(messages[messages.length - 1]).toEqual({ role: 'assistant', content: '{' }); + expect(response.message).toBe('{"a":1}'); + }); + + const tools: LLMRequest['tools'] = [ + { type: 'function', function: { name: 'lookup', description: 'Lookup', parameters: { type: 'object' } } }]; + + it.each(['claude-fable-5-1', 'claude-opus-5-5', 'claude-sonnet-5-5'])('rejects forced tool choice for %s before calling the API', async model => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const provider = new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }); + for (const toolChoice of ['required', { type: 'function', function: { name: 'lookup' } }] as const) { + await expect(provider.generateResponse({ ...request, model, tools, toolChoice })) + .rejects.toThrow(/does not support forced tool choice/); + } + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('still forwards forced tool choice to models that accept it', async () => { + for (const model of ['claude-haiku-5-5', 'claude-sonnet-4-6']) { + const fetchMock = vi.fn().mockResolvedValue(anthropicReply(model)); + vi.stubGlobal('fetch', fetchMock); + await new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }) + .generateResponse({ ...request, model, tools, toolChoice: 'required' }); + expect(sentBody(fetchMock).tool_choice).toEqual({ type: 'any' }); + } + }); + + it.each(['claude-fable-5-1', 'claude-opus-5-5', 'claude-sonnet-5-5'])('allows auto tool choice for %s', async model => { + const fetchMock = vi.fn().mockResolvedValue(anthropicReply(model)); + vi.stubGlobal('fetch', fetchMock); + await new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }).generateResponse({ ...request, model, tools, toolChoice: 'auto' }); + expect(sentBody(fetchMock).tool_choice).toEqual({ type: 'auto' }); + }); + + it.each(current.map(([model]) => model))('does not send temperature to %s', async model => { + const fetchMock = vi.fn().mockResolvedValue(anthropicReply(model)); + vi.stubGlobal('fetch', fetchMock); + await new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }).generateResponse({ ...request, model, temperature: 0.2 }); + expect(sentBody(fetchMock)).not.toHaveProperty('temperature'); + }); + + it.each([ + ['claude-fable-5-1', 16_000], + ['claude-opus-5-5', 16_000], + ['claude-sonnet-5-5', 16_000], + ['claude-haiku-5-5', 16_000], + ['claude-haiku-4-5-20251001', 1000], + ] as const)('defaults max_tokens for %s to %i when the request sets none', async (model, maxTokens) => { + const fetchMock = vi.fn().mockResolvedValue(anthropicReply(model)); + vi.stubGlobal('fetch', fetchMock); + const provider = new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }); + await provider.generateResponse({ ...request, model }); + expect(sentBody(fetchMock).max_tokens).toBe(maxTokens); + // Preflight cost estimates assume the same output budget. + const outPerToken = getCatalogEntry(model)!.capabilities.outputTokenCost / 1000; + expect(provider.estimateCost({ ...request, model })).toBeCloseTo(2 * getCatalogEntry(model)!.capabilities.inputTokenCost / 1000 + maxTokens * outPerToken, 8); + }); + + it('keeps an explicit maxTokens for always-thinking models', async () => { + const fetchMock = vi.fn().mockResolvedValue(anthropicReply('claude-opus-5-5')); + vi.stubGlobal('fetch', fetchMock); + await new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }) + .generateResponse({ ...request, model: 'claude-opus-5-5', maxTokens: 300 }); + expect(sentBody(fetchMock).max_tokens).toBe(300); + }); + + it('still sends temperature to Claude 4.6', async () => { + const fetchMock = vi.fn().mockResolvedValue(anthropicReply('claude-sonnet-4-6')); + vi.stubGlobal('fetch', fetchMock); + await new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }) + .generateResponse({ ...request, model: 'claude-sonnet-4-6', temperature: 0.2 }); + expect(sentBody(fetchMock).temperature).toBe(0.2); + }); + + it.each([ + // [model, input $/M, cache-read fraction of input] + ['claude-fable-5-1', 10, 0.025], + ['claude-opus-5-5', 4, 0.05], + ['claude-sonnet-5-5', 2, 0.05], + ['claude-sonnet-4-6', 3, 0.1], + ] as const)('prices %s cache reads at the documented rate', async (model, inPerM, readFraction) => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(anthropicReply(model, + { input_tokens: 0, output_tokens: 0, cache_read_input_tokens: 1_000_000 }))); + const response = await new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }).generateResponse({ ...request, model }); + expect(response.usage.cost).toBeCloseTo(inPerM * readFraction, 6); + }); + + it('bills Haiku 5.5 prompts over 100K tokens, cache included, at the long-prompt rate', async () => { + const provider = new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }); + const cost = async (usage: Record) => { + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(anthropicReply('claude-haiku-5-5', usage))); + return (await provider.generateResponse({ ...request, model: 'claude-haiku-5-5' })).usage.cost; + }; + // 100K input + 1K output at $0.10/$0.50 per million + expect(await cost({ input_tokens: 100_000, output_tokens: 1_000 })).toBeCloseTo(0.0105, 8); + // 60K input + 50K cache read: 110K prompt, so $0.50/$2.50 and $0.05 cache reads + expect(await cost({ input_tokens: 60_000, output_tokens: 1_000, cache_read_input_tokens: 50_000 })) + .toBeCloseTo(0.03 + 0.0025 + 0.0025, 8); + }); +}); + +function sse(text: string) { + const bytes = new TextEncoder().encode(text); + return new ReadableStream({ start(controller) { controller.enqueue(bytes); controller.close(); } }); +} +async function readAll(stream: ReadableStream): Promise { + const reader = stream.getReader(); + let out = ''; + for (let chunk = await reader.read(); !chunk.done; chunk = await reader.read()) out += chunk.value; + return out; +} +function anthropicStream(chunks: string[]): string { + const events = [ + { type: 'message_start', message: { usage: { input_tokens: 1, output_tokens: 0 } } }, + ...chunks.map(text => ({ type: 'content_block_delta', index: 0, delta: { type: 'text_delta', text } })), + { type: 'message_delta', usage: { output_tokens: 1 } }, + { type: 'message_stop' }, + ]; + return events.map(event => `data:${JSON.stringify(event)}\n\n`).join(''); +} + +describe('Anthropic JSON mode streaming', () => { + it('restores the prefilled brace on the first streamed text for Claude Haiku 4.5', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true, body: sse(anthropicStream(['"a"', ':1}'])) }); + vi.stubGlobal('fetch', fetchMock); + const stream = await new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }).streamResponse({ + ...request, model: 'claude-haiku-4-5-20251001', response_format: { type: 'json_object' } }); + expect(await readAll(stream)).toBe('{"a":1}'); + const messages = sentBody(fetchMock).messages as Array<{ role: string }>; + expect(messages[messages.length - 1].role).toBe('assistant'); + }); + + it('streams text unchanged on models that are not prefilled', async () => { + const fetchMock = vi.fn().mockResolvedValue({ ok: true, body: sse(anthropicStream(['Result: ', '{"a":1}'])) }); + vi.stubGlobal('fetch', fetchMock); + const stream = await new AnthropicProvider({ apiKey: 'k', maxRetries: 0 }).streamResponse({ + ...request, model: 'claude-sonnet-4-6', response_format: { type: 'json_object' } }); + expect(await readAll(stream)).toBe('Result: {"a":1}'); + }); +}); + +describe('explicit-only models through the factory', () => { + const tools: LLMRequest['tools'] = [ + { type: 'function', function: { name: 'lookup', description: 'Lookup', parameters: { type: 'object' } } }]; + + function factory(run = vi.fn().mockResolvedValue({ response: 'ok', usage: { prompt_tokens: 1, completion_tokens: 1 } })) { + const llm = new LLMProviders({ + anthropic: { apiKey: 'k', maxRetries: 0 }, openai: { apiKey: 'k', maxRetries: 0 }, + cloudflare: { ai: { run } as unknown as Ai, maxRetries: 0 }, stateScope: 'client', + }); + return { llm, run }; + } + + it.each([ + ['claude-opus-5-5', 'anthropic'], + ['gpt-5.5', 'openai'], + ])('serves a named %s through %s', async (model, provider) => { + const fetchMock = vi.fn().mockResolvedValue(provider === 'anthropic' ? anthropicReply(model) : openaiReply(model)); + vi.stubGlobal('fetch', fetchMock); + const { llm, run } = factory(); + const response = await llm.generateResponse({ ...request, model }); + expect(response.provider).toBe(provider); + expect(sentBody(fetchMock).model).toBe(model); + expect(run).not.toHaveBeenCalled(); + }); + + it('serves a named SEA-LION request through Workers AI', async () => { + vi.stubGlobal('fetch', vi.fn()); + const { llm, run } = factory(); + const response = await llm.generateResponse({ ...request, model: '@cf/aisingapore/gemma-sea-lion-v4-27b-it' }); + expect(response.provider).toBe('cloudflare'); + expect(run.mock.calls[0][0]).toBe('@cf/aisingapore/gemma-sea-lion-v4-27b-it'); + }); + + it.each(['anthropic', 'openai'] as const)('never picks an explicit-only %s model for an unnamed request', async provider => { + const fetchMock = vi.fn().mockImplementation(async (_url: string, init: { body: string }) => { + const model = JSON.parse(init.body).model as string; + return provider === 'anthropic' ? anthropicReply(model) : openaiReply(model); + }); + vi.stubGlobal('fetch', fetchMock); + const llm = new LLMProviders({ [provider]: { apiKey: 'k', maxRetries: 0 }, stateScope: 'client' }); + await llm.generateResponse(request); + const model = sentBody(fetchMock).model as string; + expect(getCatalogEntry(model)?.provider).toBe(provider); + expect(getCatalogEntry(model)?.explicitOnly).toBeUndefined(); + }); + + it.each(['generateResponse', 'generateResponseStream'] as const)( + 'rejects forced tool choice on claude-sonnet-5-5 via %s without calling any provider', async method => { + const fetchMock = vi.fn(); + vi.stubGlobal('fetch', fetchMock); + const { llm, run } = factory(); + await expect(llm[method]({ ...request, model: 'claude-sonnet-5-5', tools, toolChoice: 'required' })) + .rejects.toThrow(/does not support forced tool choice/); + expect(fetchMock).not.toHaveBeenCalled(); + expect(run).not.toHaveBeenCalled(); + }); + + it('allows toolChoice none on claude-sonnet-5-5 and sends no tools', async () => { + const fetchMock = vi.fn().mockResolvedValue(anthropicReply('claude-sonnet-5-5')); + vi.stubGlobal('fetch', fetchMock); + const { llm } = factory(); + await llm.generateResponse({ ...request, model: 'claude-sonnet-5-5', tools, toolChoice: 'none' }); + expect(sentBody(fetchMock)).not.toHaveProperty('tools'); + expect(sentBody(fetchMock)).not.toHaveProperty('tool_choice'); + }); + + it('records real spend in the ledger for the default gpt-4o-mini (was $0 in 1.22)', async () => { + // Responses name the dated snapshot, which 1.22 could not price. + vi.stubGlobal('fetch', vi.fn().mockResolvedValue(openaiReply('gpt-4o-mini-2024-07-18', 1000, 1000))); + const ledger = new CreditLedger(); + const llm = new LLMProviders({ openai: { apiKey: 'k', maxRetries: 0 }, stateScope: 'client', ledger }); + const response = await llm.generateResponse(request); + expect(response.provider).toBe('openai'); + expect(response.usage.cost).toBeCloseTo(0.00015 + 0.0006, 8); + expect(ledger.totalSpend()).toBeCloseTo(0.00015 + 0.0006, 8); + }); +}); + +describe('MODELS constants', () => { + it.each([ + 'GPT_6_ASTRA', 'GPT_5_6_SOL', 'GPT_5_6_TERRA', 'GPT_5_6_LUNA', 'GPT_5_5', + 'CLAUDE_FABLE_5_1', 'CLAUDE_OPUS_5_5', 'CLAUDE_SONNET_5_5', 'CLAUDE_HAIKU_5_5', + 'CF_GLM_5_3', 'CF_GLM_5_3_FLASH', 'CF_KIMI_K2_6', 'CF_DEEPSEEK_V4_PRO', 'CF_DEEPSEEK_V4_FLASH', + 'CF_GPT_OSS_120B', 'CF_GPT_OSS_20B', 'CF_GEMMA_4_26B', 'CF_QWEN_3_8_27B', 'CF_LLAMA_3_3_70B_FAST', + 'CF_LLAMA_3_1_8B_FP8', 'CF_GRANITE_4_MICRO', + ] as const)('%s names an active catalog model', key => { + expect(getCatalogEntry(MODELS[key])?.lifecycle).toBe('active'); + }); +}); diff --git a/src/index.ts b/src/index.ts index 6aae999..eee4062 100755 --- a/src/index.ts +++ b/src/index.ts @@ -536,24 +536,37 @@ export const SUPPORTED_PROVIDERS = ['openai', 'anthropic', 'cloudflare', 'cerebr */ export const MODELS = { // OpenAI models + GPT_6_ASTRA: 'gpt-6-astra', + GPT_5_6_SOL: 'gpt-5.6-sol', + GPT_5_6_TERRA: 'gpt-5.6-terra', + GPT_5_6_LUNA: 'gpt-5.6-luna', + GPT_5_5: 'gpt-5.5', /** Not accepted by the OpenAI adapter or routed by this package. OpenAI has announced shutdowns only for dated gpt-4o snapshots. */ GPT_4O: 'gpt-4o', + /** @deprecated OpenAI shuts this model down on 2027-01-14 and names gpt-6-luna as the replacement (not yet supported by this adapter). */ GPT_4O_MINI: 'gpt-4o-mini', - /** @deprecated OpenAI shuts this model down on 2026-10-23. */ + /** @deprecated OpenAI shuts this model down on 2026-10-23. Use GPT_5_6_SOL. */ GPT_4_TURBO: 'gpt-4-turbo', - /** @deprecated OpenAI shuts this model down on 2026-10-23. */ + /** @deprecated OpenAI shuts this model down on 2026-10-23. Use GPT_5_6_SOL. */ GPT_4: 'gpt-4', - /** @deprecated OpenAI shuts this model down on 2026-10-23. */ + /** @deprecated OpenAI shuts this model down on 2026-10-23. Use GPT_5_6_TERRA. */ GPT_3_5_TURBO: 'gpt-3.5-turbo', // Anthropic models // Claude model IDs are dateless from the 4.6 generation on. + CLAUDE_FABLE_5_1: 'claude-fable-5-1', + CLAUDE_OPUS_5_5: 'claude-opus-5-5', + CLAUDE_SONNET_5_5: 'claude-sonnet-5-5', + CLAUDE_HAIKU_5_5: 'claude-haiku-5-5', + /** Legacy on the Claude API; retirement not sooner than 2027-02-05. */ CLAUDE_OPUS_4_6: 'claude-opus-4-6', + /** Legacy on the Claude API; retirement not sooner than 2027-02-17. */ CLAUDE_SONNET_4_6: 'claude-sonnet-4-6', /** @deprecated Retired on the Claude API 2026-06-15. */ CLAUDE_OPUS_4: 'claude-opus-4-20250514', /** @deprecated Retired on the Claude API 2026-06-15. */ CLAUDE_SONNET_4: 'claude-sonnet-4-20250514', + /** Legacy on the Claude API; retirement not sooner than 2026-10-15. */ CLAUDE_HAIKU_4_5: 'claude-haiku-4-5-20251001', /** @deprecated Retired on the Claude API 2026-02-19. */ CLAUDE_3_7_SONNET: 'claude-3-7-sonnet-20250219', @@ -568,11 +581,28 @@ export const MODELS = { /** @deprecated Retired on the Claude API 2026-04-20; not accepted by AnthropicProvider. Use CLAUDE_HAIKU_4_5. */ CLAUDE_3_HAIKU: 'claude-3-haiku-20240307', - // Cloudflare models + // Cloudflare Workers AI models + CF_GLM_5_3: '@cf/zai-org/glm-5.3', + CF_GLM_5_3_FLASH: '@cf/zai-org/glm-5.3-flash', + CF_KIMI_K2_6: '@cf/moonshotai/kimi-k2.6', + CF_DEEPSEEK_V4_PRO: '@cf/deepseek-ai/deepseek-v4-pro-0813', + CF_DEEPSEEK_V4_FLASH: '@cf/deepseek-ai/deepseek-v4-flash-0731', + CF_GPT_OSS_120B: '@cf/openai/gpt-oss-120b', + CF_GPT_OSS_20B: '@cf/openai/gpt-oss-20b', + CF_GEMMA_4_26B: '@cf/google/gemma-4-26b-a4b-it', + CF_QWEN_3_8_27B: '@cf/qwen/qwen3.8-27b', + CF_LLAMA_3_3_70B_FAST: '@cf/meta/llama-3.3-70b-instruct-fp8-fast', + CF_LLAMA_3_1_8B_FP8: '@cf/meta/llama-3.1-8b-instruct-fp8', + CF_GRANITE_4_MICRO: '@cf/ibm-granite/granite-4.0-h-micro', + /** @deprecated Deprecated on Workers AI 2026-05-30. Use CF_LLAMA_3_1_8B_FP8. */ LLAMA_3_1_8B: '@cf/meta/llama-3.1-8b-instruct', + /** @deprecated Deprecated on Workers AI 2026-05-30. Use CF_LLAMA_3_3_70B_FAST. */ LLAMA_3_1_70B: '@cf/meta/llama-3.1-70b-instruct', + /** @deprecated Deprecated on Workers AI 2026-05-30. Use CF_LLAMA_3_1_8B_FP8. */ LLAMA_3_8B: '@cf/meta/llama-3-8b-instruct', + /** @deprecated Deprecated on Workers AI 2026-05-30. */ MISTRAL_7B: '@cf/mistral/mistral-7b-instruct-v0.1', + /** @deprecated Deprecated on Workers AI 2025-10-01. */ TINY_LLAMA: '@cf/tinyllama/tinyllama-1.1b-chat-v1.0', // Cerebras models diff --git a/src/model-catalog.ts b/src/model-catalog.ts index 0626a0a..b24cc5c 100644 --- a/src/model-catalog.ts +++ b/src/model-catalog.ts @@ -38,6 +38,12 @@ export interface ModelCatalogEntry { speedScore: number; qualityScore: number; costScore: number; + /** + * Served only when a request names this model. Recommendations and automatic + * routing skip it (for example, a model the adapter cannot yet use for every + * request shape that routing might send it). + */ + explicitOnly?: boolean; } type SelectionHealthEntry = { @@ -100,6 +106,7 @@ function entry( useCases: ModelRecommendationUseCase[], capabilities: ModelCapabilities, scores: { speed: number; quality: number; cost: number }, + options: { explicitOnly?: boolean } = {}, ): ModelCatalogEntry { return { provider, @@ -110,10 +117,66 @@ function entry( speedScore: scores.speed, qualityScore: scores.quality, costScore: scores.cost, + ...(options.explicitOnly ? { explicitOnly: true } : {}), }; } export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ + entry('openai', 'gpt-6-astra', 'active', ['HIGH_PERFORMANCE', 'TOOL_CALLING', 'LONG_CONTEXT', 'VISION'], { + maxContextLength: 1050000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsVision: true, + supportsBatching: false, + inputTokenCost: 0.01, + outputTokenCost: 0.05, + description: 'GPT-6 Astra — OpenAI\'s flagship reasoning model; prompts over 272K tokens bill 2x input and 1.5x output' + }, { speed: 3, quality: 7, cost: 1 }, { explicitOnly: true }), + entry('openai', 'gpt-5.6-sol', 'active', ['HIGH_PERFORMANCE', 'TOOL_CALLING', 'LONG_CONTEXT', 'VISION'], { + maxContextLength: 1050000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsVision: true, + supportsBatching: false, + inputTokenCost: 0.004, + outputTokenCost: 0.02, + description: 'GPT-5.6 Sol — reasoning model; promotional price stated through at least 2026-11-21; prompts over 272K tokens bill 2x input and 1.5x output' + }, { speed: 3, quality: 7, cost: 2 }, { explicitOnly: true }), + entry('openai', 'gpt-5.6-terra', 'active', ['BALANCED', 'TOOL_CALLING', 'LONG_CONTEXT', 'VISION'], { + maxContextLength: 1050000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsVision: true, + supportsBatching: false, + inputTokenCost: 0.002, + outputTokenCost: 0.012, + description: 'GPT-5.6 Terra — mid-tier reasoning model; prompts over 272K tokens bill 2x input and 1.5x output' + }, { speed: 4, quality: 6, cost: 3 }, { explicitOnly: true }), + entry('openai', 'gpt-5.6-luna', 'active', ['COST_EFFECTIVE', 'BALANCED', 'TOOL_CALLING', 'LONG_CONTEXT', 'VISION'], { + maxContextLength: 1050000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsVision: true, + supportsBatching: false, + inputTokenCost: 0.0002, + outputTokenCost: 0.0012, + description: 'GPT-5.6 Luna — low-cost reasoning model; prompts over 272K tokens bill 2x input and 1.5x output' + }, { speed: 5, quality: 5, cost: 5 }, { explicitOnly: true }), + entry('openai', 'gpt-5.5', 'active', ['HIGH_PERFORMANCE', 'TOOL_CALLING'], { + maxContextLength: 1050000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsVision: true, + supportsBatching: false, + inputTokenCost: 0.005, + outputTokenCost: 0.03, + description: 'GPT-5.5 (alias of gpt-5.5-2026-04-23) — reasoning model; prompts over 272K tokens bill 2x input and 1.5x output' + }, { speed: 3, quality: 6, cost: 1 }, { explicitOnly: true }), entry('openai', 'gpt-4o', 'retired', ['HIGH_PERFORMANCE', 'TOOL_CALLING'], { maxContextLength: 128000, supportsStreaming: true, @@ -132,45 +195,89 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ supportsBatching: false, inputTokenCost: 0.00015, outputTokenCost: 0.0006, - description: 'Modern OpenAI baseline' + description: 'GPT-4o mini (deprecated 2027-01-14, when OpenAI shuts it down; replacement gpt-6-luna)' }, { speed: 5, quality: 4, cost: 5 }), - entry('openai', 'gpt-4-turbo', 'compatibility', ['HIGH_PERFORMANCE', 'LONG_CONTEXT'], { + entry('openai', 'gpt-4-turbo', 'retired', ['HIGH_PERFORMANCE', 'LONG_CONTEXT'], { maxContextLength: 128000, supportsStreaming: true, supportsTools: true, supportsBatching: false, inputTokenCost: 0.01, outputTokenCost: 0.03, - description: 'Compatibility OpenAI GPT-4 Turbo (deprecated 2026-10-23)' + description: 'GPT-4 Turbo (deprecated 2026-10-23, when OpenAI shuts it down; replacement gpt-5.6-sol)' }, { speed: 3, quality: 4, cost: 2 }), - entry('openai', 'gpt-4', 'compatibility', ['HIGH_PERFORMANCE'], { + entry('openai', 'gpt-4', 'retired', ['HIGH_PERFORMANCE'], { maxContextLength: 8192, supportsStreaming: true, supportsTools: true, supportsBatching: false, inputTokenCost: 0.03, outputTokenCost: 0.06, - description: 'Compatibility GPT-4 (deprecated 2026-10-23)' + description: 'GPT-4 (deprecated 2026-10-23, when OpenAI shuts it down; replacement gpt-5.6-sol)' }, { speed: 2, quality: 4, cost: 1 }), - entry('openai', 'gpt-3.5-turbo', 'compatibility', ['COST_EFFECTIVE'], { + entry('openai', 'gpt-3.5-turbo', 'retired', ['COST_EFFECTIVE'], { maxContextLength: 16385, supportsStreaming: true, supportsTools: true, supportsBatching: false, inputTokenCost: 0.0005, outputTokenCost: 0.0015, - description: 'Compatibility GPT-3.5 Turbo (deprecated 2026-10-23)' + description: 'GPT-3.5 Turbo (deprecated 2026-10-23, when OpenAI shuts it down; replacement gpt-5.6-terra)' }, { speed: 4, quality: 2, cost: 3 }), - entry('openai', 'gpt-3.5-turbo-16k', 'compatibility', ['LONG_CONTEXT'], { + entry('openai', 'gpt-3.5-turbo-16k', 'retired', ['LONG_CONTEXT'], { maxContextLength: 16385, supportsStreaming: true, supportsTools: true, supportsBatching: false, inputTokenCost: 0.001, outputTokenCost: 0.002, - description: 'Compatibility GPT-3.5 Turbo 16k' + description: 'GPT-3.5 Turbo 16k (no longer a valid OpenAI model ID; gpt-3.5-turbo-16k-0613 shut down 2024-09-13)' }, { speed: 3, quality: 2, cost: 3 }), + entry('anthropic', 'claude-fable-5-1', 'active', ['HIGH_PERFORMANCE', 'LONG_CONTEXT', 'VISION', 'TOOL_CALLING'], { + maxContextLength: 1000000, + supportsStreaming: true, + supportsTools: true, + supportsVision: true, + supportsBatching: false, + supportsPromptCache: true, + inputTokenCost: 0.01, + outputTokenCost: 0.05, + description: 'Claude Fable 5.1 — top tier for demanding reasoning and long-horizon agentic work; adaptive thinking always on' + }, { speed: 2, quality: 7, cost: 1 }, { explicitOnly: true }), + entry('anthropic', 'claude-opus-5-5', 'active', ['HIGH_PERFORMANCE', 'LONG_CONTEXT', 'VISION', 'TOOL_CALLING'], { + maxContextLength: 1000000, + supportsStreaming: true, + supportsTools: true, + supportsVision: true, + supportsBatching: false, + supportsPromptCache: true, + inputTokenCost: 0.004, + outputTokenCost: 0.02, + description: 'Claude Opus 5.5 — Anthropic\'s recommended starting model; adaptive thinking always on' + }, { speed: 3, quality: 7, cost: 2 }, { explicitOnly: true }), + entry('anthropic', 'claude-sonnet-5-5', 'active', ['HIGH_PERFORMANCE', 'BALANCED', 'LONG_CONTEXT', 'VISION', 'TOOL_CALLING'], { + maxContextLength: 1000000, + supportsStreaming: true, + supportsTools: true, + supportsVision: true, + supportsBatching: false, + supportsPromptCache: true, + inputTokenCost: 0.002, + outputTokenCost: 0.01, + description: 'Claude Sonnet 5.5 — balanced speed and intelligence' + }, { speed: 4, quality: 6, cost: 4 }, { explicitOnly: true }), + entry('anthropic', 'claude-haiku-5-5', 'active', ['COST_EFFECTIVE', 'BALANCED', 'TOOL_CALLING', 'VISION', 'LONG_CONTEXT'], { + maxContextLength: 1000000, + supportsStreaming: true, + supportsTools: true, + supportsVision: true, + supportsBatching: false, + supportsPromptCache: true, + inputTokenCost: 0.0001, + outputTokenCost: 0.0005, + description: 'Claude Haiku 5.5 — fastest tier; prompts over 100K tokens bill at 5x these rates' + }, { speed: 5, quality: 5, cost: 5 }, { explicitOnly: true }), entry('anthropic', 'claude-opus-4-6', 'active', ['HIGH_PERFORMANCE', 'LONG_CONTEXT', 'VISION', 'TOOL_CALLING'], { maxContextLength: 1000000, supportsStreaming: true, @@ -179,7 +286,7 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ supportsBatching: false, inputTokenCost: 0.005, outputTokenCost: 0.025, - description: 'Claude Opus 4.6' + description: 'Claude Opus 4.6 (legacy on the Claude API; retirement not sooner than 2027-02-05)' }, { speed: 2, quality: 7, cost: 1 }), entry('anthropic', 'claude-sonnet-4-6', 'active', ['HIGH_PERFORMANCE', 'LONG_CONTEXT', 'VISION', 'TOOL_CALLING', 'BALANCED'], { maxContextLength: 1000000, @@ -189,7 +296,7 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ supportsBatching: false, inputTokenCost: 0.003, outputTokenCost: 0.015, - description: 'Claude Sonnet 4.6' + description: 'Claude Sonnet 4.6 (legacy on the Claude API; retirement not sooner than 2027-02-17)' }, { speed: 3, quality: 6, cost: 3 }), entry('anthropic', 'claude-opus-4-20250514', 'retired', ['HIGH_PERFORMANCE', 'LONG_CONTEXT'], { maxContextLength: 200000, @@ -217,7 +324,7 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ supportsBatching: false, inputTokenCost: 0.001, outputTokenCost: 0.005, - description: 'Claude Haiku 4.5' + description: 'Claude Haiku 4.5 (legacy on the Claude API; retirement not sooner than 2026-10-15, no notice yet)' }, { speed: 4, quality: 4, cost: 5 }), entry('anthropic', 'claude-3-7-sonnet-20250219', 'retired', ['HIGH_PERFORMANCE', 'LONG_CONTEXT'], { maxContextLength: 200000, @@ -265,68 +372,68 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ description: 'Claude 3 Sonnet (retired on the Claude API 2025-07-21)' }, { speed: 3, quality: 3, cost: 2 }), - entry('cloudflare', '@cf/meta/llama-3.1-8b-instruct', 'compatibility', ['COST_EFFECTIVE'], { + entry('cloudflare', '@cf/meta/llama-3.1-8b-instruct', 'retired', ['COST_EFFECTIVE'], { maxContextLength: 8192, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.000282, outputTokenCost: 0.000827, - description: 'Workers AI Llama 3.1 8B' + description: 'Workers AI Llama 3.1 8B (deprecated on Workers AI 2026-05-30; no longer in the model catalog)' }, { speed: 5, quality: 2, cost: 5 }), - entry('cloudflare', '@cf/meta/llama-3.1-70b-instruct', 'compatibility', ['HIGH_PERFORMANCE'], { + entry('cloudflare', '@cf/meta/llama-3.1-70b-instruct', 'retired', ['HIGH_PERFORMANCE'], { maxContextLength: 8192, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.0000005, outputTokenCost: 0.0000005, - description: 'Workers AI Llama 3.1 70B' + description: 'Workers AI Llama 3.1 70B (deprecated on Workers AI 2026-05-30; no longer in the model catalog)' }, { speed: 3, quality: 4, cost: 4 }), - entry('cloudflare', '@cf/meta/llama-3-8b-instruct', 'compatibility', ['COST_EFFECTIVE'], { + entry('cloudflare', '@cf/meta/llama-3-8b-instruct', 'retired', ['COST_EFFECTIVE'], { maxContextLength: 8192, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.000282, outputTokenCost: 0.000827, - description: 'Workers AI Llama 3 8B' + description: 'Workers AI Llama 3 8B (deprecated on Workers AI 2026-05-30; no longer in the model catalog)' }, { speed: 4, quality: 2, cost: 5 }), - entry('cloudflare', '@cf/meta/llama-2-7b-chat-int8', 'compatibility', ['COST_EFFECTIVE'], { + entry('cloudflare', '@cf/meta/llama-2-7b-chat-int8', 'retired', ['COST_EFFECTIVE'], { maxContextLength: 4096, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.00000005, outputTokenCost: 0.00000005, - description: 'Workers AI Llama 2 7B' + description: 'Workers AI Llama 2 7B (deprecated on Workers AI 2026-05-30; no longer in the model catalog)' }, { speed: 4, quality: 1, cost: 5 }), - entry('cloudflare', '@cf/microsoft/phi-2', 'compatibility', ['COST_EFFECTIVE'], { + entry('cloudflare', '@cf/microsoft/phi-2', 'retired', ['COST_EFFECTIVE'], { maxContextLength: 2048, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.00000002, outputTokenCost: 0.00000002, - description: 'Workers AI Phi-2' + description: 'Workers AI Phi-2 (deprecated on Workers AI 2026-05-30; no longer in the model catalog)' }, { speed: 4, quality: 1, cost: 5 }), - entry('cloudflare', '@cf/mistral/mistral-7b-instruct-v0.1', 'compatibility', ['BALANCED'], { + entry('cloudflare', '@cf/mistral/mistral-7b-instruct-v0.1', 'retired', ['BALANCED'], { maxContextLength: 8192, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.00011, outputTokenCost: 0.00019, - description: 'Workers AI Mistral 7B' + description: 'Workers AI Mistral 7B (deprecated on Workers AI 2026-05-30; no longer in the model catalog)' }, { speed: 4, quality: 2, cost: 5 }), - entry('cloudflare', '@cf/openchat/openchat-3.5-0106', 'compatibility', ['BALANCED'], { + entry('cloudflare', '@cf/openchat/openchat-3.5-0106', 'retired', ['BALANCED'], { maxContextLength: 8192, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.0000001, outputTokenCost: 0.0000001, - description: 'Workers AI OpenChat 3.5' + description: 'Workers AI OpenChat 3.5 (deprecated on Workers AI 2025-10-01; no longer in the model catalog)' }, { speed: 4, quality: 2, cost: 5 }), entry('cloudflare', '@cf/openai/gpt-oss-120b', 'active', ['TOOL_CALLING', 'HIGH_PERFORMANCE'], { maxContextLength: 128000, @@ -339,7 +446,7 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ description: 'Workers AI GPT-OSS 120B — OpenAI open-weight 117B/5.1B-active MoE, near o4-mini class (not Phi-4)' }, { speed: 4, quality: 6, cost: 4 }), entry('cloudflare', '@cf/moonshotai/kimi-k2.6', 'active', ['HIGH_PERFORMANCE', 'TOOL_CALLING', 'LONG_CONTEXT', 'VISION', 'BALANCED'], { - maxContextLength: 262100, + maxContextLength: 262144, supportsStreaming: true, supportsTools: true, toolCalling: true, @@ -350,7 +457,7 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ description: 'Workers AI Kimi K2.6 — 1T/32B-active MoE, frontier-scale agent model; vision, 262K context, tool calling (Moonshot self-reported benchmarks)' }, { speed: 3, quality: 6, cost: 5 }), entry('cloudflare', '@cf/meta/llama-3.3-70b-instruct-fp8-fast', 'active', ['COST_EFFECTIVE', 'BALANCED', 'HIGH_PERFORMANCE', 'TOOL_CALLING'], { - maxContextLength: 131072, + maxContextLength: 24000, supportsStreaming: true, supportsTools: true, toolCalling: true, @@ -369,11 +476,10 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ outputTokenCost: 0.0003, description: 'Workers AI GPT-OSS 20B — lightweight tool-calling model; cost-effective complement to gpt-oss-120b' }, { speed: 5, quality: 4, cost: 5 }), - entry('cloudflare', '@cf/qwen/qwen2.5-coder-32b-instruct', 'active', ['COST_EFFECTIVE', 'BALANCED', 'TOOL_CALLING'], { + entry('cloudflare', '@cf/qwen/qwen2.5-coder-32b-instruct', 'active', ['COST_EFFECTIVE', 'BALANCED'], { maxContextLength: 32768, supportsStreaming: true, - supportsTools: true, - toolCalling: true, + supportsTools: false, supportsBatching: true, inputTokenCost: 0.00066, outputTokenCost: 0.001, @@ -391,7 +497,7 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ description: 'Workers AI Mistral Small 3.1 24B — vision + tool calling; weakest of the active CF set' }, { speed: 4, quality: 3, cost: 5 }), entry('cloudflare', '@cf/qwen/qwen3-30b-a3b-fp8', 'active', ['BALANCED', 'HIGH_PERFORMANCE'], { - maxContextLength: 40960, + maxContextLength: 32768, supportsStreaming: true, supportsTools: true, toolCalling: true, @@ -412,7 +518,7 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ description: 'Workers AI Qwen 3.8 27B — vision-language model for reasoning and multi-turn function calling with 262K context' }, { speed: 4, quality: 5, cost: 3 }), entry('cloudflare', '@cf/meta/llama-3.2-1b-instruct', 'active', ['COST_EFFECTIVE'], { - maxContextLength: 131072, + maxContextLength: 60000, supportsStreaming: true, supportsTools: false, supportsBatching: true, @@ -421,7 +527,7 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ description: 'Workers AI Llama 3.2 1B — ultra-cheap tiny model for simple classification and summary' }, { speed: 5, quality: 2, cost: 5 }), entry('cloudflare', '@cf/meta/llama-3.2-3b-instruct', 'active', ['COST_EFFECTIVE'], { - maxContextLength: 131072, + maxContextLength: 80000, supportsStreaming: true, supportsTools: false, supportsBatching: true, @@ -429,8 +535,37 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ outputTokenCost: 0.000335, description: 'Workers AI Llama 3.2 3B — cheap small model, step up from 1B' }, { speed: 5, quality: 3, cost: 5 }), + entry('cloudflare', '@cf/meta/llama-3.1-8b-instruct-fp8', 'active', ['COST_EFFECTIVE'], { + maxContextLength: 32000, + supportsStreaming: true, + supportsTools: false, + supportsBatching: false, + inputTokenCost: 0.000152, + outputTokenCost: 0.000287, + description: 'Workers AI Llama 3.1 8B FP8 — the listed Llama 3.1 8B variant after the 2026-05-30 deprecations' + }, { speed: 5, quality: 3, cost: 5 }), + entry('cloudflare', '@cf/ibm-granite/granite-4.0-h-micro', 'active', ['COST_EFFECTIVE'], { + maxContextLength: 131000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsBatching: false, + inputTokenCost: 0.000017, + outputTokenCost: 0.000112, + description: 'Workers AI IBM Granite 4.0 H Micro — very low cost small model with function calling' + }, { speed: 5, quality: 3, cost: 5 }), + entry('cloudflare', '@cf/aisingapore/gemma-sea-lion-v4-27b-it', 'active', [], { + maxContextLength: 128000, + supportsStreaming: true, + supportsTools: false, + supportsBatching: false, + inputTokenCost: 0.000351, + outputTokenCost: 0.000555, + description: 'Workers AI SEA-LION v4 27B — Gemma-based model for Southeast Asian languages; explicit requests only' + }, { speed: 4, quality: 4, cost: 4 }, { explicitOnly: true }), entry('cloudflare', '@cf/moonshotai/kimi-k2.7-code', 'active', ['BALANCED', 'TOOL_CALLING'], { - maxContextLength: 131072, + maxContextLength: 262144, + supportsVision: true, supportsStreaming: true, supportsTools: true, toolCalling: true, @@ -451,23 +586,23 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ description: 'Workers AI GLM-4.7-Flash — 30B/3B-active MoE CoT model; emits reasoning traces, RESEARCH-only routing' }, { speed: 5, quality: 5, cost: 5 }), entry('cloudflare', 'deepseek/deepseek-v4-pro', 'active', ['HIGH_PERFORMANCE', 'BALANCED'], { - maxContextLength: 128000, + maxContextLength: 131072, supportsStreaming: true, supportsTools: false, supportsBatching: true, - inputTokenCost: 0, - outputTokenCost: 0, - description: 'Workers AI DeepSeek V4 Pro — frontier-class reasoning and coding, near gpt-oss-120b tier' - }, { speed: 3, quality: 6, cost: 5 }), + inputTokenCost: 0.00174, + outputTokenCost: 0.00348, + description: 'DeepSeek V4 Pro via the AI Gateway unified catalog (served by Fireworks; not a Workers AI model). For Workers AI use @cf/deepseek-ai/deepseek-v4-pro-0813' + }, { speed: 3, quality: 6, cost: 3 }), entry('cloudflare', '@cf/deepseek-ai/deepseek-v4-flash-0731', 'active', ['BALANCED', 'HIGH_PERFORMANCE', 'TOOL_CALLING', 'LONG_CONTEXT'], { - maxContextLength: 1310720, + maxContextLength: 1048576, supportsStreaming: true, supportsTools: true, toolCalling: true, supportsBatching: true, inputTokenCost: 0.00044, outputTokenCost: 0.00132, - description: 'Workers AI DeepSeek V4 Flash 0731 — fast hosted reasoning and agentic model; 1.31M context and function calling' + description: 'Workers AI DeepSeek V4 Flash 0731 — fast hosted reasoning and agentic model; 1M context and function calling' }, { speed: 4, quality: 6, cost: 4 }), entry('cloudflare', '@cf/deepseek-ai/deepseek-v4-pro-0813', 'active', ['HIGH_PERFORMANCE', 'TOOL_CALLING', 'LONG_CONTEXT'], { maxContextLength: 1048576, @@ -479,50 +614,50 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ outputTokenCost: 0.00396, description: 'Workers AI DeepSeek V4 Pro 0813 — hosted long-horizon reasoning and agentic model; 1M context and function calling' }, { speed: 3, quality: 6, cost: 3 }), - entry('cloudflare', '@cf/tinyllama/tinyllama-1.1b-chat-v1.0', 'compatibility', ['COST_EFFECTIVE'], { + entry('cloudflare', '@cf/tinyllama/tinyllama-1.1b-chat-v1.0', 'retired', ['COST_EFFECTIVE'], { maxContextLength: 2048, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.00000001, outputTokenCost: 0.00000001, - description: 'Workers AI TinyLlama' + description: 'Workers AI TinyLlama (deprecated on Workers AI 2025-10-01; no longer in the model catalog)' }, { speed: 5, quality: 1, cost: 5 }), - entry('cloudflare', '@cf/qwen/qwen1.5-0.5b-chat', 'compatibility', ['COST_EFFECTIVE'], { + entry('cloudflare', '@cf/qwen/qwen1.5-0.5b-chat', 'retired', ['COST_EFFECTIVE'], { maxContextLength: 2048, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.00000001, outputTokenCost: 0.00000001, - description: 'Workers AI Qwen 1.5 0.5B' + description: 'Workers AI Qwen 1.5 0.5B (deprecated on Workers AI 2025-10-01; no longer in the model catalog)' }, { speed: 5, quality: 1, cost: 5 }), - entry('cloudflare', '@cf/qwen/qwen1.5-1.8b-chat', 'compatibility', ['COST_EFFECTIVE'], { + entry('cloudflare', '@cf/qwen/qwen1.5-1.8b-chat', 'retired', ['COST_EFFECTIVE'], { maxContextLength: 4096, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.00000002, outputTokenCost: 0.00000002, - description: 'Workers AI Qwen 1.5 1.8B' + description: 'Workers AI Qwen 1.5 1.8B (deprecated on Workers AI 2025-10-01; no longer in the model catalog)' }, { speed: 5, quality: 1, cost: 5 }), - entry('cloudflare', '@cf/qwen/qwen1.5-14b-chat-awq', 'compatibility', ['BALANCED'], { + entry('cloudflare', '@cf/qwen/qwen1.5-14b-chat-awq', 'retired', ['BALANCED'], { maxContextLength: 8192, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.0000002, outputTokenCost: 0.0000002, - description: 'Workers AI Qwen 1.5 14B' + description: 'Workers AI Qwen 1.5 14B (deprecated on Workers AI 2025-10-01; no longer in the model catalog)' }, { speed: 4, quality: 3, cost: 4 }), - entry('cloudflare', '@cf/qwen/qwen1.5-7b-chat-awq', 'compatibility', ['BALANCED'], { + entry('cloudflare', '@cf/qwen/qwen1.5-7b-chat-awq', 'retired', ['BALANCED'], { maxContextLength: 8192, supportsStreaming: true, supportsTools: false, supportsBatching: true, inputTokenCost: 0.0000001, outputTokenCost: 0.0000001, - description: 'Workers AI Qwen 1.5 7B' + description: 'Workers AI Qwen 1.5 7B (deprecated on Workers AI 2025-10-01; no longer in the model catalog)' }, { speed: 4, quality: 2, cost: 4 }), entry('cloudflare', '@cf/google/gemma-4-26b-a4b-it', 'active', ['COST_EFFECTIVE', 'VISION', 'TOOL_CALLING', 'LONG_CONTEXT'], { maxContextLength: 256000, @@ -594,9 +729,9 @@ export const MODEL_CATALOG: readonly ModelCatalogEntry[] = [ supportsStreaming: false, supportsTools: false, supportsBatching: false, - inputTokenCost: 0, - outputTokenCost: 0, - description: 'Cloudflare-managed Anthropic Claude Opus 4.8 — 1M context, external billing via CF dashboard; not a native CF model' + inputTokenCost: 0.005, + outputTokenCost: 0.025, + description: 'Claude Opus 4.8 via the AI Gateway unified catalog (not a Workers AI model); 1M context. Anthropic lists Opus 4.8 as legacy' }, { speed: 2, quality: 7, cost: 3 }), entry('cloudflare', '@cf/zai-org/glm-5.2', 'active', ['HIGH_PERFORMANCE', 'TOOL_CALLING', 'LONG_CONTEXT'], { maxContextLength: 262144, @@ -955,6 +1090,7 @@ function isCompatible(entry: ModelCatalogEntry, context: ModelSelectionContext, const request = context.request; if (entry.lifecycle === 'retired') return false; + if (entry.explicitOnly) return false; // Thinking/reasoning models output chain-of-thought traces rather than direct // responses. Exclude them from every pool except RESEARCH, where the caller diff --git a/src/providers/anthropic.ts b/src/providers/anthropic.ts index 344ad8b..19c2f4d 100755 --- a/src/providers/anthropic.ts +++ b/src/providers/anthropic.ts @@ -147,9 +147,58 @@ interface AnthropicResponse { }; } +/** Models that return a 400 for a non-default temperature, top_p or top_k (Claude 4.7 and later). */ +const FIXED_SAMPLING_MODELS = new Set(['claude-fable-5-1', 'claude-opus-5-5', 'claude-sonnet-5-5', 'claude-haiku-5-5']); + +/** Cache-read price as a fraction of the input price, where it is not the standard 0.1. */ +const CACHE_READ_MULTIPLIER: Record = { + 'claude-fable-5-1': 0.025, + 'claude-opus-5-5': 0.05, + 'claude-sonnet-5-5': 0.05, +}; + +/** Prompt size (input plus cache tokens) above which a model bills at its long-prompt rate. */ +const LONG_PROMPT_THRESHOLD: Record = { 'claude-haiku-5-5': 100_000 }; +const LONG_PROMPT_MULTIPLIER = 5; + +/** + * Default max_tokens when the request sets none. Claude 5.x models think by + * default (Fable 5.1 and Opus 5.5 always), and thinking counts against + * max_tokens, so 1000 could leave no room for the answer. 16,000 stays below the + * size where non-streaming requests risk long-request timeouts. + */ +const THINKING_DEFAULT_MAX_TOKENS: Record = { + 'claude-fable-5-1': 16_000, + 'claude-opus-5-5': 16_000, + 'claude-sonnet-5-5': 16_000, + 'claude-haiku-5-5': 16_000, +}; + +function defaultMaxTokens(model: string): number { + return THINKING_DEFAULT_MAX_TOKENS[model] ?? 1000; +} + +/** Models that reject a final assistant turn (prefill) with a 400: Claude 4.6 and later. */ +const NO_PREFILL_MODELS = new Set([ + 'claude-opus-4-6', 'claude-sonnet-4-6', + 'claude-fable-5-1', 'claude-opus-5-5', 'claude-sonnet-5-5', 'claude-haiku-5-5', +]); + +/** Whether JSON mode can prefill '{' for this model. */ +function usesJsonPrefill(model: string): boolean { + return !NO_PREFILL_MODELS.has(model); +} + +/** Models that reject tool_choice "any" and "tool" with a 400. */ +const NO_FORCED_TOOL_CHOICE_MODELS = new Set(['claude-fable-5-1', 'claude-opus-5-5', 'claude-sonnet-5-5']); + export class AnthropicProvider extends BaseProvider { name = 'anthropic'; models = [ + 'claude-fable-5-1', + 'claude-opus-5-5', + 'claude-sonnet-5-5', + 'claude-haiku-5-5', 'claude-opus-4-6', 'claude-sonnet-4-6', 'claude-opus-4-20250514', @@ -194,7 +243,8 @@ export class AnthropicProvider extends BaseProvider { this.validateRequest(request); const startTime = Date.now(); - const jsonMode = request.response_format?.type === 'json_object'; + const jsonMode = request.response_format?.type === 'json_object' + && usesJsonPrefill(resolveModelId(request.model || this.getDefaultModel(request))); try { const response = await this.executeWithResiliency(async () => { @@ -250,7 +300,7 @@ export class AnthropicProvider extends BaseProvider { const inputTokens = request.messages.reduce((sum, msg) => sum + Math.ceil(msg.content.length / 4), 0 ); - const outputTokens = request.maxTokens || 1000; + const outputTokens = request.maxTokens || defaultMaxTokens(model); return this.calculateCost(inputTokens, outputTokens, model); } @@ -277,6 +327,50 @@ export class AnthropicProvider extends BaseProvider { protected getModelCapabilities(): Record { return { + 'claude-fable-5-1': { + maxContextLength: 1000000, + supportsStreaming: true, + supportsTools: true, + supportsVision: true, + supportsBatching: false, + supportsPromptCache: true, + inputTokenCost: 0.01, + outputTokenCost: 0.05, + description: 'Claude Fable 5.1 — top tier for demanding reasoning and long-horizon agentic work; adaptive thinking always on' + }, + 'claude-opus-5-5': { + maxContextLength: 1000000, + supportsStreaming: true, + supportsTools: true, + supportsVision: true, + supportsBatching: false, + supportsPromptCache: true, + inputTokenCost: 0.004, + outputTokenCost: 0.02, + description: 'Claude Opus 5.5 — Anthropic\'s recommended starting model; adaptive thinking always on' + }, + 'claude-sonnet-5-5': { + maxContextLength: 1000000, + supportsStreaming: true, + supportsTools: true, + supportsVision: true, + supportsBatching: false, + supportsPromptCache: true, + inputTokenCost: 0.002, + outputTokenCost: 0.01, + description: 'Claude Sonnet 5.5 — balanced speed and intelligence' + }, + 'claude-haiku-5-5': { + maxContextLength: 1000000, + supportsStreaming: true, + supportsTools: true, + supportsVision: true, + supportsBatching: false, + supportsPromptCache: true, + inputTokenCost: 0.0001, + outputTokenCost: 0.0005, + description: 'Claude Haiku 5.5 — fastest tier; prompts over 100K tokens bill at 5x these rates' + }, 'claude-opus-4-6': { maxContextLength: 1000000, supportsStreaming: true, @@ -286,7 +380,7 @@ export class AnthropicProvider extends BaseProvider { supportsPromptCache: true, inputTokenCost: 0.005, // $5 per 1M tokens outputTokenCost: 0.025, // $25 per 1M tokens - description: 'Claude Opus 4.6 - Latest, highest intelligence and capability' + description: 'Claude Opus 4.6 - legacy; retirement not sooner than 2027-02-05' }, 'claude-sonnet-4-6': { maxContextLength: 1000000, @@ -297,7 +391,7 @@ export class AnthropicProvider extends BaseProvider { supportsPromptCache: true, inputTokenCost: 0.003, // $3 per 1M tokens outputTokenCost: 0.015, // $15 per 1M tokens - description: 'Claude Sonnet 4.6 - Latest balanced performance model' + description: 'Claude Sonnet 4.6 - legacy; retirement not sooner than 2027-02-17' }, 'claude-opus-4-20250514': { maxContextLength: 200000, @@ -325,7 +419,7 @@ export class AnthropicProvider extends BaseProvider { supportsPromptCache: true, inputTokenCost: 0.001, // $1 per 1M tokens outputTokenCost: 0.005, // $5 per 1M tokens - description: 'Claude Haiku 4.5 - Fast and cost-effective' + description: 'Claude Haiku 4.5 - legacy; retirement not sooner than 2026-10-15' }, 'claude-3-7-sonnet-20250219': { maxContextLength: 200000, @@ -439,18 +533,23 @@ export class AnthropicProvider extends BaseProvider { messages.push(anthropicMessage); } - // Add prefilled assistant message for JSON mode - if (jsonMode) { + const model = resolveModelId(request.model || this.getDefaultModel(request)); + + // Add prefilled assistant message for JSON mode on models that accept one; + // later models rely on the system-prompt instruction alone. + if (jsonMode && usesJsonPrefill(model)) { messages.push({ role: 'assistant', content: '{' }); } - const anthropicRequest: AnthropicRequest = { - model: resolveModelId(request.model || this.getDefaultModel(request)), + model, messages, - max_tokens: request.maxTokens || 1000, - temperature: request.temperature, + max_tokens: request.maxTokens || defaultMaxTokens(model), stream: request.stream }; + // Claude 4.7 and later reject a non-default temperature with a 400. + if (!FIXED_SAMPLING_MODELS.has(anthropicRequest.model)) { + anthropicRequest.temperature = request.temperature; + } // Add system prompt if provided const systemMessage = request.messages.find(m => m.role === 'system'); @@ -653,6 +752,8 @@ export class AnthropicProvider extends BaseProvider { const anthropicRequest = { ...this.formatRequest(request), stream: true }; const model = anthropicRequest.model; + // JSON mode prefilled '{' on this model: restore it on the first text, as generateResponse does. + let restoreBrace = request.response_format?.type === 'json_object' && usesJsonPrefill(model); let sawUsage = false; let inputTokens = 0; let outputTokens = 0; @@ -754,6 +855,10 @@ export class AnthropicProvider extends BaseProvider { throw new SchemaDriftError('anthropic', 'content_block_delta.delta.text', 'string', actual); } + if (restoreBrace) { + restoreBrace = false; + if (!text.startsWith('{')) return { text: '{' + text }; + } return { text }; }, usage: currentUsage, @@ -782,13 +887,22 @@ export class AnthropicProvider extends BaseProvider { /** Accept the invalid model IDs earlier releases published, validating the real one. */ protected validateRequest(request: LLMRequest): void { super.validateRequest(request.model ? { ...request, model: resolveModelId(request.model) } : request); + const model = resolveModelId(request.model || this.getDefaultModel(request)); + const forced = request.toolChoice === 'required' || typeof request.toolChoice === 'object'; + if (forced && request.tools?.length && NO_FORCED_TOOL_CHOICE_MODELS.has(model)) { + throw new ConfigurationError( + this.name, + `${model} does not support forced tool choice; use toolChoice 'auto' and say in the prompt when to call the tool` + ); + } } /** * Token cost including prompt caching. `input` excludes cached tokens on the - * Claude API. Multipliers on the base input rate: reads 0.1x, 5-minute - * writes 1.25x, 1-hour writes 2x (platform.claude.com/docs/en/about-claude/pricing, - * checked 2026-10-15; the listed models all use these standard multipliers). + * Claude API. Multipliers on the base input rate: reads 0.1x (lower on some + * models, see CACHE_READ_MULTIPLIER), 5-minute writes 1.25x, 1-hour writes 2x + * (platform.claude.com/docs/en/about-claude/pricing, checked 2026-10-15). + * Haiku 5.5 bills every rate at 5x when the prompt, cache included, exceeds 100K tokens. */ private costWithCache( input: number, output: number, model: string, @@ -798,7 +912,10 @@ export class AnthropicProvider extends BaseProvider { const capabilities = this.getModelCapabilities()[model]; if (!capabilities) return base; const write1h = Math.min(cacheWrite1h, cacheWrite); - const cachedInputUnits = cacheRead * 0.1 + (cacheWrite - write1h) * 1.25 + write1h * 2; - return base + (cachedInputUnits / 1000) * capabilities.inputTokenCost; + const readMultiplier = CACHE_READ_MULTIPLIER[model] ?? 0.1; + const cachedInputUnits = cacheRead * readMultiplier + (cacheWrite - write1h) * 1.25 + write1h * 2; + const cost = base + (cachedInputUnits / 1000) * capabilities.inputTokenCost; + const longPrompt = input + cacheRead + cacheWrite > LONG_PROMPT_THRESHOLD[model]; + return longPrompt ? cost * LONG_PROMPT_MULTIPLIER : cost; } } diff --git a/src/providers/cloudflare.ts b/src/providers/cloudflare.ts index 707fd41..be5d95b 100755 --- a/src/providers/cloudflare.ts +++ b/src/providers/cloudflare.ts @@ -212,6 +212,9 @@ export class CloudflareProvider extends BaseProvider { '@cf/nvidia/nemotron-3-120b-a12b', '@cf/deepseek-ai/deepseek-r1-distill-qwen-32b', '@cf/qwen/qwq-32b', + '@cf/meta/llama-3.1-8b-instruct-fp8', + '@cf/ibm-granite/granite-4.0-h-micro', + '@cf/aisingapore/gemma-sea-lion-v4-27b-it', 'anthropic/claude-opus-4.8', ]; supportsStreaming = true; @@ -395,7 +398,7 @@ export class CloudflareProvider extends BaseProvider { description: 'GPT-OSS 120B - OpenAI-format tool calling on Workers AI' }, '@cf/moonshotai/kimi-k2.6': { - maxContextLength: 262100, + maxContextLength: 262144, supportsStreaming: true, supportsTools: true, toolCalling: true, @@ -448,16 +451,16 @@ export class CloudflareProvider extends BaseProvider { description: 'GLM-5.3 Flash — multimodal 320B/18B-active MoE; 1M context, reasoning, function calling, and vision' }, 'deepseek/deepseek-v4-pro': { - maxContextLength: 128000, + maxContextLength: 131072, supportsStreaming: true, supportsTools: false, supportsBatching: true, - inputTokenCost: 0, - outputTokenCost: 0, - description: 'DeepSeek V4 Pro - high-capability reasoning and coding model on Workers AI' + inputTokenCost: 0.00174, + outputTokenCost: 0.00348, + description: 'DeepSeek V4 Pro via the AI Gateway unified catalog (served by Fireworks; not a Workers AI model)' }, '@cf/deepseek-ai/deepseek-v4-flash-0731': { - maxContextLength: 1310720, + maxContextLength: 1048576, supportsStreaming: true, supportsTools: true, toolCalling: true, @@ -554,7 +557,7 @@ export class CloudflareProvider extends BaseProvider { description: 'Llama 3.2 11B Vision — image understanding' }, '@cf/meta/llama-3.3-70b-instruct-fp8-fast': { - maxContextLength: 131072, + maxContextLength: 24000, supportsStreaming: true, supportsTools: true, toolCalling: true, @@ -576,8 +579,7 @@ export class CloudflareProvider extends BaseProvider { '@cf/qwen/qwen2.5-coder-32b-instruct': { maxContextLength: 32768, supportsStreaming: true, - supportsTools: true, - toolCalling: true, + supportsTools: false, supportsBatching: true, inputTokenCost: 0.00066, outputTokenCost: 0.001, @@ -595,7 +597,7 @@ export class CloudflareProvider extends BaseProvider { description: 'Mistral Small 3.1 24B — vision + tool-calling, strong balanced model' }, '@cf/qwen/qwen3-30b-a3b-fp8': { - maxContextLength: 40960, + maxContextLength: 32768, supportsStreaming: true, supportsTools: true, toolCalling: true, @@ -616,7 +618,7 @@ export class CloudflareProvider extends BaseProvider { description: 'Qwen 3.8 27B — vision-language model for reasoning and multi-turn function calling with 262K context' }, '@cf/meta/llama-3.2-1b-instruct': { - maxContextLength: 131072, + maxContextLength: 60000, supportsStreaming: true, supportsTools: false, supportsBatching: true, @@ -625,7 +627,7 @@ export class CloudflareProvider extends BaseProvider { description: 'Llama 3.2 1B — ultra-cheap tiny model' }, '@cf/meta/llama-3.2-3b-instruct': { - maxContextLength: 131072, + maxContextLength: 80000, supportsStreaming: true, supportsTools: false, supportsBatching: true, @@ -634,7 +636,8 @@ export class CloudflareProvider extends BaseProvider { description: 'Llama 3.2 3B — cheap small model' }, '@cf/moonshotai/kimi-k2.7-code': { - maxContextLength: 131072, + maxContextLength: 262144, + supportsVision: true, supportsStreaming: true, supportsTools: true, toolCalling: true, @@ -673,13 +676,41 @@ export class CloudflareProvider extends BaseProvider { thinkingModel: true, description: 'QwQ-32B — native thinking/reasoning model; outputs chain-of-thought traces; competes with o1-mini on hard reasoning tasks' }, + '@cf/meta/llama-3.1-8b-instruct-fp8': { + maxContextLength: 32000, + supportsStreaming: true, + supportsTools: false, + supportsBatching: false, + inputTokenCost: 0.000152, + outputTokenCost: 0.000287, + description: 'Workers AI Llama 3.1 8B FP8 — the listed Llama 3.1 8B variant after the 2026-05-30 deprecations' + }, + '@cf/ibm-granite/granite-4.0-h-micro': { + maxContextLength: 131000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsBatching: false, + inputTokenCost: 0.000017, + outputTokenCost: 0.000112, + description: 'Workers AI IBM Granite 4.0 H Micro — very low cost small model with function calling' + }, + '@cf/aisingapore/gemma-sea-lion-v4-27b-it': { + maxContextLength: 128000, + supportsStreaming: true, + supportsTools: false, + supportsBatching: false, + inputTokenCost: 0.000351, + outputTokenCost: 0.000555, + description: 'Workers AI SEA-LION v4 27B — Gemma-based model for Southeast Asian languages; explicit requests only' + }, 'anthropic/claude-opus-4.8': { maxContextLength: 1000000, supportsStreaming: false, supportsTools: false, supportsBatching: false, - inputTokenCost: 0, - outputTokenCost: 0, + inputTokenCost: 0.005, + outputTokenCost: 0.025, description: 'Cloudflare-managed Anthropic Claude Opus 4.8 — 1M context frontier model; billing via Cloudflare dashboard' }, }; @@ -969,8 +1000,8 @@ export class CloudflareProvider extends BaseProvider { if (request.systemPrompt) systemParts.push(request.systemPrompt); if (systemParts.length > 0) cfRequest.system = systemParts.join('\n\n'); - if (request.temperature !== undefined) cfRequest.temperature = request.temperature; - if (request.topP !== undefined) cfRequest.top_p = request.topP; + // Claude 4.7 and later (including Opus 4.8) return a 400 for non-default + // temperature or top_p, so sampling parameters are not forwarded. return cfRequest; } diff --git a/src/providers/openai.ts b/src/providers/openai.ts index b8bf674..b0fc004 100755 --- a/src/providers/openai.ts +++ b/src/providers/openai.ts @@ -95,6 +95,7 @@ interface OpenAIRequest { messages: OpenAIMessage[]; temperature?: number; max_tokens?: number; + max_completion_tokens?: number; stream?: boolean; stream_options?: { include_usage?: boolean }; tools?: OpenAITool[]; @@ -134,9 +135,23 @@ interface OpenAIResponse { system_fingerprint?: string; } +/** Models billed at 2x input and 1.5x output for the whole request when the prompt exceeds 272K tokens. */ +const LONG_PROMPT_MODELS = new Set(['gpt-6-astra', 'gpt-5.6-sol', 'gpt-5.6-terra', 'gpt-5.6-luna', 'gpt-5.5']); +const LONG_PROMPT_THRESHOLD = 272_000; + +/** GPT-5 and later, and o-series models, are reasoning models on Chat Completions. */ +export function isOpenAIReasoningModel(model: string): boolean { + return /^(gpt-[5-9]|o\d)/.test(model); +} + export class OpenAIProvider extends BaseProvider { name = 'openai'; models = [ + 'gpt-6-astra', + 'gpt-5.6-sol', + 'gpt-5.6-terra', + 'gpt-5.6-luna', + 'gpt-5.5', 'gpt-4o-mini', 'gpt-4-turbo', 'gpt-4', @@ -237,8 +252,76 @@ export class OpenAIProvider extends BaseProvider { } } + /** + * Responses name a dated snapshot (e.g. gpt-5.5-2026-04-23), so price an + * unknown ID by its alias, and apply the long-prompt rate. + */ + protected calculateCost(inputTokens: number, outputTokens: number, model: string): number { + const capabilities = this.getModelCapabilities(); + const priced = capabilities[model] ? model : model.replace(/-\d{4}-\d{2}-\d{2}$/, ''); + const cost = super.calculateCost(inputTokens, outputTokens, priced); + if (!LONG_PROMPT_MODELS.has(priced) || inputTokens <= LONG_PROMPT_THRESHOLD) return cost; + const { inputTokenCost, outputTokenCost } = capabilities[priced]; + return (inputTokens / 1000) * inputTokenCost * 2 + (outputTokens / 1000) * outputTokenCost * 1.5; + } + protected getModelCapabilities(): Record { return { + 'gpt-6-astra': { + maxContextLength: 1050000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsVision: true, + supportsBatching: false, + inputTokenCost: 0.01, + outputTokenCost: 0.05, + description: 'GPT-6 Astra — OpenAI\'s flagship reasoning model; prompts over 272K tokens bill 2x input and 1.5x output' + }, + 'gpt-5.6-sol': { + maxContextLength: 1050000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsVision: true, + supportsBatching: false, + inputTokenCost: 0.004, + outputTokenCost: 0.02, + description: 'GPT-5.6 Sol — reasoning model; promotional price stated through at least 2026-11-21; prompts over 272K tokens bill 2x input and 1.5x output' + }, + 'gpt-5.6-terra': { + maxContextLength: 1050000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsVision: true, + supportsBatching: false, + inputTokenCost: 0.002, + outputTokenCost: 0.012, + description: 'GPT-5.6 Terra — mid-tier reasoning model; prompts over 272K tokens bill 2x input and 1.5x output' + }, + 'gpt-5.6-luna': { + maxContextLength: 1050000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsVision: true, + supportsBatching: false, + inputTokenCost: 0.0002, + outputTokenCost: 0.0012, + description: 'GPT-5.6 Luna — low-cost reasoning model; prompts over 272K tokens bill 2x input and 1.5x output' + }, + 'gpt-5.5': { + maxContextLength: 1050000, + supportsStreaming: true, + supportsTools: true, + toolCalling: true, + supportsVision: true, + supportsBatching: false, + inputTokenCost: 0.005, + outputTokenCost: 0.03, + description: 'GPT-5.5 (alias of gpt-5.5-2026-04-23) — reasoning model; prompts over 272K tokens bill 2x input and 1.5x output' + }, 'gpt-4o-mini': { maxContextLength: 128000, supportsStreaming: true, @@ -247,7 +330,7 @@ export class OpenAIProvider extends BaseProvider { supportsBatching: false, inputTokenCost: 0.00015, // $0.15 per 1M tokens outputTokenCost: 0.0006, // $0.60 per 1M tokens - description: 'GPT-4 Omni Mini - Cost-effective version' + description: 'GPT-4o mini - deprecated by OpenAI; shutdown 2027-01-14' }, 'gpt-4-turbo': { maxContextLength: 128000, @@ -256,7 +339,7 @@ export class OpenAIProvider extends BaseProvider { supportsBatching: false, inputTokenCost: 0.01, // $10 per 1M tokens outputTokenCost: 0.03, // $30 per 1M tokens - description: 'GPT-4 Turbo - High performance model' + description: 'GPT-4 Turbo - OpenAI shuts it down 2026-10-23' }, 'gpt-4': { maxContextLength: 8192, @@ -265,7 +348,7 @@ export class OpenAIProvider extends BaseProvider { supportsBatching: false, inputTokenCost: 0.03, // $30 per 1M tokens outputTokenCost: 0.06, // $60 per 1M tokens - description: 'GPT-4 - Original high-capability model' + description: 'GPT-4 - OpenAI shuts it down 2026-10-23' }, 'gpt-3.5-turbo': { maxContextLength: 16385, @@ -274,7 +357,7 @@ export class OpenAIProvider extends BaseProvider { supportsBatching: false, inputTokenCost: 0.0005, // $0.50 per 1M tokens outputTokenCost: 0.0015, // $1.50 per 1M tokens - description: 'GPT-3.5 Turbo - Fast and cost-effective' + description: 'GPT-3.5 Turbo - OpenAI shuts it down 2026-10-23' }, 'gpt-3.5-turbo-16k': { maxContextLength: 16385, @@ -283,7 +366,7 @@ export class OpenAIProvider extends BaseProvider { supportsBatching: false, inputTokenCost: 0.001, // $1 per 1M tokens outputTokenCost: 0.002, // $2 per 1M tokens - description: 'GPT-3.5 Turbo 16k - Extended context' + description: 'GPT-3.5 Turbo 16k - no longer a valid OpenAI model ID' } }; } @@ -371,14 +454,21 @@ export class OpenAIProvider extends BaseProvider { messages.push(openaiMessage); } + const model = request.model || this.getDefaultModel(request); const openaiRequest: OpenAIRequest = { - model: request.model || this.getDefaultModel(request), + model, messages, - temperature: request.temperature, - max_tokens: request.maxTokens, stream: request.stream, seed: request.seed }; + if (isOpenAIReasoningModel(model)) { + // Reasoning models take max_completion_tokens (which also covers reasoning + // tokens) and do not accept a non-default temperature. + openaiRequest.max_completion_tokens = request.maxTokens; + } else { + openaiRequest.temperature = request.temperature; + openaiRequest.max_tokens = request.maxTokens; + } // Add tools if provided if (request.tools && request.tools.length > 0) { diff --git a/src/version.ts b/src/version.ts index 72a9904..bc6231d 100644 --- a/src/version.ts +++ b/src/version.ts @@ -1,2 +1,2 @@ // Generated by scripts/sync-version.mjs; do not edit manually. -export const VERSION = '1.22.1'; +export const VERSION = '1.23.0';