From ce969ce566708b0bea252ca6787f687f579987e4 Mon Sep 17 00:00:00 2001 From: Kurt Overmier Date: Thu, 15 Oct 2026 20:11:01 -0500 Subject: [PATCH 1/3] chore: normalize factory.test.ts line endings to LF The committed blob had CRLF line endings, but .gitattributes requires LF for *.ts, so every checkout showed the file as modified. This stores the same content with LF endings; `git diff --ignore-cr-at-eol` shows no changes, and the file mode (100755) is unchanged. Co-Authored-By: Claude Opus 5.5 --- src/__tests__/factory.test.ts | 2620 ++++++++++++++++----------------- 1 file changed, 1310 insertions(+), 1310 deletions(-) diff --git a/src/__tests__/factory.test.ts b/src/__tests__/factory.test.ts index 62a9c25..ce2a333 100755 --- a/src/__tests__/factory.test.ts +++ b/src/__tests__/factory.test.ts @@ -1,1310 +1,1310 @@ -/** - * LLM Provider Factory Tests - * Tests for the provider factory with mocked providers - */ - -import { describe, it, expect, beforeEach, vi } from 'vitest'; -import { LLMProviderFactory, createCostOptimizedFactory } from '../factory'; -import { AuthenticationError, ServerError } from '../errors'; -import type { LLMRequest, LLMResponse } from '../types'; -import { OpenAIProvider } from '../providers/openai'; -import { CreditLedger } from '../utils/credit-ledger'; -import { defaultCostTracker } from '../utils/cost-tracker'; -import { defaultCircuitBreakerManager } from '../utils/circuit-breaker'; -import { defaultExhaustionRegistry } from '../utils/exhaustion'; -import { defaultLatencyHistogram } from '../utils/latency-histogram'; -import { attachStreamUsage } from '../utils/stream-usage'; - -// Mock providers -const mockOpenAIProvider = { - name: 'openai', - models: ['gpt-4', 'gpt-3.5-turbo'], - supportsStreaming: true, - supportsTools: true, - supportsBatching: true, - supportsVision: true, - generateResponse: vi.fn().mockResolvedValue({ - message: 'OpenAI response', - usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.001 }, - model: 'gpt-3.5-turbo', - provider: 'openai', - responseTime: 1000 - } as LLMResponse), - streamResponse: vi.fn(), - getProviderBalance: vi.fn(), - validateConfig: vi.fn().mockReturnValue(true), - getModels: vi.fn().mockReturnValue(['gpt-4', 'gpt-3.5-turbo']), - estimateCost: vi.fn().mockReturnValue(0.001), - healthCheck: vi.fn().mockResolvedValue(true), - getMetrics: vi.fn().mockReturnValue({ - requestCount: 1, - successCount: 1, - errorCount: 0, - averageLatency: 1000, - totalCost: 0.001, - rateLimitHits: 0, - lastUsed: Date.now() - }), - resetMetrics: vi.fn() -}; - -const mockAnthropicProvider = { - name: 'anthropic', - models: ['claude-haiku-4-5-20251001', 'claude-3-sonnet-20240229'], - supportsStreaming: true, - supportsTools: true, - supportsBatching: false, - supportsVision: true, - generateResponse: vi.fn().mockResolvedValue({ - message: 'Anthropic response', - usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.002 }, - model: 'claude-haiku-4-5-20251001', - provider: 'anthropic', - responseTime: 1200 - } as LLMResponse), - streamResponse: vi.fn(), - getProviderBalance: vi.fn(), - validateConfig: vi.fn().mockReturnValue(true), - getModels: vi.fn().mockReturnValue(['claude-haiku-4-5-20251001', 'claude-3-sonnet-20240229']), - estimateCost: vi.fn().mockReturnValue(0.002), - healthCheck: vi.fn().mockResolvedValue(true), - getMetrics: vi.fn().mockReturnValue({ - requestCount: 1, - successCount: 1, - errorCount: 0, - averageLatency: 1200, - totalCost: 0.002, - rateLimitHits: 0, - lastUsed: Date.now() - }), - resetMetrics: vi.fn() -}; - -const mockCloudflareProvider = { - name: 'cloudflare', - models: ['@cf/meta/llama-3.1-8b-instruct'], - supportsStreaming: true, - supportsTools: false, - supportsBatching: true, - supportsVision: false, - generateResponse: vi.fn().mockResolvedValue({ - message: 'Cloudflare response', - usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.0001 }, - model: '@cf/meta/llama-3.1-8b-instruct', - provider: 'cloudflare', - responseTime: 800 - } as LLMResponse), - streamResponse: vi.fn(), - getProviderBalance: vi.fn(), - validateConfig: vi.fn().mockReturnValue(true), - getModels: vi.fn().mockReturnValue(['@cf/meta/llama-3.1-8b-instruct']), - estimateCost: vi.fn().mockReturnValue(0.0001), - healthCheck: vi.fn().mockResolvedValue(true), - getMetrics: vi.fn().mockReturnValue({ - requestCount: 1, - successCount: 1, - errorCount: 0, - averageLatency: 800, - totalCost: 0.0001, - rateLimitHits: 0, - lastUsed: Date.now() - }), - resetMetrics: vi.fn() -}; - -// Mock the provider classes -vi.mock('../providers/openai', () => ({ - OpenAIProvider: vi.fn().mockImplementation(() => mockOpenAIProvider) -})); - -vi.mock('../providers/anthropic', () => ({ - AnthropicProvider: vi.fn().mockImplementation(() => mockAnthropicProvider) -})); - -vi.mock('../providers/cloudflare', () => ({ - CloudflareProvider: vi.fn().mockImplementation(() => mockCloudflareProvider) -})); - -describe('LLMProviderFactory', () => { - let factory: LLMProviderFactory; - - const testRequest: LLMRequest = { - messages: [{ role: 'user', content: 'Hello, world!' }], - maxTokens: 100, - temperature: 0.7 - }; - - beforeEach(() => { - vi.clearAllMocks(); - defaultCostTracker.reset(); - defaultCircuitBreakerManager.resetAll(); - defaultExhaustionRegistry.reset(); - defaultLatencyHistogram.reset(); - - mockOpenAIProvider.generateResponse.mockReset().mockResolvedValue({ - message: 'OpenAI response', - usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.001 }, - model: 'gpt-3.5-turbo', - provider: 'openai', - responseTime: 1000 - } as LLMResponse); - mockOpenAIProvider.streamResponse.mockReset().mockResolvedValue(new ReadableStream({ - start(controller) { - controller.enqueue('OpenAI stream'); - controller.close(); - } - })); - mockOpenAIProvider.getProviderBalance.mockReset().mockResolvedValue({ - provider: 'openai', - status: 'available', - source: 'provider_api' - }); - mockAnthropicProvider.generateResponse.mockReset().mockResolvedValue({ - message: 'Anthropic response', - usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.002 }, - model: 'claude-haiku-4-5-20251001', - provider: 'anthropic', - responseTime: 1200 - } as LLMResponse); - mockAnthropicProvider.streamResponse.mockReset().mockResolvedValue(new ReadableStream({ - start(controller) { - controller.enqueue('Anthropic stream'); - controller.close(); - } - })); - mockAnthropicProvider.getProviderBalance.mockReset().mockResolvedValue({ - provider: 'anthropic', - status: 'available', - source: 'provider_api' - }); - mockCloudflareProvider.generateResponse.mockReset().mockResolvedValue({ - message: 'Cloudflare response', - usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.0001 }, - model: '@cf/meta/llama-3.1-8b-instruct', - provider: 'cloudflare', - responseTime: 800 - } as LLMResponse); - mockCloudflareProvider.streamResponse.mockReset().mockResolvedValue(new ReadableStream({ - start(controller) { - controller.enqueue('Cloudflare stream'); - controller.close(); - } - })); - mockCloudflareProvider.getProviderBalance.mockReset().mockResolvedValue({ - provider: 'cloudflare', - status: 'unavailable', - source: 'not_supported' - }); - mockCloudflareProvider.supportsVision = false; - - factory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - anthropic: { apiKey: 'test-anthropic-key' }, - cloudflare: { ai: {} as Ai }, - defaultProvider: 'auto', - costOptimization: true, - enableCircuitBreaker: true, - }); - }); - - describe('Provider Initialization', () => { - it('should initialize all configured providers', () => { - const availableProviders = factory.getAvailableProviders(); - expect(availableProviders).toContain('openai'); - expect(availableProviders).toContain('anthropic'); - expect(availableProviders).toContain('cloudflare'); - }); - - it('should handle missing provider configurations gracefully', () => { - const factoryWithMissing = new LLMProviderFactory({ - openai: { apiKey: 'test-key' } - // Missing anthropic and cloudflare configs - }); - - const availableProviders = factoryWithMissing.getAvailableProviders(); - expect(availableProviders).toContain('openai'); - expect(availableProviders).not.toContain('anthropic'); - expect(availableProviders).not.toContain('cloudflare'); - }); - }); - - describe('Response Generation', () => { - it('should generate response using available provider', async () => { - const response = await factory.generateResponse(testRequest); - - expect(response).toBeDefined(); - expect(response.message).toBeTruthy(); - expect(response.provider).toBeOneOf(['openai', 'anthropic', 'cloudflare']); - expect(response.usage).toBeDefined(); - expect(response.responseTime).toBeGreaterThan(0); - }); - - it('should use specific provider when model is specified', async () => { - const requestWithModel: LLMRequest = { - ...testRequest, - model: 'gpt-3.5-turbo' - }; - - const response = await factory.generateResponse(requestWithModel); - expect(mockOpenAIProvider.generateResponse).toHaveBeenCalled(); - }); - - it('should fallback to other providers on failure', async () => { - // Make OpenAI provider fail - mockOpenAIProvider.generateResponse.mockRejectedValueOnce( - new Error('OpenAI temporarily unavailable') - ); - - const response = await factory.generateResponse(testRequest); - - // Should fallback to another provider - expect(response).toBeDefined(); - expect(response.provider).not.toBe('openai'); - }); - - it('should skip fully open providers and continue down the chain', async () => { - defaultCircuitBreakerManager.getBreaker('cloudflare').forceOpen(); - - const response = await factory.generateResponse(testRequest); - - expect(mockCloudflareProvider.generateResponse).not.toHaveBeenCalled(); - expect(response.provider).not.toBe('cloudflare'); - }); - }); - - describe('Cost Optimization', () => { - it('should prefer cheaper providers when cost optimization is enabled', async () => { - const response = await factory.generateResponse(testRequest); - - // Cloudflare should be preferred for cost optimization - expect(response.provider).toBe('cloudflare'); - }); - - it('should provide cost analytics', async () => { - await factory.generateResponse(testRequest); - - const analytics = factory.getCostAnalytics(); - - expect(analytics).toBeDefined(); - expect(analytics.breakdown).toBeDefined(); - expect(analytics.total).toBeCloseTo(0.0001); - expect(analytics.breakdown.cloudflare).toMatchObject({ - cost: 0.0001, - totalCost: 0.0001, - requests: 1, - requestCount: 1, - inputTokens: 10, - outputTokens: 20, - tokens: { input: 10, output: 20 } - }); - expect(analytics.breakdown.cloudflare.lastRecordedAt).toBeGreaterThan(0); - }); - }); - - describe('Health Monitoring', () => { - it('should check provider health status', async () => { - const health = await factory.getProviderHealth(); - - expect(health).toBeDefined(); - expect(health.openai).toBeDefined(); - expect(health.anthropic).toBeDefined(); - expect(health.cloudflare).toBeDefined(); - - expect(health.openai.healthy).toBe(true); - expect(health.openai.models).toEqual(['gpt-4', 'gpt-3.5-turbo']); - expect(health.openai.capabilities).toBeDefined(); - }); - - it('should handle provider health check failures', async () => { - mockOpenAIProvider.healthCheck.mockResolvedValueOnce(false); - - const health = await factory.getProviderHealth(); - expect(health.openai.healthy).toBe(false); - }); - }); - - describe('Provider Chain Building', () => { - it('should build appropriate provider chain for model-specific requests', async () => { - // Test Claude model request - const claudeRequest: LLMRequest = { - ...testRequest, - model: 'claude-haiku-4-5-20251001' - }; - - await factory.generateResponse(claudeRequest); - expect(mockAnthropicProvider.generateResponse).toHaveBeenCalled(); - }); - - it('should build cost-optimized chain for auto mode', async () => { - const autoRequest: LLMRequest = { - ...testRequest - // No specific model, should use auto selection - }; - - await factory.generateResponse(autoRequest); - - // Should prefer Cloudflare for cost optimization - expect(mockCloudflareProvider.generateResponse).toHaveBeenCalled(); - }); - - it('should demote degraded providers with critical burn rate when recommending a model', async () => { - const ledger = new CreditLedger({ - budgets: [ - { provider: 'cloudflare', monthlyBudget: 1 }, - { provider: 'openai', monthlyBudget: 10 } - ] - }); - ledger.record('cloudflare', '@cf/google/gemma-4-26b-a4b-it', 0.95, 10_000, 4_000); - - const dynamicFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - anthropic: { apiKey: 'test-anthropic-key' }, - cloudflare: { ai: {} as Ai }, - defaultProvider: 'auto', - costOptimization: true, - enableCircuitBreaker: true, - ledger - }); - - const breaker = defaultCircuitBreakerManager.getBreaker('cloudflare'); - await expect(breaker.execute(async () => { - throw new Error('degraded'); - })).rejects.toThrow('degraded'); - - expect(dynamicFactory.getRecommendedModel(testRequest)).toBe('gpt-4o-mini'); - }); - - it('should honor fallbackProvider as the next route when a rule matches', async () => { - const ruleFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - anthropic: { apiKey: 'test-anthropic-key' }, - cloudflare: { ai: {} as Ai }, - defaultProvider: 'openai', - costOptimization: false, - fallbackRules: [{ condition: 'error', fallbackProvider: 'anthropic' }] - }); - - mockOpenAIProvider.generateResponse.mockRejectedValueOnce(new Error('OpenAI down')); - - const response = await ruleFactory.generateResponse(testRequest); - - expect(response.provider).toBe('anthropic'); - expect(mockAnthropicProvider.generateResponse).toHaveBeenCalled(); - expect(mockCloudflareProvider.generateResponse).not.toHaveBeenCalled(); - }); - - it('should apply fallbackModel when routing through a fallback rule', async () => { - const ruleFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - anthropic: { apiKey: 'test-anthropic-key' }, - defaultProvider: 'openai', - costOptimization: false, - fallbackRules: [{ - condition: 'error', - fallbackProvider: 'anthropic', - fallbackModel: 'claude-haiku-4-5-20251001' - }] - }); - - mockOpenAIProvider.generateResponse.mockRejectedValueOnce(new Error('OpenAI down')); - - await ruleFactory.generateResponse({ - ...testRequest, - model: 'gpt-4' - }); - - expect(mockAnthropicProvider.generateResponse).toHaveBeenCalledWith( - expect.objectContaining({ model: 'claude-haiku-4-5-20251001' }) - ); - }); - }); - - describe('Streaming, tools, classification, and vision', () => { - async function readStream(stream: ReadableStream): Promise { - const reader = stream.getReader(); - let output = ''; - while (true) { - const { done, value } = await reader.read(); - if (done) break; - output += value; - } - return output; - } - - it('should stream through the factory and fallback before the first chunk', async () => { - const streamFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - anthropic: { apiKey: 'test-anthropic-key' }, - defaultProvider: 'openai', - costOptimization: false, - fallbackRules: [{ condition: 'error', fallbackProvider: 'anthropic' }] - }); - - mockOpenAIProvider.streamResponse.mockRejectedValueOnce(new Error('stream start failed')); - - const stream = await streamFactory.generateResponseStream(testRequest); - - expect(await readStream(stream)).toBe('Anthropic stream'); - expect(mockOpenAIProvider.streamResponse).toHaveBeenCalled(); - expect(mockAnthropicProvider.streamResponse).toHaveBeenCalled(); - }); - - it('records actual stream usage after the stream drains', async () => { - const quotaHook = { - check: vi.fn().mockResolvedValue({ allowed: true, remainingBudget: 1 }), - record: vi.fn().mockResolvedValue(undefined) - }; - const streamFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - defaultProvider: 'openai', - costOptimization: true, - quotaHook - }); - const usage: LLMResponse['usage'] = { - inputTokens: 12, - outputTokens: 34, - totalTokens: 46, - cost: 0.123 - }; - const providerStream = attachStreamUsage(new ReadableStream({ - start(controller) { - controller.enqueue('OpenAI stream'); - controller.close(); - } - }), Promise.resolve(usage)); - - mockOpenAIProvider.streamResponse.mockResolvedValueOnce(providerStream); - - const stream = await streamFactory.generateResponseStream({ - ...testRequest, - tenantId: 'tenant-stream' - }); - - expect(await readStream(stream)).toBe('OpenAI stream'); - expect(quotaHook.record).toHaveBeenCalledWith(expect.objectContaining({ - tenantId: 'tenant-stream', - provider: 'openai', - actualCost: 0.123, - inputTokens: 12, - outputTokens: 34 - })); - expect(streamFactory.getCostAnalytics().breakdown?.openai).toMatchObject({ - cost: 0.123, - tokens: { input: 12, output: 34 } - }); - }); - - it('should call quota hooks before and after successful dispatch', async () => { - const quotaHook = { - check: vi.fn().mockResolvedValue({ allowed: true, remainingBudget: 1 }), - record: vi.fn().mockResolvedValue(undefined) - }; - const quotaFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - defaultProvider: 'openai', - costOptimization: false, - quotaHook - }); - - await quotaFactory.generateResponse({ ...testRequest, tenantId: 'tenant-1' }); - - expect(quotaHook.check).toHaveBeenCalledWith(expect.objectContaining({ - tenantId: 'tenant-1', - provider: 'openai', - model: 'gpt-4o-mini' - })); - expect(quotaHook.record).toHaveBeenCalledWith(expect.objectContaining({ - tenantId: 'tenant-1', - provider: 'openai', - actualCost: 0.001, - inputTokens: 10, - outputTokens: 20 - })); - }); - - it('should deny dispatch when quota hook rejects the request', async () => { - const quotaFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - defaultProvider: 'openai', - costOptimization: false, - quotaHook: { - check: vi.fn().mockResolvedValue({ allowed: false, reason: 'budget exhausted' }), - record: vi.fn() - } - }); - - await expect(quotaFactory.generateResponse(testRequest)).rejects.toThrow('budget exhausted'); - expect(mockOpenAIProvider.generateResponse).not.toHaveBeenCalled(); - }); - - it('should execute tool loops until the final response has no tool calls', async () => { - const toolResponse: LLMResponse = { - message: '', - usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001 }, - model: 'gpt-3.5-turbo', - provider: 'openai', - responseTime: 10, - finishReason: 'tool_calls', - toolCalls: [{ - id: 'call-1', - type: 'function', - function: { name: 'lookup', arguments: '{"id":42}' } - }] - }; - const finalResponse: LLMResponse = { - message: 'done', - usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001 }, - model: 'gpt-3.5-turbo', - provider: 'openai', - responseTime: 10 - }; - mockOpenAIProvider.generateResponse - .mockResolvedValueOnce(toolResponse) - .mockResolvedValueOnce(finalResponse); - - const loopFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - defaultProvider: 'openai', - costOptimization: false - }); - const executor = { execute: vi.fn().mockResolvedValue({ value: 42 }) }; - - const response = await loopFactory.generateResponseWithTools(testRequest, executor); - - expect(response.message).toBe('done'); - expect(executor.execute).toHaveBeenCalledWith('lookup', { id: 42 }); - expect(mockOpenAIProvider.generateResponse).toHaveBeenLastCalledWith( - expect.objectContaining({ - messages: expect.arrayContaining([ - expect.objectContaining({ - toolResults: [{ id: 'call-1', output: '{"value":42}' }] - }) - ]) - }) - ); - }); - - // ── generateResponseWithTools edge cases (#28) ─────────────────────── - - function makeLoopFactory() { - return new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - defaultProvider: 'openai', - costOptimization: false - }); - } - - const toolCallResponse = (id = 'call-1'): LLMResponse => ({ - message: '', - usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.5 }, - model: 'gpt-4', - provider: 'openai', - responseTime: 10, - finishReason: 'tool_calls', - toolCalls: [{ id, type: 'function', function: { name: 'noop', arguments: '{}' } }] - }); - - const finalResponse: LLMResponse = { - message: 'done', - usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001 }, - model: 'gpt-4', - provider: 'openai', - responseTime: 10 - }; - - it('throws ToolLoopLimitError when maxIterations is exceeded', async () => { - const { ToolLoopLimitError } = await import('../errors'); - // Keep returning tool calls — never resolves naturally - mockOpenAIProvider.generateResponse.mockResolvedValue(toolCallResponse()); - - await expect( - makeLoopFactory().generateResponseWithTools( - testRequest, - { execute: vi.fn().mockResolvedValue('ok') }, - { maxIterations: 2 } - ) - ).rejects.toThrow(ToolLoopLimitError); - }); - - it('throws ToolLoopLimitError when cumulative cost exceeds maxCostUSD', async () => { - const { ToolLoopLimitError } = await import('../errors'); - // Each tool-call response costs $0.50; cap at $0.60 → second iteration should abort - mockOpenAIProvider.generateResponse - .mockResolvedValueOnce(toolCallResponse('call-1')) - .mockResolvedValueOnce(toolCallResponse('call-2')) - .mockResolvedValue(finalResponse); - - await expect( - makeLoopFactory().generateResponseWithTools( - testRequest, - { execute: vi.fn().mockResolvedValue('ok') }, - { maxCostUSD: 0.60 } - ) - ).rejects.toThrow(ToolLoopLimitError); - }); - - it('aborts immediately when abortSignal is already aborted', async () => { - const { ToolLoopAbortedError } = await import('../errors'); - const controller = new AbortController(); - controller.abort(); - - await expect( - makeLoopFactory().generateResponseWithTools( - testRequest, - { execute: vi.fn() }, - { abortSignal: controller.signal } - ) - ).rejects.toThrow(ToolLoopAbortedError); - - expect(mockOpenAIProvider.generateResponse).not.toHaveBeenCalled(); - }); - - it('records tool execution failure as a tool-result error without crashing the loop', async () => { - mockOpenAIProvider.generateResponse - .mockResolvedValueOnce(toolCallResponse()) - .mockResolvedValueOnce(finalResponse); - - const failingExecutor = { execute: vi.fn().mockRejectedValue(new Error('external service down')) }; - - const response = await makeLoopFactory().generateResponseWithTools(testRequest, failingExecutor); - - expect(response.message).toBe('done'); - // The second generateResponse call should have received the tool error as a result - const secondCallMessages = (mockOpenAIProvider.generateResponse.mock.calls[1][0] as LLMRequest).messages; - const toolResultMsg = secondCallMessages.find(m => m.toolResults && m.toolResults.length > 0); - expect(toolResultMsg?.toolResults?.[0]).toMatchObject({ - id: 'call-1', - error: 'external service down' - }); - }); - - it('feeds hallucinated tool names back as errors without executing them', async () => { - mockOpenAIProvider.generateResponse - .mockResolvedValueOnce({ - ...toolCallResponse(), - toolCalls: [{ - id: 'call-hallucinated', - type: 'function', - function: { name: 'apply_patch', arguments: '{}' } - }] - }) - .mockResolvedValueOnce(finalResponse); - const executor = { execute: vi.fn() }; - - await makeLoopFactory().generateResponseWithTools({ - ...testRequest, - tools: [{ - type: 'function', - function: { - name: 'noop', - description: 'A permitted no-op', - parameters: { type: 'object', properties: {} } - } - }] - }, executor); - - expect(executor.execute).not.toHaveBeenCalled(); - const secondCall = mockOpenAIProvider.generateResponse.mock.calls[1][0] as LLMRequest; - expect(secondCall.messages.at(-1)?.toolResults?.[0]).toMatchObject({ - id: 'call-hallucinated', - error: expect.stringContaining("unavailable tool 'apply_patch'") - }); - }); - - it('invokes onIteration callback with correct state on each iteration', async () => { - mockOpenAIProvider.generateResponse - .mockResolvedValueOnce(toolCallResponse()) - .mockResolvedValueOnce(finalResponse); - - const iterations: number[] = []; - await makeLoopFactory().generateResponseWithTools( - testRequest, - { execute: vi.fn().mockResolvedValue('result') }, - { onIteration: (iter) => { iterations.push(iter); } } - ); - - expect(iterations).toEqual([1]); - }); - - it('should classify JSON responses and expose confidence', async () => { - mockOpenAIProvider.generateResponse.mockResolvedValueOnce({ - message: '{"label":"recipe","confidence":0.92}', - usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001 }, - model: 'gpt-3.5-turbo', - provider: 'openai', - responseTime: 10 - } as LLMResponse); - - const classifyFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - defaultProvider: 'openai', - costOptimization: false - }); - - const result = await classifyFactory.classify<{ label: string; confidence: number }>('classify this'); - - expect(result.data.label).toBe('recipe'); - expect(result.confidence).toBe(0.92); - }); - - it('should route image analysis to a vision-capable provider', async () => { - const visionFactory = new LLMProviderFactory({ - anthropic: { apiKey: 'test-anthropic-key' }, - cloudflare: { ai: {} as Ai }, - costOptimization: false - }); - - await visionFactory.analyzeImage({ - image: { data: 'abc123', mimeType: 'image/jpeg' }, - prompt: 'Extract recipe text' - }); - - expect(mockAnthropicProvider.generateResponse).toHaveBeenCalledWith( - expect.objectContaining({ - images: [{ data: 'abc123', mimeType: 'image/jpeg' }], - // claude-sonnet-4-6 (quality:6) wins vision routing over haiku (quality:4) after score recalibration - model: 'claude-sonnet-4-6' - }), - expect.objectContaining({ onAttempt: expect.any(Function) }) - ); - expect(mockCloudflareProvider.generateResponse).not.toHaveBeenCalled(); - }); - - it('emits sanitized attempt and aggregate observability for image analysis', async () => { - const onImageAnalysisAttempt = vi.fn(); - const onImageAnalysisComplete = vi.fn(); - mockOpenAIProvider.generateResponse.mockImplementationOnce(async (_request, options) => { - const response = { - message: 'Cookbook page', - usage: { inputTokens: 12, outputTokens: 8, totalTokens: 20, cost: 0.003 }, - model: 'gpt-4o', - provider: 'openai', - responseTime: 4 - } as LLMResponse; - options?.onAttempt?.({ - attempt: 1, - outcome: 'success', - durationMs: 4, - willRetry: false, - response, - }); - return response; - }); - const visionFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - defaultProvider: 'openai', - costOptimization: false, - hooks: { onImageAnalysisAttempt, onImageAnalysisComplete }, - }); - - const response = await visionFactory.analyzeImage({ - image: { data: 'AQIDBA==', mimeType: 'image/jpeg' }, - prompt: 'private cookbook prompt', - model: 'gpt-4o', - requestId: 'capture-1', - metadata: { captureId: 'cap-1', pageNumber: 7, secret: 'drop-me', imageData: 'drop-me-too' }, - }); - - expect(response.usage).toMatchObject({ - cost: 0.003, - costProvenance: 'catalog_estimate', - tokenProvenance: 'provider_reported', - }); - expect(onImageAnalysisAttempt).toHaveBeenCalledWith(expect.objectContaining({ - operation: 'analyze_image', - requestId: 'capture-1', - metadata: { captureId: 'cap-1', pageNumber: 7 }, - provider: 'openai', - model: 'gpt-4o', - attempt: 1, - providerAttempt: 1, - outcome: 'success', - inputTokens: 12, - outputTokens: 8, - totalTokens: 20, - tokenProvenance: 'provider_reported', - imageCount: 1, - decodedImageBytes: 4, - costUsd: 0.003, - costProvenance: 'catalog_estimate', - })); - expect(onImageAnalysisComplete).toHaveBeenCalledWith(expect.objectContaining({ - outcome: 'success', - attempts: 1, - totalLatencyMs: 4, - catalogEstimatedCostUsd: 0.003, - unknownCostAttempts: 0, - finalProvider: 'openai', - finalModel: 'gpt-4o', - })); - expect(JSON.stringify(onImageAnalysisAttempt.mock.calls)).not.toContain('private cookbook prompt'); - expect(JSON.stringify(onImageAnalysisAttempt.mock.calls)).not.toContain('drop-me'); - }); - - it('accounts for every retry and fallback in image aggregates', async () => { - const attempts: unknown[] = []; - const completed: unknown[] = []; - const firstError = new ServerError('openai', 'raw payload must stay private'); - mockOpenAIProvider.generateResponse.mockImplementationOnce(async (_request, options) => { - options?.onAttempt?.({ attempt: 1, outcome: 'error', durationMs: 3, willRetry: true, error: firstError }); - options?.onAttempt?.({ attempt: 2, outcome: 'error', durationMs: 5, willRetry: false, error: firstError }); - throw firstError; - }); - mockAnthropicProvider.generateResponse.mockImplementationOnce(async (_request, options) => { - const response = { - message: 'fallback answer', - usage: { inputTokens: 10, outputTokens: 6, totalTokens: 16, cost: 0.002 }, - model: 'claude-sonnet-4-6', - provider: 'anthropic', - responseTime: 7, - } as LLMResponse; - options?.onAttempt?.({ attempt: 1, outcome: 'success', durationMs: 7, willRetry: false, response }); - return response; - }); - const visionFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - anthropic: { apiKey: 'test-anthropic-key' }, - defaultProvider: 'openai', - fallbackRules: [{ condition: 'error', fallbackProvider: 'anthropic' }], - costOptimization: false, - hooks: { - onImageAnalysisAttempt: event => attempts.push(event), - onImageAnalysisComplete: event => completed.push(event), - }, - }); - - await visionFactory.analyzeImage({ - image: { data: 'AQID', mimeType: 'image/jpeg' }, - prompt: 'do not emit this prompt', - model: 'gpt-4o', - }); - - expect(attempts).toHaveLength(3); - expect(attempts).toEqual([ - expect.objectContaining({ attempt: 1, provider: 'openai', providerAttempt: 1, retry: false, willRetry: true, fallback: false }), - expect.objectContaining({ attempt: 2, provider: 'openai', providerAttempt: 2, retry: true, willRetry: false, fallback: false }), - expect.objectContaining({ attempt: 3, provider: 'anthropic', providerAttempt: 1, fallback: true, fallbackFromProvider: 'openai' }), - ]); - expect(completed).toEqual([expect.objectContaining({ - outcome: 'success', - attempts: 3, - totalLatencyMs: 15, - catalogEstimatedCostUsd: 0.002, - unknownCostAttempts: 2, - finalProvider: 'anthropic', - })]); - expect(JSON.stringify({ attempts, completed })).not.toContain('raw payload must stay private'); - expect(JSON.stringify({ attempts, completed })).not.toContain('do not emit this prompt'); - }); - - it('keeps Cloudflare image cost explicitly unknown while recording decoded bytes', async () => { - const onImageAnalysisAttempt = vi.fn(); - mockCloudflareProvider.supportsVision = true; - mockCloudflareProvider.generateResponse.mockImplementationOnce(async (_request, options) => { - const response = { - message: 'page', - usage: { - inputTokens: 9, - outputTokens: 3, - totalTokens: 12, - cost: 0.0001, - costProvenance: 'catalog_estimate', - tokenProvenance: 'estimated', - }, - model: '@cf/meta/llama-3.2-11b-vision-instruct', - provider: 'cloudflare', - responseTime: 2, - } as LLMResponse; - options?.onAttempt?.({ attempt: 1, outcome: 'success', durationMs: 2, willRetry: false, response }); - return response; - }); - const visionFactory = new LLMProviderFactory({ - cloudflare: { ai: {} as Ai }, - defaultProvider: 'cloudflare', - costOptimization: false, - hooks: { onImageAnalysisAttempt }, - }); - - try { - await visionFactory.analyzeImage({ - image: { data: 'AQIDBAU=', mimeType: 'image/jpeg' }, - prompt: 'extract', - model: '@cf/meta/llama-3.2-11b-vision-instruct', - }); - } finally { - mockCloudflareProvider.supportsVision = false; - } - - expect(onImageAnalysisAttempt).toHaveBeenCalledWith(expect.objectContaining({ - provider: 'cloudflare', - decodedImageBytes: 5, - tokenProvenance: 'estimated', - costUsd: null, - costProvenance: 'unknown', - })); - }); - - it('emits only a sanitized terminal error for failed image analysis', async () => { - const events: unknown[] = []; - const authError = new AuthenticationError('openai', 'Bearer secret-value from raw response'); - mockOpenAIProvider.generateResponse.mockImplementationOnce(async (_request, options) => { - options?.onAttempt?.({ attempt: 1, outcome: 'error', durationMs: 2, willRetry: false, error: authError }); - throw authError; - }); - const visionFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-openai-key' }, - defaultProvider: 'openai', - costOptimization: false, - hooks: { - onImageAnalysisAttempt: event => events.push(event), - onImageAnalysisComplete: event => events.push(event), - }, - }); - - await expect(visionFactory.analyzeImage({ - image: { data: 'c2VjcmV0LWltYWdl', mimeType: 'image/jpeg' }, - prompt: 'secret prompt', - model: 'gpt-4o', - })).rejects.toBe(authError); - - expect(events).toHaveLength(2); - expect(events[0]).toEqual(expect.objectContaining({ - outcome: 'error', - error: { code: 'AUTHENTICATION_ERROR', category: 'authentication' }, - })); - expect(events[1]).toEqual(expect.objectContaining({ - outcome: 'error', - error: { code: 'AUTHENTICATION_ERROR', category: 'authentication' }, - })); - const serialized = JSON.stringify(events); - expect(serialized).not.toContain('secret prompt'); - expect(serialized).not.toContain('secret-value'); - expect(serialized).not.toContain('c2VjcmV0LWltYWdl'); - }); - - it('skips vision-incapable providers in the fallback chain', async () => { - const visionFactory = new LLMProviderFactory({ - cerebras: { apiKey: 'test-cerebras-key' }, - groq: { apiKey: 'test-groq-key' }, - openai: { apiKey: 'test-openai-key' }, - defaultProvider: 'cerebras', - costOptimization: false - }); - - await visionFactory.generateResponse({ - ...testRequest, - images: [{ data: 'abc123', mimeType: 'image/jpeg' }] - }); - - expect(mockOpenAIProvider.generateResponse).toHaveBeenCalledWith( - expect.objectContaining({ - images: [{ data: 'abc123', mimeType: 'image/jpeg' }] - }) - ); - }); - - it('throws when no configured provider supports image input', async () => { - const visionFactory = new LLMProviderFactory({ - cerebras: { apiKey: 'test-cerebras-key' }, - groq: { apiKey: 'test-groq-key' }, - defaultProvider: 'cerebras', - costOptimization: false - }); - - await expect(visionFactory.generateResponse({ - ...testRequest, - images: [{ data: 'abc123', mimeType: 'image/jpeg' }] - })).rejects.toThrow(/No configured providers support image input/); - }); - }); - - describe('Error Handling', () => { - it('should handle all providers failing', async () => { - // Make all providers fail - mockOpenAIProvider.generateResponse.mockRejectedValue(new Error('OpenAI failed')); - mockAnthropicProvider.generateResponse.mockRejectedValue(new Error('Anthropic failed')); - mockCloudflareProvider.generateResponse.mockRejectedValue(new Error('Cloudflare failed')); - - await expect(factory.generateResponse(testRequest)).rejects.toThrow(); - }); - - it('should not fallback for authentication errors', async () => { - const authError = new AuthenticationError('openai', 'Authentication failed'); - - mockOpenAIProvider.generateResponse.mockRejectedValueOnce(authError); - - await expect(factory.generateResponse({ - ...testRequest, - model: 'gpt-3.5-turbo' - })).rejects.toThrow('Authentication failed'); - }); - }); - - describe('Configuration Updates', () => { - it('should update configuration', () => { - factory.updateConfig({ - defaultProvider: 'anthropic', - costOptimization: false - }); - - // Configuration should be updated - // (We'd need to expose config or test behavior changes) - expect(() => factory.updateConfig({})).not.toThrow(); - }); - - it('should reset metrics and circuit breakers', () => { - factory.reset(); - - expect(mockOpenAIProvider.resetMetrics).toHaveBeenCalled(); - expect(mockAnthropicProvider.resetMetrics).toHaveBeenCalled(); - expect(mockCloudflareProvider.resetMetrics).toHaveBeenCalled(); - }); - - it('should pass maxRetries 0 to providers when factory retries are disabled', () => { - new LLMProviderFactory({ - openai: { apiKey: 'test-key' }, - enableRetries: false - }); - - expect(OpenAIProvider).toHaveBeenCalledWith( - expect.objectContaining({ - apiKey: 'test-key', - maxRetries: 0 - }) - ); - }); - - it('should preserve explicit provider maxRetries when factory retries are disabled', () => { - new LLMProviderFactory({ - openai: { apiKey: 'test-key', maxRetries: 2 }, - enableRetries: false - }); - - expect(OpenAIProvider).toHaveBeenCalledWith( - expect.objectContaining({ - apiKey: 'test-key', - maxRetries: 2 - }) - ); - }); - }); - - describe('CreditLedger integration', () => { - it('should record successful factory calls into the provided ledger even without cost optimization', async () => { - const ledger = new CreditLedger({ - budgets: [{ - provider: 'cloudflare', - monthlyBudget: 1, - rateLimits: { rpm: 10, rpd: 100, tpm: 1000, tpd: 10_000 } - }] - }); - - const ledgerFactory = new LLMProviderFactory({ - cloudflare: { ai: {} as Ai }, - costOptimization: false, - ledger - }); - - await ledgerFactory.generateResponse(testRequest); - - const accumulator = ledger.getProviderAccumulator('cloudflare'); - expect(accumulator).toMatchObject({ - spend: 0.0001, - inputTokens: 10, - outputTokens: 20, - requestCount: 1 - }); - expect(accumulator!.rateLimits.rpm!.used).toBe(1); - expect(accumulator!.rateLimits.rpd!.used).toBe(1); - expect(accumulator!.rateLimits.tpm!.used).toBe(30); - expect(accumulator!.rateLimits.tpd!.used).toBe(30); - }); - - it('should expose provider balance from the configured ledger', async () => { - const ledger = new CreditLedger({ - budgets: [{ - provider: 'cloudflare', - monthlyBudget: 1, - rateLimits: { rpm: 10 } - }] - }); - const balanceFactory = new LLMProviderFactory({ - cloudflare: { ai: {} as Ai }, - ledger - }); - - await balanceFactory.generateResponse(testRequest); - const balance = await balanceFactory.getProviderBalance('cloudflare'); - - expect(balance).toMatchObject({ - provider: 'cloudflare', - status: 'available', - source: 'ledger', - currentSpend: 0.0001, - monthlyBudget: 1, - requestCount: 1 - }); - expect((balance as { rateLimits: Record }).rateLimits.rpm.used).toBe(1); - }); - }); -}); - -describe('Cost Optimized Factory', () => { - beforeEach(() => { - vi.clearAllMocks(); - // Restore default mock implementations after previous tests may have set mockRejectedValue - mockOpenAIProvider.generateResponse.mockResolvedValue({ - message: 'OpenAI response', - usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.001 }, - model: 'gpt-3.5-turbo', - provider: 'openai', - responseTime: 1000 - } as LLMResponse); - mockCloudflareProvider.generateResponse.mockResolvedValue({ - message: 'Cloudflare response', - usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.0001 }, - model: '@cf/meta/llama-3.1-8b-instruct', - provider: 'cloudflare', - responseTime: 800 - } as LLMResponse); - }); - - it('should create cost-optimized factory with correct configuration', () => { - const factory = createCostOptimizedFactory({ - openai: { apiKey: 'test-key' }, - cloudflare: { ai: {} as Ai } - }); - - expect(factory).toBeInstanceOf(LLMProviderFactory); - // Cost optimization should be enabled by default - }); - - it('should prioritize cost-effective providers', async () => { - const factory = createCostOptimizedFactory({ - openai: { apiKey: 'test-key' }, - cloudflare: { ai: {} as Ai } - }); - - const response = await factory.generateResponse({ - messages: [{ role: 'user', content: 'Test' }] - }); - - // Should successfully route to an available provider - expect(['openai', 'cloudflare']).toContain(response.provider); - }); -}); - -describe('Response Cache Adapter', () => { - const cacheRequest: LLMRequest = { - messages: [{ role: 'user', content: 'What is 2+2?' }], - maxTokens: 50, - temperature: 0, - }; - - it('returns a cached response without calling the provider', async () => { - const cacheEvents: string[] = []; - const cachedResponse = { - message: 'cached answer', - usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0 }, - model: 'gpt-3.5-turbo', - provider: 'openai', - responseTime: 1, - }; - - const cache = { - get: vi.fn().mockResolvedValue(JSON.stringify(cachedResponse)), - put: vi.fn().mockResolvedValue(undefined), - }; - - const cachedFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-key' }, - responseCache: cache, - hooks: { - onCache: event => cacheEvents.push(`${event.layer}:${event.status}`), - }, - }); - - const response = await cachedFactory.generateResponse(cacheRequest); - expect(response.message).toBe('cached answer'); - expect(response.cache?.factory).toMatchObject({ status: 'hit' }); - expect(response.metadata?.cache).toEqual(response.cache); - expect(cacheEvents).toEqual(['factory-response:hit']); - expect(mockOpenAIProvider.generateResponse).not.toHaveBeenCalled(); - }); - - it('calls the provider on cache miss and writes the response', async () => { - const cacheEvents: string[] = []; - const cache = { - get: vi.fn().mockResolvedValue(null), - put: vi.fn().mockResolvedValue(undefined), - }; - - const cachedFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-key' }, - responseCache: cache, - responseCacheDefaultTtl: 3600, - hooks: { - onCache: event => cacheEvents.push(`${event.layer}:${event.status}`), - }, - }); - - mockOpenAIProvider.generateResponse.mockReset().mockResolvedValue({ - message: 'fresh answer', - usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001, cachedInputTokens: 2 }, - model: 'gpt-3.5-turbo', - provider: 'openai', - responseTime: 100, - } as LLMResponse); - - const response = await cachedFactory.generateResponse(cacheRequest); - expect(response.message).toBe('fresh answer'); - expect(response.cache?.providerPrefix).toMatchObject({ cachedInputTokens: 2, hit: true }); - expect(response.cache?.factory).toMatchObject({ status: 'miss', ttlSeconds: 3600 }); - expect(response.metadata?.cache).toEqual(response.cache); - expect(cacheEvents).toEqual(['provider-prefix:hit', 'factory-response:miss']); - expect(cache.get).toHaveBeenCalledOnce(); - // put is called async (fire-and-forget), so wait a tick - await new Promise(r => setTimeout(r, 0)); - expect(cache.put).toHaveBeenCalledOnce(); - expect(cache.put).toHaveBeenCalledWith( - expect.any(String), - expect.stringContaining('fresh answer'), - 3600, - ); - }); - - it('proceeds normally when cache.get throws', async () => { - const cache = { - get: vi.fn().mockRejectedValue(new Error('KV unavailable')), - put: vi.fn().mockResolvedValue(undefined), - }; - - const cachedFactory = new LLMProviderFactory({ - openai: { apiKey: 'test-key' }, - responseCache: cache, - }); - - mockOpenAIProvider.generateResponse.mockReset().mockResolvedValue({ - message: 'fallback answer', - usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001 }, - model: 'gpt-3.5-turbo', - provider: 'openai', - responseTime: 100, - } as LLMResponse); - - const response = await cachedFactory.generateResponse(cacheRequest); - expect(response.message).toBe('fallback answer'); - expect(mockOpenAIProvider.generateResponse).toHaveBeenCalled(); - }); -}); - -// Custom matcher for vitest -expect.extend({ - toBeOneOf(received: any, expected: any[]) { - const pass = expected.includes(received); - return { - pass, - message: () => `expected ${received} to be one of ${expected.join(', ')}` - }; - } -}); +/** + * LLM Provider Factory Tests + * Tests for the provider factory with mocked providers + */ + +import { describe, it, expect, beforeEach, vi } from 'vitest'; +import { LLMProviderFactory, createCostOptimizedFactory } from '../factory'; +import { AuthenticationError, ServerError } from '../errors'; +import type { LLMRequest, LLMResponse } from '../types'; +import { OpenAIProvider } from '../providers/openai'; +import { CreditLedger } from '../utils/credit-ledger'; +import { defaultCostTracker } from '../utils/cost-tracker'; +import { defaultCircuitBreakerManager } from '../utils/circuit-breaker'; +import { defaultExhaustionRegistry } from '../utils/exhaustion'; +import { defaultLatencyHistogram } from '../utils/latency-histogram'; +import { attachStreamUsage } from '../utils/stream-usage'; + +// Mock providers +const mockOpenAIProvider = { + name: 'openai', + models: ['gpt-4', 'gpt-3.5-turbo'], + supportsStreaming: true, + supportsTools: true, + supportsBatching: true, + supportsVision: true, + generateResponse: vi.fn().mockResolvedValue({ + message: 'OpenAI response', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.001 }, + model: 'gpt-3.5-turbo', + provider: 'openai', + responseTime: 1000 + } as LLMResponse), + streamResponse: vi.fn(), + getProviderBalance: vi.fn(), + validateConfig: vi.fn().mockReturnValue(true), + getModels: vi.fn().mockReturnValue(['gpt-4', 'gpt-3.5-turbo']), + estimateCost: vi.fn().mockReturnValue(0.001), + healthCheck: vi.fn().mockResolvedValue(true), + getMetrics: vi.fn().mockReturnValue({ + requestCount: 1, + successCount: 1, + errorCount: 0, + averageLatency: 1000, + totalCost: 0.001, + rateLimitHits: 0, + lastUsed: Date.now() + }), + resetMetrics: vi.fn() +}; + +const mockAnthropicProvider = { + name: 'anthropic', + models: ['claude-haiku-4-5-20251001', 'claude-3-sonnet-20240229'], + supportsStreaming: true, + supportsTools: true, + supportsBatching: false, + supportsVision: true, + generateResponse: vi.fn().mockResolvedValue({ + message: 'Anthropic response', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.002 }, + model: 'claude-haiku-4-5-20251001', + provider: 'anthropic', + responseTime: 1200 + } as LLMResponse), + streamResponse: vi.fn(), + getProviderBalance: vi.fn(), + validateConfig: vi.fn().mockReturnValue(true), + getModels: vi.fn().mockReturnValue(['claude-haiku-4-5-20251001', 'claude-3-sonnet-20240229']), + estimateCost: vi.fn().mockReturnValue(0.002), + healthCheck: vi.fn().mockResolvedValue(true), + getMetrics: vi.fn().mockReturnValue({ + requestCount: 1, + successCount: 1, + errorCount: 0, + averageLatency: 1200, + totalCost: 0.002, + rateLimitHits: 0, + lastUsed: Date.now() + }), + resetMetrics: vi.fn() +}; + +const mockCloudflareProvider = { + name: 'cloudflare', + models: ['@cf/meta/llama-3.1-8b-instruct'], + supportsStreaming: true, + supportsTools: false, + supportsBatching: true, + supportsVision: false, + generateResponse: vi.fn().mockResolvedValue({ + message: 'Cloudflare response', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.0001 }, + model: '@cf/meta/llama-3.1-8b-instruct', + provider: 'cloudflare', + responseTime: 800 + } as LLMResponse), + streamResponse: vi.fn(), + getProviderBalance: vi.fn(), + validateConfig: vi.fn().mockReturnValue(true), + getModels: vi.fn().mockReturnValue(['@cf/meta/llama-3.1-8b-instruct']), + estimateCost: vi.fn().mockReturnValue(0.0001), + healthCheck: vi.fn().mockResolvedValue(true), + getMetrics: vi.fn().mockReturnValue({ + requestCount: 1, + successCount: 1, + errorCount: 0, + averageLatency: 800, + totalCost: 0.0001, + rateLimitHits: 0, + lastUsed: Date.now() + }), + resetMetrics: vi.fn() +}; + +// Mock the provider classes +vi.mock('../providers/openai', () => ({ + OpenAIProvider: vi.fn().mockImplementation(() => mockOpenAIProvider) +})); + +vi.mock('../providers/anthropic', () => ({ + AnthropicProvider: vi.fn().mockImplementation(() => mockAnthropicProvider) +})); + +vi.mock('../providers/cloudflare', () => ({ + CloudflareProvider: vi.fn().mockImplementation(() => mockCloudflareProvider) +})); + +describe('LLMProviderFactory', () => { + let factory: LLMProviderFactory; + + const testRequest: LLMRequest = { + messages: [{ role: 'user', content: 'Hello, world!' }], + maxTokens: 100, + temperature: 0.7 + }; + + beforeEach(() => { + vi.clearAllMocks(); + defaultCostTracker.reset(); + defaultCircuitBreakerManager.resetAll(); + defaultExhaustionRegistry.reset(); + defaultLatencyHistogram.reset(); + + mockOpenAIProvider.generateResponse.mockReset().mockResolvedValue({ + message: 'OpenAI response', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.001 }, + model: 'gpt-3.5-turbo', + provider: 'openai', + responseTime: 1000 + } as LLMResponse); + mockOpenAIProvider.streamResponse.mockReset().mockResolvedValue(new ReadableStream({ + start(controller) { + controller.enqueue('OpenAI stream'); + controller.close(); + } + })); + mockOpenAIProvider.getProviderBalance.mockReset().mockResolvedValue({ + provider: 'openai', + status: 'available', + source: 'provider_api' + }); + mockAnthropicProvider.generateResponse.mockReset().mockResolvedValue({ + message: 'Anthropic response', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.002 }, + model: 'claude-haiku-4-5-20251001', + provider: 'anthropic', + responseTime: 1200 + } as LLMResponse); + mockAnthropicProvider.streamResponse.mockReset().mockResolvedValue(new ReadableStream({ + start(controller) { + controller.enqueue('Anthropic stream'); + controller.close(); + } + })); + mockAnthropicProvider.getProviderBalance.mockReset().mockResolvedValue({ + provider: 'anthropic', + status: 'available', + source: 'provider_api' + }); + mockCloudflareProvider.generateResponse.mockReset().mockResolvedValue({ + message: 'Cloudflare response', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.0001 }, + model: '@cf/meta/llama-3.1-8b-instruct', + provider: 'cloudflare', + responseTime: 800 + } as LLMResponse); + mockCloudflareProvider.streamResponse.mockReset().mockResolvedValue(new ReadableStream({ + start(controller) { + controller.enqueue('Cloudflare stream'); + controller.close(); + } + })); + mockCloudflareProvider.getProviderBalance.mockReset().mockResolvedValue({ + provider: 'cloudflare', + status: 'unavailable', + source: 'not_supported' + }); + mockCloudflareProvider.supportsVision = false; + + factory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + anthropic: { apiKey: 'test-anthropic-key' }, + cloudflare: { ai: {} as Ai }, + defaultProvider: 'auto', + costOptimization: true, + enableCircuitBreaker: true, + }); + }); + + describe('Provider Initialization', () => { + it('should initialize all configured providers', () => { + const availableProviders = factory.getAvailableProviders(); + expect(availableProviders).toContain('openai'); + expect(availableProviders).toContain('anthropic'); + expect(availableProviders).toContain('cloudflare'); + }); + + it('should handle missing provider configurations gracefully', () => { + const factoryWithMissing = new LLMProviderFactory({ + openai: { apiKey: 'test-key' } + // Missing anthropic and cloudflare configs + }); + + const availableProviders = factoryWithMissing.getAvailableProviders(); + expect(availableProviders).toContain('openai'); + expect(availableProviders).not.toContain('anthropic'); + expect(availableProviders).not.toContain('cloudflare'); + }); + }); + + describe('Response Generation', () => { + it('should generate response using available provider', async () => { + const response = await factory.generateResponse(testRequest); + + expect(response).toBeDefined(); + expect(response.message).toBeTruthy(); + expect(response.provider).toBeOneOf(['openai', 'anthropic', 'cloudflare']); + expect(response.usage).toBeDefined(); + expect(response.responseTime).toBeGreaterThan(0); + }); + + it('should use specific provider when model is specified', async () => { + const requestWithModel: LLMRequest = { + ...testRequest, + model: 'gpt-3.5-turbo' + }; + + const response = await factory.generateResponse(requestWithModel); + expect(mockOpenAIProvider.generateResponse).toHaveBeenCalled(); + }); + + it('should fallback to other providers on failure', async () => { + // Make OpenAI provider fail + mockOpenAIProvider.generateResponse.mockRejectedValueOnce( + new Error('OpenAI temporarily unavailable') + ); + + const response = await factory.generateResponse(testRequest); + + // Should fallback to another provider + expect(response).toBeDefined(); + expect(response.provider).not.toBe('openai'); + }); + + it('should skip fully open providers and continue down the chain', async () => { + defaultCircuitBreakerManager.getBreaker('cloudflare').forceOpen(); + + const response = await factory.generateResponse(testRequest); + + expect(mockCloudflareProvider.generateResponse).not.toHaveBeenCalled(); + expect(response.provider).not.toBe('cloudflare'); + }); + }); + + describe('Cost Optimization', () => { + it('should prefer cheaper providers when cost optimization is enabled', async () => { + const response = await factory.generateResponse(testRequest); + + // Cloudflare should be preferred for cost optimization + expect(response.provider).toBe('cloudflare'); + }); + + it('should provide cost analytics', async () => { + await factory.generateResponse(testRequest); + + const analytics = factory.getCostAnalytics(); + + expect(analytics).toBeDefined(); + expect(analytics.breakdown).toBeDefined(); + expect(analytics.total).toBeCloseTo(0.0001); + expect(analytics.breakdown.cloudflare).toMatchObject({ + cost: 0.0001, + totalCost: 0.0001, + requests: 1, + requestCount: 1, + inputTokens: 10, + outputTokens: 20, + tokens: { input: 10, output: 20 } + }); + expect(analytics.breakdown.cloudflare.lastRecordedAt).toBeGreaterThan(0); + }); + }); + + describe('Health Monitoring', () => { + it('should check provider health status', async () => { + const health = await factory.getProviderHealth(); + + expect(health).toBeDefined(); + expect(health.openai).toBeDefined(); + expect(health.anthropic).toBeDefined(); + expect(health.cloudflare).toBeDefined(); + + expect(health.openai.healthy).toBe(true); + expect(health.openai.models).toEqual(['gpt-4', 'gpt-3.5-turbo']); + expect(health.openai.capabilities).toBeDefined(); + }); + + it('should handle provider health check failures', async () => { + mockOpenAIProvider.healthCheck.mockResolvedValueOnce(false); + + const health = await factory.getProviderHealth(); + expect(health.openai.healthy).toBe(false); + }); + }); + + describe('Provider Chain Building', () => { + it('should build appropriate provider chain for model-specific requests', async () => { + // Test Claude model request + const claudeRequest: LLMRequest = { + ...testRequest, + model: 'claude-haiku-4-5-20251001' + }; + + await factory.generateResponse(claudeRequest); + expect(mockAnthropicProvider.generateResponse).toHaveBeenCalled(); + }); + + it('should build cost-optimized chain for auto mode', async () => { + const autoRequest: LLMRequest = { + ...testRequest + // No specific model, should use auto selection + }; + + await factory.generateResponse(autoRequest); + + // Should prefer Cloudflare for cost optimization + expect(mockCloudflareProvider.generateResponse).toHaveBeenCalled(); + }); + + it('should demote degraded providers with critical burn rate when recommending a model', async () => { + const ledger = new CreditLedger({ + budgets: [ + { provider: 'cloudflare', monthlyBudget: 1 }, + { provider: 'openai', monthlyBudget: 10 } + ] + }); + ledger.record('cloudflare', '@cf/google/gemma-4-26b-a4b-it', 0.95, 10_000, 4_000); + + const dynamicFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + anthropic: { apiKey: 'test-anthropic-key' }, + cloudflare: { ai: {} as Ai }, + defaultProvider: 'auto', + costOptimization: true, + enableCircuitBreaker: true, + ledger + }); + + const breaker = defaultCircuitBreakerManager.getBreaker('cloudflare'); + await expect(breaker.execute(async () => { + throw new Error('degraded'); + })).rejects.toThrow('degraded'); + + expect(dynamicFactory.getRecommendedModel(testRequest)).toBe('gpt-4o-mini'); + }); + + it('should honor fallbackProvider as the next route when a rule matches', async () => { + const ruleFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + anthropic: { apiKey: 'test-anthropic-key' }, + cloudflare: { ai: {} as Ai }, + defaultProvider: 'openai', + costOptimization: false, + fallbackRules: [{ condition: 'error', fallbackProvider: 'anthropic' }] + }); + + mockOpenAIProvider.generateResponse.mockRejectedValueOnce(new Error('OpenAI down')); + + const response = await ruleFactory.generateResponse(testRequest); + + expect(response.provider).toBe('anthropic'); + expect(mockAnthropicProvider.generateResponse).toHaveBeenCalled(); + expect(mockCloudflareProvider.generateResponse).not.toHaveBeenCalled(); + }); + + it('should apply fallbackModel when routing through a fallback rule', async () => { + const ruleFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + anthropic: { apiKey: 'test-anthropic-key' }, + defaultProvider: 'openai', + costOptimization: false, + fallbackRules: [{ + condition: 'error', + fallbackProvider: 'anthropic', + fallbackModel: 'claude-haiku-4-5-20251001' + }] + }); + + mockOpenAIProvider.generateResponse.mockRejectedValueOnce(new Error('OpenAI down')); + + await ruleFactory.generateResponse({ + ...testRequest, + model: 'gpt-4' + }); + + expect(mockAnthropicProvider.generateResponse).toHaveBeenCalledWith( + expect.objectContaining({ model: 'claude-haiku-4-5-20251001' }) + ); + }); + }); + + describe('Streaming, tools, classification, and vision', () => { + async function readStream(stream: ReadableStream): Promise { + const reader = stream.getReader(); + let output = ''; + while (true) { + const { done, value } = await reader.read(); + if (done) break; + output += value; + } + return output; + } + + it('should stream through the factory and fallback before the first chunk', async () => { + const streamFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + anthropic: { apiKey: 'test-anthropic-key' }, + defaultProvider: 'openai', + costOptimization: false, + fallbackRules: [{ condition: 'error', fallbackProvider: 'anthropic' }] + }); + + mockOpenAIProvider.streamResponse.mockRejectedValueOnce(new Error('stream start failed')); + + const stream = await streamFactory.generateResponseStream(testRequest); + + expect(await readStream(stream)).toBe('Anthropic stream'); + expect(mockOpenAIProvider.streamResponse).toHaveBeenCalled(); + expect(mockAnthropicProvider.streamResponse).toHaveBeenCalled(); + }); + + it('records actual stream usage after the stream drains', async () => { + const quotaHook = { + check: vi.fn().mockResolvedValue({ allowed: true, remainingBudget: 1 }), + record: vi.fn().mockResolvedValue(undefined) + }; + const streamFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + defaultProvider: 'openai', + costOptimization: true, + quotaHook + }); + const usage: LLMResponse['usage'] = { + inputTokens: 12, + outputTokens: 34, + totalTokens: 46, + cost: 0.123 + }; + const providerStream = attachStreamUsage(new ReadableStream({ + start(controller) { + controller.enqueue('OpenAI stream'); + controller.close(); + } + }), Promise.resolve(usage)); + + mockOpenAIProvider.streamResponse.mockResolvedValueOnce(providerStream); + + const stream = await streamFactory.generateResponseStream({ + ...testRequest, + tenantId: 'tenant-stream' + }); + + expect(await readStream(stream)).toBe('OpenAI stream'); + expect(quotaHook.record).toHaveBeenCalledWith(expect.objectContaining({ + tenantId: 'tenant-stream', + provider: 'openai', + actualCost: 0.123, + inputTokens: 12, + outputTokens: 34 + })); + expect(streamFactory.getCostAnalytics().breakdown?.openai).toMatchObject({ + cost: 0.123, + tokens: { input: 12, output: 34 } + }); + }); + + it('should call quota hooks before and after successful dispatch', async () => { + const quotaHook = { + check: vi.fn().mockResolvedValue({ allowed: true, remainingBudget: 1 }), + record: vi.fn().mockResolvedValue(undefined) + }; + const quotaFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + defaultProvider: 'openai', + costOptimization: false, + quotaHook + }); + + await quotaFactory.generateResponse({ ...testRequest, tenantId: 'tenant-1' }); + + expect(quotaHook.check).toHaveBeenCalledWith(expect.objectContaining({ + tenantId: 'tenant-1', + provider: 'openai', + model: 'gpt-4o-mini' + })); + expect(quotaHook.record).toHaveBeenCalledWith(expect.objectContaining({ + tenantId: 'tenant-1', + provider: 'openai', + actualCost: 0.001, + inputTokens: 10, + outputTokens: 20 + })); + }); + + it('should deny dispatch when quota hook rejects the request', async () => { + const quotaFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + defaultProvider: 'openai', + costOptimization: false, + quotaHook: { + check: vi.fn().mockResolvedValue({ allowed: false, reason: 'budget exhausted' }), + record: vi.fn() + } + }); + + await expect(quotaFactory.generateResponse(testRequest)).rejects.toThrow('budget exhausted'); + expect(mockOpenAIProvider.generateResponse).not.toHaveBeenCalled(); + }); + + it('should execute tool loops until the final response has no tool calls', async () => { + const toolResponse: LLMResponse = { + message: '', + usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001 }, + model: 'gpt-3.5-turbo', + provider: 'openai', + responseTime: 10, + finishReason: 'tool_calls', + toolCalls: [{ + id: 'call-1', + type: 'function', + function: { name: 'lookup', arguments: '{"id":42}' } + }] + }; + const finalResponse: LLMResponse = { + message: 'done', + usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001 }, + model: 'gpt-3.5-turbo', + provider: 'openai', + responseTime: 10 + }; + mockOpenAIProvider.generateResponse + .mockResolvedValueOnce(toolResponse) + .mockResolvedValueOnce(finalResponse); + + const loopFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + defaultProvider: 'openai', + costOptimization: false + }); + const executor = { execute: vi.fn().mockResolvedValue({ value: 42 }) }; + + const response = await loopFactory.generateResponseWithTools(testRequest, executor); + + expect(response.message).toBe('done'); + expect(executor.execute).toHaveBeenCalledWith('lookup', { id: 42 }); + expect(mockOpenAIProvider.generateResponse).toHaveBeenLastCalledWith( + expect.objectContaining({ + messages: expect.arrayContaining([ + expect.objectContaining({ + toolResults: [{ id: 'call-1', output: '{"value":42}' }] + }) + ]) + }) + ); + }); + + // ── generateResponseWithTools edge cases (#28) ─────────────────────── + + function makeLoopFactory() { + return new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + defaultProvider: 'openai', + costOptimization: false + }); + } + + const toolCallResponse = (id = 'call-1'): LLMResponse => ({ + message: '', + usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.5 }, + model: 'gpt-4', + provider: 'openai', + responseTime: 10, + finishReason: 'tool_calls', + toolCalls: [{ id, type: 'function', function: { name: 'noop', arguments: '{}' } }] + }); + + const finalResponse: LLMResponse = { + message: 'done', + usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001 }, + model: 'gpt-4', + provider: 'openai', + responseTime: 10 + }; + + it('throws ToolLoopLimitError when maxIterations is exceeded', async () => { + const { ToolLoopLimitError } = await import('../errors'); + // Keep returning tool calls — never resolves naturally + mockOpenAIProvider.generateResponse.mockResolvedValue(toolCallResponse()); + + await expect( + makeLoopFactory().generateResponseWithTools( + testRequest, + { execute: vi.fn().mockResolvedValue('ok') }, + { maxIterations: 2 } + ) + ).rejects.toThrow(ToolLoopLimitError); + }); + + it('throws ToolLoopLimitError when cumulative cost exceeds maxCostUSD', async () => { + const { ToolLoopLimitError } = await import('../errors'); + // Each tool-call response costs $0.50; cap at $0.60 → second iteration should abort + mockOpenAIProvider.generateResponse + .mockResolvedValueOnce(toolCallResponse('call-1')) + .mockResolvedValueOnce(toolCallResponse('call-2')) + .mockResolvedValue(finalResponse); + + await expect( + makeLoopFactory().generateResponseWithTools( + testRequest, + { execute: vi.fn().mockResolvedValue('ok') }, + { maxCostUSD: 0.60 } + ) + ).rejects.toThrow(ToolLoopLimitError); + }); + + it('aborts immediately when abortSignal is already aborted', async () => { + const { ToolLoopAbortedError } = await import('../errors'); + const controller = new AbortController(); + controller.abort(); + + await expect( + makeLoopFactory().generateResponseWithTools( + testRequest, + { execute: vi.fn() }, + { abortSignal: controller.signal } + ) + ).rejects.toThrow(ToolLoopAbortedError); + + expect(mockOpenAIProvider.generateResponse).not.toHaveBeenCalled(); + }); + + it('records tool execution failure as a tool-result error without crashing the loop', async () => { + mockOpenAIProvider.generateResponse + .mockResolvedValueOnce(toolCallResponse()) + .mockResolvedValueOnce(finalResponse); + + const failingExecutor = { execute: vi.fn().mockRejectedValue(new Error('external service down')) }; + + const response = await makeLoopFactory().generateResponseWithTools(testRequest, failingExecutor); + + expect(response.message).toBe('done'); + // The second generateResponse call should have received the tool error as a result + const secondCallMessages = (mockOpenAIProvider.generateResponse.mock.calls[1][0] as LLMRequest).messages; + const toolResultMsg = secondCallMessages.find(m => m.toolResults && m.toolResults.length > 0); + expect(toolResultMsg?.toolResults?.[0]).toMatchObject({ + id: 'call-1', + error: 'external service down' + }); + }); + + it('feeds hallucinated tool names back as errors without executing them', async () => { + mockOpenAIProvider.generateResponse + .mockResolvedValueOnce({ + ...toolCallResponse(), + toolCalls: [{ + id: 'call-hallucinated', + type: 'function', + function: { name: 'apply_patch', arguments: '{}' } + }] + }) + .mockResolvedValueOnce(finalResponse); + const executor = { execute: vi.fn() }; + + await makeLoopFactory().generateResponseWithTools({ + ...testRequest, + tools: [{ + type: 'function', + function: { + name: 'noop', + description: 'A permitted no-op', + parameters: { type: 'object', properties: {} } + } + }] + }, executor); + + expect(executor.execute).not.toHaveBeenCalled(); + const secondCall = mockOpenAIProvider.generateResponse.mock.calls[1][0] as LLMRequest; + expect(secondCall.messages.at(-1)?.toolResults?.[0]).toMatchObject({ + id: 'call-hallucinated', + error: expect.stringContaining("unavailable tool 'apply_patch'") + }); + }); + + it('invokes onIteration callback with correct state on each iteration', async () => { + mockOpenAIProvider.generateResponse + .mockResolvedValueOnce(toolCallResponse()) + .mockResolvedValueOnce(finalResponse); + + const iterations: number[] = []; + await makeLoopFactory().generateResponseWithTools( + testRequest, + { execute: vi.fn().mockResolvedValue('result') }, + { onIteration: (iter) => { iterations.push(iter); } } + ); + + expect(iterations).toEqual([1]); + }); + + it('should classify JSON responses and expose confidence', async () => { + mockOpenAIProvider.generateResponse.mockResolvedValueOnce({ + message: '{"label":"recipe","confidence":0.92}', + usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001 }, + model: 'gpt-3.5-turbo', + provider: 'openai', + responseTime: 10 + } as LLMResponse); + + const classifyFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + defaultProvider: 'openai', + costOptimization: false + }); + + const result = await classifyFactory.classify<{ label: string; confidence: number }>('classify this'); + + expect(result.data.label).toBe('recipe'); + expect(result.confidence).toBe(0.92); + }); + + it('should route image analysis to a vision-capable provider', async () => { + const visionFactory = new LLMProviderFactory({ + anthropic: { apiKey: 'test-anthropic-key' }, + cloudflare: { ai: {} as Ai }, + costOptimization: false + }); + + await visionFactory.analyzeImage({ + image: { data: 'abc123', mimeType: 'image/jpeg' }, + prompt: 'Extract recipe text' + }); + + expect(mockAnthropicProvider.generateResponse).toHaveBeenCalledWith( + expect.objectContaining({ + images: [{ data: 'abc123', mimeType: 'image/jpeg' }], + // claude-sonnet-4-6 (quality:6) wins vision routing over haiku (quality:4) after score recalibration + model: 'claude-sonnet-4-6' + }), + expect.objectContaining({ onAttempt: expect.any(Function) }) + ); + expect(mockCloudflareProvider.generateResponse).not.toHaveBeenCalled(); + }); + + it('emits sanitized attempt and aggregate observability for image analysis', async () => { + const onImageAnalysisAttempt = vi.fn(); + const onImageAnalysisComplete = vi.fn(); + mockOpenAIProvider.generateResponse.mockImplementationOnce(async (_request, options) => { + const response = { + message: 'Cookbook page', + usage: { inputTokens: 12, outputTokens: 8, totalTokens: 20, cost: 0.003 }, + model: 'gpt-4o', + provider: 'openai', + responseTime: 4 + } as LLMResponse; + options?.onAttempt?.({ + attempt: 1, + outcome: 'success', + durationMs: 4, + willRetry: false, + response, + }); + return response; + }); + const visionFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + defaultProvider: 'openai', + costOptimization: false, + hooks: { onImageAnalysisAttempt, onImageAnalysisComplete }, + }); + + const response = await visionFactory.analyzeImage({ + image: { data: 'AQIDBA==', mimeType: 'image/jpeg' }, + prompt: 'private cookbook prompt', + model: 'gpt-4o', + requestId: 'capture-1', + metadata: { captureId: 'cap-1', pageNumber: 7, secret: 'drop-me', imageData: 'drop-me-too' }, + }); + + expect(response.usage).toMatchObject({ + cost: 0.003, + costProvenance: 'catalog_estimate', + tokenProvenance: 'provider_reported', + }); + expect(onImageAnalysisAttempt).toHaveBeenCalledWith(expect.objectContaining({ + operation: 'analyze_image', + requestId: 'capture-1', + metadata: { captureId: 'cap-1', pageNumber: 7 }, + provider: 'openai', + model: 'gpt-4o', + attempt: 1, + providerAttempt: 1, + outcome: 'success', + inputTokens: 12, + outputTokens: 8, + totalTokens: 20, + tokenProvenance: 'provider_reported', + imageCount: 1, + decodedImageBytes: 4, + costUsd: 0.003, + costProvenance: 'catalog_estimate', + })); + expect(onImageAnalysisComplete).toHaveBeenCalledWith(expect.objectContaining({ + outcome: 'success', + attempts: 1, + totalLatencyMs: 4, + catalogEstimatedCostUsd: 0.003, + unknownCostAttempts: 0, + finalProvider: 'openai', + finalModel: 'gpt-4o', + })); + expect(JSON.stringify(onImageAnalysisAttempt.mock.calls)).not.toContain('private cookbook prompt'); + expect(JSON.stringify(onImageAnalysisAttempt.mock.calls)).not.toContain('drop-me'); + }); + + it('accounts for every retry and fallback in image aggregates', async () => { + const attempts: unknown[] = []; + const completed: unknown[] = []; + const firstError = new ServerError('openai', 'raw payload must stay private'); + mockOpenAIProvider.generateResponse.mockImplementationOnce(async (_request, options) => { + options?.onAttempt?.({ attempt: 1, outcome: 'error', durationMs: 3, willRetry: true, error: firstError }); + options?.onAttempt?.({ attempt: 2, outcome: 'error', durationMs: 5, willRetry: false, error: firstError }); + throw firstError; + }); + mockAnthropicProvider.generateResponse.mockImplementationOnce(async (_request, options) => { + const response = { + message: 'fallback answer', + usage: { inputTokens: 10, outputTokens: 6, totalTokens: 16, cost: 0.002 }, + model: 'claude-sonnet-4-6', + provider: 'anthropic', + responseTime: 7, + } as LLMResponse; + options?.onAttempt?.({ attempt: 1, outcome: 'success', durationMs: 7, willRetry: false, response }); + return response; + }); + const visionFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + anthropic: { apiKey: 'test-anthropic-key' }, + defaultProvider: 'openai', + fallbackRules: [{ condition: 'error', fallbackProvider: 'anthropic' }], + costOptimization: false, + hooks: { + onImageAnalysisAttempt: event => attempts.push(event), + onImageAnalysisComplete: event => completed.push(event), + }, + }); + + await visionFactory.analyzeImage({ + image: { data: 'AQID', mimeType: 'image/jpeg' }, + prompt: 'do not emit this prompt', + model: 'gpt-4o', + }); + + expect(attempts).toHaveLength(3); + expect(attempts).toEqual([ + expect.objectContaining({ attempt: 1, provider: 'openai', providerAttempt: 1, retry: false, willRetry: true, fallback: false }), + expect.objectContaining({ attempt: 2, provider: 'openai', providerAttempt: 2, retry: true, willRetry: false, fallback: false }), + expect.objectContaining({ attempt: 3, provider: 'anthropic', providerAttempt: 1, fallback: true, fallbackFromProvider: 'openai' }), + ]); + expect(completed).toEqual([expect.objectContaining({ + outcome: 'success', + attempts: 3, + totalLatencyMs: 15, + catalogEstimatedCostUsd: 0.002, + unknownCostAttempts: 2, + finalProvider: 'anthropic', + })]); + expect(JSON.stringify({ attempts, completed })).not.toContain('raw payload must stay private'); + expect(JSON.stringify({ attempts, completed })).not.toContain('do not emit this prompt'); + }); + + it('keeps Cloudflare image cost explicitly unknown while recording decoded bytes', async () => { + const onImageAnalysisAttempt = vi.fn(); + mockCloudflareProvider.supportsVision = true; + mockCloudflareProvider.generateResponse.mockImplementationOnce(async (_request, options) => { + const response = { + message: 'page', + usage: { + inputTokens: 9, + outputTokens: 3, + totalTokens: 12, + cost: 0.0001, + costProvenance: 'catalog_estimate', + tokenProvenance: 'estimated', + }, + model: '@cf/meta/llama-3.2-11b-vision-instruct', + provider: 'cloudflare', + responseTime: 2, + } as LLMResponse; + options?.onAttempt?.({ attempt: 1, outcome: 'success', durationMs: 2, willRetry: false, response }); + return response; + }); + const visionFactory = new LLMProviderFactory({ + cloudflare: { ai: {} as Ai }, + defaultProvider: 'cloudflare', + costOptimization: false, + hooks: { onImageAnalysisAttempt }, + }); + + try { + await visionFactory.analyzeImage({ + image: { data: 'AQIDBAU=', mimeType: 'image/jpeg' }, + prompt: 'extract', + model: '@cf/meta/llama-3.2-11b-vision-instruct', + }); + } finally { + mockCloudflareProvider.supportsVision = false; + } + + expect(onImageAnalysisAttempt).toHaveBeenCalledWith(expect.objectContaining({ + provider: 'cloudflare', + decodedImageBytes: 5, + tokenProvenance: 'estimated', + costUsd: null, + costProvenance: 'unknown', + })); + }); + + it('emits only a sanitized terminal error for failed image analysis', async () => { + const events: unknown[] = []; + const authError = new AuthenticationError('openai', 'Bearer secret-value from raw response'); + mockOpenAIProvider.generateResponse.mockImplementationOnce(async (_request, options) => { + options?.onAttempt?.({ attempt: 1, outcome: 'error', durationMs: 2, willRetry: false, error: authError }); + throw authError; + }); + const visionFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-openai-key' }, + defaultProvider: 'openai', + costOptimization: false, + hooks: { + onImageAnalysisAttempt: event => events.push(event), + onImageAnalysisComplete: event => events.push(event), + }, + }); + + await expect(visionFactory.analyzeImage({ + image: { data: 'c2VjcmV0LWltYWdl', mimeType: 'image/jpeg' }, + prompt: 'secret prompt', + model: 'gpt-4o', + })).rejects.toBe(authError); + + expect(events).toHaveLength(2); + expect(events[0]).toEqual(expect.objectContaining({ + outcome: 'error', + error: { code: 'AUTHENTICATION_ERROR', category: 'authentication' }, + })); + expect(events[1]).toEqual(expect.objectContaining({ + outcome: 'error', + error: { code: 'AUTHENTICATION_ERROR', category: 'authentication' }, + })); + const serialized = JSON.stringify(events); + expect(serialized).not.toContain('secret prompt'); + expect(serialized).not.toContain('secret-value'); + expect(serialized).not.toContain('c2VjcmV0LWltYWdl'); + }); + + it('skips vision-incapable providers in the fallback chain', async () => { + const visionFactory = new LLMProviderFactory({ + cerebras: { apiKey: 'test-cerebras-key' }, + groq: { apiKey: 'test-groq-key' }, + openai: { apiKey: 'test-openai-key' }, + defaultProvider: 'cerebras', + costOptimization: false + }); + + await visionFactory.generateResponse({ + ...testRequest, + images: [{ data: 'abc123', mimeType: 'image/jpeg' }] + }); + + expect(mockOpenAIProvider.generateResponse).toHaveBeenCalledWith( + expect.objectContaining({ + images: [{ data: 'abc123', mimeType: 'image/jpeg' }] + }) + ); + }); + + it('throws when no configured provider supports image input', async () => { + const visionFactory = new LLMProviderFactory({ + cerebras: { apiKey: 'test-cerebras-key' }, + groq: { apiKey: 'test-groq-key' }, + defaultProvider: 'cerebras', + costOptimization: false + }); + + await expect(visionFactory.generateResponse({ + ...testRequest, + images: [{ data: 'abc123', mimeType: 'image/jpeg' }] + })).rejects.toThrow(/No configured providers support image input/); + }); + }); + + describe('Error Handling', () => { + it('should handle all providers failing', async () => { + // Make all providers fail + mockOpenAIProvider.generateResponse.mockRejectedValue(new Error('OpenAI failed')); + mockAnthropicProvider.generateResponse.mockRejectedValue(new Error('Anthropic failed')); + mockCloudflareProvider.generateResponse.mockRejectedValue(new Error('Cloudflare failed')); + + await expect(factory.generateResponse(testRequest)).rejects.toThrow(); + }); + + it('should not fallback for authentication errors', async () => { + const authError = new AuthenticationError('openai', 'Authentication failed'); + + mockOpenAIProvider.generateResponse.mockRejectedValueOnce(authError); + + await expect(factory.generateResponse({ + ...testRequest, + model: 'gpt-3.5-turbo' + })).rejects.toThrow('Authentication failed'); + }); + }); + + describe('Configuration Updates', () => { + it('should update configuration', () => { + factory.updateConfig({ + defaultProvider: 'anthropic', + costOptimization: false + }); + + // Configuration should be updated + // (We'd need to expose config or test behavior changes) + expect(() => factory.updateConfig({})).not.toThrow(); + }); + + it('should reset metrics and circuit breakers', () => { + factory.reset(); + + expect(mockOpenAIProvider.resetMetrics).toHaveBeenCalled(); + expect(mockAnthropicProvider.resetMetrics).toHaveBeenCalled(); + expect(mockCloudflareProvider.resetMetrics).toHaveBeenCalled(); + }); + + it('should pass maxRetries 0 to providers when factory retries are disabled', () => { + new LLMProviderFactory({ + openai: { apiKey: 'test-key' }, + enableRetries: false + }); + + expect(OpenAIProvider).toHaveBeenCalledWith( + expect.objectContaining({ + apiKey: 'test-key', + maxRetries: 0 + }) + ); + }); + + it('should preserve explicit provider maxRetries when factory retries are disabled', () => { + new LLMProviderFactory({ + openai: { apiKey: 'test-key', maxRetries: 2 }, + enableRetries: false + }); + + expect(OpenAIProvider).toHaveBeenCalledWith( + expect.objectContaining({ + apiKey: 'test-key', + maxRetries: 2 + }) + ); + }); + }); + + describe('CreditLedger integration', () => { + it('should record successful factory calls into the provided ledger even without cost optimization', async () => { + const ledger = new CreditLedger({ + budgets: [{ + provider: 'cloudflare', + monthlyBudget: 1, + rateLimits: { rpm: 10, rpd: 100, tpm: 1000, tpd: 10_000 } + }] + }); + + const ledgerFactory = new LLMProviderFactory({ + cloudflare: { ai: {} as Ai }, + costOptimization: false, + ledger + }); + + await ledgerFactory.generateResponse(testRequest); + + const accumulator = ledger.getProviderAccumulator('cloudflare'); + expect(accumulator).toMatchObject({ + spend: 0.0001, + inputTokens: 10, + outputTokens: 20, + requestCount: 1 + }); + expect(accumulator!.rateLimits.rpm!.used).toBe(1); + expect(accumulator!.rateLimits.rpd!.used).toBe(1); + expect(accumulator!.rateLimits.tpm!.used).toBe(30); + expect(accumulator!.rateLimits.tpd!.used).toBe(30); + }); + + it('should expose provider balance from the configured ledger', async () => { + const ledger = new CreditLedger({ + budgets: [{ + provider: 'cloudflare', + monthlyBudget: 1, + rateLimits: { rpm: 10 } + }] + }); + const balanceFactory = new LLMProviderFactory({ + cloudflare: { ai: {} as Ai }, + ledger + }); + + await balanceFactory.generateResponse(testRequest); + const balance = await balanceFactory.getProviderBalance('cloudflare'); + + expect(balance).toMatchObject({ + provider: 'cloudflare', + status: 'available', + source: 'ledger', + currentSpend: 0.0001, + monthlyBudget: 1, + requestCount: 1 + }); + expect((balance as { rateLimits: Record }).rateLimits.rpm.used).toBe(1); + }); + }); +}); + +describe('Cost Optimized Factory', () => { + beforeEach(() => { + vi.clearAllMocks(); + // Restore default mock implementations after previous tests may have set mockRejectedValue + mockOpenAIProvider.generateResponse.mockResolvedValue({ + message: 'OpenAI response', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.001 }, + model: 'gpt-3.5-turbo', + provider: 'openai', + responseTime: 1000 + } as LLMResponse); + mockCloudflareProvider.generateResponse.mockResolvedValue({ + message: 'Cloudflare response', + usage: { inputTokens: 10, outputTokens: 20, totalTokens: 30, cost: 0.0001 }, + model: '@cf/meta/llama-3.1-8b-instruct', + provider: 'cloudflare', + responseTime: 800 + } as LLMResponse); + }); + + it('should create cost-optimized factory with correct configuration', () => { + const factory = createCostOptimizedFactory({ + openai: { apiKey: 'test-key' }, + cloudflare: { ai: {} as Ai } + }); + + expect(factory).toBeInstanceOf(LLMProviderFactory); + // Cost optimization should be enabled by default + }); + + it('should prioritize cost-effective providers', async () => { + const factory = createCostOptimizedFactory({ + openai: { apiKey: 'test-key' }, + cloudflare: { ai: {} as Ai } + }); + + const response = await factory.generateResponse({ + messages: [{ role: 'user', content: 'Test' }] + }); + + // Should successfully route to an available provider + expect(['openai', 'cloudflare']).toContain(response.provider); + }); +}); + +describe('Response Cache Adapter', () => { + const cacheRequest: LLMRequest = { + messages: [{ role: 'user', content: 'What is 2+2?' }], + maxTokens: 50, + temperature: 0, + }; + + it('returns a cached response without calling the provider', async () => { + const cacheEvents: string[] = []; + const cachedResponse = { + message: 'cached answer', + usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0 }, + model: 'gpt-3.5-turbo', + provider: 'openai', + responseTime: 1, + }; + + const cache = { + get: vi.fn().mockResolvedValue(JSON.stringify(cachedResponse)), + put: vi.fn().mockResolvedValue(undefined), + }; + + const cachedFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-key' }, + responseCache: cache, + hooks: { + onCache: event => cacheEvents.push(`${event.layer}:${event.status}`), + }, + }); + + const response = await cachedFactory.generateResponse(cacheRequest); + expect(response.message).toBe('cached answer'); + expect(response.cache?.factory).toMatchObject({ status: 'hit' }); + expect(response.metadata?.cache).toEqual(response.cache); + expect(cacheEvents).toEqual(['factory-response:hit']); + expect(mockOpenAIProvider.generateResponse).not.toHaveBeenCalled(); + }); + + it('calls the provider on cache miss and writes the response', async () => { + const cacheEvents: string[] = []; + const cache = { + get: vi.fn().mockResolvedValue(null), + put: vi.fn().mockResolvedValue(undefined), + }; + + const cachedFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-key' }, + responseCache: cache, + responseCacheDefaultTtl: 3600, + hooks: { + onCache: event => cacheEvents.push(`${event.layer}:${event.status}`), + }, + }); + + mockOpenAIProvider.generateResponse.mockReset().mockResolvedValue({ + message: 'fresh answer', + usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001, cachedInputTokens: 2 }, + model: 'gpt-3.5-turbo', + provider: 'openai', + responseTime: 100, + } as LLMResponse); + + const response = await cachedFactory.generateResponse(cacheRequest); + expect(response.message).toBe('fresh answer'); + expect(response.cache?.providerPrefix).toMatchObject({ cachedInputTokens: 2, hit: true }); + expect(response.cache?.factory).toMatchObject({ status: 'miss', ttlSeconds: 3600 }); + expect(response.metadata?.cache).toEqual(response.cache); + expect(cacheEvents).toEqual(['provider-prefix:hit', 'factory-response:miss']); + expect(cache.get).toHaveBeenCalledOnce(); + // put is called async (fire-and-forget), so wait a tick + await new Promise(r => setTimeout(r, 0)); + expect(cache.put).toHaveBeenCalledOnce(); + expect(cache.put).toHaveBeenCalledWith( + expect.any(String), + expect.stringContaining('fresh answer'), + 3600, + ); + }); + + it('proceeds normally when cache.get throws', async () => { + const cache = { + get: vi.fn().mockRejectedValue(new Error('KV unavailable')), + put: vi.fn().mockResolvedValue(undefined), + }; + + const cachedFactory = new LLMProviderFactory({ + openai: { apiKey: 'test-key' }, + responseCache: cache, + }); + + mockOpenAIProvider.generateResponse.mockReset().mockResolvedValue({ + message: 'fallback answer', + usage: { inputTokens: 5, outputTokens: 5, totalTokens: 10, cost: 0.001 }, + model: 'gpt-3.5-turbo', + provider: 'openai', + responseTime: 100, + } as LLMResponse); + + const response = await cachedFactory.generateResponse(cacheRequest); + expect(response.message).toBe('fallback answer'); + expect(mockOpenAIProvider.generateResponse).toHaveBeenCalled(); + }); +}); + +// Custom matcher for vitest +expect.extend({ + toBeOneOf(received: any, expected: any[]) { + const pass = expected.includes(received); + return { + pass, + message: () => `expected ${received} to be one of ${expected.join(', ')}` + }; + } +}); From bdf9cdbc2f79c1883a487dc2363fa8e324cc241d Mon Sep 17 00:00:00 2001 From: Kurt Overmier Date: Thu, 15 Oct 2026 20:11:15 -0500 Subject: [PATCH 2/3] docs: drop the factory.test.ts preservation step from the checkpoint Co-Authored-By: Claude Opus 5.5 --- docs/maintenance/issue-sweep.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/maintenance/issue-sweep.md b/docs/maintenance/issue-sweep.md index dfa6faf..c3520f9 100644 --- a/docs/maintenance/issue-sweep.md +++ b/docs/maintenance/issue-sweep.md @@ -6,7 +6,7 @@ Checkpoint: 2026-10-15. Repository: `Stackbilt-dev/llm-providers`. The sweep below is **merged to `main`** and was released as 1.22.0 by release PR [#121](https://github.com/Stackbilt-dev/llm-providers/pull/121) and published to npm; 1.22.1 (#122) and 1.23.0 (#123) followed. Sweep PRs: [#119](https://github.com/Stackbilt-dev/llm-providers/pull/119) (sweep plus review fixes) and [#120](https://github.com/Stackbilt-dev/llm-providers/pull/120) (#116 hook isolation). Each went through adversarial review rounds; review-driven compatibility fixes are recorded in the 1.22.0 CHANGELOG entry. The issues stay open for the remaining scope in the table. No live or billable vendor requests were made. -1. Inspect `git status --short`. Preserve the pre-existing `src/__tests__/factory.test.ts` modification; its diff is line-ending-only when checked with `--ignore-space-at-eol`. +1. Inspect `git status --short` and preserve any existing working-tree changes. (The long-standing line-ending-only `src/__tests__/factory.test.ts` diff is gone: its blob was normalized to LF. `src/errors.ts`, `src/index.ts` and `src/types.ts` are still CRLF blobs, kept clean locally by `-text` entries in `.git/info/attributes`.) 2. #112 phase 1 shipped in 1.22.1; phase 2 (current models, Workers AI deprecations, reasoning-model request parameters, `explicitOnly`) shipped in 1.23.0 via [#123](https://github.com/Stackbilt-dev/llm-providers/pull/123). Next: #124 (Anthropic `tool_use_id`, tool error text and Claude 5.x thinking blocks in tool loops; prerequisite for routing the 5.x models), then #109 atomic spend reservations (needs a design pass first). 3. Run the validation commands below after substantive code changes, then open reviewable PRs. From 332bb26225f66c0e19d2f6b9b37ae05f8d22a853 Mon Sep 17 00:00:00 2001 From: Kurt Overmier Date: Thu, 15 Oct 2026 20:15:45 -0500 Subject: [PATCH 3/3] chore: normalize errors.ts, index.ts and types.ts to LF These were the last tracked blobs stored with CRLF, despite `*.ts text eol=lf` in .gitattributes, so every clone without a local .git/info/attributes override showed them as modified. Same content with LF endings (`git diff --ignore-cr-at-eol` is empty); modes unchanged. The checkpoint's step 1 now says the repository is LF-only, that a line-ending-only diff should be restored rather than preserved, and that no local override is needed. Co-Authored-By: Claude Opus 5.5 --- docs/maintenance/issue-sweep.md | 4 +- src/errors.ts | 652 +++++++-------- src/index.ts | 1264 ++++++++++++++--------------- src/types.ts | 1318 +++++++++++++++---------------- 4 files changed, 1619 insertions(+), 1619 deletions(-) diff --git a/docs/maintenance/issue-sweep.md b/docs/maintenance/issue-sweep.md index c3520f9..d621ac2 100644 --- a/docs/maintenance/issue-sweep.md +++ b/docs/maintenance/issue-sweep.md @@ -6,7 +6,7 @@ Checkpoint: 2026-10-15. Repository: `Stackbilt-dev/llm-providers`. The sweep below is **merged to `main`** and was released as 1.22.0 by release PR [#121](https://github.com/Stackbilt-dev/llm-providers/pull/121) and published to npm; 1.22.1 (#122) and 1.23.0 (#123) followed. Sweep PRs: [#119](https://github.com/Stackbilt-dev/llm-providers/pull/119) (sweep plus review fixes) and [#120](https://github.com/Stackbilt-dev/llm-providers/pull/120) (#116 hook isolation). Each went through adversarial review rounds; review-driven compatibility fixes are recorded in the 1.22.0 CHANGELOG entry. The issues stay open for the remaining scope in the table. No live or billable vendor requests were made. -1. Inspect `git status --short` and preserve any existing working-tree changes. (The long-standing line-ending-only `src/__tests__/factory.test.ts` diff is gone: its blob was normalized to LF. `src/errors.ts`, `src/index.ts` and `src/types.ts` are still CRLF blobs, kept clean locally by `-text` entries in `.git/info/attributes`.) +1. Inspect `git status --short` and preserve any existing working-tree changes. Every tracked file is stored with LF endings, as `.gitattributes` requires, so a fresh checkout should be clean. If a file shows as modified but `git diff --ignore-cr-at-eol` is empty for it, the change is line endings only, not work: restore it with `git checkout -- ` instead of preserving it. No local `.git/info/attributes` override is needed. 2. #112 phase 1 shipped in 1.22.1; phase 2 (current models, Workers AI deprecations, reasoning-model request parameters, `explicitOnly`) shipped in 1.23.0 via [#123](https://github.com/Stackbilt-dev/llm-providers/pull/123). Next: #124 (Anthropic `tool_use_id`, tool error text and Claude 5.x thinking blocks in tool loops; prerequisite for routing the 5.x models), then #109 atomic spend reservations (needs a design pass first). 3. Run the validation commands below after substantive code changes, then open reviewable PRs. @@ -49,7 +49,7 @@ Main changes: [factory](../../src/factory.ts), [public exports](../../src/index. New utilities: [response cache identity/validation](../../src/utils/response-cache.ts), [provider SSE lifecycle](../../src/utils/provider-stream.ts), [resilience context](../../src/utils/resilience-context.ts). -New regression suites: [tenant safety](../../src/__tests__/tenant-safety.test.ts), [stream lifecycle](../../src/__tests__/stream-lifecycle.test.ts), [context ownership/recovery](../../src/__tests__/resilience-context.test.ts), [canonical identity/routing](../../src/__tests__/canonical-identity-routing.test.ts). Existing schema-drift tests now reset breaker history between cases; gateway diagnostic tests check failed-plan executability. The pre-existing factory test file was preserved. Later work added [hook isolation](../../src/__tests__/hook-isolation.test.ts) (#120, 1.22.0), [catalog accuracy](../../src/__tests__/catalog-accuracy.test.ts) (1.22.1) and [model refresh](../../src/__tests__/model-refresh.test.ts) (1.23.0). Test counts at 1.23.0: tenant safety 46, stream lifecycle 43, context ownership/recovery 14, canonical identity/routing 10, hook isolation 20, catalog accuracy 27, model refresh 125. +New regression suites: [tenant safety](../../src/__tests__/tenant-safety.test.ts), [stream lifecycle](../../src/__tests__/stream-lifecycle.test.ts), [context ownership/recovery](../../src/__tests__/resilience-context.test.ts), [canonical identity/routing](../../src/__tests__/canonical-identity-routing.test.ts). Existing schema-drift tests now reset breaker history between cases; gateway diagnostic tests check failed-plan executability. The sweep left the existing factory tests unchanged. Later work added [hook isolation](../../src/__tests__/hook-isolation.test.ts) (#120, 1.22.0), [catalog accuracy](../../src/__tests__/catalog-accuracy.test.ts) (1.22.1) and [model refresh](../../src/__tests__/model-refresh.test.ts) (1.23.0). Test counts at 1.23.0: tenant safety 46, stream lifecycle 43, context ownership/recovery 14, canonical identity/routing 10, hook isolation 20, catalog accuracy 27, model refresh 125. ## Validation evidence diff --git a/src/errors.ts b/src/errors.ts index a80f10e..fed4b4f 100755 --- a/src/errors.ts +++ b/src/errors.ts @@ -1,326 +1,326 @@ -/** - * LLM Provider Error Classes - * Structured error handling for all LLM providers - */ - -import type { LLMError } from './types.js'; - -export class LLMProviderError extends Error implements LLMError { - code: string; - provider: string; - retryable: boolean; - statusCode?: number; - rateLimited?: boolean; - quotaExceeded?: boolean; - - constructor( - message: string, - code: string, - provider: string, - retryable: boolean = false, - statusCode?: number - ) { - super(message); - this.name = 'LLMProviderError'; - this.code = code; - this.provider = provider; - this.retryable = retryable; - this.statusCode = statusCode; - } -} - -export class RateLimitError extends LLMProviderError { - constructor(provider: string, message: string = 'Rate limit exceeded') { - super(message, 'RATE_LIMIT', provider, true, 429); - this.rateLimited = true; - } -} - -export class QuotaExceededError extends LLMProviderError { - constructor(provider: string, message: string = 'Quota exceeded') { - super(message, 'QUOTA_EXCEEDED', provider, false, 429); - this.quotaExceeded = true; - } -} - -/** - * Local policy denial; never retry, fail over, or quarantine a provider. - * Extends QuotaExceededError (same code/status) so existing quota-hook denial - * checks keep matching; use `instanceof QuotaDeniedError` or `localPolicy` to - * distinguish it from vendor account exhaustion. - */ -export class QuotaDeniedError extends QuotaExceededError { - readonly localPolicy = true; - - constructor(provider: string, message: string = 'Local quota policy denied request') { - super(provider, message); - } -} - -export class AuthenticationError extends LLMProviderError { - constructor(provider: string, message: string = 'Authentication failed') { - super(message, 'AUTHENTICATION_ERROR', provider, false, 401); - } -} - -export class InvalidRequestError extends LLMProviderError { - constructor(provider: string, message: string = 'Invalid request') { - super(message, 'INVALID_REQUEST', provider, false, 400); - } -} - -export class ModelNotFoundError extends LLMProviderError { - constructor(provider: string, model: string) { - super(`Model '${model}' not found`, 'MODEL_NOT_FOUND', provider, false, 404); - } -} - -export class TimeoutError extends LLMProviderError { - constructor(provider: string, message: string = 'Request timeout') { - super(message, 'TIMEOUT', provider, true, 408); - } -} - -export class NetworkError extends LLMProviderError { - constructor(provider: string, message: string = 'Network error') { - super(message, 'NETWORK_ERROR', provider, true, 503); - } -} - -export class ServerError extends LLMProviderError { - constructor(provider: string, message: string = 'Internal server error', statusCode: number = 500) { - super(message, 'SERVER_ERROR', provider, true, statusCode); - } -} - -export class ContentFilterError extends LLMProviderError { - constructor(provider: string, message: string = 'Content filtered') { - super(message, 'CONTENT_FILTER', provider, false, 400); - } -} - -export class TokenLimitError extends LLMProviderError { - constructor(provider: string, message: string = 'Token limit exceeded') { - super(message, 'TOKEN_LIMIT', provider, false, 400); - } -} - -export class ConfigurationError extends LLMProviderError { - constructor(provider: string, message: string) { - super(message, 'CONFIGURATION_ERROR', provider, false); - } -} - -export class NoRouteError extends ConfigurationError { - constructor(message: string = 'No configured provider can serve this request') { - super('factory', message); - this.code = 'NO_ROUTE'; - } -} - -export class CfGatewayInvalidConfigError extends LLMProviderError { - /** The specific cfGateway field that was empty or missing. */ - field: 'accountId' | 'gatewayId'; - - constructor(provider: string, field: 'accountId' | 'gatewayId') { - super( - `cfGateway.${field} must be a non-empty string for the ${provider} provider`, - 'CF_GATEWAY_INVALID_CONFIG', - provider, - false - ); - this.field = field; - } -} - -export class CircuitBreakerOpenError extends LLMProviderError { - retryAfterSec: number; - consecutiveFailures: number; - - constructor(provider: string, retryAfterSec: number = 0, consecutiveFailures: number = 0) { - const retryHint = retryAfterSec > 0 ? ` Retry after ${retryAfterSec}s.` : ''; - super( - `Circuit breaker rejected request for provider: ${provider}.${retryHint}`, - 'CIRCUIT_BREAKER_OPEN', - provider, - true - ); - this.retryAfterSec = retryAfterSec; - this.consecutiveFailures = consecutiveFailures; - } -} - -export class ToolLoopLimitError extends LLMProviderError { - constructor(provider: string, message: string = 'Tool loop limit exceeded') { - super(message, 'TOOL_LOOP_LIMIT', provider, false, 400); - } -} - -export class ToolLoopAbortedError extends LLMProviderError { - constructor(provider: string, message: string = 'Tool loop aborted') { - super(message, 'TOOL_LOOP_ABORTED', provider, false, 400); - } -} - -/** - * Thrown when a provider's response envelope fails runtime schema validation. - * - * Indicates the upstream API silently changed shape - a field was renamed, - * removed, or had its type changed. Non-retryable: retrying hits the same - * broken shape. The factory treats this as fallback-eligible so traffic - * routes to a healthy provider while the drift is investigated. - */ -export class SchemaDriftError extends LLMProviderError { - path: string; - expected: string; - actual: string; - - constructor(provider: string, path: string, expected: string, actual: string) { - super( - `Response schema drift at ${path}: expected ${expected}, got ${actual}`, - 'SCHEMA_DRIFT', - provider, - false, - 422 - ); - this.path = path; - this.expected = expected; - this.actual = actual; - } -} - -/** - * Error factory for creating provider-specific errors from HTTP responses - */ -/** Shape of error response bodies across all LLM providers. */ -interface ErrorResponseBody { - message?: string; - error?: { - message?: string; - code?: string; - type?: string; - param?: string; - }; -} - -export class LLMErrorFactory { - static fromHttpResponse( - provider: string, - statusCode: number, - responseBody?: ErrorResponseBody, - message?: string - ): LLMProviderError { - const defaultMessage = message || responseBody?.message || responseBody?.error?.message || 'Unknown error'; - - switch (statusCode) { - case 400: - if (responseBody?.error?.code === 'content_filter') { - return new ContentFilterError(provider, defaultMessage); - } - if (responseBody?.error?.code === 'token_limit_exceeded') { - return new TokenLimitError(provider, defaultMessage); - } - return new InvalidRequestError(provider, defaultMessage); - - case 401: - return new AuthenticationError(provider, defaultMessage); - - case 404: { - const model = responseBody?.error?.param || 'unknown'; - return new ModelNotFoundError(provider, model); - } - - case 408: - return new TimeoutError(provider, defaultMessage); - - case 429: - if (responseBody?.error?.type === 'quota_exceeded') { - return new QuotaExceededError(provider, defaultMessage); - } - return new RateLimitError(provider, defaultMessage); - - case 500: - case 502: - case 503: - case 504: - return new ServerError(provider, defaultMessage, statusCode); - - default: - return new LLMProviderError( - defaultMessage, - 'UNKNOWN_ERROR', - provider, - statusCode >= 500, - statusCode - ); - } - } - - /** - * Create error from fetch response - */ - static async fromFetchResponse( - provider: string, - response: Response - ): Promise { - let responseBody: ErrorResponseBody; - - try { - const contentType = response.headers.get('content-type'); - if (contentType?.includes('application/json')) { - responseBody = await response.json() as ErrorResponseBody; - } else { - responseBody = { message: await response.text() }; - } - } catch { - responseBody = { message: 'Failed to parse error response' }; - } - - return this.fromHttpResponse(provider, response.status, responseBody); - } - - /** - * Check if error is retryable - */ - static isRetryable(error: Error): boolean { - if (error instanceof LLMProviderError) { - return error.retryable; - } - - // Network errors are generally retryable - if (error.message.includes('fetch')) { - return true; - } - - return false; - } - - /** - * Check if error indicates rate limiting - */ - static isRateLimited(error: Error): boolean { - return error instanceof RateLimitError; - } - - /** - * Check if error indicates quota exceeded - */ - static isQuotaExceeded(error: Error): boolean { - return error instanceof QuotaExceededError; - } - - /** - * Get retry delay from rate limit error - */ - static getRetryDelay(error: Error): number { - if (error instanceof RateLimitError) { - // Default exponential backoff starting at 1 second - return 1000; - } - if (error instanceof ServerError) { - // Server errors get shorter delays - return 500; - } - return 0; - } -} +/** + * LLM Provider Error Classes + * Structured error handling for all LLM providers + */ + +import type { LLMError } from './types.js'; + +export class LLMProviderError extends Error implements LLMError { + code: string; + provider: string; + retryable: boolean; + statusCode?: number; + rateLimited?: boolean; + quotaExceeded?: boolean; + + constructor( + message: string, + code: string, + provider: string, + retryable: boolean = false, + statusCode?: number + ) { + super(message); + this.name = 'LLMProviderError'; + this.code = code; + this.provider = provider; + this.retryable = retryable; + this.statusCode = statusCode; + } +} + +export class RateLimitError extends LLMProviderError { + constructor(provider: string, message: string = 'Rate limit exceeded') { + super(message, 'RATE_LIMIT', provider, true, 429); + this.rateLimited = true; + } +} + +export class QuotaExceededError extends LLMProviderError { + constructor(provider: string, message: string = 'Quota exceeded') { + super(message, 'QUOTA_EXCEEDED', provider, false, 429); + this.quotaExceeded = true; + } +} + +/** + * Local policy denial; never retry, fail over, or quarantine a provider. + * Extends QuotaExceededError (same code/status) so existing quota-hook denial + * checks keep matching; use `instanceof QuotaDeniedError` or `localPolicy` to + * distinguish it from vendor account exhaustion. + */ +export class QuotaDeniedError extends QuotaExceededError { + readonly localPolicy = true; + + constructor(provider: string, message: string = 'Local quota policy denied request') { + super(provider, message); + } +} + +export class AuthenticationError extends LLMProviderError { + constructor(provider: string, message: string = 'Authentication failed') { + super(message, 'AUTHENTICATION_ERROR', provider, false, 401); + } +} + +export class InvalidRequestError extends LLMProviderError { + constructor(provider: string, message: string = 'Invalid request') { + super(message, 'INVALID_REQUEST', provider, false, 400); + } +} + +export class ModelNotFoundError extends LLMProviderError { + constructor(provider: string, model: string) { + super(`Model '${model}' not found`, 'MODEL_NOT_FOUND', provider, false, 404); + } +} + +export class TimeoutError extends LLMProviderError { + constructor(provider: string, message: string = 'Request timeout') { + super(message, 'TIMEOUT', provider, true, 408); + } +} + +export class NetworkError extends LLMProviderError { + constructor(provider: string, message: string = 'Network error') { + super(message, 'NETWORK_ERROR', provider, true, 503); + } +} + +export class ServerError extends LLMProviderError { + constructor(provider: string, message: string = 'Internal server error', statusCode: number = 500) { + super(message, 'SERVER_ERROR', provider, true, statusCode); + } +} + +export class ContentFilterError extends LLMProviderError { + constructor(provider: string, message: string = 'Content filtered') { + super(message, 'CONTENT_FILTER', provider, false, 400); + } +} + +export class TokenLimitError extends LLMProviderError { + constructor(provider: string, message: string = 'Token limit exceeded') { + super(message, 'TOKEN_LIMIT', provider, false, 400); + } +} + +export class ConfigurationError extends LLMProviderError { + constructor(provider: string, message: string) { + super(message, 'CONFIGURATION_ERROR', provider, false); + } +} + +export class NoRouteError extends ConfigurationError { + constructor(message: string = 'No configured provider can serve this request') { + super('factory', message); + this.code = 'NO_ROUTE'; + } +} + +export class CfGatewayInvalidConfigError extends LLMProviderError { + /** The specific cfGateway field that was empty or missing. */ + field: 'accountId' | 'gatewayId'; + + constructor(provider: string, field: 'accountId' | 'gatewayId') { + super( + `cfGateway.${field} must be a non-empty string for the ${provider} provider`, + 'CF_GATEWAY_INVALID_CONFIG', + provider, + false + ); + this.field = field; + } +} + +export class CircuitBreakerOpenError extends LLMProviderError { + retryAfterSec: number; + consecutiveFailures: number; + + constructor(provider: string, retryAfterSec: number = 0, consecutiveFailures: number = 0) { + const retryHint = retryAfterSec > 0 ? ` Retry after ${retryAfterSec}s.` : ''; + super( + `Circuit breaker rejected request for provider: ${provider}.${retryHint}`, + 'CIRCUIT_BREAKER_OPEN', + provider, + true + ); + this.retryAfterSec = retryAfterSec; + this.consecutiveFailures = consecutiveFailures; + } +} + +export class ToolLoopLimitError extends LLMProviderError { + constructor(provider: string, message: string = 'Tool loop limit exceeded') { + super(message, 'TOOL_LOOP_LIMIT', provider, false, 400); + } +} + +export class ToolLoopAbortedError extends LLMProviderError { + constructor(provider: string, message: string = 'Tool loop aborted') { + super(message, 'TOOL_LOOP_ABORTED', provider, false, 400); + } +} + +/** + * Thrown when a provider's response envelope fails runtime schema validation. + * + * Indicates the upstream API silently changed shape - a field was renamed, + * removed, or had its type changed. Non-retryable: retrying hits the same + * broken shape. The factory treats this as fallback-eligible so traffic + * routes to a healthy provider while the drift is investigated. + */ +export class SchemaDriftError extends LLMProviderError { + path: string; + expected: string; + actual: string; + + constructor(provider: string, path: string, expected: string, actual: string) { + super( + `Response schema drift at ${path}: expected ${expected}, got ${actual}`, + 'SCHEMA_DRIFT', + provider, + false, + 422 + ); + this.path = path; + this.expected = expected; + this.actual = actual; + } +} + +/** + * Error factory for creating provider-specific errors from HTTP responses + */ +/** Shape of error response bodies across all LLM providers. */ +interface ErrorResponseBody { + message?: string; + error?: { + message?: string; + code?: string; + type?: string; + param?: string; + }; +} + +export class LLMErrorFactory { + static fromHttpResponse( + provider: string, + statusCode: number, + responseBody?: ErrorResponseBody, + message?: string + ): LLMProviderError { + const defaultMessage = message || responseBody?.message || responseBody?.error?.message || 'Unknown error'; + + switch (statusCode) { + case 400: + if (responseBody?.error?.code === 'content_filter') { + return new ContentFilterError(provider, defaultMessage); + } + if (responseBody?.error?.code === 'token_limit_exceeded') { + return new TokenLimitError(provider, defaultMessage); + } + return new InvalidRequestError(provider, defaultMessage); + + case 401: + return new AuthenticationError(provider, defaultMessage); + + case 404: { + const model = responseBody?.error?.param || 'unknown'; + return new ModelNotFoundError(provider, model); + } + + case 408: + return new TimeoutError(provider, defaultMessage); + + case 429: + if (responseBody?.error?.type === 'quota_exceeded') { + return new QuotaExceededError(provider, defaultMessage); + } + return new RateLimitError(provider, defaultMessage); + + case 500: + case 502: + case 503: + case 504: + return new ServerError(provider, defaultMessage, statusCode); + + default: + return new LLMProviderError( + defaultMessage, + 'UNKNOWN_ERROR', + provider, + statusCode >= 500, + statusCode + ); + } + } + + /** + * Create error from fetch response + */ + static async fromFetchResponse( + provider: string, + response: Response + ): Promise { + let responseBody: ErrorResponseBody; + + try { + const contentType = response.headers.get('content-type'); + if (contentType?.includes('application/json')) { + responseBody = await response.json() as ErrorResponseBody; + } else { + responseBody = { message: await response.text() }; + } + } catch { + responseBody = { message: 'Failed to parse error response' }; + } + + return this.fromHttpResponse(provider, response.status, responseBody); + } + + /** + * Check if error is retryable + */ + static isRetryable(error: Error): boolean { + if (error instanceof LLMProviderError) { + return error.retryable; + } + + // Network errors are generally retryable + if (error.message.includes('fetch')) { + return true; + } + + return false; + } + + /** + * Check if error indicates rate limiting + */ + static isRateLimited(error: Error): boolean { + return error instanceof RateLimitError; + } + + /** + * Check if error indicates quota exceeded + */ + static isQuotaExceeded(error: Error): boolean { + return error instanceof QuotaExceededError; + } + + /** + * Get retry delay from rate limit error + */ + static getRetryDelay(error: Error): number { + if (error instanceof RateLimitError) { + // Default exponential backoff starting at 1 second + return 1000; + } + if (error instanceof ServerError) { + // Server errors get shorter delays + return 500; + } + return 0; + } +} diff --git a/src/index.ts b/src/index.ts index eee4062..57914bb 100755 --- a/src/index.ts +++ b/src/index.ts @@ -1,632 +1,632 @@ -/** - * LLM Providers Package - * Unified interface for OpenAI, Anthropic, and Cloudflare AI providers - * with intelligent fallback logic and cost optimization - */ - -// Core types -export type { - LLMProvider, - LLMImageInput, - GatewayMetadata, - CfGatewayConfig, - CacheHints, - CacheObservability, - ResponseCacheAdapter, - LLMRequest, - LLMResponse, - LLMMessage, - LLMConfig, - TokenUsage, - CostProvenance, - TokenProvenance, - ProviderAttemptResult, - ProviderExecutionOptions, - Tool, - BuiltInTool, - BuiltInToolType, - BuiltInToolResult, - ToolCall, - ToolResult, - ToolChoice, - ProviderConfig, - OpenAIConfig, - AnthropicConfig, - CloudflareAIGatewayOptions, - CloudflareConfig, - CerebrasConfig, - GroqConfig, - NvidiaConfig, - ModelCapabilities, - ProviderCapabilities, - ProviderMetrics, - FallbackRule, - CircuitBreakerConfig, - CircuitBreakerState, - RetryConfig, - CostConfig, - LLMError, - StreamChunk, - StreamResponse, - StreamCompletion, - ManagedTextStream, - QuotaHook, - QuotaCheckInput, - QuotaCheckResult, - QuotaRecordInput, - ToolExecutor, - ToolLoopAbortSignal, - ToolLoopOptions, - ToolLoopState, - ClassifyOptions, - ClassifyResult, - AnalyzeImageInput, - ProviderBalance, - RateLimitBalance, - BatchRequest, - BatchResponse, - BatchJob -} from './types.js'; - -// Canonical provider contract -export { - canonicalToLLMRequest, - normalizeLLMRequest, - normalizeLegacyRequest, - normalizeCanonicalRequest, - normalizeLLMResponse, -} from './canonical.js'; -export type { - CanonicalDegradation, - CanonicalFallbackHop, - CanonicalLLMRequest, - CanonicalLLMResponse, - CanonicalMediaPart, - CanonicalMessage, - CanonicalOutputFormat, - CanonicalProviderOptions, - CanonicalRequestMetadata, - CanonicalRequirements, - CanonicalResponseError, - CanonicalRoutingMetadata, - CanonicalSamplingOptions, - CanonicalTool, - CanonicalToolMode, -} from './canonical.js'; - -// Gateway / Worker route planning -export { getGatewayRoutePlan } from './gateway-routing.js'; -export type { - GatewayRouteCachePlan, - GatewayRouteCapabilityReport, - GatewayRoutePlan, - GatewayRouteRequirements, -} from './gateway-routing.js'; - -// Provider implementations -export { BaseProvider } from './providers/base.js'; -export { OpenAIProvider } from './providers/openai.js'; -export { AnthropicProvider } from './providers/anthropic.js'; -export { CloudflareProvider } from './providers/cloudflare.js'; -export { CerebrasProvider } from './providers/cerebras.js'; -export { GroqProvider } from './providers/groq.js'; -export { NvidiaProvider } from './providers/nvidia.js'; - -// Factory pattern -export { - LLMProviderFactory, - createLLMProviderFactory, - createCostOptimizedFactory -} from './factory.js'; -export type { ProviderFactoryConfig, CostAnalytics, ProviderHealthEntry } from './factory.js'; -export { - MODEL_CATALOG, - MODEL_RECOMMENDATIONS, - PROVIDER_FALLBACK_ORDER, - getCatalogEntry, - getProviderDefaultModel, - getProviderDefaultModelForWorkload, - getProviderForCatalogModel, - getProvidersForCatalogModel, - modelSupportsBuiltInTools, - getRecommendedModel, - getRecommendedModelForWorkload, - getRoutingInfo, - inferUseCaseFromRequest, - normalizeModelWorkload, - rankModels, -} from './model-catalog.js'; -export type { - ModelCatalogEntry, - ModelLifecycle, - ModelPreferenceMap, - ModelRecommendationUseCase, - ModelSelectionContext, - ModelWorkloadClass, - ProviderName, - RoutingInfo, -} from './model-catalog.js'; - -// Local imports for use within this file -import { LLMProviderFactory } from './factory.js'; -import type { ProviderFactoryConfig, CostAnalytics, ProviderHealthEntry } from './factory.js'; -import type { - AnalyzeImageInput, - ClassifyOptions, - ClassifyResult, - LLMProvider, - LLMRequest, - LLMResponse, - ProviderBalance, - ToolExecutor, - ToolLoopOptions -} from './types.js'; -import type { ModelRecommendationUseCase } from './model-catalog.js'; -import { ConfigurationError } from './errors.js'; - -// Error classes -export { - LLMProviderError, - RateLimitError, - QuotaExceededError, - QuotaDeniedError, - AuthenticationError, - InvalidRequestError, - ModelNotFoundError, - TimeoutError, - NetworkError, - ServerError, - ContentFilterError, - TokenLimitError, - ConfigurationError, - NoRouteError, - CfGatewayInvalidConfigError, - CircuitBreakerOpenError, - SchemaDriftError, - ToolLoopLimitError, - ToolLoopAbortedError, - LLMErrorFactory -} from './errors.js'; - -// Schema validator (for custom provider authors) -export { validateSchema } from './utils/schema-validator.js'; -export type { SchemaField, SchemaFieldType } from './utils/schema-validator.js'; - -// Schema drift canary (for integration test suites and cron canaries) -export { extractShape, compareShapes, runCanaryCheck } from './utils/schema-canary.js'; -export type { ShapeMap, CanaryDiff, CanaryReport } from './utils/schema-canary.js'; - -// Image generation -export { ImageProvider, normalizeAiResponse } from './image/index.js'; -export type { ImageProviderConfig, ImageRequest, ImageResponse, ImageModelConfig, ImageModelInputFormat } from './image/index.js'; -export { IMAGE_MODELS, getImageModel } from './image/index.js'; - -// Logger -export { noopLogger, consoleLogger } from './utils/logger.js'; -export type { Logger } from './utils/logger.js'; - -// Observability hooks -export { noopHooks, composeHooks } from './utils/hooks.js'; -export type { - ObservabilityHooks, - RequestStartEvent, - RequestEndEvent, - CacheEvent, - RequestErrorEvent, - RetryEvent, - FallbackEvent, - CircuitStateChangeEvent, - QuotaExhaustedEvent, - BudgetThresholdEvent, - QuotaCheckEvent, - QuotaDeniedEvent, - ProviderBalanceEvent, - SchemaDriftEvent, - ImageAnalysisMetadata, - SanitizedTelemetryError, - ImageAnalysisAttemptEvent, - ImageAnalysisCompleteEvent, -} from './utils/hooks.js'; - -// Exhaustion registry -export { ExhaustionRegistry, defaultExhaustionRegistry } from './utils/exhaustion.js'; -export type { ExhaustionEntry, ExhaustionRegistrySnapshot } from './utils/exhaustion.js'; - -// Latency histogram -export { LatencyHistogram, defaultLatencyHistogram } from './utils/latency-histogram.js'; -export { ResilienceContext } from './utils/resilience-context.js'; -export type { LatencySummary } from './utils/latency-histogram.js'; - -// Utility classes -export { RetryManager, defaultRetryManager, withRetry, retry } from './utils/retry.js'; -export { - CircuitBreaker, - CircuitBreakerManager, - defaultCircuitBreakerManager -} from './utils/circuit-breaker.js'; -export type { CircuitBreakerManagerSnapshot, CircuitBreakerSnapshot } from './utils/circuit-breaker.js'; -export { - CostTracker, - CostOptimizer, - defaultCostTracker -} from './utils/cost-tracker.js'; -export type { - ProviderCostEntry, - ProviderCostBreakdownEntry -} from './utils/cost-tracker.js'; -export { CreditLedger } from './utils/credit-ledger.js'; -export type { - CreditLedgerSnapshot, - LedgerEvent, - ThresholdEvent, - DepletionEvent, - LedgerListener, - BudgetConfig, - ThresholdConfig, - ThresholdTier, - DepletionTier, - ProviderAccumulator, - ModelAccumulator, - RateLimitDimension, - RateLimitCheck, - SpendEntry, - BurnRate, - DepletionEstimate, - SpendSummary, -} from './utils/credit-ledger.js'; - -/** - * Overrides for `LLMProviders.fromEnv()` auto-discovery. - */ -export interface FromEnvOverrides { - stateScope?: ProviderFactoryConfig['stateScope']; - resilienceContext?: ProviderFactoryConfig['resilienceContext']; - defaultProvider?: ProviderFactoryConfig['defaultProvider']; - costOptimization?: boolean; - enableCircuitBreaker?: boolean; - enableRetries?: boolean; - fallbackRules?: ProviderFactoryConfig['fallbackRules']; - ledger?: ProviderFactoryConfig['ledger']; - quotaHook?: ProviderFactoryConfig['quotaHook']; - quotaFailPolicy?: ProviderFactoryConfig['quotaFailPolicy']; - hooks?: ProviderFactoryConfig['hooks']; -} - -/** - * Main LLMProviders class for easy usage - */ -export class LLMProviders { - private factory: LLMProviderFactory; - - constructor(config: ProviderFactoryConfig) { - this.factory = new LLMProviderFactory(config); - } - - /** - * Auto-discover providers from a Cloudflare Worker `env` object. - * - * Scans for known API-key environment variables and bindings, configures - * only the providers whose keys are present, and returns a ready-to-use - * `LLMProviders` instance. - * - * @throws ConfigurationError if no providers are detected - */ - static fromEnv( - env: Record, - overrides: FromEnvOverrides = {} - ): LLMProviders { - const config: ProviderFactoryConfig = {}; - config.stateScope = overrides.stateScope; - config.resilienceContext = overrides.resilienceContext; - - if (typeof env.ANTHROPIC_API_KEY === 'string' && env.ANTHROPIC_API_KEY) { - config.anthropic = { apiKey: env.ANTHROPIC_API_KEY }; - } - if (typeof env.OPENAI_API_KEY === 'string' && env.OPENAI_API_KEY) { - config.openai = { apiKey: env.OPENAI_API_KEY }; - } - if (typeof env.GROQ_API_KEY === 'string' && env.GROQ_API_KEY) { - config.groq = { apiKey: env.GROQ_API_KEY }; - } - if (typeof env.CEREBRAS_API_KEY === 'string' && env.CEREBRAS_API_KEY) { - config.cerebras = { apiKey: env.CEREBRAS_API_KEY }; - } - if (typeof env.NVIDIA_API_KEY === 'string' && env.NVIDIA_API_KEY) { - config.nvidia = { apiKey: env.NVIDIA_API_KEY }; - } - if (env.AI != null && typeof env.AI === 'object') { - config.cloudflare = { ai: env.AI as Ai }; - } - - const detected = [ - config.anthropic && 'anthropic', - config.openai && 'openai', - config.groq && 'groq', - config.cerebras && 'cerebras', - config.nvidia && 'nvidia', - config.cloudflare && 'cloudflare', - ].filter(Boolean) as string[]; - - if (detected.length === 0) { - throw new ConfigurationError( - 'fromEnv', - 'No LLM providers detected in env. Expected at least one of: ' + - 'ANTHROPIC_API_KEY, OPENAI_API_KEY, GROQ_API_KEY, CEREBRAS_API_KEY, NVIDIA_API_KEY, or AI binding.' - ); - } - - if (overrides.defaultProvider !== undefined) { - config.defaultProvider = overrides.defaultProvider; - } - if (overrides.costOptimization !== undefined) { - config.costOptimization = overrides.costOptimization; - } - if (overrides.enableCircuitBreaker !== undefined) { - config.enableCircuitBreaker = overrides.enableCircuitBreaker; - } - if (overrides.enableRetries !== undefined) { - config.enableRetries = overrides.enableRetries; - } - if (overrides.fallbackRules !== undefined) { - config.fallbackRules = overrides.fallbackRules; - } - if (overrides.ledger !== undefined) { - config.ledger = overrides.ledger; - } - if (overrides.quotaHook !== undefined) { - config.quotaHook = overrides.quotaHook; - } - if (overrides.quotaFailPolicy !== undefined) { - config.quotaFailPolicy = overrides.quotaFailPolicy; - } - if (overrides.hooks !== undefined) { - config.hooks = overrides.hooks; - } - - return new LLMProviders(config); - } - - /** - * Generate response with automatic provider selection and fallback - */ - async generateResponse(request: LLMRequest): Promise { - return this.factory.generateResponse(request); - } - - async generateResponseStream(request: LLMRequest): Promise> { - return this.factory.generateResponseStream(request); - } - - async generateManagedResponseStream(request: LLMRequest): Promise { - return this.factory.generateManagedResponseStream(request); - } - - getRoutingInfo(request: LLMRequest): import('./model-catalog.js').RoutingInfo { - return this.factory.getRoutingInfo(request); - } - - async generateResponseWithTools( - request: LLMRequest, - toolExecutor: ToolExecutor, - opts?: ToolLoopOptions - ): Promise { - return this.factory.generateResponseWithTools(request, toolExecutor, opts); - } - - async classify( - input: string | LLMRequest, - options?: ClassifyOptions - ): Promise> { - return this.factory.classify(input, options); - } - - async analyzeImage(input: AnalyzeImageInput): Promise { - return this.factory.analyzeImage(input); - } - - async getProviderBalance(provider?: string): Promise> { - return this.factory.getProviderBalance(provider); - } - - getRecommendedModel( - request: LLMRequest, - useCase?: ModelRecommendationUseCase - ): string { - return this.factory.getRecommendedModel(request, useCase); - } - - /** - * Get specific provider instance - */ - getProvider(name: string): LLMProvider | undefined { - return this.factory.getProvider(name); - } - - /** - * Get all available providers - */ - getAvailableProviders(): string[] { - return this.factory.getAvailableProviders(); - } - - /** - * Get provider health status - */ - async getHealth(): Promise> { - return this.factory.getProviderHealth(); - } - - /** - * Get cost analytics - */ - getCostAnalytics(): CostAnalytics { - return this.factory.getCostAnalytics(); - } - - /** - * Get latency histogram summaries for all providers - */ - getLatencyHistogram(): Record { - return this.factory.getLatencyHistogram(); - } - - /** - * Get currently quota-exhausted providers - */ - getExhaustedProviders(): string[] { - return this.factory.getExhaustedProviders(); - } - - /** - * Reset all metrics and circuit breakers - */ - reset(): void { - this.factory.reset(); - } - - /** - * Update configuration - */ - updateConfig(config: Partial): void { - this.factory.updateConfig(config); - } - -} - -/** - * Create LLMProviders instance with cost optimization - */ -export function createCostOptimizedLLMProviders( - config: ProviderFactoryConfig -): LLMProviders { - return new LLMProviders({ - ...config, - defaultProvider: 'auto', - costOptimization: true, - enableCircuitBreaker: true, - enableRetries: true, - fallbackRules: config.fallbackRules ?? [ - { condition: 'cost', threshold: 5, fallbackProvider: 'cloudflare' }, - { condition: 'rate_limit', fallbackProvider: 'cloudflare' }, - { condition: 'error', fallbackProvider: 'anthropic' } - ] - }); -} - -/** - * Create LLMProviders instance with basic configuration - */ -export function createLLMProviders(config: ProviderFactoryConfig): LLMProviders { - return new LLMProviders(config); -} - -/** - * Default export for convenience - */ -export default LLMProviders; - -/** - * Version and metadata - */ -export { VERSION } from './version.js'; -export const SUPPORTED_PROVIDERS = ['openai', 'anthropic', 'cloudflare', 'cerebras', 'groq', 'nvidia'] as const; - -/** - * Common model mappings for easy reference - */ -export const MODELS = { - // OpenAI models - GPT_6_ASTRA: 'gpt-6-astra', - GPT_5_6_SOL: 'gpt-5.6-sol', - GPT_5_6_TERRA: 'gpt-5.6-terra', - GPT_5_6_LUNA: 'gpt-5.6-luna', - GPT_5_5: 'gpt-5.5', - /** Not accepted by the OpenAI adapter or routed by this package. OpenAI has announced shutdowns only for dated gpt-4o snapshots. */ - GPT_4O: 'gpt-4o', - /** @deprecated OpenAI shuts this model down on 2027-01-14 and names gpt-6-luna as the replacement (not yet supported by this adapter). */ - GPT_4O_MINI: 'gpt-4o-mini', - /** @deprecated OpenAI shuts this model down on 2026-10-23. Use GPT_5_6_SOL. */ - GPT_4_TURBO: 'gpt-4-turbo', - /** @deprecated OpenAI shuts this model down on 2026-10-23. Use GPT_5_6_SOL. */ - GPT_4: 'gpt-4', - /** @deprecated OpenAI shuts this model down on 2026-10-23. Use GPT_5_6_TERRA. */ - GPT_3_5_TURBO: 'gpt-3.5-turbo', - - // Anthropic models - // Claude model IDs are dateless from the 4.6 generation on. - CLAUDE_FABLE_5_1: 'claude-fable-5-1', - CLAUDE_OPUS_5_5: 'claude-opus-5-5', - CLAUDE_SONNET_5_5: 'claude-sonnet-5-5', - CLAUDE_HAIKU_5_5: 'claude-haiku-5-5', - /** Legacy on the Claude API; retirement not sooner than 2027-02-05. */ - CLAUDE_OPUS_4_6: 'claude-opus-4-6', - /** Legacy on the Claude API; retirement not sooner than 2027-02-17. */ - CLAUDE_SONNET_4_6: 'claude-sonnet-4-6', - /** @deprecated Retired on the Claude API 2026-06-15. */ - CLAUDE_OPUS_4: 'claude-opus-4-20250514', - /** @deprecated Retired on the Claude API 2026-06-15. */ - CLAUDE_SONNET_4: 'claude-sonnet-4-20250514', - /** Legacy on the Claude API; retirement not sooner than 2026-10-15. */ - CLAUDE_HAIKU_4_5: 'claude-haiku-4-5-20251001', - /** @deprecated Retired on the Claude API 2026-02-19. */ - CLAUDE_3_7_SONNET: 'claude-3-7-sonnet-20250219', - /** @deprecated Retired on the Claude API 2025-10-28. */ - CLAUDE_3_5_SONNET: 'claude-3-5-sonnet-20241022', - /** @deprecated Retired on the Claude API 2026-02-19. */ - CLAUDE_3_5_HAIKU: 'claude-3-5-haiku-20241022', - /** @deprecated Retired on the Claude API 2026-01-05. */ - CLAUDE_3_OPUS: 'claude-3-opus-20240229', - /** @deprecated Retired on the Claude API 2025-07-21. */ - CLAUDE_3_SONNET: 'claude-3-sonnet-20240229', - /** @deprecated Retired on the Claude API 2026-04-20; not accepted by AnthropicProvider. Use CLAUDE_HAIKU_4_5. */ - CLAUDE_3_HAIKU: 'claude-3-haiku-20240307', - - // Cloudflare Workers AI models - CF_GLM_5_3: '@cf/zai-org/glm-5.3', - CF_GLM_5_3_FLASH: '@cf/zai-org/glm-5.3-flash', - CF_KIMI_K2_6: '@cf/moonshotai/kimi-k2.6', - CF_DEEPSEEK_V4_PRO: '@cf/deepseek-ai/deepseek-v4-pro-0813', - CF_DEEPSEEK_V4_FLASH: '@cf/deepseek-ai/deepseek-v4-flash-0731', - CF_GPT_OSS_120B: '@cf/openai/gpt-oss-120b', - CF_GPT_OSS_20B: '@cf/openai/gpt-oss-20b', - CF_GEMMA_4_26B: '@cf/google/gemma-4-26b-a4b-it', - CF_QWEN_3_8_27B: '@cf/qwen/qwen3.8-27b', - CF_LLAMA_3_3_70B_FAST: '@cf/meta/llama-3.3-70b-instruct-fp8-fast', - CF_LLAMA_3_1_8B_FP8: '@cf/meta/llama-3.1-8b-instruct-fp8', - CF_GRANITE_4_MICRO: '@cf/ibm-granite/granite-4.0-h-micro', - /** @deprecated Deprecated on Workers AI 2026-05-30. Use CF_LLAMA_3_1_8B_FP8. */ - LLAMA_3_1_8B: '@cf/meta/llama-3.1-8b-instruct', - /** @deprecated Deprecated on Workers AI 2026-05-30. Use CF_LLAMA_3_3_70B_FAST. */ - LLAMA_3_1_70B: '@cf/meta/llama-3.1-70b-instruct', - /** @deprecated Deprecated on Workers AI 2026-05-30. Use CF_LLAMA_3_1_8B_FP8. */ - LLAMA_3_8B: '@cf/meta/llama-3-8b-instruct', - /** @deprecated Deprecated on Workers AI 2026-05-30. */ - MISTRAL_7B: '@cf/mistral/mistral-7b-instruct-v0.1', - /** @deprecated Deprecated on Workers AI 2025-10-01. */ - TINY_LLAMA: '@cf/tinyllama/tinyllama-1.1b-chat-v1.0', - - // Cerebras models - CEREBRAS_LLAMA_3_1_8B: 'llama-3.1-8b', - CEREBRAS_LLAMA_3_3_70B: 'llama-3.3-70b', - CEREBRAS_ZAI_GLM_4_7: 'zai-glm-4.7', - CEREBRAS_QWEN_3_235B: 'qwen-3-235b-a22b-instruct-2507', - - // Groq models - GROQ_LLAMA_3_3_70B_VERSATILE: 'llama-3.3-70b-versatile', - GROQ_LLAMA_3_1_8B_INSTANT: 'llama-3.1-8b-instant', - GROQ_GPT_OSS_120B: 'openai/gpt-oss-120b', - GROQ_COMPOUND: 'groq/compound', - GROQ_COMPOUND_MINI: 'groq/compound-mini', - - // NVIDIA NIM models - NVIDIA_LLAMA_3_3_70B: 'meta/llama-3.3-70b-instruct', - NVIDIA_LLAMA_4_MAVERICK: 'meta/llama-4-maverick-17b-128e-instruct', - NVIDIA_NEMOTRON_70B: 'nvidia/llama-3.1-nemotron-70b-instruct', - NVIDIA_NEMOTRON_SUPER_49B: 'nvidia/llama-3.3-nemotron-super-49b-v1', - NVIDIA_NEMOTRON_ULTRA_253B: 'nvidia/llama-3.1-nemotron-ultra-253b-v1', - NVIDIA_LLAMA_3_1_70B: 'meta/llama-3.1-70b-instruct', - NVIDIA_MISTRAL_LARGE_2: 'mistralai/mistral-large-2-instruct', - NVIDIA_DEEPSEEK_V4_FLASH: 'deepseek-ai/deepseek-v4-flash', - NVIDIA_DEEPSEEK_V4_PRO: 'deepseek-ai/deepseek-v4-pro', -} as const; - +/** + * LLM Providers Package + * Unified interface for OpenAI, Anthropic, and Cloudflare AI providers + * with intelligent fallback logic and cost optimization + */ + +// Core types +export type { + LLMProvider, + LLMImageInput, + GatewayMetadata, + CfGatewayConfig, + CacheHints, + CacheObservability, + ResponseCacheAdapter, + LLMRequest, + LLMResponse, + LLMMessage, + LLMConfig, + TokenUsage, + CostProvenance, + TokenProvenance, + ProviderAttemptResult, + ProviderExecutionOptions, + Tool, + BuiltInTool, + BuiltInToolType, + BuiltInToolResult, + ToolCall, + ToolResult, + ToolChoice, + ProviderConfig, + OpenAIConfig, + AnthropicConfig, + CloudflareAIGatewayOptions, + CloudflareConfig, + CerebrasConfig, + GroqConfig, + NvidiaConfig, + ModelCapabilities, + ProviderCapabilities, + ProviderMetrics, + FallbackRule, + CircuitBreakerConfig, + CircuitBreakerState, + RetryConfig, + CostConfig, + LLMError, + StreamChunk, + StreamResponse, + StreamCompletion, + ManagedTextStream, + QuotaHook, + QuotaCheckInput, + QuotaCheckResult, + QuotaRecordInput, + ToolExecutor, + ToolLoopAbortSignal, + ToolLoopOptions, + ToolLoopState, + ClassifyOptions, + ClassifyResult, + AnalyzeImageInput, + ProviderBalance, + RateLimitBalance, + BatchRequest, + BatchResponse, + BatchJob +} from './types.js'; + +// Canonical provider contract +export { + canonicalToLLMRequest, + normalizeLLMRequest, + normalizeLegacyRequest, + normalizeCanonicalRequest, + normalizeLLMResponse, +} from './canonical.js'; +export type { + CanonicalDegradation, + CanonicalFallbackHop, + CanonicalLLMRequest, + CanonicalLLMResponse, + CanonicalMediaPart, + CanonicalMessage, + CanonicalOutputFormat, + CanonicalProviderOptions, + CanonicalRequestMetadata, + CanonicalRequirements, + CanonicalResponseError, + CanonicalRoutingMetadata, + CanonicalSamplingOptions, + CanonicalTool, + CanonicalToolMode, +} from './canonical.js'; + +// Gateway / Worker route planning +export { getGatewayRoutePlan } from './gateway-routing.js'; +export type { + GatewayRouteCachePlan, + GatewayRouteCapabilityReport, + GatewayRoutePlan, + GatewayRouteRequirements, +} from './gateway-routing.js'; + +// Provider implementations +export { BaseProvider } from './providers/base.js'; +export { OpenAIProvider } from './providers/openai.js'; +export { AnthropicProvider } from './providers/anthropic.js'; +export { CloudflareProvider } from './providers/cloudflare.js'; +export { CerebrasProvider } from './providers/cerebras.js'; +export { GroqProvider } from './providers/groq.js'; +export { NvidiaProvider } from './providers/nvidia.js'; + +// Factory pattern +export { + LLMProviderFactory, + createLLMProviderFactory, + createCostOptimizedFactory +} from './factory.js'; +export type { ProviderFactoryConfig, CostAnalytics, ProviderHealthEntry } from './factory.js'; +export { + MODEL_CATALOG, + MODEL_RECOMMENDATIONS, + PROVIDER_FALLBACK_ORDER, + getCatalogEntry, + getProviderDefaultModel, + getProviderDefaultModelForWorkload, + getProviderForCatalogModel, + getProvidersForCatalogModel, + modelSupportsBuiltInTools, + getRecommendedModel, + getRecommendedModelForWorkload, + getRoutingInfo, + inferUseCaseFromRequest, + normalizeModelWorkload, + rankModels, +} from './model-catalog.js'; +export type { + ModelCatalogEntry, + ModelLifecycle, + ModelPreferenceMap, + ModelRecommendationUseCase, + ModelSelectionContext, + ModelWorkloadClass, + ProviderName, + RoutingInfo, +} from './model-catalog.js'; + +// Local imports for use within this file +import { LLMProviderFactory } from './factory.js'; +import type { ProviderFactoryConfig, CostAnalytics, ProviderHealthEntry } from './factory.js'; +import type { + AnalyzeImageInput, + ClassifyOptions, + ClassifyResult, + LLMProvider, + LLMRequest, + LLMResponse, + ProviderBalance, + ToolExecutor, + ToolLoopOptions +} from './types.js'; +import type { ModelRecommendationUseCase } from './model-catalog.js'; +import { ConfigurationError } from './errors.js'; + +// Error classes +export { + LLMProviderError, + RateLimitError, + QuotaExceededError, + QuotaDeniedError, + AuthenticationError, + InvalidRequestError, + ModelNotFoundError, + TimeoutError, + NetworkError, + ServerError, + ContentFilterError, + TokenLimitError, + ConfigurationError, + NoRouteError, + CfGatewayInvalidConfigError, + CircuitBreakerOpenError, + SchemaDriftError, + ToolLoopLimitError, + ToolLoopAbortedError, + LLMErrorFactory +} from './errors.js'; + +// Schema validator (for custom provider authors) +export { validateSchema } from './utils/schema-validator.js'; +export type { SchemaField, SchemaFieldType } from './utils/schema-validator.js'; + +// Schema drift canary (for integration test suites and cron canaries) +export { extractShape, compareShapes, runCanaryCheck } from './utils/schema-canary.js'; +export type { ShapeMap, CanaryDiff, CanaryReport } from './utils/schema-canary.js'; + +// Image generation +export { ImageProvider, normalizeAiResponse } from './image/index.js'; +export type { ImageProviderConfig, ImageRequest, ImageResponse, ImageModelConfig, ImageModelInputFormat } from './image/index.js'; +export { IMAGE_MODELS, getImageModel } from './image/index.js'; + +// Logger +export { noopLogger, consoleLogger } from './utils/logger.js'; +export type { Logger } from './utils/logger.js'; + +// Observability hooks +export { noopHooks, composeHooks } from './utils/hooks.js'; +export type { + ObservabilityHooks, + RequestStartEvent, + RequestEndEvent, + CacheEvent, + RequestErrorEvent, + RetryEvent, + FallbackEvent, + CircuitStateChangeEvent, + QuotaExhaustedEvent, + BudgetThresholdEvent, + QuotaCheckEvent, + QuotaDeniedEvent, + ProviderBalanceEvent, + SchemaDriftEvent, + ImageAnalysisMetadata, + SanitizedTelemetryError, + ImageAnalysisAttemptEvent, + ImageAnalysisCompleteEvent, +} from './utils/hooks.js'; + +// Exhaustion registry +export { ExhaustionRegistry, defaultExhaustionRegistry } from './utils/exhaustion.js'; +export type { ExhaustionEntry, ExhaustionRegistrySnapshot } from './utils/exhaustion.js'; + +// Latency histogram +export { LatencyHistogram, defaultLatencyHistogram } from './utils/latency-histogram.js'; +export { ResilienceContext } from './utils/resilience-context.js'; +export type { LatencySummary } from './utils/latency-histogram.js'; + +// Utility classes +export { RetryManager, defaultRetryManager, withRetry, retry } from './utils/retry.js'; +export { + CircuitBreaker, + CircuitBreakerManager, + defaultCircuitBreakerManager +} from './utils/circuit-breaker.js'; +export type { CircuitBreakerManagerSnapshot, CircuitBreakerSnapshot } from './utils/circuit-breaker.js'; +export { + CostTracker, + CostOptimizer, + defaultCostTracker +} from './utils/cost-tracker.js'; +export type { + ProviderCostEntry, + ProviderCostBreakdownEntry +} from './utils/cost-tracker.js'; +export { CreditLedger } from './utils/credit-ledger.js'; +export type { + CreditLedgerSnapshot, + LedgerEvent, + ThresholdEvent, + DepletionEvent, + LedgerListener, + BudgetConfig, + ThresholdConfig, + ThresholdTier, + DepletionTier, + ProviderAccumulator, + ModelAccumulator, + RateLimitDimension, + RateLimitCheck, + SpendEntry, + BurnRate, + DepletionEstimate, + SpendSummary, +} from './utils/credit-ledger.js'; + +/** + * Overrides for `LLMProviders.fromEnv()` auto-discovery. + */ +export interface FromEnvOverrides { + stateScope?: ProviderFactoryConfig['stateScope']; + resilienceContext?: ProviderFactoryConfig['resilienceContext']; + defaultProvider?: ProviderFactoryConfig['defaultProvider']; + costOptimization?: boolean; + enableCircuitBreaker?: boolean; + enableRetries?: boolean; + fallbackRules?: ProviderFactoryConfig['fallbackRules']; + ledger?: ProviderFactoryConfig['ledger']; + quotaHook?: ProviderFactoryConfig['quotaHook']; + quotaFailPolicy?: ProviderFactoryConfig['quotaFailPolicy']; + hooks?: ProviderFactoryConfig['hooks']; +} + +/** + * Main LLMProviders class for easy usage + */ +export class LLMProviders { + private factory: LLMProviderFactory; + + constructor(config: ProviderFactoryConfig) { + this.factory = new LLMProviderFactory(config); + } + + /** + * Auto-discover providers from a Cloudflare Worker `env` object. + * + * Scans for known API-key environment variables and bindings, configures + * only the providers whose keys are present, and returns a ready-to-use + * `LLMProviders` instance. + * + * @throws ConfigurationError if no providers are detected + */ + static fromEnv( + env: Record, + overrides: FromEnvOverrides = {} + ): LLMProviders { + const config: ProviderFactoryConfig = {}; + config.stateScope = overrides.stateScope; + config.resilienceContext = overrides.resilienceContext; + + if (typeof env.ANTHROPIC_API_KEY === 'string' && env.ANTHROPIC_API_KEY) { + config.anthropic = { apiKey: env.ANTHROPIC_API_KEY }; + } + if (typeof env.OPENAI_API_KEY === 'string' && env.OPENAI_API_KEY) { + config.openai = { apiKey: env.OPENAI_API_KEY }; + } + if (typeof env.GROQ_API_KEY === 'string' && env.GROQ_API_KEY) { + config.groq = { apiKey: env.GROQ_API_KEY }; + } + if (typeof env.CEREBRAS_API_KEY === 'string' && env.CEREBRAS_API_KEY) { + config.cerebras = { apiKey: env.CEREBRAS_API_KEY }; + } + if (typeof env.NVIDIA_API_KEY === 'string' && env.NVIDIA_API_KEY) { + config.nvidia = { apiKey: env.NVIDIA_API_KEY }; + } + if (env.AI != null && typeof env.AI === 'object') { + config.cloudflare = { ai: env.AI as Ai }; + } + + const detected = [ + config.anthropic && 'anthropic', + config.openai && 'openai', + config.groq && 'groq', + config.cerebras && 'cerebras', + config.nvidia && 'nvidia', + config.cloudflare && 'cloudflare', + ].filter(Boolean) as string[]; + + if (detected.length === 0) { + throw new ConfigurationError( + 'fromEnv', + 'No LLM providers detected in env. Expected at least one of: ' + + 'ANTHROPIC_API_KEY, OPENAI_API_KEY, GROQ_API_KEY, CEREBRAS_API_KEY, NVIDIA_API_KEY, or AI binding.' + ); + } + + if (overrides.defaultProvider !== undefined) { + config.defaultProvider = overrides.defaultProvider; + } + if (overrides.costOptimization !== undefined) { + config.costOptimization = overrides.costOptimization; + } + if (overrides.enableCircuitBreaker !== undefined) { + config.enableCircuitBreaker = overrides.enableCircuitBreaker; + } + if (overrides.enableRetries !== undefined) { + config.enableRetries = overrides.enableRetries; + } + if (overrides.fallbackRules !== undefined) { + config.fallbackRules = overrides.fallbackRules; + } + if (overrides.ledger !== undefined) { + config.ledger = overrides.ledger; + } + if (overrides.quotaHook !== undefined) { + config.quotaHook = overrides.quotaHook; + } + if (overrides.quotaFailPolicy !== undefined) { + config.quotaFailPolicy = overrides.quotaFailPolicy; + } + if (overrides.hooks !== undefined) { + config.hooks = overrides.hooks; + } + + return new LLMProviders(config); + } + + /** + * Generate response with automatic provider selection and fallback + */ + async generateResponse(request: LLMRequest): Promise { + return this.factory.generateResponse(request); + } + + async generateResponseStream(request: LLMRequest): Promise> { + return this.factory.generateResponseStream(request); + } + + async generateManagedResponseStream(request: LLMRequest): Promise { + return this.factory.generateManagedResponseStream(request); + } + + getRoutingInfo(request: LLMRequest): import('./model-catalog.js').RoutingInfo { + return this.factory.getRoutingInfo(request); + } + + async generateResponseWithTools( + request: LLMRequest, + toolExecutor: ToolExecutor, + opts?: ToolLoopOptions + ): Promise { + return this.factory.generateResponseWithTools(request, toolExecutor, opts); + } + + async classify( + input: string | LLMRequest, + options?: ClassifyOptions + ): Promise> { + return this.factory.classify(input, options); + } + + async analyzeImage(input: AnalyzeImageInput): Promise { + return this.factory.analyzeImage(input); + } + + async getProviderBalance(provider?: string): Promise> { + return this.factory.getProviderBalance(provider); + } + + getRecommendedModel( + request: LLMRequest, + useCase?: ModelRecommendationUseCase + ): string { + return this.factory.getRecommendedModel(request, useCase); + } + + /** + * Get specific provider instance + */ + getProvider(name: string): LLMProvider | undefined { + return this.factory.getProvider(name); + } + + /** + * Get all available providers + */ + getAvailableProviders(): string[] { + return this.factory.getAvailableProviders(); + } + + /** + * Get provider health status + */ + async getHealth(): Promise> { + return this.factory.getProviderHealth(); + } + + /** + * Get cost analytics + */ + getCostAnalytics(): CostAnalytics { + return this.factory.getCostAnalytics(); + } + + /** + * Get latency histogram summaries for all providers + */ + getLatencyHistogram(): Record { + return this.factory.getLatencyHistogram(); + } + + /** + * Get currently quota-exhausted providers + */ + getExhaustedProviders(): string[] { + return this.factory.getExhaustedProviders(); + } + + /** + * Reset all metrics and circuit breakers + */ + reset(): void { + this.factory.reset(); + } + + /** + * Update configuration + */ + updateConfig(config: Partial): void { + this.factory.updateConfig(config); + } + +} + +/** + * Create LLMProviders instance with cost optimization + */ +export function createCostOptimizedLLMProviders( + config: ProviderFactoryConfig +): LLMProviders { + return new LLMProviders({ + ...config, + defaultProvider: 'auto', + costOptimization: true, + enableCircuitBreaker: true, + enableRetries: true, + fallbackRules: config.fallbackRules ?? [ + { condition: 'cost', threshold: 5, fallbackProvider: 'cloudflare' }, + { condition: 'rate_limit', fallbackProvider: 'cloudflare' }, + { condition: 'error', fallbackProvider: 'anthropic' } + ] + }); +} + +/** + * Create LLMProviders instance with basic configuration + */ +export function createLLMProviders(config: ProviderFactoryConfig): LLMProviders { + return new LLMProviders(config); +} + +/** + * Default export for convenience + */ +export default LLMProviders; + +/** + * Version and metadata + */ +export { VERSION } from './version.js'; +export const SUPPORTED_PROVIDERS = ['openai', 'anthropic', 'cloudflare', 'cerebras', 'groq', 'nvidia'] as const; + +/** + * Common model mappings for easy reference + */ +export const MODELS = { + // OpenAI models + GPT_6_ASTRA: 'gpt-6-astra', + GPT_5_6_SOL: 'gpt-5.6-sol', + GPT_5_6_TERRA: 'gpt-5.6-terra', + GPT_5_6_LUNA: 'gpt-5.6-luna', + GPT_5_5: 'gpt-5.5', + /** Not accepted by the OpenAI adapter or routed by this package. OpenAI has announced shutdowns only for dated gpt-4o snapshots. */ + GPT_4O: 'gpt-4o', + /** @deprecated OpenAI shuts this model down on 2027-01-14 and names gpt-6-luna as the replacement (not yet supported by this adapter). */ + GPT_4O_MINI: 'gpt-4o-mini', + /** @deprecated OpenAI shuts this model down on 2026-10-23. Use GPT_5_6_SOL. */ + GPT_4_TURBO: 'gpt-4-turbo', + /** @deprecated OpenAI shuts this model down on 2026-10-23. Use GPT_5_6_SOL. */ + GPT_4: 'gpt-4', + /** @deprecated OpenAI shuts this model down on 2026-10-23. Use GPT_5_6_TERRA. */ + GPT_3_5_TURBO: 'gpt-3.5-turbo', + + // Anthropic models + // Claude model IDs are dateless from the 4.6 generation on. + CLAUDE_FABLE_5_1: 'claude-fable-5-1', + CLAUDE_OPUS_5_5: 'claude-opus-5-5', + CLAUDE_SONNET_5_5: 'claude-sonnet-5-5', + CLAUDE_HAIKU_5_5: 'claude-haiku-5-5', + /** Legacy on the Claude API; retirement not sooner than 2027-02-05. */ + CLAUDE_OPUS_4_6: 'claude-opus-4-6', + /** Legacy on the Claude API; retirement not sooner than 2027-02-17. */ + CLAUDE_SONNET_4_6: 'claude-sonnet-4-6', + /** @deprecated Retired on the Claude API 2026-06-15. */ + CLAUDE_OPUS_4: 'claude-opus-4-20250514', + /** @deprecated Retired on the Claude API 2026-06-15. */ + CLAUDE_SONNET_4: 'claude-sonnet-4-20250514', + /** Legacy on the Claude API; retirement not sooner than 2026-10-15. */ + CLAUDE_HAIKU_4_5: 'claude-haiku-4-5-20251001', + /** @deprecated Retired on the Claude API 2026-02-19. */ + CLAUDE_3_7_SONNET: 'claude-3-7-sonnet-20250219', + /** @deprecated Retired on the Claude API 2025-10-28. */ + CLAUDE_3_5_SONNET: 'claude-3-5-sonnet-20241022', + /** @deprecated Retired on the Claude API 2026-02-19. */ + CLAUDE_3_5_HAIKU: 'claude-3-5-haiku-20241022', + /** @deprecated Retired on the Claude API 2026-01-05. */ + CLAUDE_3_OPUS: 'claude-3-opus-20240229', + /** @deprecated Retired on the Claude API 2025-07-21. */ + CLAUDE_3_SONNET: 'claude-3-sonnet-20240229', + /** @deprecated Retired on the Claude API 2026-04-20; not accepted by AnthropicProvider. Use CLAUDE_HAIKU_4_5. */ + CLAUDE_3_HAIKU: 'claude-3-haiku-20240307', + + // Cloudflare Workers AI models + CF_GLM_5_3: '@cf/zai-org/glm-5.3', + CF_GLM_5_3_FLASH: '@cf/zai-org/glm-5.3-flash', + CF_KIMI_K2_6: '@cf/moonshotai/kimi-k2.6', + CF_DEEPSEEK_V4_PRO: '@cf/deepseek-ai/deepseek-v4-pro-0813', + CF_DEEPSEEK_V4_FLASH: '@cf/deepseek-ai/deepseek-v4-flash-0731', + CF_GPT_OSS_120B: '@cf/openai/gpt-oss-120b', + CF_GPT_OSS_20B: '@cf/openai/gpt-oss-20b', + CF_GEMMA_4_26B: '@cf/google/gemma-4-26b-a4b-it', + CF_QWEN_3_8_27B: '@cf/qwen/qwen3.8-27b', + CF_LLAMA_3_3_70B_FAST: '@cf/meta/llama-3.3-70b-instruct-fp8-fast', + CF_LLAMA_3_1_8B_FP8: '@cf/meta/llama-3.1-8b-instruct-fp8', + CF_GRANITE_4_MICRO: '@cf/ibm-granite/granite-4.0-h-micro', + /** @deprecated Deprecated on Workers AI 2026-05-30. Use CF_LLAMA_3_1_8B_FP8. */ + LLAMA_3_1_8B: '@cf/meta/llama-3.1-8b-instruct', + /** @deprecated Deprecated on Workers AI 2026-05-30. Use CF_LLAMA_3_3_70B_FAST. */ + LLAMA_3_1_70B: '@cf/meta/llama-3.1-70b-instruct', + /** @deprecated Deprecated on Workers AI 2026-05-30. Use CF_LLAMA_3_1_8B_FP8. */ + LLAMA_3_8B: '@cf/meta/llama-3-8b-instruct', + /** @deprecated Deprecated on Workers AI 2026-05-30. */ + MISTRAL_7B: '@cf/mistral/mistral-7b-instruct-v0.1', + /** @deprecated Deprecated on Workers AI 2025-10-01. */ + TINY_LLAMA: '@cf/tinyllama/tinyllama-1.1b-chat-v1.0', + + // Cerebras models + CEREBRAS_LLAMA_3_1_8B: 'llama-3.1-8b', + CEREBRAS_LLAMA_3_3_70B: 'llama-3.3-70b', + CEREBRAS_ZAI_GLM_4_7: 'zai-glm-4.7', + CEREBRAS_QWEN_3_235B: 'qwen-3-235b-a22b-instruct-2507', + + // Groq models + GROQ_LLAMA_3_3_70B_VERSATILE: 'llama-3.3-70b-versatile', + GROQ_LLAMA_3_1_8B_INSTANT: 'llama-3.1-8b-instant', + GROQ_GPT_OSS_120B: 'openai/gpt-oss-120b', + GROQ_COMPOUND: 'groq/compound', + GROQ_COMPOUND_MINI: 'groq/compound-mini', + + // NVIDIA NIM models + NVIDIA_LLAMA_3_3_70B: 'meta/llama-3.3-70b-instruct', + NVIDIA_LLAMA_4_MAVERICK: 'meta/llama-4-maverick-17b-128e-instruct', + NVIDIA_NEMOTRON_70B: 'nvidia/llama-3.1-nemotron-70b-instruct', + NVIDIA_NEMOTRON_SUPER_49B: 'nvidia/llama-3.3-nemotron-super-49b-v1', + NVIDIA_NEMOTRON_ULTRA_253B: 'nvidia/llama-3.1-nemotron-ultra-253b-v1', + NVIDIA_LLAMA_3_1_70B: 'meta/llama-3.1-70b-instruct', + NVIDIA_MISTRAL_LARGE_2: 'mistralai/mistral-large-2-instruct', + NVIDIA_DEEPSEEK_V4_FLASH: 'deepseek-ai/deepseek-v4-flash', + NVIDIA_DEEPSEEK_V4_PRO: 'deepseek-ai/deepseek-v4-pro', +} as const; + diff --git a/src/types.ts b/src/types.ts index 76d7abd..232d0b6 100755 --- a/src/types.ts +++ b/src/types.ts @@ -1,659 +1,659 @@ -/** - * LLM Provider Types - * Unified types for all LLM providers with v2 architecture support - */ - -import type { Logger } from './utils/logger.js'; -import type { ObservabilityHooks } from './utils/hooks.js'; -export type { Logger, ObservabilityHooks }; - -export interface LLMMessage { - role: 'user' | 'assistant' | 'system'; - content: string; - timestamp?: string; - toolCalls?: ToolCall[]; - toolResults?: ToolResult[]; -} - -export interface LLMImageInput { - data?: string; - url?: string; - mimeType?: string; -} - -export interface GatewayMetadata { - requestId?: string; - /** cf-aig-cache-key: Cloudflare AI Gateway *response* cache key (distinct from provider prompt caching). */ - cacheKey?: string; - /** cf-aig-cache-ttl: Cloudflare AI Gateway response cache TTL in seconds. */ - cacheTtl?: number; - /** cf-aig-skip-cache: bypass the Cloudflare AI Gateway response cache for this request. */ - skipCache?: boolean; - customMetadata?: Record; -} - -/** - * First-class Cloudflare AI Gateway passthrough config attached to an HTTP - * provider via `cfGateway`. When set (and no explicit `baseUrl` override is - * present), the provider derives its base URL as - * `https://gateway.ai.cloudflare.com/v1/{accountId}/{gatewayId}/{suffix}` and - * injects `cf-aig-*` headers from `LLMRequest.gatewayMetadata`. - * - * Both fields are required; an empty string for either throws synchronously in - * the provider constructor (CF_GATEWAY_INVALID_CONFIG). - */ -export interface CfGatewayConfig { - /** Cloudflare account ID. Must be non-empty. */ - accountId: string; - /** AI Gateway ID (the named gateway within the account). Must be non-empty. */ - gatewayId: string; -} - -/** - * Provider prompt/prefix cache hints. Distinct from GatewayMetadata.cacheKey/cacheTtl, - * which control Cloudflare AI Gateway *response* caching. These hints control - * provider-side prefix/prompt caching (Anthropic cache_control, Groq/Cerebras automatic, etc.). - */ -/** - * Provider-agnostic response cache adapter for factory-level response deduplication. - * Distinct from CacheHints, which controls provider-side prefix/prompt caching. - * Works with any backing store (Cloudflare KV, Redis, in-memory map). - * - * Keys are versioned SHA-256 hashes of the complete semantic request, tenant, - * application namespace, and routing policy. Invalid/non-JSON identities bypass caching. - */ -export interface ResponseCacheAdapter { - get(key: string): Promise; - put(key: string, value: string, ttlSeconds?: number): Promise; -} - -export interface CacheHints { - /** 'off' disables cache opt-in; 'provider-prefix' bypasses factory response caching. */ - strategy?: 'off' | 'provider-prefix' | 'response' | 'both'; - /** Provider-specific cache key hint (e.g. OpenAI prompt_cache_key). */ - key?: string; - /** Desired TTL. Clamped to provider-supported values at translation time. */ - ttl?: '5m' | '1h' | '24h' | number; - /** Cloudflare Workers AI: x-session-affinity value for prefix caching. */ - sessionId?: string; - /** Which prefix to mark cacheable when explicit breakpoints are required (Anthropic). */ - cacheablePrefix?: 'auto' | 'system' | 'tools' | 'messages'; -} - -export interface CacheObservability { - /** Provider-side prompt/prefix cache behavior, normalized from TokenUsage fields. */ - providerPrefix?: { - requested?: boolean; - strategy?: CacheHints['strategy']; - sessionId?: string; - cachedInputTokens?: number; - cacheReadInputTokens?: number; - cacheWriteInputTokens?: number; - cacheCreationInputTokens?: number; - hit?: boolean; - write?: boolean; - }; - /** Cloudflare AI Gateway exact-response cache behavior when the provider exposes it. */ - aiGateway?: { - requested?: boolean; - status?: string; - cacheKey?: string; - cacheTtl?: number; - }; - /** Factory/local response cache behavior through ResponseCacheAdapter. */ - factory?: { - status: 'hit' | 'miss' | 'write' | 'error' | 'bypass'; - key?: string; - ttlSeconds?: number; - /** On hits, usage belongs to the original provider call, not this request. */ - usageKind?: 'historical'; - currentChargeUsd?: number; - error?: string; - }; -} - -export interface ToolCall { - id: string; - type: 'function'; - function: { - name: string; - arguments: string; - }; -} - -export interface ToolResult { - id: string; - output: string; - error?: string; -} - -/** Provider-agnostic constraint for caller-defined function tools. */ -export type ToolChoice = - | 'auto' - | 'none' - | 'required' - | { type: 'function'; function: { name: string } }; - -export interface LLMRequest { - messages: LLMMessage[]; - model?: string; - temperature?: number; - maxTokens?: number; - stream?: boolean; - systemPrompt?: string; - images?: LLMImageInput[]; - tools?: Tool[]; - /** - * Provider-hosted built-in tools (Groq web search, code interpreter, …). - * Only honored by providers/models that advertise `supportsBuiltInTools`; - * adapters translate to the native wire shape and gate at the boundary. - * Results are surfaced on `LLMResponse.metadata.builtInToolResults` - * (cast to `BuiltInToolResult[]`). - */ - builtInTools?: BuiltInTool[]; - toolChoice?: ToolChoice; - response_format?: - | { type: 'json_object' | 'text' } - | { type: 'json_schema'; json_schema: { name: string; schema: Record; strict?: boolean } }; - seed?: number; - gatewayMetadata?: GatewayMetadata; - cache?: CacheHints; - /** - * Cloudflare Workers AI only: LoRA adapter name or UUID to apply at inference. - * Ignored by all other providers. The package does not validate the identifier; - * it is forwarded as-is to the Workers AI binding. - */ - lora?: string; - /** Nucleus sampling cutoff. Forwarded by the Cloudflare provider; ignored by others. */ - topP?: number; - /** Penalizes repeated tokens. Forwarded by the Cloudflare provider; ignored by others. */ - frequencyPenalty?: number; - /** Provider reasoning/thinking controls. Translated to provider-native params at call time. */ - reasoning?: { - /** Token budget / quality hint. 'none' disables reasoning entirely where supported. */ - effort?: 'low' | 'medium' | 'high' | 'none'; - /** Cerebras: controls how reasoning tokens appear in the response. */ - format?: 'parsed' | 'raw' | 'hidden'; - /** Cerebras zai-glm-4.7: set false to preserve reasoning across turns (improves cache hit rate). */ - clearThinking?: boolean; - }; - /** Predicted output hint for speculative decoding (Cerebras gpt-oss-120b, zai-glm-4.7). Incompatible with tools. */ - prediction?: string; - tenantId?: string; - requestId?: string; - metadata?: Record; -} - -export interface Tool { - type: 'function'; - function: { - name: string; - description: string; - parameters: { - type: 'object'; - properties: Record; - required?: string[]; - }; - }; -} - -/** - * Provider-hosted built-in tools (e.g. Groq web search / code interpreter). - * Conceptually distinct from caller-defined function `Tool`s: the provider runs - * them server-side and returns their results inline. Identifiers are normalized - * across providers (Groq's compound vocabulary is the canonical superset); each - * adapter translates to its native wire shape and gates on model capability. - */ -export type BuiltInToolType = - | 'web_search' - | 'visit_website' - | 'browser_automation' - | 'code_interpreter' - | 'wolfram_alpha'; - -export interface BuiltInTool { - type: BuiltInToolType; -} - -/** - * A single built-in tool execution surfaced on `LLMResponse.metadata.builtInToolResults`. - * Shape mirrors Groq's `message.executed_tools[]` (verified live, issue #69): - * `type` is provider-native and open-ended (`'search'`, `'browser_search'`, - * `'browser.open'`, …) — do not treat it as a closed enum. Results carry the - * provider's web-search citations when the execution produced any. - */ -export interface BuiltInToolResult { - /** Provider-native tool kind. Open-ended — varies by provider and model. */ - type: string; - /** Tool name when the provider supplies one (Groq gpt-oss); absent on compound. */ - name?: string; - /** Raw JSON-string arguments the model passed, e.g. `'{"query":"…"}'`. */ - arguments?: string; - /** Search results / citations carried by this execution, when present. */ - results: Array<{ title: string; url: string; content: string; score: number }>; -} - -export interface LLMResponse { - id?: string; - /** Chain-of-thought / thinking trace when the provider exposes it separately. */ - reasoning?: string; - message: string; - content?: string; // Alternative to message for consistency - usage: TokenUsage; - model: string; - provider: string; - responseTime: number; - finishReason?: 'stop' | 'length' | 'tool_calls' | 'content_filter'; - toolCalls?: ToolCall[]; - cache?: CacheObservability; - metadata?: Record; -} - -export type CostProvenance = 'provider_reported' | 'catalog_estimate' | 'unknown'; -export type TokenProvenance = 'provider_reported' | 'estimated'; - -export interface TokenUsage { - inputTokens: number; - outputTokens: number; - totalTokens: number; - cost: number; // Cost in USD - /** Whether `cost` is an actual provider charge, a catalog estimate, or unavailable. */ - costProvenance?: CostProvenance; - /** Whether token counts came from the provider or were locally estimated. */ - tokenProvenance?: TokenProvenance; - /** Tokens served from provider-side prefix/prompt cache (Groq, Cerebras, OpenAI automatic). */ - cachedInputTokens?: number; - /** Anthropic: tokens read from a cache_control breakpoint (cache hit). */ - cacheReadInputTokens?: number; - /** Anthropic: tokens written to a new cache_control breakpoint (cache miss/create). */ - cacheCreationInputTokens?: number; - /** Provider-agnostic alias for cache write/create tokens. */ - cacheWriteInputTokens?: number; -} - -/** Provider execution detail exposed additively for orchestration observability. */ -export interface ProviderAttemptResult { - attempt: number; - outcome: 'success' | 'error'; - durationMs: number; - willRetry: boolean; - response?: LLMResponse; - error?: Error; -} - -export interface ProviderExecutionOptions { - onAttempt?(result: ProviderAttemptResult): void; -} - -export interface LLMProvider { - name: string; - models: string[]; - supportsStreaming: boolean; - supportsTools: boolean; - supportsBatching: boolean; - supportsVision?: boolean; - - generateResponse(request: LLMRequest, options?: ProviderExecutionOptions): Promise; - streamResponse?(request: LLMRequest): Promise>; - getProviderBalance?(): Promise; - validateConfig(): boolean; - getModels(): string[]; - estimateCost(request: LLMRequest): number; - healthCheck(): Promise; - getMetrics(): ProviderMetrics; - resetMetrics(): void; -} - -export interface LLMConfig { - provider: 'openai' | 'anthropic' | 'cloudflare' | 'cerebras' | 'groq' | 'nvidia' | 'auto'; - model: string; - temperature: number; - maxTokens: number; - apiKey?: string; - baseUrl?: string; - fallbackProvider?: 'openai' | 'anthropic' | 'cloudflare' | 'cerebras' | 'groq' | 'nvidia'; - fallbackModel?: string; - timeout?: number; - retries?: number; - retryDelay?: number; -} - -export interface ProviderConfig { - resilienceContext?: import('./utils/resilience-context.js').ResilienceContext; - /** false bypasses breaker admission and failure/success recording. */ - enableCircuitBreaker?: boolean; - apiKey?: string; - baseUrl?: string; - /** Request timeout in ms. For streams it covers opening the stream (headers), not the body. */ - timeout?: number; - /** Opt-in total deadline in ms for a streaming response, including its body. Unset means none. */ - streamBodyTimeout?: number; - maxRetries?: number; - retryDelay?: number; - rateLimitRpm?: number; - rateLimitRpd?: number; - organization?: string; - project?: string; - logger?: Logger; - hooks?: ObservabilityHooks; -} - -export interface OpenAIConfig extends ProviderConfig { - organization?: string; - project?: string; - /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ - cfGateway?: CfGatewayConfig; -} - -export interface AnthropicConfig extends ProviderConfig { - version?: string; - /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ - cfGateway?: CfGatewayConfig; -} - -export interface CloudflareAIGatewayOptions { - /** AI Gateway id for Workers AI binding calls, e.g. "default" or a custom gateway id. */ - id: string; - /** Workers AI binding Gateway response-cache key. */ - cacheKey?: string; - /** Workers AI binding Gateway response-cache TTL in seconds. */ - cacheTtl?: number; - /** Workers AI binding Gateway response-cache bypass flag. */ - skipCache?: boolean; - /** Gateway metadata visible in AI Gateway logs. */ - metadata?: Record; - collectLog?: boolean; - eventId?: string; - requestTimeoutMs?: number; -} - -export interface CloudflareConfig extends ProviderConfig { - accountId?: string; - ai?: Ai; // Cloudflare AI binding - /** Workers AI binding Gateway options passed as the third env.AI.run() argument. */ - gateway?: CloudflareAIGatewayOptions; -} - -export interface CerebrasConfig extends ProviderConfig { - // Cerebras uses OpenAI-compatible API; no extra fields needed beyond ProviderConfig - /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ - cfGateway?: CfGatewayConfig; -} - -export interface GroqConfig extends ProviderConfig { - // Groq uses OpenAI-compatible API; no extra fields needed beyond ProviderConfig - /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ - cfGateway?: CfGatewayConfig; -} - -export interface NvidiaConfig extends ProviderConfig { - // NVIDIA NIM uses OpenAI-compatible API; no extra fields needed beyond ProviderConfig - /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ - cfGateway?: CfGatewayConfig; -} - -export interface LLMError extends Error { - code: string; - provider: string; - retryable: boolean; - statusCode?: number; - rateLimited?: boolean; - quotaExceeded?: boolean; -} - -export interface CircuitBreakerConfig { - failureThreshold: number; - resetTimeout: number; - monitoringPeriod: number; - minRequests?: number; - degradationCurve?: number[]; -} - -export interface CircuitBreakerState { - state: 'CLOSED' | 'DEGRADED' | 'RECOVERING' | 'OPEN'; - failures: number; - consecutiveFailures: number; - primaryTrafficPct: number; - totalFailures: number; - totalSuccesses: number; - totalRequests: number; - lastFailure?: number; - lastSuccess?: number; - lastRequest?: number; - nextAttempt?: number; -} - -export interface RetryConfig { - maxRetries: number; - initialDelay: number; - maxDelay: number; - backoffMultiplier: number; - retryableErrors: string[]; -} - -export interface CostConfig { - inputTokenCost: number; // Cost per 1k input tokens - outputTokenCost: number; // Cost per 1k output tokens - maxMonthlyCost?: number; - alertThreshold?: number; -} - -export interface ProviderMetrics { - requestCount: number; - successCount: number; - errorCount: number; - averageLatency: number; - totalCost: number; - rateLimitHits: number; - lastUsed: number; -} - -export interface FallbackRule { - condition: 'error' | 'rate_limit' | 'cost' | 'latency'; - threshold?: number; - fallbackProvider: string; - fallbackModel?: string; -} - -export interface ModelCapabilities { - maxContextLength: number; - supportsStreaming: boolean; - supportsTools: boolean; - supportsVision?: boolean; - toolCalling?: boolean; - supportsBatching: boolean; - /** USD per 1,000 input tokens. */ - inputTokenCost: number; - /** USD per 1,000 output tokens. */ - outputTokenCost: number; - description: string; - /** Provider-side prompt/prefix caching supported for this model. */ - supportsPromptCache?: boolean; - /** - * Provider-hosted built-in tools this model can run server-side, normalized - * to `BuiltInToolType`. Absent/empty means the model is function-tools only. - * Drives capability-aware routing and boundary gating for `LLMRequest.builtInTools`. - */ - supportsBuiltInTools?: BuiltInToolType[]; - /** - * True for models that output chain-of-thought reasoning traces as part of their - * response (e.g. QwQ, DeepSeek-R1, GLM-4.7-Flash). These models are unsuitable - * for direct-response routing (summary, classification, chat) unless the caller - * explicitly handles reasoning traces. Routers must exclude thinking models from - * all non-RESEARCH use-case pools. - */ - thinkingModel?: boolean; -} - -export interface ProviderCapabilities { - models: Record; - supportsStreaming: boolean; - supportsTools: boolean; - supportsBatching: boolean; - maxBatchSize?: number; - rateLimits: { - requestsPerMinute: number; - requestsPerDay: number; - tokensPerMinute: number; - }; -} - -// Streaming support -export interface StreamChunk { - id: string; - content: string; - delta: string; - finishReason?: 'stop' | 'length' | 'tool_calls' | 'content_filter'; - usage?: Partial; -} - -export interface StreamResponse { - stream: ReadableStream; - controller: ReadableStreamDefaultController; -} - -/** Final accounting for a managed text stream, including partial output. */ -export interface StreamCompletion { - outcome: 'success' | 'error' | 'cancelled'; - emittedChars: number; - usage: TokenUsage; -} - -export interface ManagedTextStream { - stream: ReadableStream; - /** Resolves after accounting; rejects if settlement fails. Suitable for ctx.waitUntil. */ - completion: Promise; -} - -export interface QuotaCheckInput { - tenantId?: string; - provider: string; - model: string; - estimatedCost: number; - metadata?: Record; -} - -export interface QuotaCheckResult { - allowed: boolean; - reason?: string; - remainingBudget?: number; -} - -export interface QuotaRecordInput { - tenantId?: string; - provider: string; - model: string; - actualCost: number; - inputTokens?: number; - outputTokens?: number; - metadata?: Record; -} - -export interface QuotaHook { - check(input: QuotaCheckInput): Promise; - record(input: QuotaRecordInput): Promise; -} - -export interface ToolExecutor { - execute(name: string, argumentsValue: unknown): Promise; -} - -export interface ToolLoopState { - iteration: number; - cumulativeCost: number; - messageCount: number; - lastToolCalls: ToolCall[]; -} - -export interface ToolLoopAbortSignal { - abort: true; - reason?: string; -} - -export interface ToolLoopOptions { - maxIterations?: number; - maxCostUSD?: number; - /** - * Called after each iteration. Return `{ abort: true, reason? }` to stop the - * loop immediately and throw `ToolLoopAbortedError`; return void to continue. - */ - onIteration?: ( - iteration: number, - state: ToolLoopState - ) => void | ToolLoopAbortSignal | Promise; - abortSignal?: AbortSignal; -} - -export interface ClassifyOptions { - schema?: Record | { parse(data: unknown): T }; - systemPrompt?: string; - model?: string; - temperature?: number; - maxTokens?: number; - confidenceField?: string; - seed?: number; -} - -export interface ClassifyResult { - data: T; - confidence?: number; - response: LLMResponse; -} - -export interface AnalyzeImageInput { - image: LLMImageInput; - prompt: string; - model?: string; - systemPrompt?: string; - temperature?: number; - maxTokens?: number; - response_format?: LLMRequest['response_format']; - tenantId?: string; - requestId?: string; - metadata?: Record; -} - -export interface RateLimitBalance { - limit?: number; - used?: number; - remaining?: number; -} - -export interface ProviderBalance { - provider: string; - status: 'available' | 'unavailable' | 'error'; - source: 'provider_api' | 'ledger' | 'headers' | 'not_supported'; - currentSpend?: number; - monthlyBudget?: number; - remainingBudget?: number; - usedTokens?: number; - requestCount?: number; - rateLimits?: Record; - resetAt?: string; - message?: string; - raw?: unknown; -} - -// Batch processing -export interface BatchRequest { - id: string; - request: LLMRequest; - priority?: 'low' | 'normal' | 'high'; -} - -export interface BatchResponse { - id: string; - response?: LLMResponse; - error?: LLMError; - processingTime: number; -} - -export interface BatchJob { - id: string; - requests: BatchRequest[]; - status: 'pending' | 'processing' | 'completed' | 'failed'; - progress: number; - createdAt: number; - completedAt?: number; - results: BatchResponse[]; -} +/** + * LLM Provider Types + * Unified types for all LLM providers with v2 architecture support + */ + +import type { Logger } from './utils/logger.js'; +import type { ObservabilityHooks } from './utils/hooks.js'; +export type { Logger, ObservabilityHooks }; + +export interface LLMMessage { + role: 'user' | 'assistant' | 'system'; + content: string; + timestamp?: string; + toolCalls?: ToolCall[]; + toolResults?: ToolResult[]; +} + +export interface LLMImageInput { + data?: string; + url?: string; + mimeType?: string; +} + +export interface GatewayMetadata { + requestId?: string; + /** cf-aig-cache-key: Cloudflare AI Gateway *response* cache key (distinct from provider prompt caching). */ + cacheKey?: string; + /** cf-aig-cache-ttl: Cloudflare AI Gateway response cache TTL in seconds. */ + cacheTtl?: number; + /** cf-aig-skip-cache: bypass the Cloudflare AI Gateway response cache for this request. */ + skipCache?: boolean; + customMetadata?: Record; +} + +/** + * First-class Cloudflare AI Gateway passthrough config attached to an HTTP + * provider via `cfGateway`. When set (and no explicit `baseUrl` override is + * present), the provider derives its base URL as + * `https://gateway.ai.cloudflare.com/v1/{accountId}/{gatewayId}/{suffix}` and + * injects `cf-aig-*` headers from `LLMRequest.gatewayMetadata`. + * + * Both fields are required; an empty string for either throws synchronously in + * the provider constructor (CF_GATEWAY_INVALID_CONFIG). + */ +export interface CfGatewayConfig { + /** Cloudflare account ID. Must be non-empty. */ + accountId: string; + /** AI Gateway ID (the named gateway within the account). Must be non-empty. */ + gatewayId: string; +} + +/** + * Provider prompt/prefix cache hints. Distinct from GatewayMetadata.cacheKey/cacheTtl, + * which control Cloudflare AI Gateway *response* caching. These hints control + * provider-side prefix/prompt caching (Anthropic cache_control, Groq/Cerebras automatic, etc.). + */ +/** + * Provider-agnostic response cache adapter for factory-level response deduplication. + * Distinct from CacheHints, which controls provider-side prefix/prompt caching. + * Works with any backing store (Cloudflare KV, Redis, in-memory map). + * + * Keys are versioned SHA-256 hashes of the complete semantic request, tenant, + * application namespace, and routing policy. Invalid/non-JSON identities bypass caching. + */ +export interface ResponseCacheAdapter { + get(key: string): Promise; + put(key: string, value: string, ttlSeconds?: number): Promise; +} + +export interface CacheHints { + /** 'off' disables cache opt-in; 'provider-prefix' bypasses factory response caching. */ + strategy?: 'off' | 'provider-prefix' | 'response' | 'both'; + /** Provider-specific cache key hint (e.g. OpenAI prompt_cache_key). */ + key?: string; + /** Desired TTL. Clamped to provider-supported values at translation time. */ + ttl?: '5m' | '1h' | '24h' | number; + /** Cloudflare Workers AI: x-session-affinity value for prefix caching. */ + sessionId?: string; + /** Which prefix to mark cacheable when explicit breakpoints are required (Anthropic). */ + cacheablePrefix?: 'auto' | 'system' | 'tools' | 'messages'; +} + +export interface CacheObservability { + /** Provider-side prompt/prefix cache behavior, normalized from TokenUsage fields. */ + providerPrefix?: { + requested?: boolean; + strategy?: CacheHints['strategy']; + sessionId?: string; + cachedInputTokens?: number; + cacheReadInputTokens?: number; + cacheWriteInputTokens?: number; + cacheCreationInputTokens?: number; + hit?: boolean; + write?: boolean; + }; + /** Cloudflare AI Gateway exact-response cache behavior when the provider exposes it. */ + aiGateway?: { + requested?: boolean; + status?: string; + cacheKey?: string; + cacheTtl?: number; + }; + /** Factory/local response cache behavior through ResponseCacheAdapter. */ + factory?: { + status: 'hit' | 'miss' | 'write' | 'error' | 'bypass'; + key?: string; + ttlSeconds?: number; + /** On hits, usage belongs to the original provider call, not this request. */ + usageKind?: 'historical'; + currentChargeUsd?: number; + error?: string; + }; +} + +export interface ToolCall { + id: string; + type: 'function'; + function: { + name: string; + arguments: string; + }; +} + +export interface ToolResult { + id: string; + output: string; + error?: string; +} + +/** Provider-agnostic constraint for caller-defined function tools. */ +export type ToolChoice = + | 'auto' + | 'none' + | 'required' + | { type: 'function'; function: { name: string } }; + +export interface LLMRequest { + messages: LLMMessage[]; + model?: string; + temperature?: number; + maxTokens?: number; + stream?: boolean; + systemPrompt?: string; + images?: LLMImageInput[]; + tools?: Tool[]; + /** + * Provider-hosted built-in tools (Groq web search, code interpreter, …). + * Only honored by providers/models that advertise `supportsBuiltInTools`; + * adapters translate to the native wire shape and gate at the boundary. + * Results are surfaced on `LLMResponse.metadata.builtInToolResults` + * (cast to `BuiltInToolResult[]`). + */ + builtInTools?: BuiltInTool[]; + toolChoice?: ToolChoice; + response_format?: + | { type: 'json_object' | 'text' } + | { type: 'json_schema'; json_schema: { name: string; schema: Record; strict?: boolean } }; + seed?: number; + gatewayMetadata?: GatewayMetadata; + cache?: CacheHints; + /** + * Cloudflare Workers AI only: LoRA adapter name or UUID to apply at inference. + * Ignored by all other providers. The package does not validate the identifier; + * it is forwarded as-is to the Workers AI binding. + */ + lora?: string; + /** Nucleus sampling cutoff. Forwarded by the Cloudflare provider; ignored by others. */ + topP?: number; + /** Penalizes repeated tokens. Forwarded by the Cloudflare provider; ignored by others. */ + frequencyPenalty?: number; + /** Provider reasoning/thinking controls. Translated to provider-native params at call time. */ + reasoning?: { + /** Token budget / quality hint. 'none' disables reasoning entirely where supported. */ + effort?: 'low' | 'medium' | 'high' | 'none'; + /** Cerebras: controls how reasoning tokens appear in the response. */ + format?: 'parsed' | 'raw' | 'hidden'; + /** Cerebras zai-glm-4.7: set false to preserve reasoning across turns (improves cache hit rate). */ + clearThinking?: boolean; + }; + /** Predicted output hint for speculative decoding (Cerebras gpt-oss-120b, zai-glm-4.7). Incompatible with tools. */ + prediction?: string; + tenantId?: string; + requestId?: string; + metadata?: Record; +} + +export interface Tool { + type: 'function'; + function: { + name: string; + description: string; + parameters: { + type: 'object'; + properties: Record; + required?: string[]; + }; + }; +} + +/** + * Provider-hosted built-in tools (e.g. Groq web search / code interpreter). + * Conceptually distinct from caller-defined function `Tool`s: the provider runs + * them server-side and returns their results inline. Identifiers are normalized + * across providers (Groq's compound vocabulary is the canonical superset); each + * adapter translates to its native wire shape and gates on model capability. + */ +export type BuiltInToolType = + | 'web_search' + | 'visit_website' + | 'browser_automation' + | 'code_interpreter' + | 'wolfram_alpha'; + +export interface BuiltInTool { + type: BuiltInToolType; +} + +/** + * A single built-in tool execution surfaced on `LLMResponse.metadata.builtInToolResults`. + * Shape mirrors Groq's `message.executed_tools[]` (verified live, issue #69): + * `type` is provider-native and open-ended (`'search'`, `'browser_search'`, + * `'browser.open'`, …) — do not treat it as a closed enum. Results carry the + * provider's web-search citations when the execution produced any. + */ +export interface BuiltInToolResult { + /** Provider-native tool kind. Open-ended — varies by provider and model. */ + type: string; + /** Tool name when the provider supplies one (Groq gpt-oss); absent on compound. */ + name?: string; + /** Raw JSON-string arguments the model passed, e.g. `'{"query":"…"}'`. */ + arguments?: string; + /** Search results / citations carried by this execution, when present. */ + results: Array<{ title: string; url: string; content: string; score: number }>; +} + +export interface LLMResponse { + id?: string; + /** Chain-of-thought / thinking trace when the provider exposes it separately. */ + reasoning?: string; + message: string; + content?: string; // Alternative to message for consistency + usage: TokenUsage; + model: string; + provider: string; + responseTime: number; + finishReason?: 'stop' | 'length' | 'tool_calls' | 'content_filter'; + toolCalls?: ToolCall[]; + cache?: CacheObservability; + metadata?: Record; +} + +export type CostProvenance = 'provider_reported' | 'catalog_estimate' | 'unknown'; +export type TokenProvenance = 'provider_reported' | 'estimated'; + +export interface TokenUsage { + inputTokens: number; + outputTokens: number; + totalTokens: number; + cost: number; // Cost in USD + /** Whether `cost` is an actual provider charge, a catalog estimate, or unavailable. */ + costProvenance?: CostProvenance; + /** Whether token counts came from the provider or were locally estimated. */ + tokenProvenance?: TokenProvenance; + /** Tokens served from provider-side prefix/prompt cache (Groq, Cerebras, OpenAI automatic). */ + cachedInputTokens?: number; + /** Anthropic: tokens read from a cache_control breakpoint (cache hit). */ + cacheReadInputTokens?: number; + /** Anthropic: tokens written to a new cache_control breakpoint (cache miss/create). */ + cacheCreationInputTokens?: number; + /** Provider-agnostic alias for cache write/create tokens. */ + cacheWriteInputTokens?: number; +} + +/** Provider execution detail exposed additively for orchestration observability. */ +export interface ProviderAttemptResult { + attempt: number; + outcome: 'success' | 'error'; + durationMs: number; + willRetry: boolean; + response?: LLMResponse; + error?: Error; +} + +export interface ProviderExecutionOptions { + onAttempt?(result: ProviderAttemptResult): void; +} + +export interface LLMProvider { + name: string; + models: string[]; + supportsStreaming: boolean; + supportsTools: boolean; + supportsBatching: boolean; + supportsVision?: boolean; + + generateResponse(request: LLMRequest, options?: ProviderExecutionOptions): Promise; + streamResponse?(request: LLMRequest): Promise>; + getProviderBalance?(): Promise; + validateConfig(): boolean; + getModels(): string[]; + estimateCost(request: LLMRequest): number; + healthCheck(): Promise; + getMetrics(): ProviderMetrics; + resetMetrics(): void; +} + +export interface LLMConfig { + provider: 'openai' | 'anthropic' | 'cloudflare' | 'cerebras' | 'groq' | 'nvidia' | 'auto'; + model: string; + temperature: number; + maxTokens: number; + apiKey?: string; + baseUrl?: string; + fallbackProvider?: 'openai' | 'anthropic' | 'cloudflare' | 'cerebras' | 'groq' | 'nvidia'; + fallbackModel?: string; + timeout?: number; + retries?: number; + retryDelay?: number; +} + +export interface ProviderConfig { + resilienceContext?: import('./utils/resilience-context.js').ResilienceContext; + /** false bypasses breaker admission and failure/success recording. */ + enableCircuitBreaker?: boolean; + apiKey?: string; + baseUrl?: string; + /** Request timeout in ms. For streams it covers opening the stream (headers), not the body. */ + timeout?: number; + /** Opt-in total deadline in ms for a streaming response, including its body. Unset means none. */ + streamBodyTimeout?: number; + maxRetries?: number; + retryDelay?: number; + rateLimitRpm?: number; + rateLimitRpd?: number; + organization?: string; + project?: string; + logger?: Logger; + hooks?: ObservabilityHooks; +} + +export interface OpenAIConfig extends ProviderConfig { + organization?: string; + project?: string; + /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ + cfGateway?: CfGatewayConfig; +} + +export interface AnthropicConfig extends ProviderConfig { + version?: string; + /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ + cfGateway?: CfGatewayConfig; +} + +export interface CloudflareAIGatewayOptions { + /** AI Gateway id for Workers AI binding calls, e.g. "default" or a custom gateway id. */ + id: string; + /** Workers AI binding Gateway response-cache key. */ + cacheKey?: string; + /** Workers AI binding Gateway response-cache TTL in seconds. */ + cacheTtl?: number; + /** Workers AI binding Gateway response-cache bypass flag. */ + skipCache?: boolean; + /** Gateway metadata visible in AI Gateway logs. */ + metadata?: Record; + collectLog?: boolean; + eventId?: string; + requestTimeoutMs?: number; +} + +export interface CloudflareConfig extends ProviderConfig { + accountId?: string; + ai?: Ai; // Cloudflare AI binding + /** Workers AI binding Gateway options passed as the third env.AI.run() argument. */ + gateway?: CloudflareAIGatewayOptions; +} + +export interface CerebrasConfig extends ProviderConfig { + // Cerebras uses OpenAI-compatible API; no extra fields needed beyond ProviderConfig + /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ + cfGateway?: CfGatewayConfig; +} + +export interface GroqConfig extends ProviderConfig { + // Groq uses OpenAI-compatible API; no extra fields needed beyond ProviderConfig + /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ + cfGateway?: CfGatewayConfig; +} + +export interface NvidiaConfig extends ProviderConfig { + // NVIDIA NIM uses OpenAI-compatible API; no extra fields needed beyond ProviderConfig + /** Route through Cloudflare AI Gateway. Ignored when `baseUrl` is set. */ + cfGateway?: CfGatewayConfig; +} + +export interface LLMError extends Error { + code: string; + provider: string; + retryable: boolean; + statusCode?: number; + rateLimited?: boolean; + quotaExceeded?: boolean; +} + +export interface CircuitBreakerConfig { + failureThreshold: number; + resetTimeout: number; + monitoringPeriod: number; + minRequests?: number; + degradationCurve?: number[]; +} + +export interface CircuitBreakerState { + state: 'CLOSED' | 'DEGRADED' | 'RECOVERING' | 'OPEN'; + failures: number; + consecutiveFailures: number; + primaryTrafficPct: number; + totalFailures: number; + totalSuccesses: number; + totalRequests: number; + lastFailure?: number; + lastSuccess?: number; + lastRequest?: number; + nextAttempt?: number; +} + +export interface RetryConfig { + maxRetries: number; + initialDelay: number; + maxDelay: number; + backoffMultiplier: number; + retryableErrors: string[]; +} + +export interface CostConfig { + inputTokenCost: number; // Cost per 1k input tokens + outputTokenCost: number; // Cost per 1k output tokens + maxMonthlyCost?: number; + alertThreshold?: number; +} + +export interface ProviderMetrics { + requestCount: number; + successCount: number; + errorCount: number; + averageLatency: number; + totalCost: number; + rateLimitHits: number; + lastUsed: number; +} + +export interface FallbackRule { + condition: 'error' | 'rate_limit' | 'cost' | 'latency'; + threshold?: number; + fallbackProvider: string; + fallbackModel?: string; +} + +export interface ModelCapabilities { + maxContextLength: number; + supportsStreaming: boolean; + supportsTools: boolean; + supportsVision?: boolean; + toolCalling?: boolean; + supportsBatching: boolean; + /** USD per 1,000 input tokens. */ + inputTokenCost: number; + /** USD per 1,000 output tokens. */ + outputTokenCost: number; + description: string; + /** Provider-side prompt/prefix caching supported for this model. */ + supportsPromptCache?: boolean; + /** + * Provider-hosted built-in tools this model can run server-side, normalized + * to `BuiltInToolType`. Absent/empty means the model is function-tools only. + * Drives capability-aware routing and boundary gating for `LLMRequest.builtInTools`. + */ + supportsBuiltInTools?: BuiltInToolType[]; + /** + * True for models that output chain-of-thought reasoning traces as part of their + * response (e.g. QwQ, DeepSeek-R1, GLM-4.7-Flash). These models are unsuitable + * for direct-response routing (summary, classification, chat) unless the caller + * explicitly handles reasoning traces. Routers must exclude thinking models from + * all non-RESEARCH use-case pools. + */ + thinkingModel?: boolean; +} + +export interface ProviderCapabilities { + models: Record; + supportsStreaming: boolean; + supportsTools: boolean; + supportsBatching: boolean; + maxBatchSize?: number; + rateLimits: { + requestsPerMinute: number; + requestsPerDay: number; + tokensPerMinute: number; + }; +} + +// Streaming support +export interface StreamChunk { + id: string; + content: string; + delta: string; + finishReason?: 'stop' | 'length' | 'tool_calls' | 'content_filter'; + usage?: Partial; +} + +export interface StreamResponse { + stream: ReadableStream; + controller: ReadableStreamDefaultController; +} + +/** Final accounting for a managed text stream, including partial output. */ +export interface StreamCompletion { + outcome: 'success' | 'error' | 'cancelled'; + emittedChars: number; + usage: TokenUsage; +} + +export interface ManagedTextStream { + stream: ReadableStream; + /** Resolves after accounting; rejects if settlement fails. Suitable for ctx.waitUntil. */ + completion: Promise; +} + +export interface QuotaCheckInput { + tenantId?: string; + provider: string; + model: string; + estimatedCost: number; + metadata?: Record; +} + +export interface QuotaCheckResult { + allowed: boolean; + reason?: string; + remainingBudget?: number; +} + +export interface QuotaRecordInput { + tenantId?: string; + provider: string; + model: string; + actualCost: number; + inputTokens?: number; + outputTokens?: number; + metadata?: Record; +} + +export interface QuotaHook { + check(input: QuotaCheckInput): Promise; + record(input: QuotaRecordInput): Promise; +} + +export interface ToolExecutor { + execute(name: string, argumentsValue: unknown): Promise; +} + +export interface ToolLoopState { + iteration: number; + cumulativeCost: number; + messageCount: number; + lastToolCalls: ToolCall[]; +} + +export interface ToolLoopAbortSignal { + abort: true; + reason?: string; +} + +export interface ToolLoopOptions { + maxIterations?: number; + maxCostUSD?: number; + /** + * Called after each iteration. Return `{ abort: true, reason? }` to stop the + * loop immediately and throw `ToolLoopAbortedError`; return void to continue. + */ + onIteration?: ( + iteration: number, + state: ToolLoopState + ) => void | ToolLoopAbortSignal | Promise; + abortSignal?: AbortSignal; +} + +export interface ClassifyOptions { + schema?: Record | { parse(data: unknown): T }; + systemPrompt?: string; + model?: string; + temperature?: number; + maxTokens?: number; + confidenceField?: string; + seed?: number; +} + +export interface ClassifyResult { + data: T; + confidence?: number; + response: LLMResponse; +} + +export interface AnalyzeImageInput { + image: LLMImageInput; + prompt: string; + model?: string; + systemPrompt?: string; + temperature?: number; + maxTokens?: number; + response_format?: LLMRequest['response_format']; + tenantId?: string; + requestId?: string; + metadata?: Record; +} + +export interface RateLimitBalance { + limit?: number; + used?: number; + remaining?: number; +} + +export interface ProviderBalance { + provider: string; + status: 'available' | 'unavailable' | 'error'; + source: 'provider_api' | 'ledger' | 'headers' | 'not_supported'; + currentSpend?: number; + monthlyBudget?: number; + remainingBudget?: number; + usedTokens?: number; + requestCount?: number; + rateLimits?: Record; + resetAt?: string; + message?: string; + raw?: unknown; +} + +// Batch processing +export interface BatchRequest { + id: string; + request: LLMRequest; + priority?: 'low' | 'normal' | 'high'; +} + +export interface BatchResponse { + id: string; + response?: LLMResponse; + error?: LLMError; + processingTime: number; +} + +export interface BatchJob { + id: string; + requests: BatchRequest[]; + status: 'pending' | 'processing' | 'completed' | 'failed'; + progress: number; + createdAt: number; + completedAt?: number; + results: BatchResponse[]; +}