From c3719f067935bd81f2350f433cf8b612543cb7ef Mon Sep 17 00:00:00 2001 From: "user.email" Date: Sat, 17 Oct 2026 12:30:26 -0700 Subject: [PATCH] fix(inference): tolerate null role and tool_calls deltas --- src/agent/retry-policy.test.ts | 2 +- .../chat-completions-null-delta.test.ts | 227 ++++++++++++++++++ .../openai-compatible-adapter.test.ts | 33 ++- vendor/intx-inference/PATCHES.md | 26 ++ vendor/intx-inference/src/providers/openai.ts | 19 +- 5 files changed, 299 insertions(+), 8 deletions(-) create mode 100644 src/provider/chat-completions-null-delta.test.ts diff --git a/src/agent/retry-policy.test.ts b/src/agent/retry-policy.test.ts index ccd89728d..a91297fa3 100644 --- a/src/agent/retry-policy.test.ts +++ b/src/agent/retry-policy.test.ts @@ -259,7 +259,7 @@ describe("createCorbitsRetryPolicy", () => { error: { category: "protocol_mismatch", message: - "openai parseResponse: SSE chunk failed schema validation: choices0.delta.role must be a string (was null)", + "openai parseResponse: SSE chunk failed schema validation: choices[0].delta.role must be a string or null (was a number)", }, }); expect(decision).toEqual({ kind: "abort" }); diff --git a/src/provider/chat-completions-null-delta.test.ts b/src/provider/chat-completions-null-delta.test.ts new file mode 100644 index 000000000..e2513856f --- /dev/null +++ b/src/provider/chat-completions-null-delta.test.ts @@ -0,0 +1,227 @@ +// Regression coverage for the locally patched OpenAI delta schema (see +// vendor/intx-inference/PATCHES.md#providers-ts-openai-null-delta-tolerance). +// vLLM serializers behind OpenAI-compatible relays emit `"role": null` on +// streamed deltas and `"tool_calls": null` on non-tool chunks; the patch +// tolerates them (null ≡ absent) while wrong types stay rejected. +import { describe, expect, test } from "bun:test"; +import { ProtocolMismatchError } from "@intx/inference"; +import { createOpenAIAdapter } from "@intx/inference/providers"; +import type { LastCycleSource } from "@intx/types/runtime"; +import { createOpenAICompatibleAdapter } from "./openai-compatible-adapter.js"; +import { defined } from "../../testkit/defined.js"; + +const source: LastCycleSource = { + sourceId: "null-delta-test", + provider: "openai-compatible", + model: "null-delta-fixture", +}; + +const builtin = createOpenAIAdapter(source); + +describe("chat completions null delta tolerance", () => { + test("(a) null role is equivalent to an absent role", () => { + const withNull = builtin.parseResponse( + JSON.stringify({ + choices: [{ index: 0, delta: { role: null, content: "hi" } }], + }), + ); + const absent = builtin.parseResponse( + JSON.stringify({ choices: [{ index: 0, delta: { content: "hi" } }] }), + ); + expect(withNull).toEqual(absent); + const textDelta = withNull.find( + (event) => event.type === "inference.text.delta", + ); + expect( + (textDelta as unknown as { data: Record } | undefined) + ?.data, + ).toEqual({ + token: "hi", + partial: { text: "" }, + index: 0, + }); + }); + + test("(b) tool_calls null and empty array are equivalent to absent", () => { + const frames = [{ tool_calls: null }, { tool_calls: [] }, {}] as const; + const shapes = frames.map((delta) => + builtin.parseResponse(JSON.stringify({ choices: [{ index: 0, delta }] })), + ); + expect(shapes[0]).toEqual(shapes[2]); + expect(shapes[1]).toEqual(shapes[2]); + expect(shapes[2]).toEqual([]); + }); + + test("(c) wrong-type role still throws ProtocolMismatchError", () => { + expect(() => + builtin.parseResponse( + JSON.stringify({ choices: [{ index: 0, delta: { role: 42 } }] }), + ), + ).toThrow(ProtocolMismatchError); + }); + + test("(c) wrong-type tool_calls still throws ProtocolMismatchError", () => { + for (const tool_calls of [42, "nope"]) { + expect(() => + builtin.parseResponse( + JSON.stringify({ choices: [{ index: 0, delta: { tool_calls } }] }), + ), + ).toThrow(ProtocolMismatchError); + } + }); + + test("(d) incident-shape first chunk with null role and tool_calls parses", () => { + const events = builtin + .parseResponse( + JSON.stringify({ + choices: [ + { + index: 0, + delta: { role: null, tool_calls: null, content: "hello" }, + }, + ], + }), + ) + .concat( + builtin.parseResponse( + JSON.stringify({ + choices: [ + { + index: 0, + delta: { role: null, tool_calls: null, content: " world" }, + }, + ], + }), + ), + ); + const textDeltas = events.filter( + (event) => event.type === "inference.text.delta", + ); + expect( + textDeltas.map((event) => { + const { token, index, partial } = event.data as unknown as { + token: string; + index: number; + partial: { text: string }; + }; + return { token, index, partial }; + }), + ).toEqual([ + { token: "hello", partial: { text: "" }, index: 0 }, + { token: " world", partial: { text: "" }, index: 0 }, + ]); + }); + + test("(e1) non-streaming null role is equivalent to absent role", () => { + const wireUsage = { + prompt_tokens: 10, + completion_tokens: 5, + }; + const withNull = defined(builtin.parseJSONResponse)( + JSON.stringify({ + object: "chat.completion", + choices: [{ message: { role: null, content: "done" } }], + usage: wireUsage, + }), + ); + const absent = defined(builtin.parseJSONResponse)( + JSON.stringify({ + object: "chat.completion", + choices: [{ message: { content: "done" } }], + usage: wireUsage, + }), + ); + expect(withNull).toEqual(absent); + expect( + withNull.some((event) => event.type === "inference.text.delta"), + ).toBe(true); + expect(withNull.some((event) => event.type === "inference.usage")).toBe( + true, + ); + }); + + test("(e2) non-streaming null tool_calls is equivalent to absent tool_calls", () => { + const wireUsage = { + prompt_tokens: 10, + completion_tokens: 5, + }; + const withNull = defined(builtin.parseJSONResponse)( + JSON.stringify({ + object: "chat.completion", + choices: [ + { message: { role: null, tool_calls: null, content: "done" } }, + ], + usage: wireUsage, + }), + ); + const absent = defined(builtin.parseJSONResponse)( + JSON.stringify({ + object: "chat.completion", + choices: [{ message: { content: "done" } }], + usage: wireUsage, + }), + ); + expect(withNull).toEqual(absent); + }); + + test("(e3) non-streaming wrong-type role still throws ProtocolMismatchError", () => { + expect(() => + defined(builtin.parseJSONResponse)( + JSON.stringify({ + object: "chat.completion", + choices: [{ message: { role: 42, content: "done" } }], + usage: { prompt_tokens: 10, completion_tokens: 5 }, + }), + ), + ).toThrow(ProtocolMismatchError); + }); +}); + +describe("openai-compatible adapter null delta wiring parity", () => { + const wrapperSource = { + id: "test", + provider: "openai-compatible", + baseURL: "https://example.test", + apiKey: "sk-test", + model: "gpt-5.1", + } as unknown as Parameters[0]; + + test("(a) null role is equivalent to an absent role", () => { + const adapter = createOpenAICompatibleAdapter(wrapperSource); + expect( + adapter.parseResponse( + JSON.stringify({ + choices: [{ index: 0, delta: { role: null, content: "hi" } }], + }), + ), + ).toEqual( + adapter.parseResponse( + JSON.stringify({ choices: [{ index: 0, delta: { content: "hi" } }] }), + ), + ); + }); + + test("(b) tool_calls null is equivalent to absent", () => { + const adapter = createOpenAICompatibleAdapter(wrapperSource); + expect( + adapter.parseResponse( + JSON.stringify({ + choices: [{ index: 0, delta: { tool_calls: null } }], + }), + ), + ).toEqual( + adapter.parseResponse( + JSON.stringify({ choices: [{ index: 0, delta: {} }] }), + ), + ); + }); + + test("(c) wrong-type role still throws ProtocolMismatchError", () => { + const adapter = createOpenAICompatibleAdapter(wrapperSource); + expect(() => + adapter.parseResponse( + JSON.stringify({ choices: [{ index: 0, delta: { role: 42 } }] }), + ), + ).toThrow(ProtocolMismatchError); + }); +}); diff --git a/src/provider/openai-compatible-adapter.test.ts b/src/provider/openai-compatible-adapter.test.ts index 9fcc012d5..04ac703fd 100644 --- a/src/provider/openai-compatible-adapter.test.ts +++ b/src/provider/openai-compatible-adapter.test.ts @@ -109,12 +109,41 @@ describe("openai-compatible adapter SSE parse count", () => { }); describe("openai-compatible adapter null delta fields", () => { - test.each(["role", "tool_calls"])("rejects null %s", (field) => { + test.each(["role", "tool_calls"])( + "treats null %s the same as absent", + (field) => { + const adapter = createOpenAICompatibleAdapter(source); + expect( + adapter.parseResponse( + JSON.stringify({ + choices: [{ index: 0, delta: { content: "hello", [field]: null } }], + }), + ), + ).toEqual( + adapter.parseResponse( + JSON.stringify({ + choices: [{ index: 0, delta: { content: "hello" } }], + }), + ), + ); + }, + ); + + test("still rejects wrong-type role", () => { + const adapter = createOpenAICompatibleAdapter(source); + expect(() => + adapter.parseResponse( + JSON.stringify({ choices: [{ index: 0, delta: { role: 42 } }] }), + ), + ).toThrow(ProtocolMismatchError); + }); + + test("still rejects wrong-type tool_calls", () => { const adapter = createOpenAICompatibleAdapter(source); expect(() => adapter.parseResponse( JSON.stringify({ - choices: [{ index: 0, delta: { content: "hello", [field]: null } }], + choices: [{ index: 0, delta: { tool_calls: "nope" } }], }), ), ).toThrow(ProtocolMismatchError); diff --git a/vendor/intx-inference/PATCHES.md b/vendor/intx-inference/PATCHES.md index aff0b14ef..ba764992e 100644 --- a/vendor/intx-inference/PATCHES.md +++ b/vendor/intx-inference/PATCHES.md @@ -780,3 +780,29 @@ retry entirely into `harness.ts`'s `runInference` wrapper, which now emits `inference.retry` itself before the commitment boundary — see harness-ts-commitment-boundary-streaming; a reactor-level emission would double the event). + +## providers-ts-openai-null-delta-tolerance + +**Promotion candidate.** `providers/openai.ts` tolerates explicit `null` for +`role` and `tool_calls` on Chat Completions streamed deltas and the +non-streaming message (CL-3010: vLLM serializers behind OpenAI-compatible +relays — NEAR AI, Qwen3.8-Flash-Next on vast.ai 2xH200 — emit `"role": null` +on every delta and `"tool_calls": null` on non-tool chunks; the strict schema +rejected each frame, so `protocol_mismatch` aborted the turn). `null` now +means absent, mirroring how sibling(`content`/`refusal`/`reasoning*`) and +nested tool-call (`id`/`function.name`/`function.arguments`) fields already +treat it: `role` widens to `string | null`, `tool_calls` widens to +array-or-null with the streaming parser coalescing null → undefined before +iteration. Wrong-type values stay rejected (`role: 42` still fails schema +validation → `ProtocolMismatchError`, keeping the `protocol_mismatch` abort +path and its rework in `src/agent/retry-policy.test.ts` intact). No consumer +reads `delta.role`; `tool_calls: []` on every chunk (vLLM) was already +tolerated and keeps its semantics. + +**Re-carry:** spans `OpenAIToolCallDelta`/`OpenAIChunkDelta`, the streaming +`tool_calls` null-coalesce in `parseResponse`, and the mirrored +`NonStreamingMessage` widening; regression coverage lives in +`src/provider/chat-completions-null-delta.test.ts` (Corbits-side — vendored +OpenAI adapter has no co-located test file at this pin and vendor tests are +outside the repo gate). Drop when upstream opens `delta.role` / +`delta.tool_calls` (and the message schema) to null. diff --git a/vendor/intx-inference/src/providers/openai.ts b/vendor/intx-inference/src/providers/openai.ts index cb0666722..2f61008a5 100644 --- a/vendor/intx-inference/src/providers/openai.ts +++ b/vendor/intx-inference/src/providers/openai.ts @@ -507,6 +507,12 @@ const EMPTY_PARTIAL: PartialMessage = { text: "" }; // drop the whole chunk silently — taking the argument fragments with // it. Accept `string | null` here and treat null the same as the field // being absent at the consumer site. +// Locally patched — see vendor/intx-inference/PATCHES.md#providers-ts-openai-null-delta-tolerance. +// vLLM serializers behind OpenAI-compatible relays emit `role: null` on +// streamed deltas and `tool_calls: null` on non-tool chunks. The strict +// schema rejected each frame, so `protocol_mismatch` aborted the turn. +// `null` means absent here, mirroring the sibling and nested-tool-call +// fields below; wrong-type values stay rejected. const OpenAIToolCallDelta = type({ "index?": "number", "id?": "string | null", @@ -517,7 +523,7 @@ const OpenAIToolCallDelta = type({ }); const OpenAIChunkDelta = type({ - "role?": "string", + "role?": "string | null", "content?": "string | null", "reasoning_content?": "string | null", "reasoning?": "string | null", @@ -527,7 +533,7 @@ const OpenAIChunkDelta = type({ // OpenAI-compatible relays strip it before forwarding; the parser // emits refusal events only when the field is present. "refusal?": "string | null", - "tool_calls?": OpenAIToolCallDelta.array(), + "tool_calls?": OpenAIToolCallDelta.array().or("null"), }); const PromptTokensDetails = type({ "cached_tokens?": "number" }).or("null"); @@ -758,7 +764,10 @@ function parseResponse( }); } - const { tool_calls: toolCallDeltas } = delta; + // Normalize null → undefined (same idiom as below for function fields): + // the schema tolerates `tool_calls: null`, but a bare widened schema + // would hand the null to `for...of` and crash with a TypeError. + const toolCallDeltas = delta.tool_calls ?? undefined; if (toolCallDeltas !== undefined) { for (const tcDelta of toolCallDeltas) { @@ -881,12 +890,12 @@ const NonStreamingToolCall = type({ }); const NonStreamingMessage = type({ - "role?": "string", + "role?": "string | null", "content?": "string | null", "reasoning_content?": "string | null", "reasoning?": "string | null", "refusal?": "string | null", - "tool_calls?": NonStreamingToolCall.array(), + "tool_calls?": NonStreamingToolCall.array().or("null"), }); const NonStreamingCompletion = type({