From b71c88649750b19623cd6e99c65a21037ce174a8 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:01:30 -0800 Subject: [PATCH 01/11] test(server): reproduce model API evaluation routing failure --- .../src/textGeneration/TextGeneration.test.ts | 37 ++++++++++++++++++- 1 file changed, 35 insertions(+), 2 deletions(-) diff --git a/apps/server/src/textGeneration/TextGeneration.test.ts b/apps/server/src/textGeneration/TextGeneration.test.ts index 1bb849b570103..0d4ba7c3fd97e 100644 --- a/apps/server/src/textGeneration/TextGeneration.test.ts +++ b/apps/server/src/textGeneration/TextGeneration.test.ts @@ -6,7 +6,10 @@ import { type ModelEvaluationVerdict, } from "@t3tools/contracts"; import type { ProviderInstance } from "@t3tools/provider-core/server/driver"; -import { fromRunner } from "@t3tools/provider-core/server/textGenerationOperations"; +import { + decodeJsonReply, + fromRunner, +} from "@t3tools/provider-core/server/textGenerationOperations"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; @@ -45,6 +48,7 @@ const make = ( textGeneration: ProviderInstance["textGeneration"], snapshot: ServerProvider = providerSnapshot, enabled = true, + runner: TextGenerators.TextGenerators["Service"]["runner"] = () => Effect.undefined, ) => { const getInstance = vi.fn((instanceId: typeof selected) => Effect.succeed( @@ -69,7 +73,7 @@ const make = ( resolveLink: () => undefined, }), ServerSettings.layerTest({}), - Layer.mock(TextGenerators.TextGenerators)({ runner: () => Effect.undefined }), + Layer.mock(TextGenerators.TextGenerators)({ runner }), ProviderInstallationGate.layer, ), ), @@ -137,6 +141,35 @@ const evaluationVerdict: ModelEvaluationVerdict = { promptAdvice: [], }; +it.effect("judges with the selected model API without switching to an agent provider", () => + Effect.gen(function* () { + const modelSelection = { + instanceId: ProviderInstanceId.make("textgen-local"), + model: "chosen-api-model", + }; + const test = make( + fromRunner("Fixture", () => Effect.die("Unexpected agent generation")), + providerSnapshot, + true, + (instanceId) => + Effect.succeed( + instanceId === modelSelection.instanceId + ? (request) => { + expect(request.modelSelection).toEqual(modelSelection); + expect(request.prompt).toContain(evaluationInput.packet.request.text); + return decodeJsonReply(request, "Model API", JSON.stringify(evaluationVerdict)); + } + : undefined, + ), + ); + const service = yield* test.service; + expect(yield* service.generateModelEvaluation({ ...evaluationInput, modelSelection })).toEqual( + evaluationVerdict, + ); + expect(test.getInstance).not.toHaveBeenCalled(); + }), +); + it.effect.each([ "disabled-instance", "disabled-snapshot", From 928c727fb68c7f962f90ce2edb72a78c476a6f51 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:23:52 -0800 Subject: [PATCH 02/11] fix(server,web,mobile): model APIs can judge model evaluations A judge selection naming a model API failed with "No provider instance registered", and both settings screens hid model APIs from the judge picker. generateModelEvaluation now runs the selected model API directly as a blind evaluator. It has no tools or workspace, so it never falls back to another judge. Agent providers keep the exact availability checks. Web lists model APIs through the existing text generation targets and treats an enabled API as an available judge per environment. Mobile lists each enabled API's models through the same guarded command. Co-Authored-By: Claude Opus 5.5 --- .../settings/ModelEvaluationSettings.tsx | 96 ++++++++++++++++--- .../src/textGeneration/TextGeneration.test.ts | 87 +++++++++++------ .../src/textGeneration/TextGeneration.ts | 21 ++-- .../settings/ModelEvaluationSettings.tsx | 19 ++-- .../settings/useScopedModelAvailability.ts | 5 +- docs/user/model-evaluations.md | 7 +- 6 files changed, 173 insertions(+), 62 deletions(-) diff --git a/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx b/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx index 9d186387592d2..d48d9b27a8de6 100644 --- a/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx +++ b/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx @@ -2,10 +2,13 @@ import { useAtomValue } from "@effect/atom-react"; import type { MenuAction } from "@react-native-menu/menu"; import { isManagedSettingPath, + textGeneratorForSelection, type EnvironmentId, type ModelEvaluationGuidance, type ProjectId, + type ProviderInstanceId, type ServerSettingsPatch, + type TextGeneratorConfig, } from "@t3tools/contracts"; import { modelEvaluationGuidanceKey, @@ -13,7 +16,9 @@ import { modelEvaluationSelectionLabel, isModelEvaluationJudge, } from "@t3tools/client-runtime/modelEvaluations"; -import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; +import { runAtomCommand, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; +import * as Effect from "effect/Effect"; +import { AsyncResult, Atom } from "effect/reactivity"; import { useState } from "react"; import { View } from "react-native"; @@ -22,7 +27,12 @@ import { Button } from "../../components/ui/Button"; import * as Dialog from "../../components/ui/Dialog"; import { Menu } from "../../components/ui/Menu"; import { RowPressable } from "../../components/ui/RowPressable"; -import { buildModelMenuActions, buildModelOptions, groupByProvider } from "../../lib/modelOptions"; +import { + buildModelMenuActions, + buildModelOptions, + groupByProvider, + type ModelOption, +} from "../../lib/modelOptions"; import { useEnvironmentServerConfig, useProjects } from "../../state/entities"; import { modelEvaluations } from "../../state/modelEvaluations"; import { useEnvironmentQuery } from "../../state/query"; @@ -66,6 +76,26 @@ export function ModelEvaluationSettings() { ); } +// Keyed on each API's connection settings so editing one asks it again. +const apiJudgeModelsAtom = Atom.family((key: string) => { + const apis = JSON.parse(key) as ReadonlyArray; + return Atom.make((get) => + Effect.forEach( + apis, + ([environmentId, id]) => + Effect.promise(() => + runAtomCommand( + get.registry, + serverEnvironment.listTextGeneratorModels, + { environmentId, input: { id } }, + { reportFailure: false }, + ), + ).pipe(Effect.map((result) => (result._tag === "Success" ? result.value.models : []))), + { concurrency: "unbounded" }, + ), + ).pipe(Atom.setIdleTTL(5 * 60_000), Atom.withLabel(`model-evaluation-api-judges:${key}`)); +}); + function EnvironmentEvaluationSettings(props: { environmentId: EnvironmentId; label: string; @@ -79,6 +109,24 @@ function EnvironmentEvaluationSettings(props: { const projects = useProjects().filter((project) => project.environmentId === environmentId); const [chosenProjectId, setChosenProjectId] = useState(null); const [saving, setSaving] = useState(false); + // Listing an API's models needs settings write access, like choosing the judge does. + const apiJudges = ( + canWrite && config !== null + ? (Object.entries(config.settings.textGenerators) as Array< + [ProviderInstanceId, TextGeneratorConfig] + >) + : [] + ).filter(([, generator]) => generator.enabled); + const apiModelLists = useAtomValue( + apiJudgeModelsAtom( + JSON.stringify( + apiJudges.map( + ([id, generator]) => + [environmentId, id, generator.kind, generator.baseUrl, generator.apiKey] as const, + ), + ), + ), + ); if (config === null) return null; const readOnly = saving || !canWrite || props.projectSelected; const project = @@ -90,22 +138,42 @@ function EnvironmentEvaluationSettings(props: { const capable = new Set( config.providers.filter(isModelEvaluationJudge).map((provider) => provider.instanceId), ); - const models = buildModelOptions(config, selection, undefined, null).filter( - (model) => capable.has(model.selection.instanceId) && !model.isUnavailable, - ); + const listedApiModels = AsyncResult.getOrElse(apiModelLists, () => []); + const models = [ + ...buildModelOptions(config, selection, undefined, null).filter( + (model) => capable.has(model.selection.instanceId) && !model.isUnavailable, + ), + ...apiJudges.flatMap(([instanceId, generator], index) => + (listedApiModels[index] ?? []).map((model): ModelOption => ({ + key: `${instanceId}:${model.id}`, + label: model.name, + subtitle: "", + providerKey: instanceId, + providerLabel: generator.label || generator.baseUrl, + providerDriver: generator.kind, + isDefault: false, + isLegacy: false, + capabilities: null, + selection: { instanceId, model: model.id }, + })), + ), + ]; const selectedProvider = config.providers.find( (provider) => provider.instanceId === selection.instanceId, ); + const selectedApi = textGeneratorForSelection(settings, selection.instanceId); const judgeAvailable = - selectedProvider?.installed === true && - selectedProvider.status !== "error" && - selectedProvider.auth.status !== "unauthenticated" && - selectedProvider.availability !== "unavailable" && - models.some( - (model) => - model.selection.instanceId === selection.instanceId && - model.selection.model === selection.model, - ); + selectedApi !== undefined + ? selectedApi.enabled + : selectedProvider?.installed === true && + selectedProvider.status !== "error" && + selectedProvider.auth.status !== "unauthenticated" && + selectedProvider.availability !== "unavailable" && + models.some( + (model) => + model.selection.instanceId === selection.instanceId && + model.selection.model === selection.model, + ); const write = async (patch: ServerSettingsPatch) => { if (readOnly) return; setSaving(true); diff --git a/apps/server/src/textGeneration/TextGeneration.test.ts b/apps/server/src/textGeneration/TextGeneration.test.ts index 0d4ba7c3fd97e..24b6f5b020da3 100644 --- a/apps/server/src/textGeneration/TextGeneration.test.ts +++ b/apps/server/src/textGeneration/TextGeneration.test.ts @@ -12,6 +12,7 @@ import { } from "@t3tools/provider-core/server/textGenerationOperations"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as Result from "effect/Result"; import * as Schema from "effect/Schema"; import { vi } from "vite-plus/test"; @@ -123,7 +124,7 @@ const evaluationInput = { runOutcome: "completed" as const, request: { ref: "U1" as const, text: "Preserve the API." }, context: [], - evidence: [{ ref: "A1", kind: "assistant" as const, text: "The API was preserved." }], + evidence: [{ ref: "E1", kind: "assistant" as const, text: "The API was preserved." }], truncated: false, omittedItems: 0, missingArtifacts: true, @@ -135,39 +136,67 @@ const evaluationVerdict: ModelEvaluationVerdict = { taskCategory: "coding", scores: { taskCompletion: 3, correctness: null, instructionFollowing: 3, verification: null }, confidence: "low", - rationale: { text: "The reply addresses the request.", citations: ["U1", "A1"] }, + rationale: { text: "The reply addresses the request.", citations: ["U1", "E1"] }, strengths: [], weaknesses: [], promptAdvice: [], }; -it.effect("judges with the selected model API without switching to an agent provider", () => - Effect.gen(function* () { - const modelSelection = { - instanceId: ProviderInstanceId.make("textgen-local"), - model: "chosen-api-model", - }; - const test = make( - fromRunner("Fixture", () => Effect.die("Unexpected agent generation")), - providerSnapshot, - true, - (instanceId) => - Effect.succeed( - instanceId === modelSelection.instanceId - ? (request) => { - expect(request.modelSelection).toEqual(modelSelection); - expect(request.prompt).toContain(evaluationInput.packet.request.text); - return decodeJsonReply(request, "Model API", JSON.stringify(evaluationVerdict)); - } - : undefined, - ), - ); - const service = yield* test.service; - expect(yield* service.generateModelEvaluation({ ...evaluationInput, modelSelection })).toEqual( - evaluationVerdict, - ); - expect(test.getInstance).not.toHaveBeenCalled(); - }), +it.effect.each(["valid verdict", "extra verdict field", "turned off"] as const)( + "judges only with the selected model API (%s)", + (state) => + Effect.gen(function* () { + const modelSelection = { + instanceId: ProviderInstanceId.make("textgen-local"), + model: "chosen-api-model", + }; + const generateModelEvaluation = vi.fn(() => Effect.succeed(evaluationVerdict)); + const test = make( + { + ...fromRunner("Fixture", () => Effect.die("Unexpected agent generation")), + supportedOperations: ["generateModelEvaluation"], + generateModelEvaluation, + }, + providerSnapshot, + true, + (instanceId) => + Effect.succeed( + instanceId === modelSelection.instanceId + ? (request) => { + expect(request.modelSelection).toEqual(modelSelection); + expect(request.prompt).toContain(evaluationInput.packet.request.text); + return state === "turned off" + ? Effect.fail( + new TextGenerationError({ + operation: request.operation, + detail: "Model API is off.", + }), + ) + : decodeJsonReply( + request, + "Model API", + JSON.stringify( + state === "valid verdict" + ? evaluationVerdict + : { ...evaluationVerdict, extra: true }, + ), + ); + } + : undefined, + ), + ); + const service = yield* test.service; + const result = yield* service + .generateModelEvaluation({ ...evaluationInput, modelSelection }) + .pipe(Effect.result); + if (state === "valid verdict") { + expect(result).toEqual(Result.succeed(evaluationVerdict)); + } else { + expect(result._tag).toBe("Failure"); + } + expect(generateModelEvaluation).not.toHaveBeenCalled(); + expect(test.getInstance).not.toHaveBeenCalled(); + }), ); it.effect.each([ diff --git a/apps/server/src/textGeneration/TextGeneration.ts b/apps/server/src/textGeneration/TextGeneration.ts index 93dcc302f4d0c..4607cbf377cbe 100644 --- a/apps/server/src/textGeneration/TextGeneration.ts +++ b/apps/server/src/textGeneration/TextGeneration.ts @@ -188,12 +188,21 @@ export const make = Effect.gen(function* () { ); }); return TextGeneration.of({ - generateModelEvaluation: Effect.fn("TextGeneration.generateModelEvaluation")(function (input) { - return resolveInstance( - registry, - "generateModelEvaluation", - input.modelSelection.instanceId, - ).pipe(Effect.flatMap((textGeneration) => textGeneration.generateModelEvaluation(input))); + generateModelEvaluation: Effect.fn("TextGeneration.generateModelEvaluation")(function* (input) { + // A model API has no tools or workspace, so it is blind by construction. + const textGenerator = yield* textGenerators.runner(input.modelSelection.instanceId); + const judge = + textGenerator === undefined + ? yield* resolveInstance( + registry, + "generateModelEvaluation", + input.modelSelection.instanceId, + ) + : TextGenerationOperations.fromRunner("TextGenerator", textGenerator, { + supportsInvestigation: false, + supportsBlindEvaluation: true, + }); + return yield* judge.generateModelEvaluation(input); }), generateCommitMessage: Effect.fn("TextGeneration.generateCommitMessage")(function (input) { return generate( diff --git a/apps/web/src/components/settings/ModelEvaluationSettings.tsx b/apps/web/src/components/settings/ModelEvaluationSettings.tsx index 23c963530c378..e4b6611fe8dd9 100644 --- a/apps/web/src/components/settings/ModelEvaluationSettings.tsx +++ b/apps/web/src/components/settings/ModelEvaluationSettings.tsx @@ -1,6 +1,7 @@ import { useAtomValue } from "@effect/atom-react"; import { resolveProjectProviderInstanceEnabled, + textGeneratorForSelection, type EnvironmentId, type ModelEvaluationGuidance, type ProjectId, @@ -19,12 +20,12 @@ import { useState } from "react"; import { requestConfirmDialog } from "../../confirmDialog"; import { mergeEnvironmentSettings } from "../../hooks/useSettings"; import { getCustomModelOptionsByInstance } from "../../modelSelection"; -import { providerInstanceEntriesFor } from "../../providerInstances"; import { useProjects } from "../../state/entities"; import { useEnvironmentProviders } from "../../state/environments"; import { modelEvaluations } from "../../state/modelEvaluations"; import { useEnvironmentQuery } from "../../state/query"; import { EMPTY_SERVER_PROVIDERS, serverEnvironment } from "../../state/server"; +import { useTextGenerationTargets } from "../../textGenerators"; import { useAtomCommand } from "../../state/use-atom-command"; import { ProviderModelPicker } from "../chat/ProviderModelPicker"; import { TraitsPicker } from "../chat/TraitsPicker"; @@ -56,14 +57,18 @@ export function ModelEvaluationSettings() { const providers = (environment ? providersByEnvironment.get(environment.environmentId) : undefined) ?? EMPTY_SERVER_PROVIDERS; - const judgeProviders = providers.filter(isModelEvaluationJudge); const selection = settings.modelEvaluationModelSelection ?? settings.textGenerationModelSelection; - const entries = providerInstanceEntriesFor(judgeProviders, settings, null, [ - selection.instanceId, - ]); + const judgeTargets = useTextGenerationTargets({ + environmentId: environment?.environmentId ?? null, + settings, + providers: providers.filter(isModelEvaluationJudge), + projectId: null, + selections: [selection], + }); + const entries = judgeTargets.entries; const modelOptions = getCustomModelOptionsByInstance( settings, - judgeProviders, + judgeTargets.providers, selection.instanceId, selection.model, ); @@ -86,6 +91,8 @@ export function ModelEvaluationSettings() { const unavailableEnvironments = targets.flatMap((target) => { const judge = target.settings.modelEvaluationModelSelection ?? target.settings.textGenerationModelSelection; + const generator = textGeneratorForSelection(target.settings, judge.instanceId); + if (generator !== undefined) return generator.enabled ? [] : [target.label]; const provider = providersByEnvironment .get(target.environmentId) ?.find( diff --git a/apps/web/src/components/settings/useScopedModelAvailability.ts b/apps/web/src/components/settings/useScopedModelAvailability.ts index 1c2a17de7364b..a88f4eb2588b2 100644 --- a/apps/web/src/components/settings/useScopedModelAvailability.ts +++ b/apps/web/src/components/settings/useScopedModelAvailability.ts @@ -47,10 +47,7 @@ export function useScopedModelDisabledReason( const unavailable = `This model is unavailable on ${environment?.label ?? "a selected environment"}. Select that environment to choose its model separately.`; const generator = textGeneratorForSelection(candidate.settings, instanceId); if (shownGenerator !== undefined || generator !== undefined) { - if ( - requiredOperations.includes("generateReviewGuide") || - requiredOperations.includes("generateModelEvaluation") - ) + if (requiredOperations.includes("generateReviewGuide")) return "Choose a coding agent for this setting."; // Only the shown environment's model list is loaded; elsewhere it must reach the same API. if ( diff --git a/docs/user/model-evaluations.md b/docs/user/model-evaluations.md index c58d0a7bb0b46..bb1dfa268cc8c 100644 --- a/docs/user/model-evaluations.md +++ b/docs/user/model-evaluations.md @@ -6,9 +6,10 @@ model call. Use **Judge model** to select the provider and model that scores tas of the model doing the work. **Use default** returns it to the environment's text generation model. Only providers that can judge without tools or access to the project are available. The evaluator uses the same background generation system as thread titles, with its own optional -model choice. Claude and Pi can serve as judges when available. A supported judge can -evaluate work from any harness, including Codex; the harness that produced the work does not need -to support judging. Choose an available judge before enabling the setting. +model choice. Claude, Pi, and model APIs added under **Settings → Providers** can serve as judges +when available. A supported judge can evaluate work from any harness, including Codex; the harness +that produced the work does not need to support judging. Choose an available judge before enabling +the setting. The selected judge is used for new evaluations. If it is unavailable, the evaluation fails visibly instead of switching to another model. Claude judges require a CLI that supports `--safe-mode`, which keeps account authentication while disabling custom instructions and extensions. From 3e955db41cf53e3ee3084c65bf664425b25f57f5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:30:07 -0800 Subject: [PATCH 03/11] feat(cursor): support isolated model evaluation judges --- .../src/server/textGeneration.test.ts | 155 +++++++++++++++++- .../src/server/textGeneration.ts | 57 ++++++- 2 files changed, 208 insertions(+), 4 deletions(-) diff --git a/packages/provider-cursor/src/server/textGeneration.test.ts b/packages/provider-cursor/src/server/textGeneration.test.ts index bfb02df390d83..fb8c1fb3f1a27 100644 --- a/packages/provider-cursor/src/server/textGeneration.test.ts +++ b/packages/provider-cursor/src/server/textGeneration.test.ts @@ -1,4 +1,4 @@ -import type { RunResult } from "@cursor/sdk"; +import type { AgentOptions, RunResult, SDKMessage } from "@cursor/sdk"; import { ProviderInstanceId, TextGenerationError } from "@t3tools/contracts"; import { CursorSettings } from "../settings.ts"; import { describe, expect, it } from "@effect/vitest"; @@ -13,7 +13,8 @@ import { makeCursorTextGeneration } from "./textGeneration.ts"; import * as CursorSdk from "./CursorSdk.ts"; const cursorSdkMock = vi.hoisted(() => ({ - create: vi.fn<(options: unknown) => Promise>(), + messages: [] as SDKMessage[], + create: vi.fn<(options: AgentOptions) => Promise>(), send: vi.fn<(prompt: string) => Promise>(), close: vi.fn(), cancel: vi.fn(async () => {}), @@ -47,15 +48,20 @@ const cursorSettings = decodeCursorSettings({ enabled: true }); beforeEach(() => { hasCustomPolicy = false; + cursorSdkMock.messages = []; cursorSdkMock.create.mockReset(); cursorSdkMock.send.mockReset(); cursorSdkMock.create.mockImplementation(async (options) => { cursorSdkMock.send.mockImplementation(async (prompt) => ({ status: "running", cancel: cursorSdkMock.cancel, + stream: async function* () { + yield* cursorSdkMock.messages; + }, wait: () => cursorSdkMock.prompt(prompt, options), })); return { + model: options.model, close: cursorSdkMock.close, [Symbol.asyncDispose]: async () => { cursorSdkMock.close(); @@ -77,7 +83,152 @@ beforeEach(() => { }); }); +const evaluationInput = { + cwd: "/isolated-evaluator", + modelSelection: createModelSelection(ProviderInstanceId.make("cursor"), "composer-2", [ + { id: "fastMode", value: false }, + ]), + packet: { + scope: "turn" as const, + runOutcome: "completed" as const, + request: { ref: "U1" as const, text: "Repair the issue." }, + context: [], + evidence: [{ ref: "E1", kind: "assistant" as const, text: "Fixed." }], + truncated: false, + omittedItems: 0, + missingArtifacts: false, + }, +}; +const evaluationVerdict = { + outcome: "insufficient_evidence", + taskCategory: "other", + confidence: "low", + scores: { + taskCompletion: null, + correctness: null, + instructionFollowing: null, + verification: null, + }, + rationale: { text: "No verifiable outcome.", citations: ["E1"] }, + strengths: [], + weaknesses: [], + promptAdvice: [], +}; +const evaluationResult: RunResult = { + id: "run-evaluation", + status: "finished", + model: { id: "composer-2", params: [{ id: "fast", value: "false" }] }, + result: JSON.stringify(evaluationVerdict), +}; + describe("CursorTextGeneration", () => { + it.effect("evaluates with the selected model, no tools, and no ambient settings", () => + Effect.gen(function* () { + cursorSdkMock.prompt.mockResolvedValueOnce(evaluationResult); + const generation = yield* makeCursorTextGeneration(cursorSettings, { CURSOR_API_KEY: "key" }); + expect(generation.supportedOperations).toContain("generateModelEvaluation"); + expect(yield* generation.generateModelEvaluation(evaluationInput)).toEqual(evaluationVerdict); + expect(cursorSdkMock.create).toHaveBeenCalledExactlyOnceWith( + expect.objectContaining({ + model: evaluationResult.model, + tools: [], + local: expect.objectContaining({ + cwd: "/isolated-text-generation", + settingSources: [], + sandboxOptions: { enabled: true }, + enableAgentRetries: false, + }), + }), + ); + expect(cursorSdkMock.close).toHaveBeenCalledOnce(); + expect(cursorSdkMock.storeDispose).toHaveBeenCalledOnce(); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect.each(["auto", "default", ""])( + "rejects the unpinned judge model %s before starting Cursor", + (model) => + Effect.gen(function* () { + const generation = yield* makeCursorTextGeneration(cursorSettings, { + CURSOR_API_KEY: "key", + }); + const error = yield* generation + .generateModelEvaluation({ + ...evaluationInput, + modelSelection: { ...evaluationInput.modelSelection, model }, + }) + .pipe(Effect.flip); + expect(error.detail).toContain("explicit model"); + expect(cursorSdkMock.create).not.toHaveBeenCalled(); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect.each([ + undefined, + { id: "different-model" }, + { id: "composer-2", params: [{ id: "fast", value: "true" }] }, + { id: "composer-2" }, + ])("rejects an evaluation without confirmation of the exact model and options (%j)", (model) => + Effect.gen(function* () { + const result = { ...evaluationResult }; + if (model) result.model = model; + else delete result.model; + cursorSdkMock.prompt.mockResolvedValueOnce(result); + const generation = yield* makeCursorTextGeneration(cursorSettings, { CURSOR_API_KEY: "key" }); + const error = yield* generation.generateModelEvaluation(evaluationInput).pipe(Effect.flip); + expect(error.detail).toContain("selected model and options"); + expect(cursorSdkMock.create).toHaveBeenCalledOnce(); + expect(cursorSdkMock.close).toHaveBeenCalledOnce(); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect.each([ + { + type: "tool_call", + agent_id: "agent-1", + run_id: "run-1", + name: "read", + call_id: "tool-1", + status: "running", + }, + { + type: "assistant", + agent_id: "agent-1", + run_id: "run-1", + message: { + role: "assistant", + content: [{ type: "tool_use", id: "tool-1", name: "read", input: {} }], + }, + }, + { type: "request", agent_id: "agent-1", run_id: "run-1", request_id: "request-1" }, + { type: "task", agent_id: "agent-1", run_id: "run-1", status: "running" }, + { type: "system", agent_id: "agent-1", run_id: "run-1", tools: ["read"] }, + ] satisfies SDKMessage[])("rejects forbidden evaluation activity (%j)", (event) => + Effect.gen(function* () { + cursorSdkMock.messages = [event]; + cursorSdkMock.prompt.mockResolvedValueOnce(evaluationResult); + const generation = yield* makeCursorTextGeneration(cursorSettings, { CURSOR_API_KEY: "key" }); + const error = yield* generation.generateModelEvaluation(evaluationInput).pipe(Effect.flip); + expect(error.detail).toContain("tool"); + expect(cursorSdkMock.prompt).not.toHaveBeenCalled(); + expect(cursorSdkMock.cancel).toHaveBeenCalledOnce(); + expect(cursorSdkMock.close).toHaveBeenCalledOnce(); + expect(cursorSdkMock.storeDispose).toHaveBeenCalledOnce(); + }).pipe(Effect.provide(layerTest)), + ); + + it.effect("does not retry a judge without sandboxing", () => + Effect.gen(function* () { + cursorSdkMock.create.mockRejectedValueOnce(new Error("sandboxing is not supported")); + const generation = yield* makeCursorTextGeneration(cursorSettings, { CURSOR_API_KEY: "key" }); + const error = yield* generation.generateModelEvaluation(evaluationInput).pipe(Effect.flip); + expect(error.detail).toContain("could not be created"); + expect(cursorSdkMock.create).toHaveBeenCalledOnce(); + expect(cursorSdkMock.send).not.toHaveBeenCalled(); + expect(cursorSdkMock.storeDispose).toHaveBeenCalledOnce(); + }).pipe(Effect.provide(layerTest)), + ); + it.effect("resolves the browser credential for every request after an account change", () => Effect.gen(function* () { let apiKey = "first-browser-key"; diff --git a/packages/provider-cursor/src/server/textGeneration.ts b/packages/provider-cursor/src/server/textGeneration.ts index 4790b897fb610..dacf913308ad7 100644 --- a/packages/provider-cursor/src/server/textGeneration.ts +++ b/packages/provider-cursor/src/server/textGeneration.ts @@ -84,6 +84,13 @@ export const makeCursorTextGeneration = Effect.fn("makeCursorTextGeneration")(fu )( function* (input) { const { operation, prompt, modelSelection } = input; + const evaluating = operation === "generateModelEvaluation"; + if (evaluating && ["", "auto", "default"].includes(modelSelection.model.trim())) { + return yield* new TextGenerationError({ + operation, + detail: "Cursor evaluation requires an explicit model. Choose a named judge model.", + }); + } const apiKey = yield* resolveCursorApiKey(operation); const sdkFailure = (detail: string) => (cause: unknown) => @@ -118,13 +125,14 @@ export const makeCursorTextGeneration = Effect.fn("makeCursorTextGeneration")(fu apiKey, mode: "plan", model: cursorSdkModelSelection(modelSelection), + ...(evaluating ? { tools: [] } : {}), local: { cwd, store, autoReview: false, sandboxOptions: { enabled: true }, settingSources: [], - enableAgentRetries: true, + enableAgentRetries: !evaluating, }, } satisfies AgentOptions; const createCursorAgent = (sandboxEnabled: boolean) => @@ -151,7 +159,10 @@ export const makeCursorTextGeneration = Effect.fn("makeCursorTextGeneration")(fu // Empty cwd and setting sources isolate the run from user projects even if sandboxing fails. const agent = yield* Effect.acquireRelease( createCursorAgent(true).pipe( - Effect.catchIf(cursorSandboxUnsupported, () => createCursorAgent(false)), + Effect.catchIf( + (cause) => !evaluating && cursorSandboxUnsupported(cause), + () => createCursorAgent(false), + ), ), (agent) => Effect.tryPromise({ @@ -176,6 +187,31 @@ export const makeCursorTextGeneration = Effect.fn("makeCursorTextGeneration")(fu }).pipe(Effect.timeout("5 seconds"), Effect.ignore({ log: true })) : Effect.void, ); + if (evaluating) { + yield* Effect.tryPromise({ + try: async () => { + for await (const event of run.stream()) { + if ( + event.type === "tool_call" || + event.type === "request" || + event.type === "task" || + (event.type === "system" && (event.tools?.length ?? 0) > 0) || + (event.type === "assistant" && + event.message.content.some((block) => block.type === "tool_use")) + ) { + throw new TextGenerationError({ + operation, + detail: "Cursor attempted tool work or user input during model evaluation.", + }); + } + } + }, + catch: (cause) => + isTextGenerationError(cause) + ? cause + : sdkFailure("Cursor evaluation event stream failed.")(cause), + }); + } return yield* Effect.tryPromise({ try: () => run.wait(), catch: sdkFailure("Cursor SDK run failed."), @@ -206,6 +242,22 @@ export const makeCursorTextGeneration = Effect.fn("makeCursorTextGeneration")(fu }); } + if ( + evaluating && + (promptResult.model?.id !== agentOptions.model.id || + agentOptions.model.params?.some( + (requested) => + !promptResult.model?.params?.some( + (actual) => actual.id === requested.id && actual.value === requested.value, + ), + )) + ) { + return yield* new TextGenerationError({ + operation, + detail: "Cursor did not confirm the selected model and options for evaluation.", + }); + } + return yield* TextGenerationOperations.decodeJsonReply(input, "Cursor SDK", rawResult); }, (effect, input) => { @@ -229,5 +281,6 @@ export const makeCursorTextGeneration = Effect.fn("makeCursorTextGeneration")(fu return TextGenerationOperations.fromRunner("CursorTextGeneration", runCursorJson, { supportsInvestigation: false, + supportsBlindEvaluation: true, }); }); From a6b126f80830694d648524a16e7879bdfbaafdd4 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:35:40 -0800 Subject: [PATCH 04/11] fix(server): run exact Codex evaluation judges in isolation --- .../settings/ModelEvaluationSettings.tsx | 17 +- .../CodexTextGeneration.test.ts | 117 ++++++++- .../src/textGeneration/CodexTextGeneration.ts | 242 +++++++++++++++++- .../src/textGeneration/TextGeneration.ts | 1 - .../src/server/v2/textGeneration.ts | 47 +++- 5 files changed, 400 insertions(+), 24 deletions(-) diff --git a/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx b/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx index d48d9b27a8de6..53912b4faaf34 100644 --- a/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx +++ b/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx @@ -18,6 +18,7 @@ import { } from "@t3tools/client-runtime/modelEvaluations"; import { runAtomCommand, squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime"; import * as Effect from "effect/Effect"; +import * as Hash from "effect/Hash"; import { AsyncResult, Atom } from "effect/reactivity"; import { useState } from "react"; import { View } from "react-native"; @@ -76,9 +77,10 @@ export function ModelEvaluationSettings() { ); } -// Keyed on each API's connection settings so editing one asks it again. const apiJudgeModelsAtom = Atom.family((key: string) => { - const apis = JSON.parse(key) as ReadonlyArray; + const apis = JSON.parse(key) as ReadonlyArray< + readonly [EnvironmentId, ProviderInstanceId, number] + >; return Atom.make((get) => Effect.forEach( apis, @@ -93,7 +95,7 @@ const apiJudgeModelsAtom = Atom.family((key: string) => { ).pipe(Effect.map((result) => (result._tag === "Success" ? result.value.models : []))), { concurrency: "unbounded" }, ), - ).pipe(Atom.setIdleTTL(5 * 60_000), Atom.withLabel(`model-evaluation-api-judges:${key}`)); + ).pipe(Atom.setIdleTTL(5 * 60_000), Atom.withLabel("model-evaluation-api-judges")); }); function EnvironmentEvaluationSettings(props: { @@ -109,9 +111,11 @@ function EnvironmentEvaluationSettings(props: { const projects = useProjects().filter((project) => project.environmentId === environmentId); const [chosenProjectId, setChosenProjectId] = useState(null); const [saving, setSaving] = useState(false); - // Listing an API's models needs settings write access, like choosing the judge does. + const canListModels = useAtomValue( + serverEnvironment.listTextGeneratorModels.permissionAtom(environmentId), + ); const apiJudges = ( - canWrite && config !== null + canListModels && config !== null ? (Object.entries(config.settings.textGenerators) as Array< [ProviderInstanceId, TextGeneratorConfig] >) @@ -121,8 +125,7 @@ function EnvironmentEvaluationSettings(props: { apiJudgeModelsAtom( JSON.stringify( apiJudges.map( - ([id, generator]) => - [environmentId, id, generator.kind, generator.baseUrl, generator.apiKey] as const, + ([id, generator]) => [environmentId, id, Hash.string(JSON.stringify(generator))] as const, ), ), ), diff --git a/apps/server/src/textGeneration/CodexTextGeneration.test.ts b/apps/server/src/textGeneration/CodexTextGeneration.test.ts index 62a9189269237..b6083e7ad5a04 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.test.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.test.ts @@ -48,12 +48,14 @@ interface FakeCodexInput { output: string; lifecycle?: { port: number; command: "exec" | "mcp list" }; exitCode?: number; + events?: ReadonlyArray; stderr?: string; requireImage?: boolean; requireServiceTier?: string; requireReasoningEffort?: string; forbidReasoningEffort?: boolean; requireArg?: string; + requiredArgs?: ReadonlyArray; forbidArg?: string; disabledFeatures?: ReadonlyArray; stdinMustContain?: string; @@ -70,6 +72,7 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { requireReasoningEffort: input.requireReasoningEffort ?? null, forbidReasoningEffort: input.forbidReasoningEffort ?? false, requireArg: input.requireArg ?? null, + requiredArgs: input.requiredArgs ?? [], forbidArg: input.forbidArg ?? null, disabledFeatures: input.disabledFeatures ?? [], stdinMustContain: input.stdinMustContain ?? null, @@ -79,6 +82,7 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { requireInstructions: input.requireInstructions ?? null, stderr: input.stderr ?? null, output: input.output, + events: input.events ?? [], exitCode: input.exitCode ?? 0, }); return Effect.gen(function* () { @@ -139,6 +143,7 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { "}", 'if (check.requiredCwd !== null && process.cwd() !== check.requiredCwd) fail("cannot inspect checkout", 10);', 'if (check.requireInstructions !== null && !instructions.includes(check.requireInstructions)) fail("missing writer task", 11);', + 'for (const arg of check.requiredArgs) if (!originalArgs.includes(` ${arg} `)) fail("missing arg: " + arg, 8);', "if (check.requireArg !== null && !originalArgs.includes(` ${check.requireArg} `)) {", ' fail("missing arg: " + check.requireArg, 8);', "}", @@ -169,6 +174,7 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { "}", 'if (check.stderr !== null) process.stderr.write(check.stderr + "\\n");', 'if (outputPath !== null) NodeFS.writeFileSync(outputPath, check.output + "\\n");', + 'for (const event of check.events) process.stdout.write(JSON.stringify(event) + "\\n");', "process.exitCode = check.exitCode;", "", ].join("\n"), @@ -185,6 +191,7 @@ function withFakeCodexEnv( defaultModel?: string; customModels?: ReadonlyArray; managedRuntime?: boolean; + configToml?: string; }, effectFn: (textGeneration: TextGeneration.TextGeneration["Service"]) => Effect.Effect, ) { @@ -192,6 +199,8 @@ function withFakeCodexEnv( const fs = yield* FileSystem.FileSystem; const tempDir = yield* fs.makeTempDirectoryScoped({ prefix: "t3code-codex-text-" }); const codexPath = yield* makeFakeCodexBinary(tempDir, input); + if (input.configToml !== undefined) + yield* fs.writeFileString(`${tempDir}/config.toml`, input.configToml); const config = decodeCodexSettings({ binaryPath: codexPath, launchArgs: input.launchArgs, @@ -200,7 +209,7 @@ function withFakeCodexEnv( }); const textGeneration = yield* makeCodexTextGeneration( config, - input.environment === undefined ? undefined : { ...process.env, ...input.environment }, + { ...process.env, CODEX_HOME: tempDir, ...input.environment }, Effect.succeed( (input.models ?? []).map((slug) => ({ slug, @@ -213,7 +222,7 @@ function withFakeCodexEnv( input.resolvedRuntime || input.managedRuntime ? Effect.succeed({ config, - environment: input.environment ?? process.env, + environment: { ...process.env, CODEX_HOME: tempDir, ...input.environment }, revision: "test", }) : undefined, @@ -298,6 +307,110 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => { ), ); + it.effect.each([ + "valid verdict", + "tool activity", + "MCP discovery unavailable", + "extra field", + ] as const)("uses an isolated exact Codex judge (%s)", (state) => { + const verdict = { + outcome: "insufficient_evidence", + taskCategory: "other", + confidence: "low", + scores: { + taskCompletion: null, + correctness: null, + instructionFollowing: null, + verification: null, + }, + rationale: { text: "No execution evidence was supplied.", citations: ["U1"] }, + strengths: [], + weaknesses: [], + promptAdvice: [], + }; + return withFakeCodexEnv( + { + output: JSON.stringify({ + ...verdict, + ...(state === "extra field" ? { invented: true } : {}), + }), + models: ["other-default-model"], + defaultModel: "other-default-model", + requireArg: "--model selected-judge", + requireReasoningEffort: "high", + requireServiceTier: "priority", + disabledFeatures: [ + "plugins", + "apps", + "shell_tool", + "unified_exec", + "hooks", + "multi_agent", + "view_image", + "image_generation", + "browser_use", + "computer_use", + "skill_search", + "tool_suggest", + "sleep_tool", + ], + mcpServers: state === "MCP discovery unavailable" ? 2 : [], + events: + state === "tool activity" + ? [ + { + type: "item.completed", + item: { type: "command_execution", command: "cat /secret" }, + }, + ] + : [ + { + type: "item.completed", + item: { type: "agent_message", text: JSON.stringify(verdict) }, + }, + ], + }, + (runtime) => + Effect.gen(function* () { + const exit = yield* runtime + .generateModelEvaluation({ + cwd: process.cwd(), + modelSelection: createModelSelection( + ProviderInstanceId.make("codex"), + "selected-judge", + [ + { id: "reasoningEffort", value: "high" }, + { id: "serviceTier", value: "priority" }, + ], + ), + packet: { + scope: "turn", + runOutcome: "completed", + request: { ref: "U1", text: "Explain the fix." }, + context: [], + evidence: [], + truncated: false, + omittedItems: 0, + missingArtifacts: true, + }, + }) + .pipe(Effect.exit); + if (state === "valid verdict" || state === "MCP discovery unavailable") { + expect(Exit.isSuccess(exit) ? exit.value : exit).toEqual(verdict); + } else { + expect(Exit.isFailure(exit)).toBe(true); + expect(Exit.isFailure(exit) ? Cause.pretty(exit.cause) : "").toContain( + state === "tool activity" + ? "tool activity" + : state === "extra field" + ? "invalid structured output" + : "MCP", + ); + } + }), + ); + }); + it.effect("preserves a service tier when an existing runtime resolves its executable", () => withFakeCodexEnv( { diff --git a/apps/server/src/textGeneration/CodexTextGeneration.ts b/apps/server/src/textGeneration/CodexTextGeneration.ts index 839fea3de5147..a86b86fb61a32 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.ts @@ -5,6 +5,8 @@ import * as Path from "effect/Path"; import * as Schema from "effect/Schema"; import * as Scope from "effect/Scope"; import * as Stream from "effect/Stream"; +import { parse, stringify, type TomlTable } from "smol-toml"; +import { tokenizeCliArgs } from "@t3tools/shared/cliArgs"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; import { @@ -45,6 +47,58 @@ const CODEX_NO_PLUGIN_ARGS = [ "--config", "features.apps=false", ] as const; +const CODEX_JUDGE_FEATURES = [ + "plugins", + "apps", + "shell_tool", + "unified_exec", + "hooks", + "multi_agent", + "multi_agent_v2", + "view_image", + "image_generation", + "browser_use", + "computer_use", + "skill_search", + "tool_suggest", + "sleep_tool", + "code_mode", + "code_mode_host", + "artifact", + "memories", + "goals", + "default_mode_request_user_input", + "in_app_browser", + "in_app_local_automation", + "workspace_dependencies", +] as const; +const CODEX_ROUTING_KEYS = new Set([ + "model_provider", + "model_providers", + "openai_base_url", + "chatgpt_base_url", + "cli_auth_credentials_store", + "forced_login_method", + "forced_chatgpt_workspace_id", +]); + +const routingOverrides = (table: TomlTable, prefix = ""): ReadonlyArray => + Object.entries(table).flatMap(([key, value]) => { + if (prefix === "" && !CODEX_ROUTING_KEYS.has(key)) return []; + const name = prefix === "" ? key : `${prefix}.${JSON.stringify(key)}`; + return value !== null && + typeof value === "object" && + !Array.isArray(value) && + !(value instanceof Date) + ? routingOverrides(value as TomlTable, name) + : [ + "--config", + `${name}=${stringify({ value }) + .trim() + .replace(/^value\s*=\s*/, "")}`, + ]; + }); + const CodexMcpServers = Schema.fromJsonString( Schema.Array(Schema.Struct({ name: Schema.String, enabled: Schema.Boolean })), ); @@ -218,6 +272,113 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func Effect.orElseSucceed((): ReadonlyArray => []), ); + const evaluationLaunchArgs = Effect.fn("CodexTextGeneration.evaluationLaunchArgs")(function* ( + homePath: string, + launchArgs: string, + ) { + const args = tokenizeCliArgs(launchArgs); + const overrides: string[] = []; + let profile: string | undefined; + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === "--profile" || arg === "-p") profile = args[++index]; + else if (arg?.startsWith("--profile=") || arg?.startsWith("-p=")) + profile = arg.slice(arg.indexOf("=") + 1); + else if ( + arg === "--config" || + arg === "-c" || + arg?.startsWith("--config=") || + arg?.startsWith("-c=") + ) { + const value = + arg === "--config" || arg === "-c" ? args[++index] : arg.slice(arg.indexOf("=") + 1); + if (value && CODEX_ROUTING_KEYS.has(value.split(/[.=]/)[0] ?? "")) + overrides.push("--config", value); + } else if (arg === "--oss") overrides.push(arg); + else if (arg === "--local-provider" && args[index + 1]) overrides.push(arg, args[++index]!); + } + const readRouting = Effect.fn("CodexTextGeneration.readRouting")(function* (name: string) { + const filePath = path.join(homePath, name); + if (!(yield* fileSystem.exists(filePath))) return []; + const source = yield* fileSystem.readFileString(filePath); + return yield* Effect.try({ + try: () => routingOverrides(parse(source)), + catch: (cause) => + new TextGenerationError({ + operation: "generateModelEvaluation", + detail: "Invalid Codex routing configuration.", + cause, + }), + }); + }); + return [ + ...(yield* readRouting("config.toml")), + ...(profile === undefined ? [] : yield* readRouting(`${profile}.config.toml`)), + ...overrides, + "--ignore-user-config", + "--ignore-rules", + "--json", + "--config", + "project_doc_max_bytes=0", + "--config", + "tools.experimental_request_user_input.enabled=false", + "--config", + "tools.update_plan.enabled=false", + "--config", + "skills.include_instructions=false", + "--config", + "features.skip_host_skill_discovery=true", + "--config", + 'web_search="disabled"', + ...CODEX_JUDGE_FEATURES.flatMap((feature) => ["--config", `features.${feature}=false`]), + ]; + }); + + const isolatedEvaluatorHome = Effect.fn("CodexTextGeneration.isolatedEvaluatorHome")(function* ( + originalHome: string, + operation: TextGenerationOperations.Operation, + ) { + const evaluationHome = yield* fileSystem + .makeTempDirectoryScoped({ prefix: "t3-codex-evaluator-" }) + .pipe( + Effect.mapError( + (cause) => + new TextGenerationError({ + operation, + detail: "Failed to create the Codex evaluator home.", + cause, + }), + ), + ); + for (const name of ["auth.json", "models_cache.json"]) { + const source = path.join(originalHome, name); + if ( + yield* fileSystem.exists(source).pipe( + Effect.mapError( + (cause) => + new TextGenerationError({ + operation, + detail: "Failed to locate the Codex evaluator login.", + cause, + }), + ), + ) + ) { + yield* fileSystem.copyFile(source, path.join(evaluationHome, name)).pipe( + Effect.mapError( + (cause) => + new TextGenerationError({ + operation, + detail: "Failed to prepare the Codex evaluator login.", + cause, + }), + ), + ); + } + } + return evaluationHome; + }); + const runCodexJson = Effect.fn("runCodexJson")(function* ({ operation, cwd, @@ -264,11 +425,13 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func )?.slug ?? listed.find((candidate) => codexModelFamily(candidate.slug) === requestedFamily)?.slug; const model = - matchedModel ?? - listed.find((candidate) => candidate.isDefault)?.slug ?? - listed.find((candidate) => !candidate.isLegacy)?.slug ?? - listed[0]?.slug ?? - requestedModel; + operation === "generateModelEvaluation" + ? requestedModel + : (matchedModel ?? + listed.find((candidate) => candidate.isDefault)?.slug ?? + listed.find((candidate) => !candidate.isLegacy)?.slug ?? + listed[0]?.slug ?? + requestedModel); // The selection's options were chosen for the requested model and may not fit a fallback. const fellBack = matchedModel === undefined && model !== requestedModel; const launchArgs = resolveCodexLaunchArgs(effectiveConfig.launchArgs, effectiveEnvironment); @@ -293,17 +456,34 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func } : {}), }; - const mcpServersOff = yield* listEnabledMcpServers({ - binary, - cwd, - env, - launchArgs: codexMcpListLaunchArgs(launchArgs), - }).pipe(Effect.map(codexMcpServersOff)); + const originalHome = env.CODEX_HOME ?? path.join(yield* HostProcess.HomeDirectory, ".codex"); + const judgeArgs = + operation === "generateModelEvaluation" + ? yield* evaluationLaunchArgs(originalHome, launchArgs).pipe( + Effect.mapError( + (cause) => + new TextGenerationError({ + operation, + detail: "Failed to isolate the Codex evaluator configuration.", + cause, + }), + ), + ) + : undefined; + const mcpServersOff = judgeArgs + ? null + : yield* listEnabledMcpServers({ + binary, + cwd, + env, + launchArgs: codexMcpListLaunchArgs(launchArgs), + }).pipe(Effect.map(codexMcpServersOff)); + if (judgeArgs) env.CODEX_HOME = yield* isolatedEvaluatorHome(originalHome, operation); const spawnCommand = yield* resolveSpawnCommand( binary, [ "exec", - ...codexExecLaunchArgs(launchArgs), + ...(judgeArgs ?? codexExecLaunchArgs(launchArgs)), // A user's MCP server could act on the text-generation prompt. ...CODEX_NO_PLUGIN_ARGS, // Memory candidates cite the supplied exchange; checkout instructions add unrelated input. @@ -360,6 +540,38 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func { concurrency: "unbounded" }, ); + if (judgeArgs) { + const events = yield* Effect.try({ + try: () => + stdout + .split("\n") + .filter((line) => line.trim().length > 0) + .map((line) => JSON.parse(line) as unknown), + catch: (cause) => + new TextGenerationError({ + operation, + detail: "Codex returned invalid evaluation events.", + cause, + }), + }); + for (const event of events) { + if (event !== null && typeof event === "object" && "item" in event) { + const item = event.item; + if ( + item === null || + typeof item !== "object" || + !("type" in item) || + !["agent_message", "reasoning", "error"].includes(String(item.type)) + ) { + return yield* new TextGenerationError({ + operation, + detail: "Codex evaluator attempted tool activity.", + }); + } + } + } + } + if (exitCode !== 0) { const stderrDetail = stderr.trim(); const stdoutDetail = stdout.trim(); @@ -393,7 +605,10 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func ), ); - const decodeOutput = Schema.decodeEffect(Schema.fromJsonString(outputSchemaJson)); + const decodeOutput = Schema.decodeEffect( + Schema.fromJsonString(outputSchemaJson), + operation === "generateModelEvaluation" ? { onExcessProperty: "error" } : undefined, + ); return yield* fileSystem.readFileString(outputPath).pipe( Effect.mapError( @@ -421,5 +636,6 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func return TextGenerationOperations.fromRunner("CodexTextGeneration", runCodexJson, { supportsInvestigation: true, + supportsBlindEvaluation: true, }); }); diff --git a/apps/server/src/textGeneration/TextGeneration.ts b/apps/server/src/textGeneration/TextGeneration.ts index 4607cbf377cbe..163ac5d3bd84c 100644 --- a/apps/server/src/textGeneration/TextGeneration.ts +++ b/apps/server/src/textGeneration/TextGeneration.ts @@ -189,7 +189,6 @@ export const make = Effect.gen(function* () { }); return TextGeneration.of({ generateModelEvaluation: Effect.fn("TextGeneration.generateModelEvaluation")(function* (input) { - // A model API has no tools or workspace, so it is blind by construction. const textGenerator = yield* textGenerators.runner(input.modelSelection.instanceId); const judge = textGenerator === undefined diff --git a/packages/provider-opencode/src/server/v2/textGeneration.ts b/packages/provider-opencode/src/server/v2/textGeneration.ts index c627ef9a38f97..bca9a5fabd58f 100644 --- a/packages/provider-opencode/src/server/v2/textGeneration.ts +++ b/packages/provider-opencode/src/server/v2/textGeneration.ts @@ -5,6 +5,7 @@ import { getModelSelectionStringOptionValue } from "@t3tools/shared/model"; import * as NodeURL from "node:url"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as SecureRandom from "effect/crypto/SecureRandom"; import * as FileSystem from "effect/FileSystem"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; @@ -14,6 +15,8 @@ import * as ProviderHost from "@t3tools/provider-core/server/ProviderHost"; import * as OpenCode2Server from "./OpenCode2Server.ts"; import * as OpenCodeRuntime from "../OpenCodeRuntime.ts"; import * as TextGenerationOperations from "@t3tools/provider-core/server/textGenerationOperations"; +import * as OpenCode2Client from "./OpenCode2Client.ts"; +import * as BlindEvaluation from "../blindEvaluation.ts"; const isTextGenerationError = Schema.is(TextGenerationError); @@ -33,6 +36,7 @@ const runOnServer = Effect.fn("Opencode2TextGeneration.runOnServer")(function* ( ) { const { resolveAttachmentPath } = yield* ProviderHost.ProviderHost; const fileSystem = yield* FileSystem.FileSystem; + const blindEvaluation = input.operation === "generateModelEvaluation"; const { client } = connection; const parsed = OpenCodeRuntime.parseOpenCodeModelSlug(input.modelSelection.model); if (parsed === null) { @@ -44,7 +48,15 @@ const runOnServer = Effect.fn("Opencode2TextGeneration.runOnServer")(function* ( const variant = getModelSelectionStringOptionValue(input.modelSelection, "variant"); const outcome = yield* Deferred.make(); let sessionId: string | undefined; + let executionModelMatched = !blindEvaluation; const texts = new Map(); + const successfulOutcome = (): Outcome => + executionModelMatched + ? { _tag: "text", text: [...texts.values()].join("\n").trim() } + : { + _tag: "failed", + detail: "OpenCode did not confirm the requested model and variant for blind evaluation.", + }; // Subscribed first: the stream is volatile, and the reply arrives on it. const events = yield* connection.events; yield* events.pipe( @@ -56,7 +68,7 @@ const runOnServer = Effect.fn("Opencode2TextGeneration.runOnServer")(function* ( return Deferred.succeed( outcome, event.executionType === "session.execution.succeeded" - ? { _tag: "text", text: [...texts.values()].join("\n").trim() } + ? successfulOutcome() : { _tag: "failed", detail: "OpenCode ended the generation in a way this version cannot read.", @@ -72,7 +84,40 @@ const runOnServer = Effect.fn("Opencode2TextGeneration.runOnServer")(function* ( ) { return Effect.void; } + if (blindEvaluation && event.type.startsWith("session.tool.")) { + return Deferred.succeed(outcome, { + _tag: "failed", + detail: "OpenCode reported tool activity during blind evaluation.", + cause: event, + }); + } switch (event.type) { + case "session.step.started": { + if (!blindEvaluation) return Effect.void; + const model = event.data.model; + if ( + model.providerID !== parsed.providerID || + model.id !== parsed.modelID || + (variant !== undefined && model.variant !== variant) + ) { + return Deferred.succeed(outcome, { + _tag: "failed", + detail: "OpenCode executed blind evaluation with a different model or variant.", + cause: event, + }); + } + executionModelMatched = true; + return Effect.void; + } + case "session.step.ended": + if (blindEvaluation && event.data.finish === "tool-calls") { + return Deferred.succeed(outcome, { + _tag: "failed", + detail: "OpenCode reported tool activity during blind evaluation.", + cause: event, + }); + } + return Effect.void; case "session.text.ended": texts.set(`${event.data.assistantMessageID}:${event.data.ordinal}`, event.data.text); return Effect.void; From 1a8629f2339451c099e09e41bddd2a7ee6b12b21 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 14:52:49 -0800 Subject: [PATCH 05/11] fix(opencode): isolate blind evaluation judges --- .../src/server/blindEvaluation.ts | 167 +++++++++++ .../src/server/driver.test.ts | 4 + .../provider-opencode/src/server/driver.ts | 16 +- .../src/server/textGeneration.test.ts | 190 ++++++++++-- .../src/server/textGeneration.ts | 69 ++++- .../src/server/v2/textGeneration.test.ts | 282 ++++++++++++++++++ .../src/server/v2/textGeneration.ts | 107 +++++-- 7 files changed, 769 insertions(+), 66 deletions(-) create mode 100644 packages/provider-opencode/src/server/blindEvaluation.ts create mode 100644 packages/provider-opencode/src/server/v2/textGeneration.test.ts diff --git a/packages/provider-opencode/src/server/blindEvaluation.ts b/packages/provider-opencode/src/server/blindEvaluation.ts new file mode 100644 index 0000000000000..e8c87dc47cb25 --- /dev/null +++ b/packages/provider-opencode/src/server/blindEvaluation.ts @@ -0,0 +1,167 @@ +import { + TextGenerationError, + type ModelSelection, + type TextGenerationOperation, +} from "@t3tools/contracts"; +import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; +import { fromLenientJson } from "@t3tools/shared/schemaJson"; +import * as Effect from "effect/Effect"; +import * as FileSystem from "effect/FileSystem"; +import * as Option from "effect/Option"; +import * as Path from "effect/Path"; +import * as Schema from "effect/Schema"; + +const Config = Schema.Record(Schema.String, Schema.Unknown); +type Config = typeof Config.Type; +const decodeConfig = Schema.decodeUnknownOption(fromLenientJson(Config)); +const encodeConfig = Schema.encodeSync(Schema.fromJsonString(Config)); +const isConfig = Schema.is(Config); +const isTextGenerationError = Schema.is(TextGenerationError); +const ROUTING_KEYS = ["provider", "providers", "enabled_providers", "disabled_providers"] as const; + +function readRoutingConfig(content: string): Config | undefined { + const decoded = decodeConfig(content); + if (Option.isNone(decoded)) return undefined; + return Object.fromEntries( + ROUTING_KEYS.flatMap((key) => { + const value = decoded.value[key]; + return value === undefined ? [] : [[key, value]]; + }), + ); +} + +function mergeConfig(target: Config, source: Config): Config { + const result = { ...target }; + for (const [key, value] of Object.entries(source)) { + const previous = result[key]; + result[key] = isConfig(previous) && isConfig(value) ? mergeConfig(previous, value) : value; + } + return result; +} + +export const makeBlindEvaluationEnvironment = Effect.fn("OpenCode.makeBlindEvaluationEnvironment")( + function* (input: { + readonly operation: TextGenerationOperation; + readonly directory: string; + readonly environment: NodeJS.ProcessEnv; + }) { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const invalid = () => + new TextGenerationError({ + operation: input.operation, + detail: "OpenCode configuration could not be isolated for blind evaluation.", + }); + const readConfig = (filePath: string) => + fs.readFileString(filePath).pipe( + Effect.catchIf( + (cause) => cause.reason._tag === "NotFound", + () => Effect.succeed("{}"), + ), + Effect.flatMap((content) => { + const config = readRoutingConfig(content); + return config === undefined ? Effect.fail(invalid()) : Effect.succeed(config); + }), + Effect.mapError((cause) => (isTextGenerationError(cause) ? cause : invalid())), + ); + + let routing: Config = {}; + const home = input.environment.HOME ?? input.environment.USERPROFILE; + const configHome = input.environment.XDG_CONFIG_HOME; + const globalDirectory = + configHome !== undefined && configHome.trim().length > 0 + ? path.join(configHome, "opencode") + : home === undefined + ? undefined + : path.join(home, ".config", "opencode"); + const directories = [ + ...(globalDirectory === undefined ? [] : [globalDirectory]), + ...(input.environment.OPENCODE_CONFIG_DIR === undefined + ? [] + : [ + path.resolve( + input.directory, + expandHomePath(input.environment.OPENCODE_CONFIG_DIR, home ?? ""), + ), + ]), + ]; + for (const directory of directories) { + for (const name of ["config.json", "opencode.json", "opencode.jsonc"]) { + routing = mergeConfig(routing, yield* readConfig(path.join(directory, name))); + } + } + if (input.environment.OPENCODE_CONFIG !== undefined) { + const configuredPath = path.resolve( + input.directory, + expandHomePath(input.environment.OPENCODE_CONFIG, home ?? ""), + ); + routing = mergeConfig(routing, yield* readConfig(configuredPath)); + } + if (input.environment.OPENCODE_CONFIG_CONTENT !== undefined) { + const content = readRoutingConfig(input.environment.OPENCODE_CONFIG_CONTENT); + if (content === undefined) return yield* invalid(); + routing = mergeConfig(routing, content); + } + + const temporary = yield* fs.makeTempDirectoryScoped({ prefix: "t3-opencode-evaluation-" }); + const xdgConfigHome = path.join(temporary, "xdg-config"); + const configDirectory = path.join(temporary, "config"); + yield* fs.makeDirectory(xdgConfigHome, { recursive: true }); + yield* fs.makeDirectory(configDirectory, { recursive: true }); + const { + OPENCODE_CONFIG: _config, + OPENCODE_CONFIG_DIR: _configDirectory, + OPENCODE_PERMISSION: _permission, + XDG_CONFIG_HOME: _xdgConfigHome, + ...preserved + } = input.environment; + return { + ...preserved, + XDG_CONFIG_HOME: xdgConfigHome, + OPENCODE_CONFIG_DIR: configDirectory, + OPENCODE_CONFIG_CONTENT: encodeConfig(routing), + OPENCODE_DISABLE_PROJECT_CONFIG: "1", + OPENCODE_DISABLE_CLAUDE_CODE: "1", + OPENCODE_DISABLE_CLAUDE_CODE_PROMPT: "1", + OPENCODE_DISABLE_CLAUDE_CODE_SKILLS: "1", + } satisfies NodeJS.ProcessEnv; + }, + (effect) => + effect.pipe( + Effect.mapError((cause) => + isTextGenerationError(cause) + ? cause + : new TextGenerationError({ + operation: "generateModelEvaluation", + detail: "OpenCode configuration could not be isolated for blind evaluation.", + cause, + }), + ), + ), +); + +export const validateBlindEvaluationSelection = ( + modelSelection: ModelSelection, + operation: TextGenerationOperation, +) => { + const unsupported = modelSelection.options?.find( + (option, index) => option.id !== "variant" || typeof option.value !== "string" || index > 0, + ); + return unsupported === undefined + ? Effect.void + : Effect.fail( + new TextGenerationError({ + operation, + detail: `OpenCode blind evaluation does not support the '${unsupported.id}' model option.`, + }), + ); +}; + +export const hasBlindEvaluationPartActivity = (parts: ReadonlyArray) => + parts.some( + (part) => + part === null || + typeof part !== "object" || + !("type" in part) || + (part.type !== "text" && part.type !== "reasoning"), + ); diff --git a/packages/provider-opencode/src/server/driver.test.ts b/packages/provider-opencode/src/server/driver.test.ts index 6d74dc8dd2542..72ad66964bb5e 100644 --- a/packages/provider-opencode/src/server/driver.test.ts +++ b/packages/provider-opencode/src/server/driver.test.ts @@ -39,6 +39,7 @@ import * as ProviderMaintenance from "@t3tools/provider-core/server/maintenanceR import * as OpenCodeRuntime from "./OpenCodeRuntime.ts"; import { OpenCodeDriver, isLocalOpenCodeServerUrl, openCodeUpdateFor } from "./driver.ts"; import * as OpenCodeServerLedger from "./OpenCodeServerLedger.ts"; +import * as OpenCode2Client from "./v2/OpenCode2Client.ts"; import { OPENCODE_2_RESPONSES, OPENCODE_2_WORKSPACE_RESPONSES, @@ -86,6 +87,7 @@ const layer = Layer.mergeAll( ProviderEventLoggers.NoOpProviderEventLoggers, ), Layer.succeed(OpenCodeRuntime.OpenCodeRuntime, openCode2Runtime), + OpenCode2Client.layer.pipe(Layer.provide(FetchHttpClient.layer)), ).pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))); const create = (config: Partial, http: HttpClient.HttpClient, enabled = true) => @@ -243,6 +245,7 @@ it.live("discovers models through a home-relative OpenCode wrapper", () => ProviderEventLoggers.NoOpProviderEventLoggers, ), OpenCodeRuntime.layer.pipe(Layer.provide(OpenCodeServerLedger.layerTest)), + OpenCode2Client.layer, ).pipe( Layer.provideMerge( Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, FetchHttpClient.layer), @@ -1011,6 +1014,7 @@ const layerUpdate = Layer.mergeAll( ProviderEventLoggers.NoOpProviderEventLoggers, ), Layer.succeed(OpenCodeRuntime.OpenCodeRuntime, changingRuntime), + OpenCode2Client.layer.pipe(Layer.provide(FetchHttpClient.layer)), ).pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))); const openCode1Provider = (id: string, models: ReadonlyArray) => ({ diff --git a/packages/provider-opencode/src/server/driver.ts b/packages/provider-opencode/src/server/driver.ts index cbc8557441f96..2c02cb30a4c53 100644 --- a/packages/provider-opencode/src/server/driver.ts +++ b/packages/provider-opencode/src/server/driver.ts @@ -258,6 +258,7 @@ export type OpenCodeDriverEnv = | HttpClient.HttpClient | ProviderLatestVersions.ProviderLatestVersions | OpenCodeRuntime.OpenCodeRuntime + | OpenCode2Client.OpenCode2Client | Path.Path; export const OpenCodeDriver: ProviderDriver< @@ -464,12 +465,15 @@ export const OpenCodeDriver: ProviderDriver< }); const textGeneration = selectOpenCodeRuntimeTextGeneration( resolveRuntime, - yield* makeOpenCodeTextGeneration(effectiveConfig).pipe( - Effect.provideService(OpenCodeServerOwner.OpenCodeServerOwner, serverOwner), - ), - yield* OpenCode2TextGeneration.make().pipe( - Effect.provideService(OpenCode2Server.OpenCode2Server, openCode2Server), - ), + yield* makeOpenCodeTextGeneration(effectiveConfig, { + environment: processEnv, + directory: host.paths.cwd, + }).pipe(Effect.provideService(OpenCodeServerOwner.OpenCodeServerOwner, serverOwner)), + yield* OpenCode2TextGeneration.make({ + binaryPath: effectiveConfig.binaryPath, + environment: processEnv, + directory: host.paths.cwd, + }).pipe(Effect.provideService(OpenCode2Server.OpenCode2Server, openCode2Server)), ); const checkUsageLimits = readOpenCodeGoUsageLimits({ diff --git a/packages/provider-opencode/src/server/textGeneration.test.ts b/packages/provider-opencode/src/server/textGeneration.test.ts index 872263da2a07c..323bfdc8c413b 100644 --- a/packages/provider-opencode/src/server/textGeneration.test.ts +++ b/packages/provider-opencode/src/server/textGeneration.test.ts @@ -24,12 +24,14 @@ import type { ProviderTextGenerationRuntime } from "@t3tools/provider-core/serve const runtimeMock = { state: { startCalls: [] as string[], + startEnvironments: [] as Array, promptUrls: [] as string[], promptParts: [] as ReadonlyArray[], promptInputs: [] as Array[0]>, authHeaders: [] as Array, closeCalls: [] as string[], sessionCreateCalls: 0, + sessionCreateInputs: [] as Array[0]>, sessionCalls: [] as string[], onSessionCreate: undefined as ((signal?: AbortSignal) => Promise) | undefined, onPrompt: undefined as @@ -41,17 +43,30 @@ const runtimeMock = { sessionResult: undefined as { data?: { id: string } } | undefined, promptRequestError: undefined as unknown, promptResult: undefined as - | { data?: { info?: { error?: unknown }; parts?: Array } } + | { + data?: { + info?: { + error?: unknown; + providerID?: string; + modelID?: string; + variant?: string; + agent?: string; + }; + parts?: Array; + }; + } | undefined, }, reset() { this.state.startCalls.length = 0; + this.state.startEnvironments.length = 0; this.state.promptUrls.length = 0; this.state.promptParts.length = 0; this.state.promptInputs.length = 0; this.state.authHeaders.length = 0; this.state.closeCalls.length = 0; this.state.sessionCreateCalls = 0; + this.state.sessionCreateInputs.length = 0; this.state.sessionCalls.length = 0; this.state.onSessionCreate = undefined; this.state.onPrompt = undefined; @@ -70,6 +85,7 @@ const OpenCodeRuntimeTestDouble: OpenCodeRuntime.OpenCodeRuntime["Service"] = { const index = runtimeMock.state.startCalls.length + 1; const url = `http://127.0.0.1:${4_300 + index}`; runtimeMock.state.startCalls.push(binaryPath); + runtimeMock.state.startEnvironments.push(environment); yield* Effect.addFinalizer(() => Effect.sync(() => { runtimeMock.state.closeCalls.push(url); @@ -112,13 +128,22 @@ const OpenCodeRuntimeTestDouble: OpenCodeRuntime.OpenCodeRuntime["Service"] = { provider: { list: async () => ({ data: { - all: [{ id: "openai", models: { "gpt-5": { id: "gpt-5" } } }], + all: [ + { + id: "openai", + models: { "gpt-5": { id: "gpt-5", variants: { high: {} } } }, + }, + ], }, }), }, session: { - create: async (_input: unknown, options?: { readonly signal?: AbortSignal }) => { + create: async ( + input: Parameters[0], + options?: { readonly signal?: AbortSignal }, + ) => { runtimeMock.state.sessionCreateCalls += 1; + runtimeMock.state.sessionCreateInputs.push(input); if (runtimeMock.state.onSessionCreate) { return runtimeMock.state.onSessionCreate(options?.signal); } @@ -195,6 +220,30 @@ const DEFAULT_COMMIT_MESSAGE_INPUT = { stagedPatch: "diff --git a/README.md b/README.md", modelSelection: DEFAULT_TEST_MODEL_SELECTION, }; +const DEFAULT_EVALUATION_INPUT = { + cwd: "/isolated-evaluator", + modelSelection: DEFAULT_TEST_MODEL_SELECTION, + packet: { + scope: "turn" as const, + runOutcome: "completed" as const, + request: { ref: "U1" as const, text: "Fix the issue." }, + context: [], + evidence: [{ ref: "E1", kind: "assistant" as const, text: "Fixed." }], + truncated: false, + omittedItems: 0, + missingArtifacts: false, + }, +}; +const VALID_EVALUATION = { + outcome: "scored", + taskCategory: "coding", + scores: { taskCompletion: 4, correctness: 4, instructionFollowing: 4, verification: 4 }, + confidence: "high", + rationale: { text: "The requested fix is complete.", citations: ["U1", "E1"] }, + strengths: [], + weaknesses: [], + promptAdvice: [], +}; const OPENCODE_TEXT_GENERATION_IDLE_TTL_MS = 30_000; @@ -243,9 +292,10 @@ function withOpenCodeTextGeneration( : {}), ...(environment ? { environment } : {}), }); - const textGeneration = yield* OpenCodeTextGeneration.makeOpenCodeTextGeneration(settings).pipe( - Effect.provideService(OpenCodeServerOwner.OpenCodeServerOwner, serverOwner), - ); + const textGeneration = yield* OpenCodeTextGeneration.makeOpenCodeTextGeneration( + settings, + environment ? { environment } : {}, + ).pipe(Effect.provideService(OpenCodeServerOwner.OpenCodeServerOwner, serverOwner)); return yield* effectFn(textGeneration); }).pipe(Effect.scoped); } @@ -420,32 +470,112 @@ it.layer(layerOpenCodeTextGenerationTest)("OpenCodeTextGeneration", (it) => { ), ); - it.effect( - "rejects blind evaluation before opening a server session with inherited configuration", - () => - withOpenCodeTextGeneration(DEFAULT_OPENCODE_SETTINGS, (textGeneration) => + it.effect("isolates blind evaluation from shared servers, tools, and custom instructions", () => + withOpenCodeTextGeneration( + EXISTING_SERVER_OPENCODE_SETTINGS, + (textGeneration) => Effect.gen(function* () { - const error = yield* textGeneration - .generateModelEvaluation({ - cwd: "/isolated-evaluator", - modelSelection: DEFAULT_TEST_MODEL_SELECTION, - packet: { - scope: "turn", - runOutcome: "completed", - request: { ref: "U1", text: "Fix the issue." }, - context: [], - evidence: [{ ref: "E1", kind: "assistant", text: "Fixed." }], - truncated: false, - omittedItems: 0, - missingArtifacts: false, - }, - }) - .pipe(Effect.flip); - expect(error.detail).toContain("cannot isolate"); - expect(textGeneration.supportedOperations).not.toContain("generateModelEvaluation"); - expect(runtimeMock.state.sessionCreateCalls).toBe(0); + runtimeMock.state.promptResult = { + data: { + info: { providerID: "openai", modelID: "gpt-5", variant: "high" }, + parts: [{ type: "text", text: JSON.stringify(VALID_EVALUATION) }], + }, + }; + + const verdict = yield* textGeneration.generateModelEvaluation({ + ...DEFAULT_EVALUATION_INPUT, + modelSelection: { + ...DEFAULT_TEST_MODEL_SELECTION, + options: [{ id: "variant", value: "high" }], + }, + }); + + expect(verdict).toEqual(VALID_EVALUATION); + expect(textGeneration.supportedOperations).toContain("generateModelEvaluation"); + expect(runtimeMock.state.promptUrls).toEqual(["http://127.0.0.1:4301"]); + expect(runtimeMock.state.promptInputs[0]).toMatchObject({ + model: { providerID: "openai", modelID: "gpt-5" }, + variant: "high", + }); + expect(runtimeMock.state.sessionCreateInputs[0]?.permission).toEqual([ + { permission: "*", pattern: "*", action: "deny" }, + ]); + const environment = runtimeMock.state.startEnvironments[0]; + expect(environment?.XDG_CONFIG_HOME).not.toBe("/user-config"); + expect(environment?.OPENCODE_CONFIG).toBeUndefined(); + expect(environment?.OPENCODE_DISABLE_PROJECT_CONFIG).toBe("1"); + expect(environment?.OPENCODE_DISABLE_CLAUDE_CODE).toBe("1"); + expect(JSON.parse(environment?.OPENCODE_CONFIG_CONTENT ?? "null")).toEqual({ + provider: { custom: { options: { baseURL: "http://model.internal" } } }, + enabled_providers: ["custom"], + }); + expect(runtimeMock.state.closeCalls).toEqual(["http://127.0.0.1:4301"]); }), - ), + { + XDG_CONFIG_HOME: "/user-config", + OPENCODE_CONFIG: "/user-config/opencode.json", + OPENCODE_CONFIG_CONTENT: `{ + // Routing survives, but customization does not. + "provider": { "custom": { "options": { "baseURL": "http://model.internal" } } }, + "enabled_providers": ["custom"], + "instructions": ["/secret/instructions.md"], + "plugin": ["secret-plugin"], + "mcp": { "private": { "type": "remote", "url": "http://private" } } + }`, + }, + ), + ); + + it.effect("fails blind evaluation instead of dropping an unavailable or invalid variant", () => + withOpenCodeTextGeneration(DEFAULT_OPENCODE_SETTINGS, (textGeneration) => + Effect.gen(function* () { + const error = yield* textGeneration + .generateModelEvaluation({ + ...DEFAULT_EVALUATION_INPUT, + modelSelection: { + ...DEFAULT_TEST_MODEL_SELECTION, + options: [{ id: "variant", value: "stale" }], + }, + }) + .pipe(Effect.flip); + const invalidOption = yield* textGeneration + .generateModelEvaluation({ + ...DEFAULT_EVALUATION_INPUT, + modelSelection: { + ...DEFAULT_TEST_MODEL_SELECTION, + options: [{ id: "variant", value: true }], + }, + }) + .pipe(Effect.flip); + + expect(error.detail).toContain("variant"); + expect(invalidOption.detail).toContain("variant"); + expect(runtimeMock.state.promptUrls).toEqual([]); + }), + ), + ); + + it.effect("fails blind evaluation when OpenCode returns tool activity", () => + withOpenCodeTextGeneration(DEFAULT_OPENCODE_SETTINGS, (textGeneration) => + Effect.gen(function* () { + runtimeMock.state.promptResult = { + data: { + info: { providerID: "openai", modelID: "gpt-5" }, + parts: [ + { type: "tool", tool: "read", state: { status: "completed" } }, + { type: "text", text: JSON.stringify(VALID_EVALUATION) }, + ], + }, + }; + + const error = yield* textGeneration + .generateModelEvaluation(DEFAULT_EVALUATION_INPUT) + .pipe(Effect.flip); + + expect(error.detail).toContain("tool activity"); + expect(runtimeMock.state.sessionCalls).toEqual(["delete http://127.0.0.1:4301/session"]); + }), + ), ); it.effect("gives up and aborts the session when a prompt never settles", () => diff --git a/packages/provider-opencode/src/server/textGeneration.ts b/packages/provider-opencode/src/server/textGeneration.ts index 4c86d085bcf72..4dc72a1086425 100644 --- a/packages/provider-opencode/src/server/textGeneration.ts +++ b/packages/provider-opencode/src/server/textGeneration.ts @@ -9,6 +9,7 @@ import * as ProviderHost from "@t3tools/provider-core/server/ProviderHost"; import * as TextGenerationOperations from "@t3tools/provider-core/server/textGenerationOperations"; import * as OpenCodeRuntime from "./OpenCodeRuntime.ts"; import * as OpenCodeServerOwner from "./OpenCodeServerOwner.ts"; +import * as BlindEvaluation from "./blindEvaluation.ts"; const GENERATION_TIMEOUT = "3 minutes"; @@ -178,7 +179,13 @@ function getOpenCodeTextResponse(parts: ReadonlyArray | undefined): str export const makeOpenCodeTextGeneration = Effect.fn("makeOpenCodeTextGeneration")(function* ( openCodeSettings: OpenCodeSettings, + options: { + readonly environment?: NodeJS.ProcessEnv; + readonly directory?: string; + } = {}, ) { + const processEnvironment = options.environment ?? process.env; + const processDirectory = options.directory ?? process.cwd(); const host = yield* ProviderHost.ProviderHost; const openCodeRuntime = yield* OpenCodeRuntime.OpenCodeRuntime; const serverOwner = yield* OpenCodeServerOwner.OpenCodeServerOwner; @@ -186,6 +193,13 @@ export const makeOpenCodeTextGeneration = Effect.fn("makeOpenCodeTextGeneration" const runOpenCodeJson = Effect.fn("runOpenCodeJson")(function* ( input: TextGenerationOperations.Request, ) { + const blindEvaluation = input.operation === "generateModelEvaluation"; + if (blindEvaluation) { + yield* BlindEvaluation.validateBlindEvaluationSelection( + input.modelSelection, + input.operation, + ); + } const parsedModel = OpenCodeRuntime.parseOpenCodeModelSlug(input.modelSelection.model); if (!parsedModel) { return yield* new TextGenerationError({ @@ -216,10 +230,12 @@ export const makeOpenCodeTextGeneration = Effect.fn("makeOpenCodeTextGeneration" client.session.create( { title: `T3 Code ${input.operation}`, - // OpenCode Zen refuses free-model requests that carry no tools, so keep read. + // OpenCode Zen refuses ordinary free-model requests that carry no tools. permission: [ { permission: "*", pattern: "*", action: "deny" }, - { permission: "read", pattern: "*", action: "allow" }, + ...(!blindEvaluation + ? [{ permission: "read", pattern: "*", action: "allow" as const }] + : []), ], }, { signal }, @@ -243,11 +259,21 @@ export const makeOpenCodeTextGeneration = Effect.fn("makeOpenCodeTextGeneration" ).pipe(Effect.timeout("5 seconds"), Effect.ignore({ log: "Warn" })), ); const selectedAgent = getModelSelectionStringOptionValue(input.modelSelection, "agent"); + const requestedVariant = getModelSelectionStringOptionValue( + input.modelSelection, + "variant", + ); const selectedVariant = yield* OpenCodeRuntime.resolveOpenCodeVariant( client, parsedModel, - getModelSelectionStringOptionValue(input.modelSelection, "variant"), + requestedVariant, ); + if (blindEvaluation && selectedVariant !== requestedVariant) { + return yield* new TextGenerationError({ + operation: input.operation, + detail: "OpenCode does not support the requested model variant for blind evaluation.", + }); + } const promptContext = { operation: input.operation, cwd: input.cwd, @@ -303,7 +329,25 @@ export const makeOpenCodeTextGeneration = Effect.fn("makeOpenCodeTextGeneration" cause: promptError, }); } + const responseInfo = result.data?.info; + if ( + blindEvaluation && + (responseInfo?.providerID !== parsedModel.providerID || + responseInfo.modelID !== parsedModel.modelID || + (requestedVariant !== undefined && responseInfo.variant !== requestedVariant)) + ) { + return yield* new TextGenerationError({ + operation: input.operation, + detail: "OpenCode executed blind evaluation with a different model or variant.", + }); + } const responseParts = result.data?.parts ?? []; + if (blindEvaluation && BlindEvaluation.hasBlindEvaluationPartActivity(responseParts)) { + return yield* new TextGenerationError({ + operation: input.operation, + detail: "OpenCode reported tool activity during blind evaluation.", + }); + } const rawText = getOpenCodeTextResponse(responseParts); if (rawText.length === 0) { return yield* new OpenCodeTextGenerationEmptyOutputError({ @@ -359,8 +403,22 @@ export const makeOpenCodeTextGeneration = Effect.fn("makeOpenCodeTextGeneration" }), ); - const serverOutput = - openCodeSettings.serverUrl.length > 0 + const isolatedOutput = Effect.gen(function* () { + const environment = yield* BlindEvaluation.makeBlindEvaluationEnvironment({ + operation: input.operation, + directory: processDirectory, + environment: processEnvironment, + }); + const server = yield* openCodeRuntime.startOpenCodeServerProcess({ + binaryPath: openCodeSettings.binaryPath, + directory: input.cwd, + environment, + }); + return yield* runAgainstServer(server); + }).pipe(Effect.scoped); + const serverOutput = blindEvaluation + ? isolatedOutput + : openCodeSettings.serverUrl.length > 0 ? openCodeRuntime .connectToOpenCodeServer({ binaryPath: openCodeSettings.binaryPath, @@ -400,5 +458,6 @@ export const makeOpenCodeTextGeneration = Effect.fn("makeOpenCodeTextGeneration" return TextGenerationOperations.fromRunner("OpenCodeTextGeneration", runOpenCodeJson, { supportsInvestigation: false, + supportsBlindEvaluation: true, }); }); diff --git a/packages/provider-opencode/src/server/v2/textGeneration.test.ts b/packages/provider-opencode/src/server/v2/textGeneration.test.ts new file mode 100644 index 0000000000000..ff7e4ec64257f --- /dev/null +++ b/packages/provider-opencode/src/server/v2/textGeneration.test.ts @@ -0,0 +1,282 @@ +import { ProviderInstanceId } from "@t3tools/contracts"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; +import * as NodeServices from "@effect/platform-node/NodeServices"; +import { it } from "@effect/vitest"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as Stream from "effect/Stream"; +import { beforeEach, expect } from "vite-plus/test"; + +import * as TestProviderHost from "@t3tools/provider-testing/TestProviderHost"; +import * as OpenCodeRuntime from "../OpenCodeRuntime.ts"; +import * as OpenCode2Client from "./OpenCode2Client.ts"; +import * as OpenCode2Server from "./OpenCode2Server.ts"; +import * as OpenCode2TextGeneration from "./textGeneration.ts"; + +const SESSION_ID = "ses_blind_evaluation"; +const OWNED_SERVER_URL = "http://127.0.0.1:4302"; + +const state = { + startEnvironments: [] as Array, + closeCalls: [] as string[], + connectUrls: [] as string[], + sessionInputs: [] as unknown[], + promptInputs: [] as unknown[], + permissionReplies: [] as unknown[], + events: [] as OpenCode2Client.OpenCode2StreamEvent[], +}; + +let promptSent = Promise.withResolvers(); +const runtime = { + startOpenCodeServerProcess: ({ + environment, + verify, + }: { + environment?: NodeJS.ProcessEnv; + verify?: (url: string) => Effect.Effect; + }) => + Effect.gen(function* () { + state.startEnvironments.push(environment); + if (verify !== undefined) yield* verify(OWNED_SERVER_URL); + yield* Effect.addFinalizer(() => + Effect.sync(() => { + state.closeCalls.push(OWNED_SERVER_URL); + }), + ); + return { + url: OWNED_SERVER_URL, + version: "2.2.6", + isRunning: Effect.succeed(true), + exitCode: Effect.never, + }; + }), + connectToOpenCodeServer: () => Effect.die(new Error("A shared server must not be used")), + runOpenCodeCommand: () => Effect.die(new Error("Not used")), + createOpenCodeSdkClient: () => Effect.die(new Error("Not used")), + loadOpenCodeInventory: () => Effect.die(new Error("Not used")), + loadOpenCodeSkills: () => Effect.die(new Error("Not used")), +} as unknown as OpenCodeRuntime.OpenCodeRuntime["Service"]; + +const client = { + connect: ({ baseUrl }: { baseUrl: string }) => { + state.connectUrls.push(baseUrl); + return Effect.succeed({ + client: { + server: { info: () => Effect.succeed({ version: "2.2.6" }) }, + session: { + create: (input: unknown) => + Effect.sync(() => { + state.sessionInputs.push(input); + return { + id: SESSION_ID, + model: { providerID: "openai", id: "gpt-5", variant: "high" }, + }; + }), + prompt: (input: unknown) => + Effect.sync(() => { + state.promptInputs.push(input); + promptSent.resolve(); + }), + remove: () => Effect.void, + interrupt: () => Effect.void, + }, + permission: { + reply: (input: unknown) => + Effect.sync(() => { + state.permissionReplies.push(input); + }), + }, + }, + events: Effect.succeed( + Stream.unwrap( + Effect.map( + Effect.promise(() => promptSent.promise), + () => Stream.fromIterable(state.events), + ), + ), + ), + imageJobs: () => Effect.succeed([]), + readImage: () => Effect.die(new Error("Not used")), + } as unknown as OpenCode2Client.OpenCode2Api); + }, +} satisfies OpenCode2Client.OpenCode2Client["Service"]; + +const sharedServer = OpenCode2Server.OpenCode2Server.of({ + withConnection: () => Effect.die(new Error("A shared server must not be used")), +}); + +const layer = Layer.mergeAll( + Layer.succeed(OpenCodeRuntime.OpenCodeRuntime, runtime), + Layer.succeed(OpenCode2Client.OpenCode2Client, client), + Layer.succeed(OpenCode2Server.OpenCode2Server, sharedServer), +).pipe( + Layer.provideMerge(TestProviderHost.layer()), + Layer.provideMerge(NodeServices.layer), + Layer.provideMerge(NodeCrypto.layer), +); + +const evaluationInput = { + cwd: "/isolated-evaluator", + modelSelection: { + instanceId: ProviderInstanceId.make("opencode"), + model: "openai/gpt-5", + options: [{ id: "variant", value: "high" }], + }, + packet: { + scope: "turn" as const, + runOutcome: "completed" as const, + request: { ref: "U1" as const, text: "Fix the issue." }, + context: [], + evidence: [{ ref: "E1", kind: "assistant" as const, text: "Fixed." }], + truncated: false, + omittedItems: 0, + missingArtifacts: false, + }, +}; +const validEvaluation = { + outcome: "scored", + taskCategory: "coding", + scores: { taskCompletion: 4, correctness: 4, instructionFollowing: 4, verification: 4 }, + confidence: "high", + rationale: { text: "The requested fix is complete.", citations: ["U1", "E1"] }, + strengths: [], + weaknesses: [], + promptAdvice: [], +}; + +const make = () => + OpenCode2TextGeneration.make({ + binaryPath: "fake-opencode", + directory: process.cwd(), + environment: { + XDG_CONFIG_HOME: "/user-config", + OPENCODE_CONFIG: "/user-config/opencode.json", + OPENCODE_CONFIG_CONTENT: JSON.stringify({ + providers: { custom: { options: { baseURL: "http://model.internal" } } }, + instructions: ["/secret/instructions.md"], + plugins: ["secret-plugin"], + mcp: { private: { type: "remote", url: "http://private" } }, + }), + }, + }).pipe(Effect.provideService(OpenCode2Server.OpenCode2Server, sharedServer)); + +beforeEach(() => { + promptSent = Promise.withResolvers(); + state.startEnvironments.length = 0; + state.closeCalls.length = 0; + state.connectUrls.length = 0; + state.sessionInputs.length = 0; + state.promptInputs.length = 0; + state.permissionReplies.length = 0; + state.events = [ + { + type: "session.step.started", + data: { + sessionID: SESSION_ID, + model: { providerID: "openai", id: "gpt-5", variant: "high" }, + }, + }, + { + type: "session.text.ended", + data: { + sessionID: SESSION_ID, + assistantMessageID: "msg_blind_evaluation", + ordinal: 0, + text: JSON.stringify(validEvaluation), + }, + }, + { + type: "session.execution.succeeded", + data: { sessionID: SESSION_ID }, + }, + ] as OpenCode2Client.OpenCode2StreamEvent[]; +}); + +it.layer(layer)("OpenCode2TextGeneration blind evaluation", (it) => { + it.effect("uses an owned deny-all server and the exact requested model variant", () => + Effect.gen(function* () { + const textGeneration = yield* make(); + const verdict = yield* textGeneration.generateModelEvaluation(evaluationInput); + + expect(verdict).toEqual(validEvaluation); + expect(textGeneration.supportedOperations).toContain("generateModelEvaluation"); + expect(state.connectUrls).toEqual([OWNED_SERVER_URL]); + expect(state.sessionInputs[0]).toMatchObject({ + model: { providerID: "openai", id: "gpt-5", variant: "high" }, + permissions: [{ action: "*", resource: "*", effect: "deny" }], + }); + const environment = state.startEnvironments[0]; + expect(environment?.XDG_CONFIG_HOME).not.toBe("/user-config"); + expect(environment?.OPENCODE_CONFIG).toBeUndefined(); + expect(JSON.parse(environment?.OPENCODE_CONFIG_CONTENT ?? "null")).toEqual({ + providers: { custom: { options: { baseURL: "http://model.internal" } } }, + }); + expect(state.closeCalls).toEqual([OWNED_SERVER_URL]); + }).pipe(Effect.scoped), + ); + + it.effect("fails when execution reports a different model", () => + Effect.gen(function* () { + state.events[0] = { + type: "session.step.started", + data: { + sessionID: SESSION_ID, + model: { providerID: "openai", id: "gpt-5-mini", variant: "high" }, + }, + } as OpenCode2Client.OpenCode2StreamEvent; + const textGeneration = yield* make(); + + const error = yield* textGeneration + .generateModelEvaluation(evaluationInput) + .pipe(Effect.flip); + + expect(error.detail).toContain("different model or variant"); + }).pipe(Effect.scoped), + ); + + it.effect("fails when a tool event is observed", () => + Effect.gen(function* () { + state.events.splice(1, 0, { + type: "session.tool.called", + data: { + sessionID: SESSION_ID, + id: "tool_read", + assistantMessageID: "msg_blind_evaluation", + input: { filePath: "/secret" }, + executed: true, + }, + } as unknown as OpenCode2Client.OpenCode2StreamEvent); + const textGeneration = yield* make(); + + const error = yield* textGeneration + .generateModelEvaluation(evaluationInput) + .pipe(Effect.flip); + + expect(error.detail).toContain("tool activity"); + }).pipe(Effect.scoped), + ); + + it.effect("rejects a permission request and fails the evaluation", () => + Effect.gen(function* () { + state.events.splice(1, 0, { + type: "permission.asked", + data: { sessionID: SESSION_ID, id: "permission_read" }, + } as unknown as OpenCode2Client.OpenCode2StreamEvent); + const textGeneration = yield* make(); + + const error = yield* textGeneration + .generateModelEvaluation(evaluationInput) + .pipe(Effect.flip); + + expect(error.detail).toContain("permission"); + expect(state.permissionReplies).toEqual([ + { + sessionID: SESSION_ID, + requestID: "permission_read", + decision: "reject", + message: "Tools are not available while generating text.", + }, + ]); + }).pipe(Effect.scoped), + ); +}); diff --git a/packages/provider-opencode/src/server/v2/textGeneration.ts b/packages/provider-opencode/src/server/v2/textGeneration.ts index bca9a5fabd58f..c277e3b5316f3 100644 --- a/packages/provider-opencode/src/server/v2/textGeneration.ts +++ b/packages/provider-opencode/src/server/v2/textGeneration.ts @@ -70,7 +70,7 @@ const runOnServer = Effect.fn("Opencode2TextGeneration.runOnServer")(function* ( event.executionType === "session.execution.succeeded" ? successfulOutcome() : { - _tag: "failed", + _tag: "failed" as const, detail: "OpenCode ended the generation in a way this version cannot read.", cause: event, }, @@ -129,12 +129,20 @@ const runOnServer = Effect.fn("Opencode2TextGeneration.runOnServer")(function* ( decision: "reject", message: "Tools are not available while generating text.", }) - .pipe(Effect.ignore({ log: true })); + .pipe( + Effect.ignore({ log: true }), + Effect.andThen( + blindEvaluation + ? Deferred.succeed(outcome, { + _tag: "failed", + detail: "OpenCode requested tool permission during blind evaluation.", + cause: event, + }) + : Effect.void, + ), + ); case "session.execution.succeeded": - return Deferred.succeed(outcome, { - _tag: "text", - text: [...texts.values()].join("\n").trim(), - }); + return Deferred.succeed(outcome, successfulOutcome()); case "session.execution.failed": return Deferred.succeed(outcome, { _tag: "failed", @@ -172,7 +180,9 @@ const runOnServer = Effect.fn("Opencode2TextGeneration.runOnServer")(function* ( id: Model.ID.make(parsed.modelID), ...(variant === undefined ? {} : { variant: Model.VariantID.make(variant) }), }), - permissions: GENERATION_PERMISSIONS, + permissions: blindEvaluation + ? [{ action: "*", resource: "*", effect: "deny" as const }] + : GENERATION_PERMISSIONS, }); sessionId = session.id; yield* Effect.addFinalizer(() => @@ -230,28 +240,75 @@ const runOnServer = Effect.fn("Opencode2TextGeneration.runOnServer")(function* ( return result.text; }, Effect.scoped); -export const make = Effect.fn("OpenCode2TextGeneration.make")(function* () { +export const make = Effect.fn("OpenCode2TextGeneration.make")(function* ( + options: { + readonly binaryPath?: string; + readonly environment?: NodeJS.ProcessEnv; + readonly directory?: string; + } = {}, +) { + const binaryPath = options.binaryPath ?? "opencode"; + const processEnvironment = options.environment ?? process.env; + const processDirectory = options.directory ?? process.cwd(); const server = yield* OpenCode2Server.OpenCode2Server; + const openCodeRuntime = yield* OpenCodeRuntime.OpenCodeRuntime; + const openCode2Client = yield* OpenCode2Client.OpenCode2Client; + const secureRandom = yield* SecureRandom.SecureRandom; const host = yield* ProviderHost.ProviderHost; const fileSystem = yield* FileSystem.FileSystem; - const run: TextGenerationOperations.Runner = (input) => - server - .withConnection((connection) => runOnServer(connection, input)) - .pipe( - Effect.provideService(ProviderHost.ProviderHost, host), - Effect.provideService(FileSystem.FileSystem, fileSystem), - Effect.mapError((cause) => - isTextGenerationError(cause) - ? cause - : new TextGenerationError({ - operation: input.operation, - detail: "OpenCode text generation failed.", - cause, - }), - ), - Effect.flatMap((raw) => TextGenerationOperations.decodeJsonReply(input, "OpenCode", raw)), - ); + const runIsolated = Effect.fn("OpenCode2TextGeneration.runIsolated")(function* < + S extends Schema.Top, + >(input: TextGenerationOperations.Request) { + const environment = yield* BlindEvaluation.makeBlindEvaluationEnvironment({ + operation: input.operation, + directory: processDirectory, + environment: processEnvironment, + }); + const isolatedServer = yield* OpenCode2Server.make({ + binaryPath, + serverUrl: "", + serverPassword: "", + directory: input.cwd, + environment, + }); + const raw = yield* isolatedServer.withConnection((connection) => + runOnServer(connection, input), + ); + return yield* TextGenerationOperations.decodeJsonReply(input, "OpenCode", raw); + }, Effect.scoped); + const run: TextGenerationOperations.Runner = (input) => { + const blindEvaluation = input.operation === "generateModelEvaluation"; + const generated = blindEvaluation + ? BlindEvaluation.validateBlindEvaluationSelection( + input.modelSelection, + input.operation, + ).pipe(Effect.andThen(runIsolated(input))) + : server + .withConnection((connection) => runOnServer(connection, input)) + .pipe( + Effect.flatMap((raw) => + TextGenerationOperations.decodeJsonReply(input, "OpenCode", raw), + ), + ); + return generated.pipe( + Effect.provideService(ProviderHost.ProviderHost, host), + Effect.provideService(FileSystem.FileSystem, fileSystem), + Effect.provideService(OpenCodeRuntime.OpenCodeRuntime, openCodeRuntime), + Effect.provideService(OpenCode2Client.OpenCode2Client, openCode2Client), + Effect.provideService(SecureRandom.SecureRandom, secureRandom), + Effect.mapError((cause) => + isTextGenerationError(cause) + ? cause + : new TextGenerationError({ + operation: input.operation, + detail: "OpenCode text generation failed.", + cause, + }), + ), + ); + }; return TextGenerationOperations.fromRunner("OpenCode2TextGeneration", run, { supportsInvestigation: false, + supportsBlindEvaluation: true, }); }); From 829c4beaf8f1cc468ea791c08064690d2bc97b0e Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:11:03 -0800 Subject: [PATCH 06/11] fix(evaluations): preserve judge credentials and validate complete verdicts --- .../settings/ModelEvaluationSettings.tsx | 5 +- .../CodexTextGeneration.test.ts | 23 +- .../src/textGeneration/CodexTextGeneration.ts | 272 ++++++++++-------- docs/user/model-evaluations.md | 8 +- .../src/server/modelEvaluation.test.ts | 3 + .../src/server/modelEvaluation.ts | 3 + .../src/server/driver.test.ts | 4 - .../provider-opencode/src/server/driver.ts | 14 +- .../model-evaluation-api-judge.e2e.ts | 109 +++++++ 9 files changed, 307 insertions(+), 134 deletions(-) create mode 100644 scripts/scenarios/model-evaluation-api-judge.e2e.ts diff --git a/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx b/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx index 53912b4faaf34..5734512a174cf 100644 --- a/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx +++ b/apps/mobile/src/features/settings/ModelEvaluationSettings.tsx @@ -144,7 +144,10 @@ function EnvironmentEvaluationSettings(props: { const listedApiModels = AsyncResult.getOrElse(apiModelLists, () => []); const models = [ ...buildModelOptions(config, selection, undefined, null).filter( - (model) => capable.has(model.selection.instanceId) && !model.isUnavailable, + (model) => + capable.has(model.selection.instanceId) && + !model.isUnavailable && + textGeneratorForSelection(settings, model.selection.instanceId) === undefined, ), ...apiJudges.flatMap(([instanceId, generator], index) => (listedApiModels[index] ?? []).map((model): ModelOption => ({ diff --git a/apps/server/src/textGeneration/CodexTextGeneration.test.ts b/apps/server/src/textGeneration/CodexTextGeneration.test.ts index b6083e7ad5a04..c08926f603fcd 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.test.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.test.ts @@ -63,6 +63,7 @@ interface FakeCodexInput { mcpServers?: ReadonlyArray<{ name: string; enabled: boolean }> | number; requiredCwd?: string; requireInstructions?: string; + requireIsolatedHome?: boolean; } function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { @@ -80,6 +81,7 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { mcpServers: input.mcpServers ?? [], requiredCwd: input.requiredCwd ?? null, requireInstructions: input.requireInstructions ?? null, + requireIsolatedHome: input.requireIsolatedHome ?? false, stderr: input.stderr ?? null, output: input.output, events: input.events ?? [], @@ -134,6 +136,16 @@ function makeFakeCodexBinary(dir: string, input: FakeCodexInput) { ' const last = args.filter(arg => arg.startsWith("features." + name + "=")).at(-1);', ' if (last !== "features." + name + "=false") fail("feature not disabled: " + name, 8);', "}", + "if (check.requireIsolatedHome) {", + " const home = process.env.CODEX_HOME;", + ' if (NodeFS.existsSync(home + "/AGENTS.md")) fail("inherited global instructions", 12);', + ' const config = NodeFS.readFileSync(home + "/config.toml", "utf8");', + ' if (!config.includes(\'model_provider = "private"\') || !config.includes(\'base_url = "http://127.0.0.1:1/v1"\') || config.includes("developer_instructions")) fail("incorrect isolated routing", 13);', + ' if (!NodeFS.lstatSync(home + "/auth.json").isSymbolicLink() || !NodeFS.lstatSync(home + "/auth.lock").isSymbolicLink()) fail("credential identity was split", 14);', + ' const auth = NodeFS.realpathSync(home + "/auth.json");', + ' NodeFS.writeFileSync(auth + ".new", "rotated-login");', + ' NodeFS.renameSync(auth + ".new", auth);', + "}", "const chunks = [];", "for await (const chunk of process.stdin) chunks.push(chunk);", 'const stdinContent = Buffer.concat(chunks).toString("utf8");', @@ -207,6 +219,8 @@ function withFakeCodexEnv( setupMode: input.resolvedRuntime ?? (input.managedRuntime ? "managed" : "existing"), customModels: input.customModels, }); + yield* fs.writeFileString(`${tempDir}/AGENTS.md`, "Unrelated user instructions."); + yield* fs.writeFileString(`${tempDir}/auth.json`, "original-login"); const textGeneration = yield* makeCodexTextGeneration( config, { ...process.env, CODEX_HOME: tempDir, ...input.environment }, @@ -227,7 +241,10 @@ function withFakeCodexEnv( }) : undefined, ); - return yield* effectFn(textGeneration); + const result = yield* effectFn(textGeneration); + if (input.requireIsolatedHome) + expect(yield* fs.readFileString(`${tempDir}/auth.json`)).toBe("rotated-login"); + return result; }).pipe(Effect.scoped); } @@ -337,6 +354,10 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => { models: ["other-default-model"], defaultModel: "other-default-model", requireArg: "--model selected-judge", + configToml: + 'model_provider = "private"\ndeveloper_instructions = "Do not inherit"\n[model_providers.private]\nbase_url = "http://127.0.0.1:1/v1"\nwire_api = "responses"\n', + requireIsolatedHome: true, + requireReasoningEffort: "high", requireServiceTier: "priority", disabledFeatures: [ diff --git a/apps/server/src/textGeneration/CodexTextGeneration.ts b/apps/server/src/textGeneration/CodexTextGeneration.ts index a86b86fb61a32..1790efecd8ad5 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.ts @@ -82,22 +82,27 @@ const CODEX_ROUTING_KEYS = new Set([ "forced_chatgpt_workspace_id", ]); -const routingOverrides = (table: TomlTable, prefix = ""): ReadonlyArray => - Object.entries(table).flatMap(([key, value]) => { - if (prefix === "" && !CODEX_ROUTING_KEYS.has(key)) return []; - const name = prefix === "" ? key : `${prefix}.${JSON.stringify(key)}`; - return value !== null && +const routingConfig = (table: TomlTable): TomlTable => + Object.fromEntries(Object.entries(table).filter(([key]) => CODEX_ROUTING_KEYS.has(key))); + +const mergeRoutingConfig = (base: TomlTable, override: TomlTable): TomlTable => { + const result = { ...base }; + for (const [key, value] of Object.entries(override)) { + const previous = result[key]; + result[key] = + previous !== null && + typeof previous === "object" && + !Array.isArray(previous) && + !(previous instanceof Date) && + value !== null && typeof value === "object" && !Array.isArray(value) && !(value instanceof Date) - ? routingOverrides(value as TomlTable, name) - : [ - "--config", - `${name}=${stringify({ value }) - .trim() - .replace(/^value\s*=\s*/, "")}`, - ]; - }); + ? mergeRoutingConfig(previous as TomlTable, value as TomlTable) + : value; + } + return result; +}; const CodexMcpServers = Schema.fromJsonString( Schema.Array(Schema.Struct({ name: Schema.String, enabled: Schema.Boolean })), @@ -272,71 +277,90 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func Effect.orElseSucceed((): ReadonlyArray => []), ); - const evaluationLaunchArgs = Effect.fn("CodexTextGeneration.evaluationLaunchArgs")(function* ( - homePath: string, - launchArgs: string, - ) { - const args = tokenizeCliArgs(launchArgs); - const overrides: string[] = []; - let profile: string | undefined; - for (let index = 0; index < args.length; index++) { - const arg = args[index]; - if (arg === "--profile" || arg === "-p") profile = args[++index]; - else if (arg?.startsWith("--profile=") || arg?.startsWith("-p=")) - profile = arg.slice(arg.indexOf("=") + 1); - else if ( - arg === "--config" || - arg === "-c" || - arg?.startsWith("--config=") || - arg?.startsWith("-c=") - ) { - const value = - arg === "--config" || arg === "-c" ? args[++index] : arg.slice(arg.indexOf("=") + 1); - if (value && CODEX_ROUTING_KEYS.has(value.split(/[.=]/)[0] ?? "")) - overrides.push("--config", value); - } else if (arg === "--oss") overrides.push(arg); - else if (arg === "--local-provider" && args[index + 1]) overrides.push(arg, args[++index]!); - } - const readRouting = Effect.fn("CodexTextGeneration.readRouting")(function* (name: string) { - const filePath = path.join(homePath, name); - if (!(yield* fileSystem.exists(filePath))) return []; - const source = yield* fileSystem.readFileString(filePath); - return yield* Effect.try({ - try: () => routingOverrides(parse(source)), - catch: (cause) => - new TextGenerationError({ - operation: "generateModelEvaluation", - detail: "Invalid Codex routing configuration.", - cause, - }), + const evaluationConfiguration = Effect.fn("CodexTextGeneration.evaluationConfiguration")( + function* (homePath: string, launchArgs: string) { + const invalid = (cause: unknown) => + new TextGenerationError({ + operation: "generateModelEvaluation", + detail: "Failed to isolate the Codex evaluator configuration.", + cause, + }); + const parseConfig = (source: string) => + Effect.try({ try: () => parse(source), catch: invalid }); + const readConfig = Effect.fn("CodexTextGeneration.readRouting")(function* (name: string) { + const filePath = path.join(homePath, name); + if (!(yield* fileSystem.exists(filePath))) return {}; + return yield* parseConfig(yield* fileSystem.readFileString(filePath)); }); - }); - return [ - ...(yield* readRouting("config.toml")), - ...(profile === undefined ? [] : yield* readRouting(`${profile}.config.toml`)), - ...overrides, - "--ignore-user-config", - "--ignore-rules", - "--json", - "--config", - "project_doc_max_bytes=0", - "--config", - "tools.experimental_request_user_input.enabled=false", - "--config", - "tools.update_plan.enabled=false", - "--config", - "skills.include_instructions=false", - "--config", - "features.skip_host_skill_discovery=true", - "--config", - 'web_search="disabled"', - ...CODEX_JUDGE_FEATURES.flatMap((feature) => ["--config", `features.${feature}=false`]), - ]; - }); + const original = yield* readConfig("config.toml"); + const args = tokenizeCliArgs(launchArgs); + const overrides: TomlTable[] = []; + const switches: string[] = []; + let profile = typeof original.profile === "string" ? original.profile : undefined; + for (let index = 0; index < args.length; index++) { + const arg = args[index]; + if (arg === "--profile" || arg === "-p") profile = args[++index]; + else if (arg?.startsWith("--profile=") || arg?.startsWith("-p=")) + profile = arg.slice(arg.indexOf("=") + 1); + else if ( + arg === "--config" || + arg === "-c" || + arg?.startsWith("--config=") || + arg?.startsWith("-c=") + ) { + const value = + arg === "--config" || arg === "-c" ? args[++index] : arg.slice(arg.indexOf("=") + 1); + if (value && CODEX_ROUTING_KEYS.has(value.split(/[.=]/)[0] ?? "")) + overrides.push(routingConfig(yield* parseConfig(value))); + } else if (arg === "--oss") switches.push(arg); + else if (arg === "--local-provider" && args[index + 1]) switches.push(arg, args[++index]!); + } + const routing = [ + routingConfig(original), + ...(profile === undefined + ? [] + : [routingConfig(yield* readConfig(`${profile}.config.toml`))]), + ...overrides, + ].reduce(mergeRoutingConfig, {}); + if ( + routing.cli_auth_credentials_store !== undefined && + routing.cli_auth_credentials_store !== "file" + ) { + return yield* new TextGenerationError({ + operation: "generateModelEvaluation", + detail: + "Codex blind evaluation requires file-backed login or a model API. Keyring login cannot be isolated.", + }); + } + const source = yield* Effect.try({ try: () => stringify(routing), catch: invalid }); + return { + source, + args: [ + ...switches, + "--ignore-rules", + "--json", + "--config", + "project_doc_max_bytes=0", + "--config", + "tools.experimental_request_user_input.enabled=false", + "--config", + "tools.update_plan.enabled=false", + "--config", + "skills.include_instructions=false", + "--config", + "features.skip_host_skill_discovery=true", + "--config", + 'web_search="disabled"', + ...CODEX_JUDGE_FEATURES.flatMap((feature) => ["--config", `features.${feature}=false`]), + ], + }; + }, + ); const isolatedEvaluatorHome = Effect.fn("CodexTextGeneration.isolatedEvaluatorHome")(function* ( originalHome: string, operation: TextGenerationOperations.Operation, + configuration: string, ) { const evaluationHome = yield* fileSystem .makeTempDirectoryScoped({ prefix: "t3-codex-evaluator-" }) @@ -350,32 +374,39 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func }), ), ); - for (const name of ["auth.json", "models_cache.json"]) { - const source = path.join(originalHome, name); - if ( - yield* fileSystem.exists(source).pipe( - Effect.mapError( - (cause) => - new TextGenerationError({ - operation, - detail: "Failed to locate the Codex evaluator login.", - cause, - }), - ), - ) - ) { - yield* fileSystem.copyFile(source, path.join(evaluationHome, name)).pipe( - Effect.mapError( - (cause) => - new TextGenerationError({ - operation, - detail: "Failed to prepare the Codex evaluator login.", - cause, - }), - ), - ); + yield* fileSystem + .writeFileString(path.join(evaluationHome, "config.toml"), configuration, { mode: 0o600 }) + .pipe( + Effect.mapError( + (cause) => + new TextGenerationError({ + operation, + detail: "Failed to prepare the Codex evaluator configuration.", + cause, + }), + ), + ); + const prepare = Effect.gen(function* () { + if (yield* fileSystem.exists(path.join(originalHome, "auth.json"))) { + // Codex follows credential links on atomic refresh; the lock must share the same identity. + for (const name of ["auth.json", "auth.lock"]) { + yield* fileSystem.symlink(path.join(originalHome, name), path.join(evaluationHome, name)); + } } - } + const cache = path.join(originalHome, "models_cache.json"); + if (yield* fileSystem.exists(cache)) + yield* fileSystem.copyFile(cache, path.join(evaluationHome, "models_cache.json")); + }); + yield* prepare.pipe( + Effect.mapError( + (cause) => + new TextGenerationError({ + operation, + detail: "Failed to prepare the Codex evaluator login.", + cause, + }), + ), + ); return evaluationHome; }); @@ -441,7 +472,8 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func : getModelSelectionStringOptionValue(modelSelection, "reasoningEffort")) ?? DEFAULT_TEXT_GENERATION_REASONING_EFFORT; const serviceTier = - effectiveConfig.setupMode === "managed" || fellBack + (effectiveConfig.setupMode === "managed" && operation !== "generateModelEvaluation") || + fellBack ? undefined : getCodexServiceTierOptionValue(modelSelection); const binary = effectiveConfig.binaryPath || "codex"; @@ -457,9 +489,9 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func : {}), }; const originalHome = env.CODEX_HOME ?? path.join(yield* HostProcess.HomeDirectory, ".codex"); - const judgeArgs = + const judgeConfiguration = operation === "generateModelEvaluation" - ? yield* evaluationLaunchArgs(originalHome, launchArgs).pipe( + ? yield* evaluationConfiguration(originalHome, launchArgs).pipe( Effect.mapError( (cause) => new TextGenerationError({ @@ -470,7 +502,7 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func ), ) : undefined; - const mcpServersOff = judgeArgs + const mcpServersOff = judgeConfiguration ? null : yield* listEnabledMcpServers({ binary, @@ -478,12 +510,17 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func env, launchArgs: codexMcpListLaunchArgs(launchArgs), }).pipe(Effect.map(codexMcpServersOff)); - if (judgeArgs) env.CODEX_HOME = yield* isolatedEvaluatorHome(originalHome, operation); + if (judgeConfiguration) + env.CODEX_HOME = yield* isolatedEvaluatorHome( + originalHome, + operation, + judgeConfiguration.source, + ); const spawnCommand = yield* resolveSpawnCommand( binary, [ "exec", - ...(judgeArgs ?? codexExecLaunchArgs(launchArgs)), + ...(judgeConfiguration?.args ?? codexExecLaunchArgs(launchArgs)), // A user's MCP server could act on the text-generation prompt. ...CODEX_NO_PLUGIN_ARGS, // Memory candidates cite the supplied exchange; checkout instructions add unrelated input. @@ -540,7 +577,19 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func { concurrency: "unbounded" }, ); - if (judgeArgs) { + if (exitCode !== 0) { + const stderrDetail = stderr.trim(); + const stdoutDetail = stdout.trim(); + const detail = stderrDetail.length > 0 ? stderrDetail : stdoutDetail; + return yield* new TextGenerationError({ + operation, + detail: + detail.length > 0 + ? `Codex CLI command failed: ${detail}` + : `Codex CLI command failed with code ${exitCode}.`, + }); + } + if (judgeConfiguration) { const events = yield* Effect.try({ try: () => stdout @@ -571,19 +620,6 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func } } } - - if (exitCode !== 0) { - const stderrDetail = stderr.trim(); - const stdoutDetail = stdout.trim(); - const detail = stderrDetail.length > 0 ? stderrDetail : stdoutDetail; - return yield* new TextGenerationError({ - operation, - detail: - detail.length > 0 - ? `Codex CLI command failed: ${detail}` - : `Codex CLI command failed with code ${exitCode}.`, - }); - } }); const cleanup = Effect.all([removeTempFileDir(schemaPath), removeTempFileDir(outputPath)], { diff --git a/docs/user/model-evaluations.md b/docs/user/model-evaluations.md index bb1dfa268cc8c..2a3595c938999 100644 --- a/docs/user/model-evaluations.md +++ b/docs/user/model-evaluations.md @@ -6,10 +6,10 @@ model call. Use **Judge model** to select the provider and model that scores tas of the model doing the work. **Use default** returns it to the environment's text generation model. Only providers that can judge without tools or access to the project are available. The evaluator uses the same background generation system as thread titles, with its own optional -model choice. Claude, Pi, and model APIs added under **Settings → Providers** can serve as judges -when available. A supported judge can evaluate work from any harness, including Codex; the harness -that produced the work does not need to support judging. Choose an available judge before enabling -the setting. +model choice. Claude, Codex, Cursor, OpenCode, Pi, and model APIs added under **Settings → Providers** +can serve as judges when available. A supported judge can evaluate work from any harness. The +harness that produced the work does not need to support judging. Choose an available judge before +enabling the setting. Codex judges require file-backed login or API authentication. The selected judge is used for new evaluations. If it is unavailable, the evaluation fails visibly instead of switching to another model. Claude judges require a CLI that supports `--safe-mode`, which keeps account authentication while disabling custom instructions and extensions. diff --git a/packages/provider-core/src/server/modelEvaluation.test.ts b/packages/provider-core/src/server/modelEvaluation.test.ts index 2972cd64ebb0a..0b7f6d0260857 100644 --- a/packages/provider-core/src/server/modelEvaluation.test.ts +++ b/packages/provider-core/src/server/modelEvaluation.test.ts @@ -174,6 +174,9 @@ it("separates identities from evidence and preserves tool outcomes and task data ]) { expect(prompt.toLowerCase()).not.toContain(identity.toLowerCase()); } + expect(prompt).toContain('"taskCategory"'); + expect(prompt).toContain('"scores"'); + expect(prompt).toContain('"additionalProperties":false'); expect(prompt).toContain("2 tests failed"); expect(prompt).toContain('\\"exitCode\\":1'); expect(result.packet.evidence.map((entry) => entry.ref)).toEqual(["E1", "E2", "E3"]); diff --git a/packages/provider-core/src/server/modelEvaluation.ts b/packages/provider-core/src/server/modelEvaluation.ts index 542fb4dc83d60..1722d292ea839 100644 --- a/packages/provider-core/src/server/modelEvaluation.ts +++ b/packages/provider-core/src/server/modelEvaluation.ts @@ -10,6 +10,8 @@ import * as DateTime from "effect/DateTime"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; +import { toJsonSchemaObject } from "./textGenerationUtils.ts"; + export const MODEL_EVALUATION_RUBRIC_VERSION = "1"; const decodeModelEvaluationVerdict = Schema.decodeUnknownOption(ModelEvaluationVerdict, { onExcessProperty: "error", @@ -410,6 +412,7 @@ export function buildModelEvaluationPrompt(packet: BlindModelEvaluationPacket) { "rationale and each observation have text (at most 600 characters) and citations (up to 5 refs from this packet). Cite evidence for every observation; never invent refs. Each of strengths, weaknesses, promptAdvice has at most 3 observations.", "Describe what worked or failed in this observed task. Prompt advice must follow from concrete evidence, such as a missed constraint or a successful explicit acceptance check. Do not invent model capacity, token limits, preferred prompt length, latency, cost, general ability, or comparative rankings from one sample.", "Return only JSON, without markdown fences.", + `Required JSON Schema: ${JSON.stringify(toJsonSchemaObject(ModelEvaluationVerdict))}`, JSON.stringify(packet), ].join("\n\n"), }; diff --git a/packages/provider-opencode/src/server/driver.test.ts b/packages/provider-opencode/src/server/driver.test.ts index 72ad66964bb5e..6d74dc8dd2542 100644 --- a/packages/provider-opencode/src/server/driver.test.ts +++ b/packages/provider-opencode/src/server/driver.test.ts @@ -39,7 +39,6 @@ import * as ProviderMaintenance from "@t3tools/provider-core/server/maintenanceR import * as OpenCodeRuntime from "./OpenCodeRuntime.ts"; import { OpenCodeDriver, isLocalOpenCodeServerUrl, openCodeUpdateFor } from "./driver.ts"; import * as OpenCodeServerLedger from "./OpenCodeServerLedger.ts"; -import * as OpenCode2Client from "./v2/OpenCode2Client.ts"; import { OPENCODE_2_RESPONSES, OPENCODE_2_WORKSPACE_RESPONSES, @@ -87,7 +86,6 @@ const layer = Layer.mergeAll( ProviderEventLoggers.NoOpProviderEventLoggers, ), Layer.succeed(OpenCodeRuntime.OpenCodeRuntime, openCode2Runtime), - OpenCode2Client.layer.pipe(Layer.provide(FetchHttpClient.layer)), ).pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))); const create = (config: Partial, http: HttpClient.HttpClient, enabled = true) => @@ -245,7 +243,6 @@ it.live("discovers models through a home-relative OpenCode wrapper", () => ProviderEventLoggers.NoOpProviderEventLoggers, ), OpenCodeRuntime.layer.pipe(Layer.provide(OpenCodeServerLedger.layerTest)), - OpenCode2Client.layer, ).pipe( Layer.provideMerge( Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, FetchHttpClient.layer), @@ -1014,7 +1011,6 @@ const layerUpdate = Layer.mergeAll( ProviderEventLoggers.NoOpProviderEventLoggers, ), Layer.succeed(OpenCodeRuntime.OpenCodeRuntime, changingRuntime), - OpenCode2Client.layer.pipe(Layer.provide(FetchHttpClient.layer)), ).pipe(Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))); const openCode1Provider = (id: string, models: ReadonlyArray) => ({ diff --git a/packages/provider-opencode/src/server/driver.ts b/packages/provider-opencode/src/server/driver.ts index 2c02cb30a4c53..ec8d0dca94eee 100644 --- a/packages/provider-opencode/src/server/driver.ts +++ b/packages/provider-opencode/src/server/driver.ts @@ -258,7 +258,6 @@ export type OpenCodeDriverEnv = | HttpClient.HttpClient | ProviderLatestVersions.ProviderLatestVersions | OpenCodeRuntime.OpenCodeRuntime - | OpenCode2Client.OpenCode2Client | Path.Path; export const OpenCodeDriver: ProviderDriver< @@ -390,6 +389,9 @@ export const OpenCodeDriver: ProviderDriver< }), ), ); + const openCode2Client = yield* OpenCode2Client.make.pipe( + Effect.provideService(HttpClient.HttpClient, httpClient), + ); const openCode2Server = yield* OpenCode2Server.make({ binaryPath: effectiveConfig.binaryPath, serverUrl: effectiveConfig.serverUrl, @@ -398,10 +400,7 @@ export const OpenCodeDriver: ProviderDriver< environment: processEnv, }).pipe( Effect.provideService(OpenCodeRuntime.OpenCodeRuntime, openCodeRuntime), - Effect.provideService( - OpenCode2Client.OpenCode2Client, - yield* OpenCode2Client.make.pipe(Effect.provideService(HttpClient.HttpClient, httpClient)), - ), + Effect.provideService(OpenCode2Client.OpenCode2Client, openCode2Client), ); const orchestrationAdapter = selectOpenCodeRuntimeAdapter({ probe: runtimeProbe, @@ -473,7 +472,10 @@ export const OpenCodeDriver: ProviderDriver< binaryPath: effectiveConfig.binaryPath, environment: processEnv, directory: host.paths.cwd, - }).pipe(Effect.provideService(OpenCode2Server.OpenCode2Server, openCode2Server)), + }).pipe( + Effect.provideService(OpenCode2Server.OpenCode2Server, openCode2Server), + Effect.provideService(OpenCode2Client.OpenCode2Client, openCode2Client), + ), ); const checkUsageLimits = readOpenCodeGoUsageLimits({ diff --git a/scripts/scenarios/model-evaluation-api-judge.e2e.ts b/scripts/scenarios/model-evaluation-api-judge.e2e.ts new file mode 100644 index 0000000000000..f83a432585726 --- /dev/null +++ b/scripts/scenarios/model-evaluation-api-judge.e2e.ts @@ -0,0 +1,109 @@ +import { expect } from "e2e"; +// @effect-diagnostics-next-line nodeBuiltinImport:off - The fixture owns a disposable HTTP listener. +import * as NodeHttp from "node:http"; + +import { test, WEB } from "./t3.ts"; + +test( + "A connected model API can be selected as the evaluation judge", + WEB, + async ({ t3, app, screen }) => { + const judged = Promise.withResolvers(); + await using api = NodeHttp.createServer(async (request, response) => { + response.setHeader("content-type", "application/json"); + if (request.url === "/v1/models") { + response.end(JSON.stringify({ data: [{ id: "judge-fixture-model", object: "model" }] })); + return; + } + if (request.url === "/v1/chat/completions") { + const chunks: Buffer[] = []; + for await (const chunk of request) chunks.push(Buffer.from(chunk)); + const body = Buffer.concat(chunks).toString("utf8"); + response.end( + JSON.stringify({ + choices: [ + { + finish_reason: "stop", + message: { + content: JSON.stringify({ + outcome: "insufficient_evidence", + taskCategory: "other", + confidence: "low", + scores: { + taskCompletion: null, + correctness: null, + instructionFollowing: null, + verification: null, + }, + rationale: { + text: "The fixture judge received the task evidence.", + citations: ["U1"], + }, + strengths: [], + weaknesses: [], + promptAdvice: [], + }), + }, + }, + ], + }), + ); + judged.resolve(body); + return; + } + response.writeHead(404).end(); + }); + await new Promise((resolve, reject) => { + api.once("error", reject); + api.listen(0, "127.0.0.1", resolve); + }); + const address = api.address(); + if (address === null || typeof address === "string") throw new Error("No model API port."); + + const workspace = await t3.project("judge-fixture"); + await t3.open(); + await expect(screen.getByRole("button", "New thread", { exact: true })).toBeVisible(); + await app.open("/settings/providers"); + await screen.getByRole("button", "Add generator").tap(); + await screen.getByRole("radio", "Custom").tap(); + await screen.getByRole("textbox", "Base URL").fill(`http://127.0.0.1:${address.port}/v1`); + await screen.getByRole("textbox", "Name (optional)").fill("Scenario API"); + await screen.getByRole("button", "Connect", { exact: true }).tap(); + await screen.getByRole("button", "Done", { exact: true }).tap(); + + await app.open("/settings/labs"); + const judge = screen.getByRole("button", "Judge model", { exact: true }); + await expect(judge).toBeEnabled(); + await app.screenshot("judge-api-selection"); + await judge.tap(); + await screen.getByRole("button", "Scenario API", { exact: true }).tap(); + await screen.getByRole("option", "judge-fixture-model", { exact: false }).tap(); + await expect(judge).toHaveText(/judge-fixture-model$/); + const enabled = screen.getByRole("switch", "Blind model evaluations"); + await expect(enabled).toBeEnabled(); + await enabled.tap(); + await expect(enabled).toBeChecked(); + + await t3.thread({ + project: workspace, + title: "Evaluated task", + messages: ["Explain the project."], + model: "multi-turn", + }); + expect(JSON.parse(await judged.promise).model).toBe("judge-fixture-model"); + + await app.open("/settings/labs"); + await expect(judge).toHaveText(/judge-fixture-model$/); + await expect(enabled).toBeChecked(); + await app.screenshot("judge-api-saved"); + await screen.getByRole("combobox", "Evaluation project on", { exact: false }).tap(); + await screen.getByRole("option", "judge-fixture", { exact: true }).tap(); + await screen.getByText("Recent evaluations (1)", { exact: true }).tap(); + await expect( + screen.getByText("The fixture judge received the task evidence.", { exact: true }), + ).toBeVisible(); + await app.screenshot("judge-api-evaluation"); + await screen.getByRole("button", "Use default", { exact: true }).tap(); + await expect(judge).toHaveText(/gpt-6-luna$/); + }, +); From bdddbfb5242f2759dbf0ae5355161a4d528524e9 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:27:43 -0800 Subject: [PATCH 07/11] fix(evaluations): preserve provider routing and reject external judges --- .../CodexTextGeneration.test.ts | 3 +- .../src/textGeneration/CodexTextGeneration.ts | 21 ++++----- .../OpenCode2TextGeneration.test.ts | 13 +++++- docs/user/model-evaluations.md | 2 + .../src/server/blindEvaluation.ts | 45 ++++++++++++++----- .../provider-opencode/src/server/driver.ts | 1 + .../src/server/textGeneration.test.ts | 11 ++++- .../src/server/textGeneration.ts | 3 +- .../src/server/v2/textGeneration.test.ts | 37 +++++++++++++-- .../src/server/v2/textGeneration.ts | 3 +- 10 files changed, 105 insertions(+), 34 deletions(-) diff --git a/apps/server/src/textGeneration/CodexTextGeneration.test.ts b/apps/server/src/textGeneration/CodexTextGeneration.test.ts index c08926f603fcd..a67f05960e57f 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.test.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.test.ts @@ -355,9 +355,8 @@ it.layer(layerCodexTextGenerationTest)("CodexTextGeneration", (it) => { defaultModel: "other-default-model", requireArg: "--model selected-judge", configToml: - 'model_provider = "private"\ndeveloper_instructions = "Do not inherit"\n[model_providers.private]\nbase_url = "http://127.0.0.1:1/v1"\nwire_api = "responses"\n', + 'profile = "private"\nmodel_provider = "wrong"\ndeveloper_instructions = "Do not inherit"\n[profiles.private]\nmodel_provider = "private"\n[model_providers.private]\nbase_url = "http://127.0.0.1:1/v1"\nwire_api = "responses"\n', requireIsolatedHome: true, - requireReasoningEffort: "high", requireServiceTier: "priority", disabledFeatures: [ diff --git a/apps/server/src/textGeneration/CodexTextGeneration.ts b/apps/server/src/textGeneration/CodexTextGeneration.ts index 1790efecd8ad5..a8b839a83535e 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.ts @@ -74,6 +74,7 @@ const CODEX_JUDGE_FEATURES = [ ] as const; const CODEX_ROUTING_KEYS = new Set([ "model_provider", + "oss_provider", "model_providers", "openai_base_url", "chatgpt_base_url", @@ -85,21 +86,15 @@ const CODEX_ROUTING_KEYS = new Set([ const routingConfig = (table: TomlTable): TomlTable => Object.fromEntries(Object.entries(table).filter(([key]) => CODEX_ROUTING_KEYS.has(key))); +const isTomlTable = (value: unknown): value is TomlTable => + value !== null && typeof value === "object" && !Array.isArray(value) && !(value instanceof Date); + const mergeRoutingConfig = (base: TomlTable, override: TomlTable): TomlTable => { const result = { ...base }; for (const [key, value] of Object.entries(override)) { const previous = result[key]; result[key] = - previous !== null && - typeof previous === "object" && - !Array.isArray(previous) && - !(previous instanceof Date) && - value !== null && - typeof value === "object" && - !Array.isArray(value) && - !(value instanceof Date) - ? mergeRoutingConfig(previous as TomlTable, value as TomlTable) - : value; + isTomlTable(previous) && isTomlTable(value) ? mergeRoutingConfig(previous, value) : value; } return result; }; @@ -314,9 +309,15 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func overrides.push(routingConfig(yield* parseConfig(value))); } else if (arg === "--oss") switches.push(arg); else if (arg === "--local-provider" && args[index + 1]) switches.push(arg, args[++index]!); + else if (arg?.startsWith("--local-provider=")) switches.push(arg); } const routing = [ routingConfig(original), + ...(profile !== undefined && + isTomlTable(original.profiles) && + isTomlTable(original.profiles[profile]) + ? [routingConfig(original.profiles[profile])] + : []), ...(profile === undefined ? [] : [routingConfig(yield* readConfig(`${profile}.config.toml`))]), diff --git a/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts b/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts index 869cd1b4fb5e7..efad62a46f1cb 100644 --- a/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts +++ b/apps/server/src/textGeneration/OpenCode2TextGeneration.test.ts @@ -1,5 +1,6 @@ import { assert, it } from "@effect/vitest"; import * as NodeServices from "@effect/platform-node/NodeServices"; +import * as NodeCrypto from "@effect/platform-node/NodeCrypto"; import * as NodeURL from "node:url"; import { ChatAttachmentId, ChatImageAttachment, ProviderInstanceId } from "@t3tools/contracts"; import * as Deferred from "effect/Deferred"; @@ -7,6 +8,7 @@ import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as FetchHttpClient from "effect/http/FetchHttpClient"; import { TestClock } from "effect/testing"; import * as ProviderHost from "@t3tools/provider-core/server/ProviderHost"; @@ -14,9 +16,18 @@ import * as TestProviderHost from "@t3tools/provider-testing/TestProviderHost"; import * as OpenCode2AdapterV2Testkit from "../orchestration-v2/Adapters/OpenCode2AdapterV2.testkit.ts"; import * as OpenCode2Server from "@t3tools/provider-opencode/server/v2/OpenCode2Server"; import * as OpenCode2TextGeneration from "@t3tools/provider-opencode/server/v2/textGeneration"; +import * as OpenCode2Client from "@t3tools/provider-opencode/server/v2/OpenCode2Client"; +import * as OpenCodeRuntime from "@t3tools/provider-opencode/server/OpenCodeRuntime"; +import * as OpenCodeServerLedger from "@t3tools/provider-opencode/server/OpenCodeServerLedger"; import { OPENCODE2_TITLE_GENERATION } from "./OpenCode2TextGeneration.fixture.ts"; -const layer = TestProviderHost.layer().pipe(Layer.provideMerge(NodeServices.layer)); +const layer = Layer.mergeAll( + TestProviderHost.layer(), + OpenCodeRuntime.layer.pipe(Layer.provide(OpenCodeServerLedger.layerTest)), + OpenCode2Client.layer, +).pipe( + Layer.provideMerge(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer, FetchHttpClient.layer)), +); it.layer(layer)("OpenCode2TextGeneration", (it) => { it.effect.each([false, true] as const)( diff --git a/docs/user/model-evaluations.md b/docs/user/model-evaluations.md index 2a3595c938999..319c350f7d512 100644 --- a/docs/user/model-evaluations.md +++ b/docs/user/model-evaluations.md @@ -10,6 +10,8 @@ model choice. Claude, Codex, Cursor, OpenCode, Pi, and model APIs added under ** can serve as judges when available. A supported judge can evaluate work from any harness. The harness that produced the work does not need to support judging. Choose an available judge before enabling the setting. Codex judges require file-backed login or API authentication. +OpenCode judges require a local provider connection. Externally configured OpenCode servers +cannot supply an isolated judge, and models that require tools may reject evaluation calls. The selected judge is used for new evaluations. If it is unavailable, the evaluation fails visibly instead of switching to another model. Claude judges require a CLI that supports `--safe-mode`, which keeps account authentication while disabling custom instructions and extensions. diff --git a/packages/provider-opencode/src/server/blindEvaluation.ts b/packages/provider-opencode/src/server/blindEvaluation.ts index e8c87dc47cb25..17ffffce7bae5 100644 --- a/packages/provider-opencode/src/server/blindEvaluation.ts +++ b/packages/provider-opencode/src/server/blindEvaluation.ts @@ -5,6 +5,7 @@ import { } from "@t3tools/contracts"; import { expandHomePath } from "@t3tools/provider-core/server/pathExpansion"; import { fromLenientJson } from "@t3tools/shared/schemaJson"; +import * as HostProcess from "@t3tools/shared/HostProcess"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Option from "effect/Option"; @@ -19,13 +20,24 @@ const isConfig = Schema.is(Config); const isTextGenerationError = Schema.is(TextGenerationError); const ROUTING_KEYS = ["provider", "providers", "enabled_providers", "disabled_providers"] as const; -function readRoutingConfig(content: string): Config | undefined { +function readRoutingConfig( + content: string, + resolveFile: (file: string) => string, +): Config | undefined { const decoded = decodeConfig(content); if (Option.isNone(decoded)) return undefined; + const rewrite = (value: unknown): unknown => { + if (typeof value === "string") + return value.replace(/\{file:([^}]+)\}/g, (_, file: string) => `{file:${resolveFile(file)}}`); + if (Array.isArray(value)) return value.map(rewrite); + if (isConfig(value)) + return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, rewrite(item)])); + return value; + }; return Object.fromEntries( ROUTING_KEYS.flatMap((key) => { const value = decoded.value[key]; - return value === undefined ? [] : [[key, value]]; + return value === undefined ? [] : [[key, rewrite(value)]]; }), ); } @@ -47,6 +59,10 @@ export const makeBlindEvaluationEnvironment = Effect.fn("OpenCode.makeBlindEvalu }) { const fs = yield* FileSystem.FileSystem; const path = yield* Path.Path; + const home = + input.environment.HOME ?? input.environment.USERPROFILE ?? (yield* HostProcess.HomeDirectory); + const resolveFile = (directory: string) => (file: string) => + path.resolve(directory, expandHomePath(file, home)); const invalid = () => new TextGenerationError({ operation: input.operation, @@ -59,29 +75,26 @@ export const makeBlindEvaluationEnvironment = Effect.fn("OpenCode.makeBlindEvalu () => Effect.succeed("{}"), ), Effect.flatMap((content) => { - const config = readRoutingConfig(content); + const config = readRoutingConfig(content, resolveFile(path.dirname(filePath))); return config === undefined ? Effect.fail(invalid()) : Effect.succeed(config); }), Effect.mapError((cause) => (isTextGenerationError(cause) ? cause : invalid())), ); let routing: Config = {}; - const home = input.environment.HOME ?? input.environment.USERPROFILE; const configHome = input.environment.XDG_CONFIG_HOME; const globalDirectory = configHome !== undefined && configHome.trim().length > 0 ? path.join(configHome, "opencode") - : home === undefined - ? undefined - : path.join(home, ".config", "opencode"); + : path.join(home, ".config", "opencode"); const directories = [ - ...(globalDirectory === undefined ? [] : [globalDirectory]), + globalDirectory, ...(input.environment.OPENCODE_CONFIG_DIR === undefined ? [] : [ path.resolve( input.directory, - expandHomePath(input.environment.OPENCODE_CONFIG_DIR, home ?? ""), + expandHomePath(input.environment.OPENCODE_CONFIG_DIR, home), ), ]), ]; @@ -93,12 +106,15 @@ export const makeBlindEvaluationEnvironment = Effect.fn("OpenCode.makeBlindEvalu if (input.environment.OPENCODE_CONFIG !== undefined) { const configuredPath = path.resolve( input.directory, - expandHomePath(input.environment.OPENCODE_CONFIG, home ?? ""), + expandHomePath(input.environment.OPENCODE_CONFIG, home), ); routing = mergeConfig(routing, yield* readConfig(configuredPath)); } if (input.environment.OPENCODE_CONFIG_CONTENT !== undefined) { - const content = readRoutingConfig(input.environment.OPENCODE_CONFIG_CONTENT); + const content = readRoutingConfig( + input.environment.OPENCODE_CONFIG_CONTENT, + resolveFile(input.directory), + ); if (content === undefined) return yield* invalid(); routing = mergeConfig(routing, content); } @@ -117,6 +133,11 @@ export const makeBlindEvaluationEnvironment = Effect.fn("OpenCode.makeBlindEvalu } = input.environment; return { ...preserved, + HOME: temporary, + ...(input.environment.USERPROFILE === undefined ? {} : { USERPROFILE: temporary }), + XDG_DATA_HOME: input.environment.XDG_DATA_HOME ?? path.join(home, ".local", "share"), + XDG_STATE_HOME: path.join(temporary, "state"), + XDG_CACHE_HOME: path.join(temporary, "cache"), XDG_CONFIG_HOME: xdgConfigHome, OPENCODE_CONFIG_DIR: configDirectory, OPENCODE_CONFIG_CONTENT: encodeConfig(routing), @@ -163,5 +184,5 @@ export const hasBlindEvaluationPartActivity = (parts: ReadonlyArray) => part === null || typeof part !== "object" || !("type" in part) || - (part.type !== "text" && part.type !== "reasoning"), + !["text", "reasoning", "step-start", "step-finish", "snapshot"].includes(String(part.type)), ); diff --git a/packages/provider-opencode/src/server/driver.ts b/packages/provider-opencode/src/server/driver.ts index ec8d0dca94eee..e2dcc93cbb7ff 100644 --- a/packages/provider-opencode/src/server/driver.ts +++ b/packages/provider-opencode/src/server/driver.ts @@ -470,6 +470,7 @@ export const OpenCodeDriver: ProviderDriver< }).pipe(Effect.provideService(OpenCodeServerOwner.OpenCodeServerOwner, serverOwner)), yield* OpenCode2TextGeneration.make({ binaryPath: effectiveConfig.binaryPath, + serverUrl: effectiveConfig.serverUrl, environment: processEnv, directory: host.paths.cwd, }).pipe( diff --git a/packages/provider-opencode/src/server/textGeneration.test.ts b/packages/provider-opencode/src/server/textGeneration.test.ts index 323bfdc8c413b..fd51b652befc6 100644 --- a/packages/provider-opencode/src/server/textGeneration.test.ts +++ b/packages/provider-opencode/src/server/textGeneration.test.ts @@ -472,13 +472,19 @@ it.layer(layerOpenCodeTextGenerationTest)("OpenCodeTextGeneration", (it) => { it.effect("isolates blind evaluation from shared servers, tools, and custom instructions", () => withOpenCodeTextGeneration( - EXISTING_SERVER_OPENCODE_SETTINGS, + DEFAULT_OPENCODE_SETTINGS, (textGeneration) => Effect.gen(function* () { runtimeMock.state.promptResult = { data: { info: { providerID: "openai", modelID: "gpt-5", variant: "high" }, - parts: [{ type: "text", text: JSON.stringify(VALID_EVALUATION) }], + parts: [ + { type: "step-start" }, + { type: "reasoning", text: "Check the evidence." }, + { type: "text", text: JSON.stringify(VALID_EVALUATION) }, + { type: "step-finish", reason: "stop" }, + { type: "snapshot", snapshot: "unchanged" }, + ], }, }; @@ -794,6 +800,7 @@ it.layer(layerOpenCodeTextGenerationExistingServerTest)( it.effect("reuses a configured OpenCode server URL without spawning or applying idle TTL", () => withOpenCodeTextGeneration(EXISTING_SERVER_OPENCODE_SETTINGS, (textGeneration) => Effect.gen(function* () { + expect(textGeneration.supportedOperations).not.toContain("generateModelEvaluation"); yield* textGeneration.generateCommitMessage({ cwd: process.cwd(), branch: "feature/opencode-reuse", diff --git a/packages/provider-opencode/src/server/textGeneration.ts b/packages/provider-opencode/src/server/textGeneration.ts index 4dc72a1086425..74e45d8fac2fc 100644 --- a/packages/provider-opencode/src/server/textGeneration.ts +++ b/packages/provider-opencode/src/server/textGeneration.ts @@ -230,7 +230,6 @@ export const makeOpenCodeTextGeneration = Effect.fn("makeOpenCodeTextGeneration" client.session.create( { title: `T3 Code ${input.operation}`, - // OpenCode Zen refuses ordinary free-model requests that carry no tools. permission: [ { permission: "*", pattern: "*", action: "deny" }, ...(!blindEvaluation @@ -458,6 +457,6 @@ export const makeOpenCodeTextGeneration = Effect.fn("makeOpenCodeTextGeneration" return TextGenerationOperations.fromRunner("OpenCodeTextGeneration", runOpenCodeJson, { supportsInvestigation: false, - supportsBlindEvaluation: true, + supportsBlindEvaluation: openCodeSettings.serverUrl.trim().length === 0, }); }); diff --git a/packages/provider-opencode/src/server/v2/textGeneration.test.ts b/packages/provider-opencode/src/server/v2/textGeneration.test.ts index ff7e4ec64257f..6699af516b31d 100644 --- a/packages/provider-opencode/src/server/v2/textGeneration.test.ts +++ b/packages/provider-opencode/src/server/v2/textGeneration.test.ts @@ -4,6 +4,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { it } from "@effect/vitest"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; +import * as FileSystem from "effect/FileSystem"; import * as Stream from "effect/Stream"; import { beforeEach, expect } from "vite-plus/test"; @@ -144,13 +145,15 @@ const validEvaluation = { promptAdvice: [], }; -const make = () => +const make = (configPath = "/user-config/opencode.json", serverUrl = "") => OpenCode2TextGeneration.make({ binaryPath: "fake-opencode", + serverUrl, directory: process.cwd(), environment: { + HOME: "/original-home", XDG_CONFIG_HOME: "/user-config", - OPENCODE_CONFIG: "/user-config/opencode.json", + OPENCODE_CONFIG: configPath, OPENCODE_CONFIG_CONTENT: JSON.stringify({ providers: { custom: { options: { baseURL: "http://model.internal" } } }, instructions: ["/secret/instructions.md"], @@ -195,7 +198,14 @@ beforeEach(() => { it.layer(layer)("OpenCode2TextGeneration blind evaluation", (it) => { it.effect("uses an owned deny-all server and the exact requested model variant", () => Effect.gen(function* () { - const textGeneration = yield* make(); + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-opencode-routing-" }); + const configPath = `${directory}/opencode.json`; + yield* fs.writeFileString( + configPath, + JSON.stringify({ providers: { custom: { settings: { apiKey: "{file:./key}" } } } }), + ); + const textGeneration = yield* make(configPath); const verdict = yield* textGeneration.generateModelEvaluation(evaluationInput); expect(verdict).toEqual(validEvaluation); @@ -206,15 +216,34 @@ it.layer(layer)("OpenCode2TextGeneration blind evaluation", (it) => { permissions: [{ action: "*", resource: "*", effect: "deny" }], }); const environment = state.startEnvironments[0]; + expect(environment?.HOME).not.toBe("/original-home"); + expect(environment?.XDG_DATA_HOME).toBe("/original-home/.local/share"); expect(environment?.XDG_CONFIG_HOME).not.toBe("/user-config"); expect(environment?.OPENCODE_CONFIG).toBeUndefined(); expect(JSON.parse(environment?.OPENCODE_CONFIG_CONTENT ?? "null")).toEqual({ - providers: { custom: { options: { baseURL: "http://model.internal" } } }, + providers: { + custom: { + options: { baseURL: "http://model.internal" }, + settings: { apiKey: `{file:${directory}/key}` }, + }, + }, }); expect(state.closeCalls).toEqual([OWNED_SERVER_URL]); }).pipe(Effect.scoped), ); + it.effect("does not advertise judges from an external server with separate credentials", () => + Effect.gen(function* () { + const textGeneration = yield* make(undefined, "https://remote.example.com"); + expect(textGeneration.supportedOperations).not.toContain("generateModelEvaluation"); + const error = yield* textGeneration + .generateModelEvaluation(evaluationInput) + .pipe(Effect.flip); + expect(error.detail).toContain("cannot isolate"); + expect(state.connectUrls).toEqual([]); + }).pipe(Effect.scoped), + ); + it.effect("fails when execution reports a different model", () => Effect.gen(function* () { state.events[0] = { diff --git a/packages/provider-opencode/src/server/v2/textGeneration.ts b/packages/provider-opencode/src/server/v2/textGeneration.ts index c277e3b5316f3..9e00e67c86c51 100644 --- a/packages/provider-opencode/src/server/v2/textGeneration.ts +++ b/packages/provider-opencode/src/server/v2/textGeneration.ts @@ -243,6 +243,7 @@ const runOnServer = Effect.fn("Opencode2TextGeneration.runOnServer")(function* ( export const make = Effect.fn("OpenCode2TextGeneration.make")(function* ( options: { readonly binaryPath?: string; + readonly serverUrl?: string; readonly environment?: NodeJS.ProcessEnv; readonly directory?: string; } = {}, @@ -309,6 +310,6 @@ export const make = Effect.fn("OpenCode2TextGeneration.make")(function* ( }; return TextGenerationOperations.fromRunner("OpenCode2TextGeneration", run, { supportsInvestigation: false, - supportsBlindEvaluation: true, + supportsBlindEvaluation: !options.serverUrl?.trim(), }); }); From 895471a45dbef1e2808a4cc195f74d490fdfccd5 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:29:24 -0800 Subject: [PATCH 08/11] fix(server): narrow selected Codex profile before extracting routing --- apps/server/src/textGeneration/CodexTextGeneration.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/apps/server/src/textGeneration/CodexTextGeneration.ts b/apps/server/src/textGeneration/CodexTextGeneration.ts index a8b839a83535e..28cd4236f4feb 100644 --- a/apps/server/src/textGeneration/CodexTextGeneration.ts +++ b/apps/server/src/textGeneration/CodexTextGeneration.ts @@ -311,13 +311,13 @@ export const makeCodexTextGeneration = Effect.fn("makeCodexTextGeneration")(func else if (arg === "--local-provider" && args[index + 1]) switches.push(arg, args[++index]!); else if (arg?.startsWith("--local-provider=")) switches.push(arg); } + const profileRouting = + profile !== undefined && isTomlTable(original.profiles) + ? original.profiles[profile] + : undefined; const routing = [ routingConfig(original), - ...(profile !== undefined && - isTomlTable(original.profiles) && - isTomlTable(original.profiles[profile]) - ? [routingConfig(original.profiles[profile])] - : []), + ...(isTomlTable(profileRouting) ? [routingConfig(profileRouting)] : []), ...(profile === undefined ? [] : [routingConfig(yield* readConfig(`${profile}.config.toml`))]), From b905c499f24cdbfa77e59e60f84166667036736f Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Thu, 12 Nov 2026 15:31:32 -0800 Subject: [PATCH 09/11] test(evaluations): assert connected API appears in judge picker --- scripts/scenarios/model-evaluation-api-judge.e2e.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/scripts/scenarios/model-evaluation-api-judge.e2e.ts b/scripts/scenarios/model-evaluation-api-judge.e2e.ts index f83a432585726..448b837e6da1b 100644 --- a/scripts/scenarios/model-evaluation-api-judge.e2e.ts +++ b/scripts/scenarios/model-evaluation-api-judge.e2e.ts @@ -76,7 +76,9 @@ test( await expect(judge).toBeEnabled(); await app.screenshot("judge-api-selection"); await judge.tap(); - await screen.getByRole("button", "Scenario API", { exact: true }).tap(); + const apiProvider = screen.getByRole("button", "Scenario API", { exact: true }); + await expect(apiProvider).toBeVisible(); + await apiProvider.tap(); await screen.getByRole("option", "judge-fixture-model", { exact: false }).tap(); await expect(judge).toHaveText(/judge-fixture-model$/); const enabled = screen.getByRole("switch", "Blind model evaluations"); From e4eebb995ec5af2beb51fab7e112ed6f36dac18a Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Fri, 13 Nov 2026 00:16:37 -0800 Subject: [PATCH 10/11] fix(evaluations): preserve native OpenCode configuration Match v1 and v2 configuration precedence and filenames, and ignore empty override flags. Keep keyboard palette coverage deterministic by seeding its remembered-project prerequisite without removing navigation assertions. --- .../CommandPalette.browser.test.tsx | 31 ++- .../src/server/blindEvaluation.ts | 40 ++-- .../src/server/textGeneration.test.ts | 186 ++++++++++++------ .../src/server/textGeneration.ts | 1 + .../src/server/v2/textGeneration.test.ts | 89 ++++++++- .../src/server/v2/textGeneration.ts | 1 + 6 files changed, 257 insertions(+), 91 deletions(-) diff --git a/apps/web/src/components/CommandPalette.browser.test.tsx b/apps/web/src/components/CommandPalette.browser.test.tsx index 0e75db7824084..5772d40da8fa2 100644 --- a/apps/web/src/components/CommandPalette.browser.test.tsx +++ b/apps/web/src/components/CommandPalette.browser.test.tsx @@ -1,10 +1,16 @@ import { EnvironmentId, ProjectId, ThreadId } from "@t3tools/contracts"; +import { scopedProjectKey } from "@t3tools/client-runtime/environment"; import { describe, expect, it, vi } from "vite-plus/test"; import { page, userEvent } from "vite-plus/test/browser"; import { act, render } from "../../test/render"; import { openCommandPalette } from "../commandPaletteBus"; import { useCommandPaletteStore } from "../commandPaletteStore"; +import { + LAST_USED_PROJECT_STORAGE_KEY, + lastUsedProjectKeySchema, +} from "../hooks/useHandleNewThread"; +import { removeLocalStorageItem, setLocalStorageItem } from "../hooks/useLocalStorage"; import { makeThreadFixture } from "../test-fixtures"; import type { Project, Thread } from "../types"; import { CommandPalette } from "./CommandPalette"; @@ -98,16 +104,33 @@ describe("CommandPalette open intents", () => { updatedAt: "2026-03-01T00:00:00.000Z", scripts: [], }); - setup({ - projects: [project("acme", "Acme"), project("beta", "Beta"), project("gamma", "Gamma")], - }); + const acme = project("acme", "Acme"); + setup({ projects: [acme, project("beta", "Beta"), project("gamma", "Gamma")] }); + // Keep the starting row independent of remembered project preferences. + const previousLastUsedProject = window.localStorage.getItem(LAST_USED_PROJECT_STORAGE_KEY); + using _rememberedProject = { + [Symbol.dispose]() { + if (previousLastUsedProject === null) { + removeLocalStorageItem(LAST_USED_PROJECT_STORAGE_KEY); + } else { + window.localStorage.setItem(LAST_USED_PROJECT_STORAGE_KEY, previousLastUsedProject); + } + }, + }; + setLocalStorageItem( + LAST_USED_PROJECT_STORAGE_KEY, + scopedProjectKey({ environmentId, projectId: acme.id }), + lastUsedProjectKeySchema, + ); await using _screen = await render({null}); openCommandPalette({}); await expect .element(page.getByPlaceholder("Search commands, projects, and threads...")) .toHaveFocus(); - await expect.element(page.getByRole("option").first()).toHaveAttribute("data-highlighted"); + await expect + .element(page.getByRole("option", { name: /^New thread in Acme\b/ })) + .toHaveAttribute("data-highlighted"); await userEvent.keyboard("{ArrowDown}"); await expect .element(page.getByRole("option", { name: /^New thread in\.\.\./ })) diff --git a/packages/provider-opencode/src/server/blindEvaluation.ts b/packages/provider-opencode/src/server/blindEvaluation.ts index 17ffffce7bae5..c96cf14cb724c 100644 --- a/packages/provider-opencode/src/server/blindEvaluation.ts +++ b/packages/provider-opencode/src/server/blindEvaluation.ts @@ -53,6 +53,7 @@ function mergeConfig(target: Config, source: Config): Config { export const makeBlindEvaluationEnvironment = Effect.fn("OpenCode.makeBlindEvaluationEnvironment")( function* (input: { + readonly generation: "v1" | "v2"; readonly operation: TextGenerationOperation; readonly directory: string; readonly environment: NodeJS.ProcessEnv; @@ -82,35 +83,34 @@ export const makeBlindEvaluationEnvironment = Effect.fn("OpenCode.makeBlindEvalu ); let routing: Config = {}; + const configuredDirectory = input.environment.OPENCODE_CONFIG_DIR + ? path.resolve(input.directory, expandHomePath(input.environment.OPENCODE_CONFIG_DIR, home)) + : undefined; const configHome = input.environment.XDG_CONFIG_HOME; + // 2.x replaces the global directory; 1.x loads this directory after the explicit config file. const globalDirectory = - configHome !== undefined && configHome.trim().length > 0 - ? path.join(configHome, "opencode") - : path.join(home, ".config", "opencode"); - const directories = [ - globalDirectory, - ...(input.environment.OPENCODE_CONFIG_DIR === undefined - ? [] - : [ - path.resolve( - input.directory, - expandHomePath(input.environment.OPENCODE_CONFIG_DIR, home), - ), - ]), - ]; - for (const directory of directories) { - for (const name of ["config.json", "opencode.json", "opencode.jsonc"]) { - routing = mergeConfig(routing, yield* readConfig(path.join(directory, name))); - } + input.generation === "v2" && configuredDirectory !== undefined + ? configuredDirectory + : configHome !== undefined && configHome.trim().length > 0 + ? path.join(configHome, "opencode") + : path.join(home, ".config", "opencode"); + const configNames = ["opencode.json", "opencode.jsonc"]; + for (const name of [...(input.generation === "v1" ? ["config.json"] : []), ...configNames]) { + routing = mergeConfig(routing, yield* readConfig(path.join(globalDirectory, name))); } - if (input.environment.OPENCODE_CONFIG !== undefined) { + if (input.environment.OPENCODE_CONFIG) { const configuredPath = path.resolve( input.directory, expandHomePath(input.environment.OPENCODE_CONFIG, home), ); routing = mergeConfig(routing, yield* readConfig(configuredPath)); } - if (input.environment.OPENCODE_CONFIG_CONTENT !== undefined) { + if (input.generation === "v1" && configuredDirectory !== undefined) { + for (const name of configNames) { + routing = mergeConfig(routing, yield* readConfig(path.join(configuredDirectory, name))); + } + } + if (input.environment.OPENCODE_CONFIG_CONTENT) { const content = readRoutingConfig( input.environment.OPENCODE_CONFIG_CONTENT, resolveFile(input.directory), diff --git a/packages/provider-opencode/src/server/textGeneration.test.ts b/packages/provider-opencode/src/server/textGeneration.test.ts index fd51b652befc6..c5f17d44523bc 100644 --- a/packages/provider-opencode/src/server/textGeneration.test.ts +++ b/packages/provider-opencode/src/server/textGeneration.test.ts @@ -9,6 +9,7 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; +import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; import * as TestClock from "effect/testing/TestClock"; @@ -282,20 +283,21 @@ function withOpenCodeTextGeneration( settings: OpenCodeSettings, effectFn: (textGeneration: ProviderTextGenerationRuntime) => Effect.Effect, environment?: NodeJS.ProcessEnv, + directory = process.cwd(), ) { return Effect.gen(function* () { const serverOwner = yield* OpenCodeServerOwner.make({ binaryPath: settings.binaryPath, - directory: process.cwd(), + directory, ...(settingsSecretLiteral(settings.serverPassword) ? { serverPassword: settingsSecretLiteral(settings.serverPassword) } : {}), ...(environment ? { environment } : {}), }); - const textGeneration = yield* OpenCodeTextGeneration.makeOpenCodeTextGeneration( - settings, - environment ? { environment } : {}, - ).pipe(Effect.provideService(OpenCodeServerOwner.OpenCodeServerOwner, serverOwner)); + const textGeneration = yield* OpenCodeTextGeneration.makeOpenCodeTextGeneration(settings, { + directory, + ...(environment ? { environment } : {}), + }).pipe(Effect.provideService(OpenCodeServerOwner.OpenCodeServerOwner, serverOwner)); return yield* effectFn(textGeneration); }).pipe(Effect.scoped); } @@ -470,66 +472,134 @@ it.layer(layerOpenCodeTextGenerationTest)("OpenCodeTextGeneration", (it) => { ), ); - it.effect("isolates blind evaluation from shared servers, tools, and custom instructions", () => - withOpenCodeTextGeneration( - DEFAULT_OPENCODE_SETTINGS, - (textGeneration) => - Effect.gen(function* () { - runtimeMock.state.promptResult = { - data: { - info: { providerID: "openai", modelID: "gpt-5", variant: "high" }, - parts: [ - { type: "step-start" }, - { type: "reasoning", text: "Check the evidence." }, - { type: "text", text: JSON.stringify(VALID_EVALUATION) }, - { type: "step-finish", reason: "stop" }, - { type: "snapshot", snapshot: "unchanged" }, - ], - }, - }; + it.effect.each([ + "configured overrides", + "empty file override", + "empty directory override", + "empty inline override", + ])( + "isolates blind evaluation from shared servers, tools, and custom instructions (%s)", + (overrides) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-opencode-routing-" }); + const configHome = `${directory}/global`; + const configDirectory = `${directory}/override`; + const configFile = `${directory}/explicit.json`; + yield* fs.makeDirectory(`${configHome}/opencode`, { recursive: true }); + yield* fs.makeDirectory(configDirectory); + yield* fs.writeFileString( + `${configHome}/opencode/config.json`, + JSON.stringify({ + provider: { openai: { options: { apiKey: "global-key" } } }, + enabled_providers: ["openai"], + disabled_providers: ["global-disabled"], + }), + ); + yield* fs.writeFileString( + `${directory}/opencode.json`, + JSON.stringify({ + provider: { openai: { options: { baseURL: "http://ignored-cwd.internal" } } }, + }), + ); + yield* fs.writeFileString( + configFile, + JSON.stringify({ + provider: { openai: { options: { baseURL: "http://explicit.internal" } } }, + }), + ); + yield* fs.writeFileString( + `${configDirectory}/config.json`, + JSON.stringify({ disabled_providers: ["ignored-directory-file"] }), + ); + yield* fs.writeFileString( + `${configDirectory}/opencode.json`, + JSON.stringify({ + provider: { openai: { options: { baseURL: "http://directory-json.internal" } } }, + }), + ); + yield* fs.writeFileString( + `${configDirectory}/opencode.jsonc`, + JSON.stringify({ + provider: { openai: { options: { baseURL: "http://directory-jsonc.internal" } } }, + }), + ); + return yield* withOpenCodeTextGeneration( + DEFAULT_OPENCODE_SETTINGS, + (textGeneration) => + Effect.gen(function* () { + runtimeMock.state.promptResult = { + data: { + info: { providerID: "openai", modelID: "gpt-5", variant: "high" }, + parts: [ + { type: "step-start" }, + { type: "reasoning", text: "Check the evidence." }, + { type: "text", text: JSON.stringify(VALID_EVALUATION) }, + { type: "step-finish", reason: "stop" }, + { type: "snapshot", snapshot: "unchanged" }, + ], + }, + }; - const verdict = yield* textGeneration.generateModelEvaluation({ - ...DEFAULT_EVALUATION_INPUT, - modelSelection: { - ...DEFAULT_TEST_MODEL_SELECTION, - options: [{ id: "variant", value: "high" }], - }, - }); + const verdict = yield* textGeneration.generateModelEvaluation({ + ...DEFAULT_EVALUATION_INPUT, + modelSelection: { + ...DEFAULT_TEST_MODEL_SELECTION, + options: [{ id: "variant", value: "high" }], + }, + }); - expect(verdict).toEqual(VALID_EVALUATION); - expect(textGeneration.supportedOperations).toContain("generateModelEvaluation"); - expect(runtimeMock.state.promptUrls).toEqual(["http://127.0.0.1:4301"]); - expect(runtimeMock.state.promptInputs[0]).toMatchObject({ - model: { providerID: "openai", modelID: "gpt-5" }, - variant: "high", - }); - expect(runtimeMock.state.sessionCreateInputs[0]?.permission).toEqual([ - { permission: "*", pattern: "*", action: "deny" }, - ]); - const environment = runtimeMock.state.startEnvironments[0]; - expect(environment?.XDG_CONFIG_HOME).not.toBe("/user-config"); - expect(environment?.OPENCODE_CONFIG).toBeUndefined(); - expect(environment?.OPENCODE_DISABLE_PROJECT_CONFIG).toBe("1"); - expect(environment?.OPENCODE_DISABLE_CLAUDE_CODE).toBe("1"); - expect(JSON.parse(environment?.OPENCODE_CONFIG_CONTENT ?? "null")).toEqual({ - provider: { custom: { options: { baseURL: "http://model.internal" } } }, - enabled_providers: ["custom"], - }); - expect(runtimeMock.state.closeCalls).toEqual(["http://127.0.0.1:4301"]); - }), - { - XDG_CONFIG_HOME: "/user-config", - OPENCODE_CONFIG: "/user-config/opencode.json", - OPENCODE_CONFIG_CONTENT: `{ + expect(verdict).toEqual(VALID_EVALUATION); + expect(textGeneration.supportedOperations).toContain("generateModelEvaluation"); + expect(runtimeMock.state.promptUrls).toEqual(["http://127.0.0.1:4301"]); + expect(runtimeMock.state.promptInputs[0]).toMatchObject({ + model: { providerID: "openai", modelID: "gpt-5" }, + variant: "high", + }); + expect(runtimeMock.state.sessionCreateInputs[0]?.permission).toEqual([ + { permission: "*", pattern: "*", action: "deny" }, + ]); + const environment = runtimeMock.state.startEnvironments[0]; + expect(environment?.XDG_CONFIG_HOME).not.toBe(configHome); + expect(environment?.OPENCODE_CONFIG).toBeUndefined(); + expect(environment?.OPENCODE_DISABLE_PROJECT_CONFIG).toBe("1"); + expect(environment?.OPENCODE_DISABLE_CLAUDE_CODE).toBe("1"); + expect(JSON.parse(environment?.OPENCODE_CONFIG_CONTENT ?? "null")).toEqual({ + provider: { + openai: { + options: { + baseURL: + overrides === "empty directory override" + ? "http://explicit.internal" + : "http://directory-jsonc.internal", + apiKey: overrides === "empty inline override" ? "global-key" : "inline-key", + }, + }, + }, + enabled_providers: ["openai"], + disabled_providers: ["global-disabled"], + }); + expect(runtimeMock.state.closeCalls).toEqual(["http://127.0.0.1:4301"]); + }), + { + XDG_CONFIG_HOME: configHome, + OPENCODE_CONFIG: overrides === "empty file override" ? "" : configFile, + OPENCODE_CONFIG_DIR: overrides === "empty directory override" ? "" : configDirectory, + OPENCODE_CONFIG_CONTENT: + overrides === "empty inline override" + ? "" + : `{ // Routing survives, but customization does not. - "provider": { "custom": { "options": { "baseURL": "http://model.internal" } } }, - "enabled_providers": ["custom"], + "provider": { "openai": { "options": { "apiKey": "inline-key" } } }, + "enabled_providers": ["openai"], "instructions": ["/secret/instructions.md"], "plugin": ["secret-plugin"], "mcp": { "private": { "type": "remote", "url": "http://private" } } }`, - }, - ), + }, + directory, + ); + }).pipe(Effect.scoped), ); it.effect("fails blind evaluation instead of dropping an unavailable or invalid variant", () => diff --git a/packages/provider-opencode/src/server/textGeneration.ts b/packages/provider-opencode/src/server/textGeneration.ts index 74e45d8fac2fc..efc0e5483e75d 100644 --- a/packages/provider-opencode/src/server/textGeneration.ts +++ b/packages/provider-opencode/src/server/textGeneration.ts @@ -404,6 +404,7 @@ export const makeOpenCodeTextGeneration = Effect.fn("makeOpenCodeTextGeneration" const isolatedOutput = Effect.gen(function* () { const environment = yield* BlindEvaluation.makeBlindEvaluationEnvironment({ + generation: "v1", operation: input.operation, directory: processDirectory, environment: processEnvironment, diff --git a/packages/provider-opencode/src/server/v2/textGeneration.test.ts b/packages/provider-opencode/src/server/v2/textGeneration.test.ts index 6699af516b31d..316049dfaa946 100644 --- a/packages/provider-opencode/src/server/v2/textGeneration.test.ts +++ b/packages/provider-opencode/src/server/v2/textGeneration.test.ts @@ -145,11 +145,16 @@ const validEvaluation = { promptAdvice: [], }; -const make = (configPath = "/user-config/opencode.json", serverUrl = "") => +const make = ( + configPath = "/user-config/opencode.json", + serverUrl = "", + environment: NodeJS.ProcessEnv = {}, + directory = process.cwd(), +) => OpenCode2TextGeneration.make({ binaryPath: "fake-opencode", serverUrl, - directory: process.cwd(), + directory, environment: { HOME: "/original-home", XDG_CONFIG_HOME: "/user-config", @@ -160,6 +165,7 @@ const make = (configPath = "/user-config/opencode.json", serverUrl = "") => plugins: ["secret-plugin"], mcp: { private: { type: "remote", url: "http://private" } }, }), + ...environment, }, }).pipe(Effect.provideService(OpenCode2Server.OpenCode2Server, sharedServer)); @@ -196,16 +202,67 @@ beforeEach(() => { }); it.layer(layer)("OpenCode2TextGeneration blind evaluation", (it) => { - it.effect("uses an owned deny-all server and the exact requested model variant", () => + it.effect.each([ + "configured overrides", + "empty file override", + "empty directory override", + "empty inline override", + ])("uses an owned deny-all server and the exact requested model variant (%s)", (overrides) => Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; const directory = yield* fs.makeTempDirectoryScoped({ prefix: "t3-opencode-routing-" }); - const configPath = `${directory}/opencode.json`; + const configHome = `${directory}/global`; + const configDirectory = `${directory}/override`; + const configPath = `${directory}/explicit.json`; + yield* fs.makeDirectory(`${configHome}/opencode`, { recursive: true }); + yield* fs.makeDirectory(configDirectory); + yield* fs.writeFileString( + `${configHome}/opencode/opencode.json`, + JSON.stringify({ providers: { ambient: { settings: { apiKey: "ignored-global-key" } } } }), + ); + yield* fs.writeFileString( + `${configDirectory}/config.json`, + JSON.stringify({ + providers: { ignored: { settings: { apiKey: "ignored-directory-key" } } }, + }), + ); + yield* fs.writeFileString( + `${directory}/opencode.json`, + JSON.stringify({ providers: { cwd: { settings: { apiKey: "ignored-cwd-key" } } } }), + ); + yield* fs.writeFileString( + `${configDirectory}/opencode.json`, + JSON.stringify({ + providers: { + custom: { + settings: { + baseURL: "http://directory.internal", + headers: { "x-config-source": "directory" }, + }, + }, + }, + }), + ); yield* fs.writeFileString( configPath, - JSON.stringify({ providers: { custom: { settings: { apiKey: "{file:./key}" } } } }), + JSON.stringify({ + providers: { + custom: { + settings: { apiKey: "{file:./key}", baseURL: "http://explicit.internal" }, + }, + }, + }), + ); + const textGeneration = yield* make( + overrides === "empty file override" ? "" : configPath, + "", + { + XDG_CONFIG_HOME: configHome, + OPENCODE_CONFIG_DIR: overrides === "empty directory override" ? "" : configDirectory, + ...(overrides === "empty inline override" ? { OPENCODE_CONFIG_CONTENT: "" } : {}), + }, + directory, ); - const textGeneration = yield* make(configPath); const verdict = yield* textGeneration.generateModelEvaluation(evaluationInput); expect(verdict).toEqual(validEvaluation); @@ -218,13 +275,27 @@ it.layer(layer)("OpenCode2TextGeneration blind evaluation", (it) => { const environment = state.startEnvironments[0]; expect(environment?.HOME).not.toBe("/original-home"); expect(environment?.XDG_DATA_HOME).toBe("/original-home/.local/share"); - expect(environment?.XDG_CONFIG_HOME).not.toBe("/user-config"); + expect(environment?.XDG_CONFIG_HOME).not.toBe(configHome); expect(environment?.OPENCODE_CONFIG).toBeUndefined(); expect(JSON.parse(environment?.OPENCODE_CONFIG_CONTENT ?? "null")).toEqual({ providers: { + ...(overrides === "empty directory override" + ? { ambient: { settings: { apiKey: "ignored-global-key" } } } + : {}), custom: { - options: { baseURL: "http://model.internal" }, - settings: { apiKey: `{file:${directory}/key}` }, + ...(overrides === "empty inline override" + ? {} + : { options: { baseURL: "http://model.internal" } }), + settings: { + ...(overrides === "empty file override" ? {} : { apiKey: `{file:${directory}/key}` }), + baseURL: + overrides === "empty file override" + ? "http://directory.internal" + : "http://explicit.internal", + ...(overrides === "empty directory override" + ? {} + : { headers: { "x-config-source": "directory" } }), + }, }, }, }); diff --git a/packages/provider-opencode/src/server/v2/textGeneration.ts b/packages/provider-opencode/src/server/v2/textGeneration.ts index 9e00e67c86c51..d38c7a1f3ce17 100644 --- a/packages/provider-opencode/src/server/v2/textGeneration.ts +++ b/packages/provider-opencode/src/server/v2/textGeneration.ts @@ -261,6 +261,7 @@ export const make = Effect.fn("OpenCode2TextGeneration.make")(function* ( S extends Schema.Top, >(input: TextGenerationOperations.Request) { const environment = yield* BlindEvaluation.makeBlindEvaluationEnvironment({ + generation: "v2", operation: input.operation, directory: processDirectory, environment: processEnvironment, From 9565de63db039897e1da9d3b231df203497c6c75 Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Fri, 13 Nov 2026 15:21:45 -0800 Subject: [PATCH 11/11] test(web): drop the palette test workaround that main's leak fix replaces Co-Authored-By: Claude Opus 5.5 --- .../CommandPalette.browser.test.tsx | 33 ++++--------------- 1 file changed, 6 insertions(+), 27 deletions(-) diff --git a/apps/web/src/components/CommandPalette.browser.test.tsx b/apps/web/src/components/CommandPalette.browser.test.tsx index 53017a4da524d..adfd7f506a833 100644 --- a/apps/web/src/components/CommandPalette.browser.test.tsx +++ b/apps/web/src/components/CommandPalette.browser.test.tsx @@ -1,16 +1,12 @@ import { EnvironmentId, ProjectId, ThreadId } from "@t3tools/contracts"; -import { scopedProjectKey } from "@t3tools/client-runtime/environment"; import { describe, expect, it, vi } from "vite-plus/test"; import { page, userEvent } from "vite-plus/test/browser"; import { act, render } from "../../test/render"; import { openCommandPalette } from "../commandPaletteBus"; import { useCommandPaletteStore } from "../commandPaletteStore"; -import { - LAST_USED_PROJECT_STORAGE_KEY, - lastUsedProjectKeySchema, -} from "../hooks/useHandleNewThread"; -import { removeLocalStorageItem, setLocalStorageItem } from "../hooks/useLocalStorage"; +import { LAST_USED_PROJECT_STORAGE_KEY } from "../hooks/useHandleNewThread"; +import { removeLocalStorageItem } from "../hooks/useLocalStorage"; import { makeThreadFixture } from "../test-fixtures"; import type { Project, Thread } from "../types"; import { CommandPalette } from "./CommandPalette"; @@ -107,33 +103,16 @@ describe("CommandPalette open intents", () => { updatedAt: "2026-03-01T00:00:00.000Z", scripts: [], }); - const acme = project("acme", "Acme"); - setup({ projects: [acme, project("beta", "Beta"), project("gamma", "Gamma")] }); - // Keep the starting row independent of remembered project preferences. - const previousLastUsedProject = window.localStorage.getItem(LAST_USED_PROJECT_STORAGE_KEY); - using _rememberedProject = { - [Symbol.dispose]() { - if (previousLastUsedProject === null) { - removeLocalStorageItem(LAST_USED_PROJECT_STORAGE_KEY); - } else { - window.localStorage.setItem(LAST_USED_PROJECT_STORAGE_KEY, previousLastUsedProject); - } - }, - }; - setLocalStorageItem( - LAST_USED_PROJECT_STORAGE_KEY, - scopedProjectKey({ environmentId, projectId: acme.id }), - lastUsedProjectKeySchema, - ); + setup({ + projects: [project("acme", "Acme"), project("beta", "Beta"), project("gamma", "Gamma")], + }); await using _screen = await render({null}); openCommandPalette({}); await expect .element(page.getByPlaceholder("Search commands, projects, and threads...")) .toHaveFocus(); - await expect - .element(page.getByRole("option", { name: /^New thread in Acme\b/ })) - .toHaveAttribute("data-highlighted"); + await expect.element(page.getByRole("option").first()).toHaveAttribute("data-highlighted"); await userEvent.keyboard("{ArrowDown}"); await expect .element(page.getByRole("option", { name: /^New thread in\.\.\./ }))