fix(evaluations): support connected APIs and isolated provider judges

apps/mobile/src/features/settings/ModelEvaluationSettings.tsx

@@ -1,11 +1,14 @@
-import { useAtomValue } from "@effect/atom-react";
+import { useAtomRefresh, useAtomValue } from "@effect/atom-react";
 import type { MenuAction } from "@react-native-menu/menu";
 import {
   isManagedSettingPath,
+  textGeneratorForSelection,
   type EnvironmentId,
   type ModelEvaluationGuidance,
   type ProjectId,
+  type ProviderInstanceId,
   type ServerSettingsPatch,
+  type TextGeneratorConfig,
 } from "@t3tools/contracts";
 import {
   modelEvaluationGuidanceKey,
@@ -14,6 +17,8 @@ import {
   isModelEvaluationJudge,
 } from "@t3tools/client-runtime/modelEvaluations";
 import { squashAtomCommandFailure } from "@t3tools/client-runtime/state/runtime";
+import * as Hash from "effect/Hash";
+import { AsyncResult } from "effect/reactivity";
 import { useState } from "react";
 import { View } from "react-native";
 
@@ -22,14 +27,20 @@ import { Button } from "../../components/ui/Button";
 import * as Dialog from "../../components/ui/Dialog";
 import { Menu } from "../../components/ui/Menu";
 import { RowPressable } from "../../components/ui/RowPressable";
-import { buildModelMenuActions, buildModelOptions, groupByProvider } from "../../lib/modelOptions";
+import {
+  buildModelMenuActions,
+  buildModelOptions,
+  groupByProvider,
+  type ModelOption,
+} from "../../lib/modelOptions";
 import { useEnvironmentServerConfig, useProjects } from "../../state/entities";
 import { modelEvaluations } from "../../state/modelEvaluations";
 import { useEnvironmentQuery } from "../../state/query";
 import { serverEnvironment } from "../../state/server";
 import { useAtomCommand } from "../../state/use-atom-command";
 import { SettingsSection } from "./components/SettingsSection";
 import { SettingsSwitchRow } from "./components/SettingsSwitchRow";
+import { apiJudgeModelsAtom } from "./modelEvaluationApiModels";
 import { useSettingsEnvironmentFilter } from "./settings-environment-filter";
 
 export function ModelEvaluationSettings() {
@@ -79,6 +90,25 @@ function EnvironmentEvaluationSettings(props: {
   const projects = useProjects().filter((project) => project.environmentId === environmentId);
   const [chosenProjectId, setChosenProjectId] = useState<ProjectId | null>(null);
   const [saving, setSaving] = useState(false);
+  const canListModels = useAtomValue(
+    serverEnvironment.listTextGeneratorModels.permissionAtom(environmentId),
+  );
+  const apiJudges = (
+    canListModels && config !== null
+      ? (Object.entries(config.settings.textGenerators) as Array<
+          [ProviderInstanceId, TextGeneratorConfig]
+        >)
+      : []
+  ).filter(([, generator]) => generator.enabled);
+  const apiModelsAtom = apiJudgeModelsAtom(
+    JSON.stringify(
+      apiJudges.map(
+        ([id, generator]) => [environmentId, id, Hash.string(JSON.stringify(generator))] as const,
+      ),
+    ),
+  );
+  const apiModelLists = useAtomValue(apiModelsAtom);
+  const refreshApiModels = useAtomRefresh(apiModelsAtom);
   if (config === null) return null;
   const readOnly = saving || !canWrite || props.projectSelected;
   const project =
@@ -90,22 +120,49 @@ function EnvironmentEvaluationSettings(props: {
   const capable = new Set(
     config.providers.filter(isModelEvaluationJudge).map((provider) => provider.instanceId),
   );
-  const models = buildModelOptions(config, selection, undefined, null).filter(
-    (model) => capable.has(model.selection.instanceId) && !model.isUnavailable,
+  const listedApiModels = apiModelLists.map((result) => AsyncResult.getOrElse(result, () => []));
+  const apiModelsLoading = apiModelLists.some((result) => result.waiting);
+  const failedApis = apiJudges.flatMap(([, generator], index) =>
+    apiModelLists[index]?._tag === "Failure" ? [generator.label || generator.baseUrl] : [],
   );
+  const models = [
+    ...buildModelOptions(config, selection, undefined, null).filter(
+      (model) =>
+        capable.has(model.selection.instanceId) &&
+        !model.isUnavailable &&
+        textGeneratorForSelection(settings, model.selection.instanceId) === undefined,
+    ),
+    ...apiJudges.flatMap(([instanceId, generator], index) =>
+      (listedApiModels[index] ?? []).map((model): ModelOption => ({
+        key: `${instanceId}:${model.id}`,
+        label: model.name,
+        subtitle: "",
+        providerKey: instanceId,
+        providerLabel: generator.label || generator.baseUrl,
+        providerDriver: generator.kind,
+        isDefault: false,
+        isLegacy: false,
+        capabilities: null,
+        selection: { instanceId, model: model.id },
+      })),
+    ),
+  ];
   const selectedProvider = config.providers.find(
     (provider) => provider.instanceId === selection.instanceId,
   );
+  const selectedApi = textGeneratorForSelection(settings, selection.instanceId);
   const judgeAvailable =
-    selectedProvider?.installed === true &&
-    selectedProvider.status !== "error" &&
-    selectedProvider.auth.status !== "unauthenticated" &&
-    selectedProvider.availability !== "unavailable" &&
-    models.some(
-      (model) =>
-        model.selection.instanceId === selection.instanceId &&
-        model.selection.model === selection.model,
-    );
+    selectedApi !== undefined
+      ? selectedApi.enabled
+      : selectedProvider?.installed === true &&
+        selectedProvider.status !== "error" &&
+        selectedProvider.auth.status !== "unauthenticated" &&
+        selectedProvider.availability !== "unavailable" &&
+        models.some(
+          (model) =>
+            model.selection.instanceId === selection.instanceId &&
+            model.selection.model === selection.model,
+        );
   const write = async (patch: ServerSettingsPatch) => {
     if (readOnly) return;
     setSaving(true);
@@ -156,6 +213,21 @@ function EnvironmentEvaluationSettings(props: {
           if (model) void write({ modelEvaluationModelSelection: model.selection });
         }}
       />
+      {apiJudges.length > 0 ? (
+        <View className="gap-2 px-4 py-2">
+          {failedApis.length > 0 ? (
+            <Text className="text-sm text-foreground-muted">
+              Could not load models from {failedApis.join(", ")}.
+            </Text>
+          ) : null}
+          <Button
+            label={apiModelsLoading ? "Loading API models…" : "Refresh API models"}
+            tone="text"
+            disabled={apiModelsLoading}
+            onPress={refreshApiModels}
+          />
+        </View>
+      ) : null}
       {!judgeAvailable ? (
         <Text className="px-4 py-2 text-sm text-foreground-muted">
           Choose an available judge before enabling.

apps/mobile/src/features/settings/modelEvaluationApiModels.test.ts

@@ -0,0 +1,87 @@
+import { it } from "@effect/vitest";
+import {
+  EnvironmentId,
+  ProviderInstanceId,
+  type ListTextGeneratorModelsInput,
+  type ListTextGeneratorModelsResult,
+} from "@t3tools/contracts";
+import type { AtomCommandResult } from "@t3tools/client-runtime/state/runtime";
+import * as Cause from "effect/Cause";
+import * as Effect from "effect/Effect";
+import { AsyncResult, Atom, AtomRegistry } from "effect/reactivity";
+import { expect, vi } from "vite-plus/test";
+
+type ModelListResult = AtomCommandResult<ListTextGeneratorModelsResult, string>;
+type Request = { environmentId: EnvironmentId; input: ListTextGeneratorModelsInput };
+
+const state = vi.hoisted(() => ({
+  listModels: (_request: Request): Promise<ModelListResult> => {
+    throw new Error("model discovery not set up");
+  },
+}));
+
+vi.mock("../../state/server", () => ({
+  serverEnvironment: {
+    listTextGeneratorModels: {
+      label: "list models",
+      run: (_registry: unknown, request: Request) => state.listModels(request),
+    },
+  },
+}));
+
+import { apiJudgeModelsAtom } from "./modelEvaluationApiModels";
+
+function setup(ids: readonly string[], listModels: typeof state.listModels) {
+  state.listModels = listModels;
+  const registry = AtomRegistry.make();
+  const catalog = apiJudgeModelsAtom(
+    JSON.stringify(
+      ids.map((id) => [EnvironmentId.make("judge-env"), ProviderInstanceId.make(id), 0]),
+    ),
+  );
+  registry.mount(catalog);
+  const result = (index: number) => Atom.map(catalog, (results) => results[index]!);
+  return {
+    registry,
+    catalog,
+    read: (index: number) =>
+      AtomRegistry.getResult(registry, result(index), { suspendOnWaiting: true }),
+    [Symbol.dispose]: () => registry.dispose(),
+  };
+}
+
+it.effect("keeps discovery failures visible and can reload unchanged settings after recovery", () =>
+  Effect.gen(function* () {
+    using h = setup(["offline-api"], async () => AsyncResult.failure(Cause.fail("offline")));
+
+    const failed = yield* Effect.exit(h.read(0));
+    expect(failed._tag).toBe("Failure");
+    expect(h.registry.get(h.catalog)[0]?._tag).toBe("Failure");
+
+    state.listModels = async ({ input }) =>
+      AsyncResult.success({
+        models: input.refresh ? [{ id: "recovered-model", name: "Recovered model" }] : [],
+      });
+    h.registry.refresh(h.catalog);
+
+    expect(yield* h.read(0)).toEqual([{ id: "recovered-model", name: "Recovered model" }]);
+  }),
+);
+
+it.effect("shows each API's models while another discovery request is still pending", () =>
+  Effect.gen(function* () {
+    const slow = Promise.withResolvers<ModelListResult>();
+    using h = setup(["slow-api", "ready-api"], async ({ input }) =>
+      input.id === "slow-api"
+        ? slow.promise
+        : AsyncResult.success({ models: [{ id: "ready-model", name: "Ready model" }] }),
+    );
+
+    expect(yield* h.read(1)).toEqual([{ id: "ready-model", name: "Ready model" }]);
+    expect(h.registry.get(h.catalog)[0]?.waiting).toBe(true);
+
+    slow.resolve(AsyncResult.success({ models: [{ id: "slow-model", name: "Slow model" }] }));
+    expect(yield* h.read(0)).toEqual([{ id: "slow-model", name: "Slow model" }]);
+    expect(h.registry.get(h.catalog)[1]?._tag).toBe("Success");
+  }),
+);

apps/mobile/src/features/settings/modelEvaluationApiModels.ts

@@ -0,0 +1,40 @@
+import type { EnvironmentId, ProviderInstanceId } from "@t3tools/contracts";
+import { runAtomCommand } from "@t3tools/client-runtime/state/runtime";
+import * as Effect from "effect/Effect";
+import * as Option from "effect/Option";
+import { Atom } from "effect/reactivity";
+
+import { serverEnvironment } from "../../state/server";
+
+type ApiJudgeKey = readonly [EnvironmentId, ProviderInstanceId, number];
+
+const apiJudgeModelListAtom = Atom.family((key: string) => {
+  const [environmentId, id] = JSON.parse(key) as ApiJudgeKey;
+  return Atom.make((get) => {
+    const refresh = Option.isSome(get.self());
+    return Effect.promise(() =>
+      runAtomCommand(
+        get.registry,
+        serverEnvironment.listTextGeneratorModels,
+        { environmentId, input: { id, ...(refresh ? { refresh: true } : {}) } },
+        { reportFailure: false },
+      ),
+    ).pipe(
+      Effect.flatMap((result) =>
+        result._tag === "Success"
+          ? Effect.succeed(result.value.models)
+          : Effect.failCause(result.cause),
+      ),
+    );
+  }).pipe(Atom.setIdleTTL(5 * 60_000));
+});
+
+export const apiJudgeModelsAtom = Atom.family((key: string) => {
+  const requests = (JSON.parse(key) as ReadonlyArray<ApiJudgeKey>).map((api) =>
+    apiJudgeModelListAtom(JSON.stringify(api)),
+  );
+  return Atom.readable(
+    (get) => requests.map((request) => get(request)),
+    (refresh) => requests.forEach(refresh),
+  ).pipe(Atom.withLabel("model-evaluation-api-judges"));
+});