diff --git a/.fallowrc.json b/.fallowrc.json index ec6d9858..0a169870 100644 --- a/.fallowrc.json +++ b/.fallowrc.json @@ -23,6 +23,17 @@ "examples/agent-dir/tools/*.ts", "registry/**/*.ts" ], + "ignoreUnresolvedImports": [ + "../dist/index.js" + ], + "ignoreExports": [ + { + "file": "packages/create-lousho-agent/src/index.ts", + "exports": [ + "main" + ] + } + ], "ignoreDependencies": [ "@ai-sdk/anthropic", "@ai-sdk/openai", diff --git a/CHANGELOG.md b/CHANGELOG.md index c571344c..813b14ba 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,12 @@ opt-out. - **Behavior change:** `POST /chat` to a session paused on an approval is a `409` JSON body (`LOUSHO_SESSION_AWAITING_APPROVAL`, with `approvalId` and `approvalIds`) instead of a `200` stream ending in a generic error. - A decision on an approval whose winning decision is still streaming is a `409` (`LOUSHO_APPROVAL_CONFLICT`), as docs/auth.md says, instead of a `404`. - The session routes' `400` bodies carry a `code`: `LOUSHO_REQUEST_INVALID` (invalid JSON, missing or empty fields), `LOUSHO_APPROVAL_DECISION_INVALID` (an approval body with neither `approved` nor `answer`), `LOUSHO_SESSION_ID_INVALID`. +- `lousho chat` prints a failed turn with its error code, the provider's failure category and HTTP status, and the fix (`error: User not found. (auth-failure, HTTP 401)` then `[LOUSHO_PROVIDER_REQUEST_FAILED] ...`), as docs/cli.md says, instead of the bare provider message. The terminal chat of a project scaffolded by `lousho init` / `create-lousho-agent` does the same. +- A `.env` saved with a UTF-8 byte order mark (PowerShell 5.1's `Set-Content -Encoding utf8`) no longer loses its first variable: the `lousho` commands and the scaffolded `src/index.ts` drop the mark before parsing. Variables already set in the environment still win. +- `lousho doctor src/agent.ts` loads a `.ts`/`.js` agent module the way `lousho dev` does instead of failing with `LOUSHO_SPEC_UNSUPPORTED_FORMAT`, and a missing key for the module's provider is a failure. A scaffolded project's `npm run doctor` now runs `lousho doctor src/agent.ts`, so it fails when the agent's key is missing. +- `lousho doctor --ping` checks OpenRouter at `/api/v1/key`, which needs the key (its `/models` answers without one, so a wrong key passed); a key the provider rejects is now a failure, and a ping of a custom `OPENAI_BASE_URL` says it shows reachability only. +- `lousho doctor` fix lines use PowerShell syntax (`$env:NAME=""`) on Windows. +- The `lousho eval` table shows the scores of passing scored gates (such as `t.check('words', n, atLeast(3))`), not only in `--json`. ### Documentation diff --git a/bin/lousho.js b/bin/lousho.js index 09a6bc24..3a1f125e 100755 --- a/bin/lousho.js +++ b/bin/lousho.js @@ -1,7 +1,9 @@ #!/usr/bin/env node 'use strict'; +const fs = require('node:fs'); const path = require('node:path'); +const { parseEnv } = require('node:util'); const { pathToFileURL } = require('node:url'); const USAGE = [ @@ -24,8 +26,8 @@ const USAGE = [ ' Open Agent Forge, the visual agent builder.', ' lousho mcp [--http --port N --host H]', ' Serve an agent as an MCP server (stdio by default).', - ' lousho doctor [agent.yaml|json|agent-dir] [--json] [--ping]', - ' Check Node, peer dependencies, API keys and (optionally) an agent spec.', + ' lousho doctor [agent.yaml|json|agent-dir|agent.ts] [--json] [--ping]', + ' Check Node, peer dependencies, API keys and (optionally) an agent spec, directory or module.', ' lousho eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] [--record | --replay | --drift [--drift-usage]] [--url [--token ]] [--config vitest.config.ts] [--timeout ]', ' Run *.eval.ts files under your vitest and report the results.', ' lousho traces [traceId|prefix] [--dir D] [--limit N] [--json] [--content]', @@ -73,10 +75,21 @@ const COMMANDS = new Map([ ['traces', { module: 'traces', run: 'runTraces' }], ]); -/** Loads ./.env into process.env (variables already set are kept), like `node --env-file-if-exists=.env`. */ +const BYTE_ORDER_MARK = 0xfeff; + +/** + * Loads ./.env into process.env (variables already set are kept), like `node --env-file-if-exists=.env`. + * Read by hand, not with process.loadEnvFile(): that keeps a leading byte order mark (PowerShell 5.1's + * `Set-Content -Encoding utf8` writes one), so the first key would be named with a U+FEFF prefix and + * silently missing. + */ function loadDotEnv() { try { - process.loadEnvFile(path.resolve('.env')); + const text = fs.readFileSync(path.resolve('.env'), 'utf8'); + const vars = parseEnv(text.charCodeAt(0) === BYTE_ORDER_MARK ? text.slice(1) : text); + for (const [name, value] of Object.entries(vars)) { + if (process.env[name] === undefined) process.env[name] = value; + } } catch (error) { if (error.code !== 'ENOENT') console.error(`lousho: could not load .env: ${error.message}`); } diff --git a/docs/cli.md b/docs/cli.md index 0eb458d1..ae94b580 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -8,7 +8,7 @@ Run it with `npx lousho ` inside a project that has the SDK installed. | Command | What it does | Details | | ------- | ------------ | ------- | | `lousho init [dir]` | Scaffold a project: an agent with an example tool, an offline test, `.env.example`; installs dependencies and runs `git init`. | [Installation](./installation.md#scaffolding-a-new-project) | -| `lousho doctor [spec\|agent-dir] [--json] [--ping]` | Check Node, peer packages, provider keys, and optionally a spec file or agent directory; `--ping` also checks each configured provider endpoint answers. Prints a fix for every problem. | [Installation](./installation.md#troubleshooting-lousho-doctor) | +| `lousho doctor [spec\|agent-dir\|agent.ts] [--json] [--ping]` | Check Node, peer packages, provider keys, and optionally a spec file, agent directory or agent module (its provider's key is then required); `--ping` also checks each configured provider endpoint answers and accepts the key. Prints a fix for every problem. | [Installation](./installation.md#troubleshooting-lousho-doctor) | | `lousho dev ` | Local dev server for a spec file, an agent directory or a TS agent: chat UI with a session per tab and streamed events, hot reload on save. | [Below](#lousho-dev) | | `lousho chat ` | Terminal REPL for a spec file, an agent directory or a TS agent: streams replies, shows tool calls, asks for approvals and questions. | [Below](#lousho-chat) | | `lousho acp ` | Serve a spec file, an agent directory or a TS agent to an editor (Zed and other ACP clients) over the Agent Client Protocol on stdio. | [ACP](./acp.md) | @@ -33,7 +33,7 @@ lousho add [--registry ] [--dir ] [--yes] [--allo lousho build --target= [--out=] (or --agent=) lousho studio [--port N] [--host H] [--prod|--dev] lousho mcp [--http --port N --host H] -lousho doctor [agent.yaml|json|agent-dir] [--json] [--ping] +lousho doctor [agent.yaml|json|agent-dir|agent.ts] [--json] [--ping] lousho eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] [--record | --replay | --drift [--drift-usage]] [--url [--token ]] [--config vitest.config.ts] [--timeout ] lousho traces [traceId|prefix] [--dir D] [--limit N] [--json] [--content] lousho --version @@ -59,7 +59,8 @@ value, an extra argument, or a missing `--target` or agent path. `lousho build` **`.env`.** The commands that run an agent (`dev`, `chat`, `acp`, `mcp`), `doctor`, `eval` (judge evals need provider keys) and `build` load `./.env` from the working directory when it exists, like `node --env-file-if-exists=.env`: a variable already set in the environment -wins over the file. +wins over the file. A UTF-8 byte order mark at the start of the file (which +PowerShell 5.1's `Set-Content -Encoding utf8` writes) is ignored. `lousho init` is the exception: it parses its own flags, and a bad flag or value fails with `InitUsageError` — its message printed as-is, exit code 2 — @@ -216,7 +217,9 @@ Done, the report is on its way. - After each turn `[usage]` shows the tokens and, when every model used has known pricing, the cost (`~` marks estimated tokens). - Errors from the SDK print with their `[LOUSHO_...]` code and fix, and the - session continues. + session continues. A provider failure also names its category and HTTP + status, e.g. `error: User not found. (auth-failure, HTTP 401)` for a wrong + API key. - `--traces` or `--traces=` writes the session's runs to local trace files, as for `lousho dev`; a line on stderr says where. diff --git a/docs/installation.md b/docs/installation.md index b61e941d..04072848 100644 --- a/docs/installation.md +++ b/docs/installation.md @@ -204,6 +204,11 @@ npm install ./lousho-build-ai-agent-.tgz ai zod To scaffold a new project the same way, from the checkout: `node bin/lousho.js init ../my-agent --sdk-path .`. +On Windows, `npx C:\abs\path\create-lousho-agent-.tgz` silently does +nothing (an npm quirk with absolute drive-letter paths). Use a relative path +(`npx ./create-lousho-agent-.tgz my-agent`) or a `file:` spec +(`npx file:C:/abs/path/create-lousho-agent-.tgz my-agent`) instead. + `npm install github:LinuxDevil/agent-sdk` does **not** work: `dist/` is not in git and the repository has no `prepare` build step, so the install has no entry points. @@ -287,13 +292,24 @@ What it checks: 5. With a spec path (`lousho doctor agent.yaml`): the spec is validated with field paths for every error, its provider package and key are checked (missing ones become failures), its `tools` must be built-in tools, and any - `mcpServers` command must be resolvable. + `mcpServers` command must be resolvable. An agent directory or a `.ts`/`.js` + agent module (`lousho doctor src/agent.ts`, what a scaffolded project's + `npm run doctor` runs) is loaded the way `lousho dev` loads it, and its + model's provider package and key are checked the same way. Without a path + a missing key is only a warning, since doctor cannot know which provider + your agent uses. 6. Ollama reachability, only when the spec uses Ollama or `OLLAMA_HOST` is set. 7. Docker availability, a warning only when the spec uses a sandboxed tool. -8. With `--ping`: `GET /models` for each provider whose variable is set - (OpenAI at `OPENAI_BASE_URL` when set, Anthropic, OpenRouter), with a 5 s - timeout. An unreachable server or a rejected key is a warning. Keys are sent - to their own provider only and never printed. +8. With `--ping`: a `GET` for each provider whose variable is set, with a 5 s + timeout, to an endpoint that needs the key: `/models` for OpenAI and + Anthropic, `/api/v1/key` for OpenRouter (its `/models` answers without a + key). A rejected key (HTTP 401/403) is a failure; an unreachable server is a + warning. With `OPENAI_BASE_URL` set, `/models` is pinged and the line + says it shows reachability only, since a local server may not check keys. + Keys are sent to their own provider only and never printed. + +On Windows the fix lines use PowerShell syntax (`$env:OPENAI_API_KEY=""`) +instead of `export`. The exit code is `1` if any check fails and `0` otherwise (warnings do not fail), so it can gate CI. Add `--json` for machine-readable output. Colour is diff --git a/llms-full.txt b/llms-full.txt index 5335153e..40a0f329 100644 --- a/llms-full.txt +++ b/llms-full.txt @@ -563,6 +563,11 @@ npm install ./lousho-build-ai-agent-.tgz ai zod To scaffold a new project the same way, from the checkout: `node bin/lousho.js init ../my-agent --sdk-path .`. +On Windows, `npx C:\abs\path\create-lousho-agent-.tgz` silently does +nothing (an npm quirk with absolute drive-letter paths). Use a relative path +(`npx ./create-lousho-agent-.tgz my-agent`) or a `file:` spec +(`npx file:C:/abs/path/create-lousho-agent-.tgz my-agent`) instead. + `npm install github:LinuxDevil/agent-sdk` does **not** work: `dist/` is not in git and the repository has no `prepare` build step, so the install has no entry points. @@ -646,13 +651,24 @@ What it checks: 5. With a spec path (`lousho doctor agent.yaml`): the spec is validated with field paths for every error, its provider package and key are checked (missing ones become failures), its `tools` must be built-in tools, and any - `mcpServers` command must be resolvable. + `mcpServers` command must be resolvable. An agent directory or a `.ts`/`.js` + agent module (`lousho doctor src/agent.ts`, what a scaffolded project's + `npm run doctor` runs) is loaded the way `lousho dev` loads it, and its + model's provider package and key are checked the same way. Without a path + a missing key is only a warning, since doctor cannot know which provider + your agent uses. 6. Ollama reachability, only when the spec uses Ollama or `OLLAMA_HOST` is set. 7. Docker availability, a warning only when the spec uses a sandboxed tool. -8. With `--ping`: `GET /models` for each provider whose variable is set - (OpenAI at `OPENAI_BASE_URL` when set, Anthropic, OpenRouter), with a 5 s - timeout. An unreachable server or a rejected key is a warning. Keys are sent - to their own provider only and never printed. +8. With `--ping`: a `GET` for each provider whose variable is set, with a 5 s + timeout, to an endpoint that needs the key: `/models` for OpenAI and + Anthropic, `/api/v1/key` for OpenRouter (its `/models` answers without a + key). A rejected key (HTTP 401/403) is a failure; an unreachable server is a + warning. With `OPENAI_BASE_URL` set, `/models` is pinged and the line + says it shows reachability only, since a local server may not check keys. + Keys are sent to their own provider only and never printed. + +On Windows the fix lines use PowerShell syntax (`$env:OPENAI_API_KEY=""`) +instead of `export`. The exit code is `1` if any check fails and `0` otherwise (warnings do not fail), so it can gate CI. Add `--json` for machine-readable output. Colour is @@ -4747,7 +4763,7 @@ Run it with `npx lousho ` inside a project that has the SDK installed. | Command | What it does | Details | | ------- | ------------ | ------- | | `lousho init [dir]` | Scaffold a project: an agent with an example tool, an offline test, `.env.example`; installs dependencies and runs `git init`. | [Installation](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/installation.md#scaffolding-a-new-project) | -| `lousho doctor [spec\|agent-dir] [--json] [--ping]` | Check Node, peer packages, provider keys, and optionally a spec file or agent directory; `--ping` also checks each configured provider endpoint answers. Prints a fix for every problem. | [Installation](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/installation.md#troubleshooting-lousho-doctor) | +| `lousho doctor [spec\|agent-dir\|agent.ts] [--json] [--ping]` | Check Node, peer packages, provider keys, and optionally a spec file, agent directory or agent module (its provider's key is then required); `--ping` also checks each configured provider endpoint answers and accepts the key. Prints a fix for every problem. | [Installation](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/installation.md#troubleshooting-lousho-doctor) | | `lousho dev ` | Local dev server for a spec file, an agent directory or a TS agent: chat UI with a session per tab and streamed events, hot reload on save. | [Below](#lousho-dev) | | `lousho chat ` | Terminal REPL for a spec file, an agent directory or a TS agent: streams replies, shows tool calls, asks for approvals and questions. | [Below](#lousho-chat) | | `lousho acp ` | Serve a spec file, an agent directory or a TS agent to an editor (Zed and other ACP clients) over the Agent Client Protocol on stdio. | [ACP](https://github.com/LinuxDevil/agent-sdk/blob/main/docs/acp.md) | @@ -4772,7 +4788,7 @@ lousho add [--registry ] [--dir ] [--yes] [--allo lousho build --target= [--out=] (or --agent=) lousho studio [--port N] [--host H] [--prod|--dev] lousho mcp [--http --port N --host H] -lousho doctor [agent.yaml|json|agent-dir] [--json] [--ping] +lousho doctor [agent.yaml|json|agent-dir|agent.ts] [--json] [--ping] lousho eval [globs...] [--tag t] [--junit path] [--json path] [--strict] [--judge] [--record | --replay | --drift [--drift-usage]] [--url [--token ]] [--config vitest.config.ts] [--timeout ] lousho traces [traceId|prefix] [--dir D] [--limit N] [--json] [--content] lousho --version @@ -4798,7 +4814,8 @@ value, an extra argument, or a missing `--target` or agent path. `lousho build` **`.env`.** The commands that run an agent (`dev`, `chat`, `acp`, `mcp`), `doctor`, `eval` (judge evals need provider keys) and `build` load `./.env` from the working directory when it exists, like `node --env-file-if-exists=.env`: a variable already set in the environment -wins over the file. +wins over the file. A UTF-8 byte order mark at the start of the file (which +PowerShell 5.1's `Set-Content -Encoding utf8` writes) is ignored. `lousho init` is the exception: it parses its own flags, and a bad flag or value fails with `InitUsageError` — its message printed as-is, exit code 2 — @@ -4955,7 +4972,9 @@ Done, the report is on its way. - After each turn `[usage]` shows the tokens and, when every model used has known pricing, the cost (`~` marks estimated tokens). - Errors from the SDK print with their `[LOUSHO_...]` code and fix, and the - session continues. + session continues. A provider failure also names its category and HTTP + status, e.g. `error: User not found. (auth-failure, HTTP 401)` for a wrong + API key. - `--traces` or `--traces=` writes the session's runs to local trace files, as for `lousho dev`; a line on stderr says where. diff --git a/packages/create-lousho-agent/README.md b/packages/create-lousho-agent/README.md index 68b92cea..5d3e594f 100644 --- a/packages/create-lousho-agent/README.md +++ b/packages/create-lousho-agent/README.md @@ -32,6 +32,8 @@ With npm, put the options after `--`: npm create lousho-agent my-agent -- --yes --provider anthropic --template tools ``` +Running a packed `.tgz` of this package on Windows: `npx C:\abs\path\create-lousho-agent-.tgz` silently does nothing (an npm quirk with absolute drive-letter paths). Use a relative path (`npx ./create-lousho-agent-.tgz my-agent`) or a `file:` spec (`npx file:C:/abs/path/create-lousho-agent-.tgz my-agent`). + This package is a thin wrapper: it runs `lousho init` from the SDK, so `npx @lousho/build-ai-agent init --help` lists every option. ## Links diff --git a/src/cli/bin.test.ts b/src/cli/bin.test.ts index 920d4d99..906fe10b 100644 --- a/src/cli/bin.test.ts +++ b/src/cli/bin.test.ts @@ -21,7 +21,7 @@ describe('bin/lousho.js', () => { it('keeps the top-level usage in step with the commands', () => { const banner = lousho('--help').stdout; - expect(banner).toContain('lousho doctor [agent.yaml|json|agent-dir] [--json] [--ping]'); + expect(banner).toContain('lousho doctor [agent.yaml|json|agent-dir|agent.ts] [--json] [--ping]'); // One-line descriptions and the build targets (a usage line alone said neither). expect(banner).toContain('Chat with an agent in the terminal.'); expect(banner).toContain('--target='); @@ -85,4 +85,24 @@ describe('bin/lousho.js loads ./.env for the agent commands', () => { } } ); + + it.skipIf(!fs.existsSync(path.resolve(__dirname, '..', '..', 'dist', 'cli', 'doctor.mjs')))( + 'keeps the first key of a .env saved with a byte order mark, and a variable already set wins (needs npm run build)', + () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'lousho-dotenv-bom-')); + try { + // What PowerShell 5.1's `Set-Content -Encoding utf8` writes. + fs.writeFileSync(path.join(dir, '.env'), String.fromCharCode(0xfeff) + 'OPENAI_API_KEY=sk-from-dotenv-file\nANTHROPIC_API_KEY=from-file\n'); + const env: NodeJS.ProcessEnv = { ...process.env, ANTHROPIC_API_KEY: '' }; + delete env.OPENAI_API_KEY; + const res = spawnSync(process.execPath, [BIN, 'doctor', '--json'], { cwd: dir, env, encoding: 'utf8' }); + const checks = (JSON.parse(res.stdout) as { checks: { id: string; status: string }[] }).checks; + expect(checks.find((c) => c.id === 'env.openai')?.status).toBe('ok'); + // Set (even to empty) in the environment: the file does not override it. + expect(checks.find((c) => c.id === 'env.anthropic')?.status).toBe('warn'); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } + } + ); }); diff --git a/src/cli/chat.test.ts b/src/cli/chat.test.ts index f62597b8..50114557 100644 --- a/src/cli/chat.test.ts +++ b/src/cli/chat.test.ts @@ -1,4 +1,5 @@ import { describe, it, expect } from 'vitest'; +import { APICallError } from 'ai'; import { memoryStore } from '../storage/agentStore'; import { SignInRequired } from '../oauth/signIn'; import { fakeOAuthServer, githubProvider } from '../oauth/__fixtures__/fakeOAuth'; @@ -258,6 +259,15 @@ describe('lousho chat REPL', () => { expect(out).toContain('Again.'); }); + it('prints a provider failure with its code, category, HTTP status and fix', async () => { + const unauthorized = new APICallError({ message: 'User not found.', url: 'https://openrouter.ai/api/v1/chat/completions', requestBodyValues: {}, statusCode: 401 }); + const { err } = await converse(['hi', '/quit'], [{ error: unauthorized }]); + expect(err).toContain('error: User not found. (auth-failure, HTTP 401)'); + expect(err).toContain('[LOUSHO_PROVIDER_REQUEST_FAILED] Check the provider status, your API key'); + // Printed once, not once for the `error` event and again for the rejected turn. + expect(err.match(/User not found/g)).toHaveLength(1); + }); + it('streams the continuation after y, and asks again when it pauses a second time', async () => { const { out } = await converse(['ping twice', 'y', 'y', '/quit'], [{ toolCalls: [{ name: 'ping' }] }, { toolCalls: [{ name: 'ping' }] }, 'Both done.']); expect(out.match(/Approve ping\(\{\}\)\? \[y\/N\] /g)).toHaveLength(2); diff --git a/src/cli/chatRepl.ts b/src/cli/chatRepl.ts index d43c81a0..219fd54e 100644 --- a/src/cli/chatRepl.ts +++ b/src/cli/chatRepl.ts @@ -7,7 +7,7 @@ import * as readline from 'node:readline'; import type { Readable, Writable } from 'node:stream'; import type { SimpleAgent } from '../createAgent'; -import type { AgentEvent, AgentEventOf, AgentEventType, AgentEventUsage } from '../execution/agentEvents'; +import type { AgentEvent, AgentEventError, AgentEventOf, AgentEventType, AgentEventUsage } from '../execution/agentEvents'; import type { ApprovalQuestion, PendingApproval } from '../execution/ApprovalGate'; import { textOf } from '../providers/content'; import type { Message } from '../providers/llm'; @@ -16,7 +16,8 @@ import { assertSessionId } from '../session/sessionStore'; import { memoryStore, type AgentStore } from '../storage/agentStore'; import { newId } from '../utils/id'; import { hasOwnStore } from './devReload'; -import { SDKError } from '../execution/errors'; +import { SDKError, type CompactedProviderError } from '../execution/errors'; +import { errorHelp } from '../utils/errorCodes'; import { awaitSignInGate } from '../oauth/signInPending'; export interface ChatReplOptions { @@ -51,8 +52,34 @@ function show(value: unknown): string { return clip(typeof value === 'string' ? value : (JSON.stringify(value) ?? 'undefined')); } +/** `[code] hint (docs)`, the help line every SDKError carries. */ +function helpLine(code: string, hint = errorHelp(code)?.hint, docs = errorHelp(code)?.docs): string { + return [`[${code}]`, hint, docs && `(${docs})`].filter(Boolean).join(' '); +} + +/** + * `error: `, plus for an SDK error what kind of failure it was (a + * provider error's category and HTTP status) and its `[code] hint` line: a bare + * "User not found." from a provider does not say the API key is wrong. + */ function describeError(error: unknown): string { - return `error: ${error instanceof Error ? error.message : String(error)}`; + if (!(error instanceof SDKError)) return `error: ${error instanceof Error ? error.message : String(error)}`; + const { compacted, statusCode } = error as { compacted?: CompactedProviderError; statusCode?: number }; + const category = compacted?.category; + const status = statusCode ?? compacted?.statusCode; + const facts = [category && category !== 'unknown' && category, status !== undefined && `HTTP ${status}`].filter(Boolean); + return `error: ${error.detail}${facts.length ? ` (${facts.join(', ')})` : ''}\n${helpLine(error.code, error.hint, error.docs)}`; +} + +/** An `error` event's error: its message and, when it has a code, the code's help line. */ +function describeEventError({ message, code }: AgentEventError): string { + return `error: ${message}${code && !message.includes(`[${code}]`) ? `\n${helpLine(code)}` : ''}`; +} + +/** What failed the run behind `events` (its `result`'s rejection), when it is a run that failed. */ +async function runFailure(events: AsyncIterable): Promise { + const result = (events as { result?: Promise }).result; + return result?.then(() => undefined, (error: unknown) => error ?? new Error('The run failed.')); } function usageLine({ inputTokens, outputTokens, estimated, costUsd }: AgentEventUsage): string { @@ -88,6 +115,19 @@ export async function runChatRepl(options: ChatReplOptions): Promise { lineOpen = false; }; + let heldErrors: AgentEventError[] = []; + /** + * Prints the held `error` events, then `failure` (the run's rejection) when the run failed. The events + * of that failure (the step's raw error and the run's compacted one share its message) are not repeated. + */ + const flushErrors = (failure: unknown) => { + const held = heldErrors; + heldErrors = []; + const failed = failure instanceof SDKError ? [failure.detail, failure.message] : failure instanceof Error ? [failure.message] : []; + for (const error of held) if (!failed.includes(error.message)) errorOutput.write(`${describeEventError(error)}\n`); + if (failure !== undefined) fail(failure); + }; + type Printers = { [K in AgentEventType]?: (event: AgentEventOf) => void }; const printers: Printers = { 'text.delta': (event) => { @@ -114,9 +154,11 @@ export async function runChatRepl(options: ChatReplOptions): Promise { }, 'tool.done': (event) => say(dim(` -> ${show(event.result)}`)), 'tool.error': (event) => say(dim(` -> error: ${event.error.message}`)), + // Held until the run ends (see render()): the error that failed the run is printed from the run's + // rejection, which has the provider's category and HTTP status that the event leaves out. error: (event) => { endLine(); - errorOutput.write(`error: ${event.error.message}\n`); + heldErrors.push(event.error); }, 'run.done': (event) => { endLine(); @@ -129,9 +171,11 @@ export async function runChatRepl(options: ChatReplOptions): Promise { async function render(events: AsyncIterable): Promise | undefined> { let paused: AgentEventOf<'approval.requested'> | undefined; for await (const event of events) { + if (event.type === 'run.done' && !event.subagent) flushErrors(event.finishReason === 'error' ? await runFailure(events) : undefined); (printers[event.type] as ((event: AgentEvent) => void) | undefined)?.(event); if (event.type === 'approval.requested') paused = event; } + flushErrors(undefined); return paused; } diff --git a/src/cli/devReload.ts b/src/cli/devReload.ts index 1b4773fb..eb85cb17 100644 --- a/src/cli/devReload.ts +++ b/src/cli/devReload.ts @@ -94,6 +94,8 @@ export function detectTarget(rawPath: string, command = 'dev'): DevTarget { ); } +let loads = 0; + /** Agents built with their own `store` (a config export's, or `overrides.store`): `/chat` sessions use it, not the dev store. */ const storeOwners = new WeakSet(); @@ -111,14 +113,39 @@ function isAgentConfig(value: unknown): value is CreateAgentConfig { return typeof value === 'object' && value !== null && ('instructions' in value || 'prompt' in value); } -async function loadModuleAgent(file: string, token: string, overrides: CreateAgentConfig, command = 'dev'): Promise { +/** The module's default (or `agent`) export, imported afresh (`?t=` cache-bust). */ +async function importAgentExport(file: string, token: string): Promise { let mod: Record; try { mod = (await import(`${pathToFileURL(file).href}?t=${token}`)) as Record; } catch (error) { throw explainImportError(file, error); } - const exported = mod.default ?? mod.agent; + return mod.default ?? mod.agent; +} + +function notAnAgent(file: string, command: string): SDKError { + return new SDKError( + `lousho ${command}: ${file} must export a SimpleAgent or a createAgent() config as its default export (or as 'agent').`, + 'LOUSHO_CONFIG_INVALID', + { hint: "Add 'export default createAgent({ ... })', or export the createAgent() options object." } + ); +} + +/** + * The createAgent() options an agent module exports, imported as `lousho dev` imports it but + * without building the agent (`lousho doctor` inspects them); undefined for an exported agent + * that was not built by createAgent(). Throws like `lousho dev` for a module that exports no agent. + */ +export async function agentModuleConfig(file: string, command = 'dev'): Promise { + const exported = await importAgentExport(file, `${Date.now()}-${loads++}`); + if (isSimpleAgent(exported)) return createAgentConfigOf(exported); + if (isAgentConfig(exported)) return exported; + throw notAnAgent(file, command); +} + +async function loadModuleAgent(file: string, token: string, overrides: CreateAgentConfig, command = 'dev'): Promise { + const exported = await importAgentExport(file, token); if (isSimpleAgent(exported)) { // A module can export an already-built agent. Its createAgent() options are // remembered by createAgentConfigOf(), so an `overrides.exporter` (--traces) @@ -142,15 +169,9 @@ async function loadModuleAgent(file: string, token: string, overrides: CreateAge if (exported.store) storeOwners.add(agent); return agent; } - throw new SDKError( - `lousho ${command}: ${file} must export a SimpleAgent or a createAgent() config as its default export (or as 'agent').`, - 'LOUSHO_CONFIG_INVALID', - { hint: "Add 'export default createAgent({ ... })', or export the createAgent() options object." } - ); + throw notAnAgent(file, command); } -let loads = 0; - /** * Builds the agent a target describes. Directory and module loads import the * edited files afresh (`?t=` cache-bust); they also `ready()` the agent so MCP diff --git a/src/cli/doctor.ts b/src/cli/doctor.ts index ae152eb9..f82cd233 100644 --- a/src/cli/doctor.ts +++ b/src/cli/doctor.ts @@ -1,5 +1,5 @@ /** - * `lousho doctor [agent.yaml|json] [--json] [--ping]` - first-run diagnostic. + * `lousho doctor [agent.yaml|json|agent-dir|agent.ts] [--json] [--ping]` - first-run diagnostic. * * Thin wrapper over the pure core in doctorCore.ts: builds the real * DoctorEnvironment (process, fs, module resolution, fetch, Docker), @@ -12,6 +12,7 @@ import { MissingPeerDependencyError, loadOptionalPeer } from '../providers/optio import { findSdkRoot } from '../deploy/bundle'; import { loadSpec } from '../spec/loadSpec'; import { resolveAgentDir } from '../agentDir'; +import { agentModuleConfig } from './devReload'; import { resolveSpecTool } from '../spec/specToAgent'; import { unknownPermissionToolMessage, unknownPermissionTools } from '../execution/permissions'; import { exitCodeFor, parseCommand, type CommandSpec } from './args'; @@ -24,22 +25,22 @@ const DOCKER_TIMEOUT_MS = 2000; export interface DoctorArgs { specPath?: string; json: boolean; - /** `--ping`: GET `/models` for each configured provider. */ + /** `--ping`: GET an endpoint of each configured provider (one that checks the key where there is one). */ ping?: boolean; /** `-h` / `--help` was given: print the usage, run nothing. */ help?: boolean; } -const USAGE = 'Usage: lousho doctor [agent.yaml|json|agent-dir] [--json] [--ping]'; +const USAGE = 'Usage: lousho doctor [agent.yaml|json|agent-dir|agent.ts] [--json] [--ping]'; const HELP = `${USAGE} -Checks Node, the SDK's peer packages, provider API keys and, given a path, an agent spec or agent directory. +Checks Node, the SDK's peer packages, provider API keys and, given a path, an agent spec, agent directory or agent module. Prints a fix for every problem; exits 1 when a check fails. ./.env is loaded when it exists. Options: --json print the report as JSON - --ping also check that each configured provider endpoint answers + --ping also check that each configured provider endpoint answers and accepts the key -h, --help show this help`; const SPEC: CommandSpec = { @@ -110,6 +111,14 @@ function readSdkManifest(): SdkManifest { return JSON.parse(fs.readFileSync(path.join(findSdkRoot(), 'package.json'), 'utf8')); } +/** The names of a createAgent() `tools` option: tools, arrays and records of them (nested); none for a function of the run. */ +function toolNames(tools: unknown): string[] { + if (Array.isArray(tools)) return tools.flatMap(toolNames); + if (typeof tools !== 'object' || tools === null) return []; + const { name } = tools as { name?: unknown }; + return typeof name === 'string' ? [name] : Object.values(tools).flatMap(toolNames); +} + /** The real DoctorEnvironment for the current process. */ export function buildEnvironment(args: DoctorArgs, cwd: string = process.cwd()): DoctorEnvironment { const resolveSpecFile = (file: string) => path.resolve(cwd, file); @@ -125,6 +134,12 @@ export function buildEnvironment(args: DoctorArgs, cwd: string = process.cwd()): const permissionIssues = unknownPermissionTools(config.permissions, manifest.tools, { lenient: true }).map(unknownPermissionToolMessage); return { name: manifest.name, model: typeof config.model === 'string' ? config.model : undefined, tools: manifest.tools, permissionIssues }; }, + loadAgentModule: async (file) => { + const config = await agentModuleConfig(file, 'doctor'); + if (!config) return { name: path.basename(file), tools: [] }; + return { name: config.name ?? 'agent', model: typeof config.model === 'string' ? config.model : undefined, tools: toolNames(config.tools) }; + }, + platform: process.platform, loadSpec, resolveTool: resolveSpecTool, commandExists: (command) => commandExists(command, process.env), diff --git a/src/cli/doctorChecks.ts b/src/cli/doctorChecks.ts index bd34bbf5..ee86c673 100644 --- a/src/cli/doctorChecks.ts +++ b/src/cli/doctorChecks.ts @@ -218,10 +218,15 @@ function checkApiKey(env: DoctorEnvironment, info: ProviderInfo, needs: SpecNeed ...base, status: needed ? 'fail' : 'warn', finding: `${needed ? 'not set, and the agent spec needs it' : 'not set'}${baseUrlNote(env, info.name)}`, - fix: `Set ${info.envKey} in your environment, e.g. export ${info.envKey}=`, + fix: `Set ${info.envKey} in your environment, e.g. ${setEnvExample(env, info.envKey)}`, }; } +/** How to set `name` in the user's shell: PowerShell on Windows (where `export` does nothing), POSIX elsewhere. */ +function setEnvExample(env: DoctorEnvironment, name: string): string { + return env.platform === 'win32' ? `$env:${name}="" (PowerShell), or add it to .env` : `export ${name}=`; +} + /** Which provider `createAgent()` would pick with no model or provider argument. */ function checkDefaultProvider(env: DoctorEnvironment): DoctorCheck { const base = { id: 'env.default', title: 'Default provider for createAgent()' }; @@ -271,38 +276,53 @@ interface PingTarget { envKey: string; /** The env var that moves the endpoint, named in the fix. */ urlVar?: string; - base: string; + /** The URL to GET. */ + url: string; headers: Record; + /** True when the endpoint rejects a bad key, so a 200 proves the key; false for reachability only. */ + checksKey: boolean; } -/** The `/models` endpoints of the providers configured in the environment (Ollama has its own check). */ +/** + * The endpoints to ping for the providers configured in the environment (Ollama has its own check): + * one that needs the key where the provider has one (OpenRouter's /models answers without a key, its /key does not). + */ function pingTargets(env: DoctorEnvironment): PingTarget[] { const vars = env.env; const bearer = (key: string | undefined): Record => (key ? { Authorization: `Bearer ${key}` } : {}); const targets: PingTarget[] = []; if (vars.OPENAI_API_KEY || vars.OPENAI_BASE_URL) { const base = (vars.OPENAI_BASE_URL || DEFAULT_OPENAI_URL).replace(/\/+$/, ''); - targets.push({ provider: 'openai', envKey: 'OPENAI_API_KEY', urlVar: 'OPENAI_BASE_URL', base, headers: bearer(vars.OPENAI_API_KEY) }); + // A local or compatible server may serve /models to anyone; OpenAI's own needs the key. + const checksKey = !vars.OPENAI_BASE_URL && Boolean(vars.OPENAI_API_KEY); + targets.push({ provider: 'openai', envKey: 'OPENAI_API_KEY', urlVar: 'OPENAI_BASE_URL', url: `${base}/models`, headers: bearer(vars.OPENAI_API_KEY), checksKey }); } if (vars.ANTHROPIC_API_KEY) { const headers = { 'x-api-key': vars.ANTHROPIC_API_KEY, 'anthropic-version': '2023-06-01' }; - targets.push({ provider: 'anthropic', envKey: 'ANTHROPIC_API_KEY', base: 'https://api.anthropic.com/v1', headers }); + targets.push({ provider: 'anthropic', envKey: 'ANTHROPIC_API_KEY', url: 'https://api.anthropic.com/v1/models', headers, checksKey: true }); } if (vars.OPENROUTER_API_KEY) { - targets.push({ provider: 'openrouter', envKey: 'OPENROUTER_API_KEY', base: 'https://openrouter.ai/api/v1', headers: bearer(vars.OPENROUTER_API_KEY) }); + const url = 'https://openrouter.ai/api/v1/key'; + targets.push({ provider: 'openrouter', envKey: 'OPENROUTER_API_KEY', url, headers: bearer(vars.OPENROUTER_API_KEY), checksKey: true }); } return targets; } async function ping(env: DoctorEnvironment, target: PingTarget): Promise { - const url = `${target.base}/models`; + const { url } = target; const base = { id: `ping.${target.provider}`, title: `${target.provider} endpoint (GET ${displayUrl(url) ?? 'invalid URL'})` }; const where = target.urlVar ? `, and that ${target.urlVar} points at it` : ''; try { const response = await env.fetch(url, { signal: AbortSignal.timeout(PING_TIMEOUT_MS), headers: target.headers }); - if (response.ok) return { ...base, status: 'ok', finding: `reachable (HTTP ${response.status})` }; + if (response.ok) { + const finding = target.checksKey + ? `reachable, key accepted (HTTP ${response.status})` + : `reachable (HTTP ${response.status}; reachability only, the key is not checked)`; + return { ...base, status: 'ok', finding }; + } + // Every model call would be refused the same way. if (response.status === 401 || response.status === 403) { - return { ...base, status: 'warn', finding: `reachable, but the key was rejected (HTTP ${response.status})`, fix: `Check ${target.envKey}.` }; + return { ...base, status: 'fail', finding: `reachable, but the key was rejected (HTTP ${response.status})`, fix: `Check ${target.envKey}.` }; } return { ...base, status: 'warn', finding: `answered HTTP ${response.status}`, fix: `Check the server is up${where}.` }; } catch (error) { @@ -310,7 +330,7 @@ async function ping(env: DoctorEnvironment, target: PingTarget): Promise/models` for each configured provider. Never prints a key. */ +/** `--ping`: GET an endpoint of each configured provider, one that checks the key where there is one. Never prints a key. */ export async function checkPings(env: DoctorEnvironment): Promise { const targets = pingTargets(env); if (targets.length === 0) { diff --git a/src/cli/doctorCore.test.ts b/src/cli/doctorCore.test.ts index bbffb66f..f6ee6e39 100644 --- a/src/cli/doctorCore.test.ts +++ b/src/cli/doctorCore.test.ts @@ -447,7 +447,7 @@ describe('OpenAI base URL and --ping', () => { expect(report.checks.find((c) => c.id === 'ping.openai')).toMatchObject({ status: 'ok', title: 'openai endpoint (GET http://localhost:1234/v1/models)', - finding: 'reachable (HTTP 200)', + finding: 'reachable (HTTP 200; reachability only, the key is not checked)', }); }); @@ -459,9 +459,25 @@ describe('OpenAI base URL and --ping', () => { fix: expect.stringContaining('OPENAI_BASE_URL'), }); const rejected = makeEnv({ env: { OPENROUTER_API_KEY: SECRET }, fetch: async () => ({ ok: false, status: 401 }) }); - const result = await runDoctor(rejected, { ping: true }).then((r) => r.checks.find((c) => c.id === 'ping.openrouter')); - expect(result).toMatchObject({ status: 'warn', fix: 'Check OPENROUTER_API_KEY.' }); + const report = await runDoctor(rejected, { ping: true }); + const result = report.checks.find((c) => c.id === 'ping.openrouter'); + // A key the provider rejects is a failure: every model call would fail the same way. + expect(result).toMatchObject({ status: 'fail', fix: 'Check OPENROUTER_API_KEY.' }); expect(result?.finding).toContain('key was rejected'); + expect(report.exitCode).toBe(1); + }); + + it('--ping asks OpenRouter for the key itself (its /models answers without one)', async () => { + const fetch = vi.fn(async (_url: string, _init: { signal: AbortSignal; headers?: Record }) => ({ ok: true, status: 200 })); + const report = await runDoctor(makeEnv({ env: { OPENROUTER_API_KEY: SECRET }, fetch }), { ping: true }); + expect(fetch.mock.calls.map(([url]) => url)).toEqual(['https://openrouter.ai/api/v1/key']); + expect(fetch.mock.calls[0][1].headers).toEqual({ Authorization: `Bearer ${SECRET}` }); + expect(report.checks.find((c) => c.id === 'ping.openrouter')).toMatchObject({ status: 'ok', finding: 'reachable, key accepted (HTTP 200)' }); + }); + + it('--ping says when an endpoint does not prove the key (a custom OPENAI_BASE_URL)', async () => { + const report = await runDoctor(makeEnv({ env: { OPENAI_BASE_URL: 'http://localhost:1234/v1' } }), { ping: true }); + expect(report.checks.find((c) => c.id === 'ping.openai')?.finding).toBe('reachable (HTTP 200; reachability only, the key is not checked)'); }); it('--ping with no provider configured says so', async () => { @@ -592,7 +608,60 @@ describe('agent directory', () => { }); }); +describe('agent module', () => { + const moduleEnv = (overrides: Partial = {}) => + makeEnv({ + specPath: '/p/src/agent.ts', + loadAgentModule: async () => ({ name: 'agent', model: 'openrouter/openai/gpt-4o-mini', tools: ['current_time'] }), + ...overrides, + }); + + it('loads a .ts module like `lousho dev` does instead of reading it as a spec file', async () => { + const loadSpec = vi.fn(); + const env = moduleEnv({ loadSpec }); + expect(await check(env, 'spec')).toMatchObject({ status: 'ok', title: 'Agent module', finding: "'agent' loads (1 tool(s): current_time)" }); + expect(await check(env, 'spec.provider')).toMatchObject({ status: 'ok', finding: "'openrouter' with model 'openai/gpt-4o-mini'" }); + expect(loadSpec).not.toHaveBeenCalled(); + }); + + it("fails when the agent's provider key is missing", async () => { + const report = await runDoctor(moduleEnv()); + expect(report.checks.find((c) => c.id === 'env.openrouter')).toMatchObject({ status: 'fail', finding: expect.stringContaining('the agent spec needs it') }); + expect(report.exitCode).toBe(1); + }); + + it('reports a module that fails to load, naming the file', async () => { + const env = moduleEnv({ + loadAgentModule: async () => { + throw new Error('it must export a SimpleAgent or a createAgent() config'); + }, + }); + expect(await check(env, 'spec')).toMatchObject({ status: 'fail', title: 'Agent module', fix: 'Fix the file named above: /p/src/agent.ts.' }); + }); +}); + +describe('fix lines use the shell of the platform', () => { + it('PowerShell syntax on Windows, export elsewhere', async () => { + expect((await check(makeEnv({ platform: 'win32' }), 'env.anthropic')).fix).toBe( + 'Set ANTHROPIC_API_KEY in your environment, e.g. $env:ANTHROPIC_API_KEY="" (PowerShell), or add it to .env' + ); + expect((await check(makeEnv({ platform: 'linux' }), 'env.anthropic')).fix).toBe( + 'Set ANTHROPIC_API_KEY in your environment, e.g. export ANTHROPIC_API_KEY=' + ); + }); +}); + describe('spec load failure fix text', () => { + it('an unsupported file type lists what doctor accepts', async () => { + const env = makeEnv({ + specPath: 'agent.toml', + loadSpec: () => { + throw Object.assign(new Error('unsupported'), { code: 'LOUSHO_SPEC_UNSUPPORTED_FORMAT' }); + }, + }); + expect((await check(env, 'spec')).fix).toBe('Pass an agent spec (.yaml/.yml/.json), an agent directory, or a .ts/.js module that exports an agent.'); + }); + it('a missing spec says to check the path, not to fix fields', async () => { const env = makeEnv({ specPath: 'nope.yaml', diff --git a/src/cli/doctorCore.ts b/src/cli/doctorCore.ts index af1f45c4..e36ce4ab 100644 --- a/src/cli/doctorCore.ts +++ b/src/cli/doctorCore.ts @@ -22,7 +22,7 @@ function summarize(checks: DoctorCheck[]): Record { } export interface DoctorOptions { - /** `--ping`: GET `/models` for each configured provider. */ + /** `--ping`: GET an endpoint of each configured provider (one that checks the key where there is one). */ ping?: boolean; } diff --git a/src/cli/doctorSpec.ts b/src/cli/doctorSpec.ts index a8071cc8..7a5744ba 100644 --- a/src/cli/doctorSpec.ts +++ b/src/cli/doctorSpec.ts @@ -1,7 +1,7 @@ /** * `lousho doctor `: validates the spec with the SDK's own loader * and checks what the spec references (provider, built-in tools, MCP servers). - * `lousho doctor ` loads the directory and checks its model's provider. + * `lousho doctor ` and `lousho doctor ` load the agent and check its model's provider. */ import * as fs from 'node:fs'; import * as nodePath from 'node:path'; @@ -89,11 +89,17 @@ function checkPolicy(spec: AgentSpec): DoctorCheck[] { }); } -/** The fix line for a spec that failed to load: a missing file, a config of an agent directory, or bad fields. */ +const TARGET_HINT = 'Pass an agent spec (.yaml/.yml/.json), an agent directory, or a .ts/.js module that exports an agent.'; +/** What `lousho dev` loads as an agent module (src/cli/devReload.ts). */ +const MODULE_FILE = /\.[cm]?[tj]s$/i; + +/** The fix line for a spec that failed to load: a missing file, a config of an agent directory, an unsupported file type, or bad fields. */ function loadFix(file: string, error: unknown): string { - if ((error as { code?: unknown }).code === 'LOUSHO_SPEC_NOT_FOUND') { - return `Check the path: ${file} does not exist (pass an agent.yaml / agent.json spec or an agent directory).`; + const code = (error as { code?: unknown }).code; + if (code === 'LOUSHO_SPEC_NOT_FOUND') { + return `Check the path: ${file} does not exist (pass an agent.yaml / agent.json spec, an agent directory or an agent.ts module).`; } + if (code === 'LOUSHO_SPEC_UNSUPPORTED_FORMAT') return TARGET_HINT; const dir = nodePath.dirname(file); if (/^agent\.(json|ya?ml)$/i.test(nodePath.basename(file)) && fs.existsSync(nodePath.join(dir, 'instructions.md'))) { return `${file} looks like the config of an agent directory, which is not an agent spec: run \`lousho doctor ${dir}\` instead.`; @@ -112,17 +118,27 @@ function loadFailure(file: string, error: unknown): SpecInspection { return { checks: [check], needs: NO_SPEC_NEEDS }; } -/** `lousho doctor `: loads the directory (config, instructions, tools, ...) and checks its model's provider. */ -async function inspectAgentDir(env: DoctorEnvironment, dir: string): Promise { - const title = 'Agent directory'; - if (!env.loadAgentDir) { - return { checks: [{ id: 'spec', status: 'warn', title, finding: 'agent directories cannot be inspected here', fix: 'Pass a spec file instead.' }], needs: NO_SPEC_NEEDS }; +interface AgentLoader { + title: string; + load: ((path: string) => Promise) | undefined; + /** What cannot be inspected without `load`, e.g. 'agent directories'. */ + what: string; + fix: string; +} + +/** + * `lousho doctor ` or `lousho doctor `: loads the agent like `lousho dev` + * does (config, instructions, tools, ...) and checks its model's provider. + */ +async function inspectLoadedAgent(path: string, { title, load, what, fix }: AgentLoader): Promise { + if (!load) { + return { checks: [{ id: 'spec', status: 'warn', title, finding: `${what} cannot be inspected here`, fix: 'Pass a spec file instead.' }], needs: NO_SPEC_NEEDS }; } let summary: AgentDirSummary; try { - summary = await env.loadAgentDir(dir); + summary = await load(path); } catch (error) { - const check: DoctorCheck = { id: 'spec', status: 'fail', title, finding: message(error), fix: `Fix the file named above in ${dir}.` }; + const check: DoctorCheck = { id: 'spec', status: 'fail', title, finding: message(error), fix }; return { checks: [check], needs: NO_SPEC_NEEDS }; } const tools = summary.tools.length === 0 ? 'no tools' : `${summary.tools.length} tool(s): ${summary.tools.join(', ')}`; @@ -149,7 +165,12 @@ async function inspectAgentDir(env: DoctorEnvironment, dir: string): Promise { const path = env.specPath; if (!path) return { checks: [], needs: NO_SPEC_NEEDS }; - if (env.isDirectory?.(path)) return inspectAgentDir(env, path); + if (env.isDirectory?.(path)) { + return inspectLoadedAgent(path, { title: 'Agent directory', load: env.loadAgentDir, what: 'agent directories', fix: `Fix the file named above in ${path}.` }); + } + if (MODULE_FILE.test(path)) { + return inspectLoadedAgent(path, { title: 'Agent module', load: env.loadAgentModule, what: 'agent modules', fix: `Fix the file named above: ${path}.` }); + } let spec: AgentSpec; try { diff --git a/src/cli/doctorTypes.ts b/src/cli/doctorTypes.ts index 405020ae..bb0a9550 100644 --- a/src/cli/doctorTypes.ts +++ b/src/cli/doctorTypes.ts @@ -27,7 +27,7 @@ export interface SdkManifest { peerDependencies?: Record; } -/** What the doctor reports about a loaded agent directory. */ +/** What the doctor reports about a loaded agent directory or agent module. */ export interface AgentDirSummary { name: string; /** The config's `provider/model` string; undefined when `agent.ts` sets a provider instance. */ @@ -54,6 +54,10 @@ export interface DoctorEnvironment { isDirectory?(path: string): boolean; /** Loads an agent directory like `lousho dev` does (no provider is created); throws with the file at fault. */ loadAgentDir?(dir: string): Promise; + /** Loads a .ts/.js agent module like `lousho dev` does (no provider is created); throws with the file at fault. */ + loadAgentModule?(file: string): Promise; + /** `process.platform`; on 'win32' fix lines use PowerShell syntax. Absent: POSIX shell syntax. */ + platform?: string; /** Loads and validates a spec (the SDK's own loader); throws with field paths on invalid input. */ loadSpec(path: string): AgentSpec; /** Resolves a built-in tool name; throws if unknown. */ diff --git a/src/cli/evalReport.test.ts b/src/cli/evalReport.test.ts index f8a18dac..43d21ed4 100644 --- a/src/cli/evalReport.test.ts +++ b/src/cli/evalReport.test.ts @@ -107,6 +107,18 @@ describe('renderTable', () => { `); }); + it('shows the scores of passing scored gates, also whole-number ones', () => { + const scoredGates: EvalResult = { + ...passing, + assertions: [ + { name: 'completed()', kind: 'gate', passed: true, score: 1, threshold: 1 }, + { name: 'words', kind: 'gate', passed: true, score: 5, threshold: 3 }, + { name: 'faithful', kind: 'gate', passed: true, score: 1, threshold: 0.8 }, + ], + }; + expect(renderTable([scoredGates], false).split('\n')[1]).toContain('words=5, faithful=1'); + }); + it('marks soft failures as failing under --strict', () => { expect(renderTable([passing, softFail], true)).toMatchInlineSnapshot(` "EVAL CASE RESULT SCORES DURATION diff --git a/src/cli/evalReport.ts b/src/cli/evalReport.ts index e3e2ddfb..c3a3a049 100644 --- a/src/cli/evalReport.ts +++ b/src/cli/evalReport.ts @@ -61,9 +61,14 @@ function formatScore(assertion: AssertionResult): string { return `${assertion.name}=${score}`; } -/** Scored assertions (anything that is not a plain yes/no gate) for the table's SCORES column. */ +/** A plain yes/no check: scored 1 or 0 against the threshold 1 (or none). */ +function isYesNo({ score, threshold }: AssertionResult): boolean { + return (score === 0 || score === 1) && (threshold === undefined || threshold === 1); +} + +/** Scored assertions (anything that is not a plain yes/no check, passing gates included) for the table's SCORES column. */ function scoresColumn(result: EvalResult): string { - const scored = result.assertions.filter((a) => a.kind === 'soft' || !Number.isInteger(a.score) || a.name === 'score'); + const scored = result.assertions.filter((a) => a.kind === 'soft' || !isYesNo(a) || a.name === 'score'); return scored.map(formatScore).join(', ') || '-'; } diff --git a/src/cli/init/__snapshots__/templates.test.ts.snap b/src/cli/init/__snapshots__/templates.test.ts.snap index 9506381f..862facb3 100644 --- a/src/cli/init/__snapshots__/templates.test.ts.snap +++ b/src/cli/init/__snapshots__/templates.test.ts.snap @@ -95,7 +95,7 @@ exports[`renderProject > snapshot: minimal package.json 1`] = ` "dev:ui": "lousho dev src/agent.ts", "test": "vitest run", "typecheck": "tsc --noEmit", - "doctor": "lousho doctor", + "doctor": "lousho doctor src/agent.ts", "build": "lousho build src/agent.ts --target=node-server" }, "dependencies": { @@ -466,7 +466,7 @@ exports[`renderProject > snapshot: tools package.json 1`] = ` "dev:ui": "lousho dev src/agent.ts", "test": "vitest run", "typecheck": "tsc --noEmit", - "doctor": "lousho doctor", + "doctor": "lousho doctor src/agent.ts", "build": "lousho build src/agent.ts --target=node-server" }, "dependencies": { diff --git a/src/cli/init/templates.test.ts b/src/cli/init/templates.test.ts index 29d22f2a..333333ab 100644 --- a/src/cli/init/templates.test.ts +++ b/src/cli/init/templates.test.ts @@ -162,6 +162,25 @@ describe('renderProject', () => { expect(index).toContain('console.error(`\\nerror: ${errorText(error)}`);'); }); + it('the terminal chat prints an SDK error with its code, HTTP status and fix', () => { + const index = renderProject(config('openai', 'minimal'))['src/index.ts']!; + expect(index).toContain('error instanceof SDKError'); + expect(index).toContain('[${error.code}]'); + expect(index).toContain('HTTP ${statusCode}'); + }); + + it('the terminal chat loads a .env saved with a byte order mark without losing its first key', () => { + const index = renderProject(config('openai', 'minimal'))['src/index.ts']!; + expect(index).not.toContain('process.loadEnvFile()'); + expect(index).toContain("parseEnv(readFileSync('.env', 'utf8').replace(/^\\uFEFF/, ''))"); + expect(index).toContain('if (process.env[name] === undefined)'); + }); + + it('the doctor script checks the agent module, so a missing provider key fails it', () => { + const pkg = JSON.parse(renderProject(config('openai', 'minimal'))['package.json']!); + expect(pkg.scripts.doctor).toBe('lousho doctor src/agent.ts'); + }); + it('the terminal chat streams text deltas and waits for the turn in flight at end of input', () => { const index = renderProject(config('openai', 'minimal'))['src/index.ts']!; expect(index).toContain("event.type === 'text.delta'"); diff --git a/src/cli/init/templates.ts b/src/cli/init/templates.ts index 1687c1da..29c71095 100644 --- a/src/cli/init/templates.ts +++ b/src/cli/init/templates.ts @@ -70,7 +70,7 @@ function packageJson(config: ProjectConfig): string { ...(yaml ? {} : { 'dev:ui': 'lousho dev src/agent.ts' }), test: 'vitest run', typecheck: 'tsc --noEmit', - doctor: yaml ? 'lousho doctor agent.yaml' : 'lousho doctor', + doctor: yaml ? 'lousho doctor agent.yaml' : 'lousho doctor src/agent.ts', // A self-contained HTTP server in .lousho/build/node-server (--target=docker adds a Dockerfile). build: `lousho build ${yaml ? 'agent.yaml' : 'src/agent.ts'} --target=node-server`, }, @@ -174,17 +174,31 @@ export function buildAgent(provider?: LLMProvider) { } function indexSource(): string { - return `import { createInterface } from 'node:readline/promises'; + return `import { readFileSync } from 'node:fs'; +import { createInterface } from 'node:readline/promises'; +import { parseEnv } from 'node:util'; +import { SDKError } from '@lousho/build-ai-agent'; import { buildAgent } from './agent.js'; -// Load .env when there is one; variables already set in the environment win. +// Load .env when there is one; variables already set in the environment win. The byte order mark +// some editors write (PowerShell's Set-Content -Encoding utf8) is dropped, or the first key is lost. try { - process.loadEnvFile(); + const vars = parseEnv(readFileSync('.env', 'utf8').replace(/^\\uFEFF/, '')); + for (const [name, value] of Object.entries(vars)) { + if (process.env[name] === undefined) process.env[name] = value; + } } catch (error) { if ((error as NodeJS.ErrnoException).code !== 'ENOENT') throw error; } -const errorText = (error: unknown) => (error instanceof Error ? error.message : String(error)); +// An SDK error says what to do about it: its code and fix, and a provider's HTTP status +// (a bare "User not found." from the provider means the API key is wrong). +const errorText = (error: unknown) => { + if (!(error instanceof SDKError)) return error instanceof Error ? error.message : String(error); + const { statusCode, compacted } = error as SDKError & { statusCode?: number; compacted?: { category: string } }; + const facts = [compacted && compacted.category !== 'unknown' && compacted.category, statusCode && \`HTTP \${statusCode}\`].filter(Boolean); + return \`\${error.detail}\${facts.length ? \` (\${facts.join(', ')})\` : ''}\\n[\${error.code}] \${error.hint ?? ''}\`; +}; let agent: ReturnType; try {