diff --git a/command-snapshot.json b/command-snapshot.json index 39455470..c3080f7d 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -376,6 +376,38 @@ "flags": ["api-name", "api-version", "concise", "flags-dir", "json", "skip-retrieve", "target-org", "verbose"], "plugin": "@salesforce/plugin-agent" }, + { + "alias": [], + "command": "agent:scorer:generate-metadata-file", + "flagAliases": [], + "flagChars": ["o"], + "flags": [ + "agent-api-name", + "api-name", + "api-version", + "description", + "engine-type", + "flags-dir", + "json", + "label", + "lightning-type", + "output-dir", + "preview", + "spec", + "spec-schema", + "status", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:scorer:run", + "flagAliases": [], + "flagChars": ["o"], + "flags": ["api-name", "api-version", "data", "file", "flags-dir", "json", "scorer-version", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, { "alias": [], "command": "agent:test:create", diff --git a/messages/agent.scorer.generate-metadata-file.md b/messages/agent.scorer.generate-metadata-file.md new file mode 100644 index 00000000..93fa6477 --- /dev/null +++ b/messages/agent.scorer.generate-metadata-file.md @@ -0,0 +1,113 @@ +# summary + +Scaffold an AiAgentScorerDefinition metadata XML file (an agent scorer), interactively or from a spec file. This is a one-time starter, not the source of truth: once the AiAgentScorerDefinition XML is created (or retrieved from an org), edit it directly. + +# description + +Creates an AiAgentScorerDefinition metadata XML file either interactively (prompting for each field) or from a YAML spec file. + +Run with no flags to start the interactive interview. The command prompts you for the scorer's lightning type, optional output labels, engine type, and agent associations. + +Alternatively, provide a --spec flag pointing to a YAML file that defines the scorer. This is useful for repeatable automation or when the scorer has many output values. + +Use --preview to see the generated XML without writing it to disk. + +This command is a one-shot scaffolding helper: it generates the scorer definition (and, for PromptTemplate scorers, its prompt template) metadata XML to give you a fast start. The generated XML — not this command and not the spec — is the source of truth. Once a definition exists locally, whether you created it here or retrieved it from an org, it has no connection back to the spec: make every further change (add a version, promote or archive a version, toggle an agent association, or edit the prompt rubric) directly in the metadata XML. If you already know the XML structure you can author it by hand and skip this command entirely; it exists because the XML is intricate and encodes rules the spec cannot fully capture. + +# flags.api-name.summary + +API name of the scorer definition. + +# flags.agent-api-name.summary + +API name of the agent to associate with this scorer. + +# flags.lightning-type.summary + +Lightning type the scorer's value conforms to (for example, lightning**textType or lightning**numberType). + +# flags.label.summary + +Display label for the scorer version. + +# flags.description.summary + +Description of what this scorer evaluates. + +# flags.engine-type.summary + +Engine type for scoring (Manual or PromptTemplate). + +# flags.status.summary + +Initial status of the scorer version (Available or Draft). + +# flags.spec.summary + +Path to a scorer spec YAML file. Bypasses interactive prompts. + +# flags.spec-schema.summary + +Output the JSON Schema for the --spec YAML file and exit. + +# flags.output-dir.summary + +Output directory for the generated metadata XML files (scorer definition and prompt template). + +# flags.preview.summary + +Preview the generated XML without writing to disk. + +# examples + +- Show the JSON Schema for the spec YAML file: + + <%= config.bin %> <%= command.id %> --spec-schema + +- Create a scorer interactively: + + <%= config.bin %> <%= command.id %> + +- Create a scorer from a spec file: + + <%= config.bin %> <%= command.id %> --spec specs/expert-analysis-scorer.yaml + +- Preview the XML that would be generated: + + <%= config.bin %> <%= command.id %> --spec specs/expert-analysis-scorer.yaml --preview + +- Create a manual scorer with flags (non-interactive): + + <%= config.bin %> <%= command.id %> --api-name Expert_Analysis --lightning-type lightning\_\_textType --engine-type Manual --label Expert_Analysis --agent-api-name My_Agent --status Available + +- Create a prompt-based scorer (generates both scorer definition and prompt template): + + <%= config.bin %> <%= command.id %> --api-name sentiment_analysis --lightning-type lightning\_\_textType --engine-type PromptTemplate --label sentiment_analysis --agent-api-name My_Agent + +# error.missingRequiredFlags + +Missing required flags: %s. When using --json, all required flags must be provided. + +# error.invalidSpecYaml + +Could not parse the --spec file as YAML: %s + +# error.invalidSpecShape + +The --spec file must define a YAML object matching the scorer spec schema (see --spec-schema). Received: %s + +# error.noAgentsInOrg + +No agents found in the org. Deploy an agent first, or specify one with --agent-api-name. + +# error.scorerExists + +A scorer named '%s' already exists in this project. `generate-metadata-file` only scaffolds a new scorer and never overwrites an existing one. Edit its metadata XML directly instead (%s) — that is where you add a new version, promote or archive a version, toggle an agent association, or change the prompt rubric. To scaffold a different scorer, choose a new API name. + +# info.scaffoldIntro + +Heads up: this command only scaffolds the scorer's AiAgentScorerDefinition metadata XML to get you started — the generated file, not this interview, is the source of truth. After it's written, make any further changes directly in the AiAgentScorerDefinition XML. + +# info.editXmlDirectly + +Done. This scorer is now defined by its metadata XML, which is the source of truth from here on — the spec is no longer connected to it. Make any further change (add a version, promote or archive a version, toggle an agent association, or edit the prompt rubric) directly in the generated XML file. diff --git a/messages/agent.scorer.run.md b/messages/agent.scorer.run.md new file mode 100644 index 00000000..da92a736 --- /dev/null +++ b/messages/agent.scorer.run.md @@ -0,0 +1,59 @@ +# summary + +Run an agent scorer against an STDM session and print its score. + +# description + +Runs a scorer that is already authored in your project metadata, referenced by its API name, against a single STDM (Session Trace Data Model) session, then prints the resulting score, outcome labels, and explanation. + +The scorer is resolved from your project's package directories by API name. If no scorer with that API name exists locally, the command errors — author it first with `sf agent scorer generate-metadata-file`. + +Provide the session either inline as a JSON string with --data, or as a path to a local JSON file with --file. Exactly one of the two is required. + +To help you hand-construct a valid session, run this command with --help: the full JSON Schema for the session object is printed under the --data flag. + +# flags.api-name.summary + +API name of the scorer to run. Must match a scorer authored in this project's metadata. + +# flags.scorer-version.summary + +Version number of the scorer to run. Omit to run the highest-numbered Available version; the command errors if none is Available. A Draft version must be selected explicitly; an Archived version cannot be run. + +# flags.data.summary + +Inline STDM session JSON to score. + +# flags.data.description + +Inline STDM session JSON to score. The value must be a JSON object matching the following JSON Schema (the shape of the scorer's Input:Session value): + +# flags.file.summary + +Path to a local JSON file containing the STDM session to score. + +# examples + +- Run a scorer against a session stored in a local file: + + <%= config.bin %> <%= command.id %> --api-name Sentiment_Scorer --file ./session.json + +- Run a scorer against an inline session JSON string: + + <%= config.bin %> <%= command.id %> --api-name Sentiment_Scorer --data '{"sessionState":{"sessionId":"1","startTimestamp":"2026-01-01T00:00:00Z","channel":"web"},"actors":[],"metrics":{"durationMs":0,"turns":0},"runs":[]}' + +- Show the session JSON Schema in the help output: + + <%= config.bin %> <%= command.id %> --help + +# error.invalidSessionJson + +Could not parse the STDM session as JSON: %s + +# error.invalidSessionShape + +The STDM session must be a JSON object matching the session schema (see --help). Received: %s + +# error.scorerRunFailed + +Scorer '%s' did not produce a valid score: %s. The command exits non-zero so a scripted loop won't treat a failed evaluation as a passing one; the full result (including any explanation) is in the error's "data" field when run with --json. diff --git a/package.json b/package.json index 17160dc7..7660a49e 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,10 @@ "description": "Command to validate an Agent Script file.", "external": true }, + "scorer": { + "description": "Commands to scaffold and run agent scorers.", + "external": true + }, "adl": { "description": "Commands to manage Agentforce Data Libraries.", "external": true, diff --git a/schemas/agent-mcp-create.json b/schemas/agent-mcp-create.json index a4c5b095..a6ea809a 100644 --- a/schemas/agent-mcp-create.json +++ b/schemas/agent-mcp-create.json @@ -15,10 +15,7 @@ } } }, - "required": [ - "server", - "assets" - ], + "required": ["server", "assets"], "additionalProperties": false }, "McpServerOutput": { @@ -61,12 +58,7 @@ "type": "string" } }, - "required": [ - "id", - "name", - "type", - "status" - ], + "required": ["id", "name", "type", "status"], "additionalProperties": false }, "McpServerType": { @@ -86,17 +78,12 @@ "type": "string" } }, - "required": [ - "authType" - ], + "required": ["authType"], "additionalProperties": false }, "McpAuthType": { "type": "string", - "enum": [ - "OAUTH", - "NO_AUTH" - ] + "enum": ["OAUTH", "NO_AUTH"] }, "McpFetchedAsset": { "type": "object", @@ -124,21 +111,17 @@ }, "status": { "type": "string" + }, + "securityWarning": { + "type": "string" } }, - "required": [ - "name", - "kind" - ], + "required": ["name", "kind"], "additionalProperties": false }, "McpAssetKind": { "type": "string", - "enum": [ - "MCP_TOOL", - "MCP_PROMPT", - "MCP_RESOURCE" - ] + "enum": ["MCP_TOOL", "MCP_PROMPT", "MCP_RESOURCE"] } } -} \ No newline at end of file +} diff --git a/schemas/agent-mcp-fetch.json b/schemas/agent-mcp-fetch.json index b99f8e9c..b15d7242 100644 --- a/schemas/agent-mcp-fetch.json +++ b/schemas/agent-mcp-fetch.json @@ -15,9 +15,7 @@ } } }, - "required": [ - "assets" - ], + "required": ["assets"], "additionalProperties": false }, "McpFetchedAsset": { @@ -46,21 +44,17 @@ }, "status": { "type": "string" + }, + "securityWarning": { + "type": "string" } }, - "required": [ - "name", - "kind" - ], + "required": ["name", "kind"], "additionalProperties": false }, "McpAssetKind": { "type": "string", - "enum": [ - "MCP_TOOL", - "MCP_PROMPT", - "MCP_RESOURCE" - ] + "enum": ["MCP_TOOL", "MCP_PROMPT", "MCP_RESOURCE"] } } -} \ No newline at end of file +} diff --git a/schemas/agent-scorer-generate__metadata__file.json b/schemas/agent-scorer-generate__metadata__file.json new file mode 100644 index 00000000..3ed1b068 --- /dev/null +++ b/schemas/agent-scorer-generate__metadata__file.json @@ -0,0 +1,45 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentScorerGenerateMetadataFileCommandResult", + "definitions": { + "AgentScorerGenerateMetadataFileCommandResult": { + "anyOf": [ + { + "$ref": "#/definitions/AgentScorerGenerateMetadataFileResult" + }, + { + "$ref": "#/definitions/ScorerSpecSchemaResult" + } + ], + "description": "Everything `run` can resolve to: a scaffolded/previewed scorer, or (under `--spec-schema --json`) the schema." + }, + "AgentScorerGenerateMetadataFileResult": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "apiName": { + "type": "string" + }, + "contents": { + "type": "string" + }, + "promptTemplatePath": { + "type": "string" + }, + "guidance": { + "type": "string", + "description": "Guidance surfaced to a caller running with --json (where `this.log` output is suppressed): a written scorer's metadata XML is the source of truth, so any further change is made directly in the XML." + } + }, + "required": ["path", "apiName", "contents"], + "additionalProperties": false + }, + "ScorerSpecSchemaResult": { + "type": "object", + "additionalProperties": {}, + "description": "The scorer spec JSON Schema payload. Under `--spec-schema --json` (where `styledJSON` is suppressed) it is returned as the command result so the schema still rides in the standard {status, result} envelope for machine consumers, matching what `--spec-schema` prints without `--json`." + } + } +} diff --git a/schemas/agent-scorer-run.json b/schemas/agent-scorer-run.json new file mode 100644 index 00000000..808b4ff7 --- /dev/null +++ b/schemas/agent-scorer-run.json @@ -0,0 +1,50 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentScorerRunResult", + "definitions": { + "AgentScorerRunResult": { + "type": "object", + "additionalProperties": false, + "properties": { + "scorerApiName": { + "type": "string" + }, + "scorerVersion": { + "type": "number", + "description": "The version number that was actually run (highest Available by default, or the requested --scorer-version)." + }, + "ok": { + "type": "boolean" + }, + "output": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "The score: one or more labels, or a typed value matching the scorer's lightning type." + }, + "explanation": { + "type": "string" + }, + "raw": { + "type": "string", + "description": "Raw engine output (for debugging / loose formats)." + }, + "error": { + "type": "string" + } + }, + "required": ["ok", "scorerApiName"] + } + } +} diff --git a/src/commands/agent/scorer/generate-metadata-file.ts b/src/commands/agent/scorer/generate-metadata-file.ts new file mode 100644 index 00000000..37ddebcd --- /dev/null +++ b/src/commands/agent/scorer/generate-metadata-file.ts @@ -0,0 +1,460 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { join, resolve } from 'node:path'; +import { readFileSync, existsSync } from 'node:fs'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, EnvironmentVariable } from '@salesforce/core'; +import { + Agent, + type ScorerSpec, + createScorerDefinition, + labelToApiName, + scorerSpecJsonSchema, + type SupportedLightningType, + SUPPORTED_LIGHTNING_TYPES, + SCORER_API_NAME_MAX_LENGTH, + SCORER_API_NAME_PATTERN, + SCORER_ENGINE_TYPES, + SCORER_STATUSES, + SCORER_OUTCOME_TYPES, + SCORER_INPUT_SCOPES, +} from '@salesforce/agents'; +import { confirm, select, input as inquirerInput } from '@inquirer/prompts'; +import YAML from 'yaml'; +import { FlaggablePrompt, makeFlags, promptForFlag } from '../../../flags.js'; +import { theme } from '../../../inquirer-theme.js'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.scorer.generate-metadata-file'); + +export type AgentScorerGenerateMetadataFileResult = { + path: string; + apiName: string; + contents: string; + promptTemplatePath?: string; + /** + * Guidance surfaced to a caller running with --json (where `this.log` output is suppressed): a written + * scorer's metadata XML is the source of truth, so any further change is made directly in the XML. + */ + guidance?: string; +}; + +/** + * The scorer spec JSON Schema payload. Under `--spec-schema --json` (where `styledJSON` is suppressed) it is + * returned as the command result so the schema still rides in the standard {status, result} envelope for + * machine consumers, matching what `--spec-schema` prints without `--json`. + */ +export type ScorerSpecSchemaResult = Record; + +/** Everything `run` can resolve to: a scaffolded/previewed scorer, or (under `--spec-schema --json`) the schema. */ +export type AgentScorerGenerateMetadataFileCommandResult = + | AgentScorerGenerateMetadataFileResult + | ScorerSpecSchemaResult; + +const FLAGGABLE_PROMPTS = { + label: { + message: messages.getMessage('flags.label.summary'), + promptMessage: 'Scorer label (display name)', + validate: (d: string): boolean | string => d.length > 0 || 'Label cannot be empty', + required: true, + }, + 'api-name': { + message: messages.getMessage('flags.api-name.summary'), + promptMessage: 'Scorer API name', + validate: (d: string): boolean | string => { + if (!d.length) return 'API name cannot be empty'; + if (d.length > SCORER_API_NAME_MAX_LENGTH) + return `API name cannot exceed ${SCORER_API_NAME_MAX_LENGTH} characters`; + if (!SCORER_API_NAME_PATTERN.test(d)) return 'Must start with letter, only alphanumerics and underscores'; + return true; + }, + required: true, + }, + 'lightning-type': { + message: messages.getMessage('flags.lightning-type.summary'), + promptMessage: 'Select the lightning type this scorer produces', + options: SUPPORTED_LIGHTNING_TYPES, + validate: (d: string): boolean | string => + (SUPPORTED_LIGHTNING_TYPES as readonly string[]).includes(d) || 'Invalid lightning type', + required: true, + }, + description: { + message: messages.getMessage('flags.description.summary'), + promptMessage: 'Description (optional, press Enter to skip)', + validate: (): boolean | string => true, + }, + 'engine-type': { + message: messages.getMessage('flags.engine-type.summary'), + promptMessage: 'Scoring engine type', + options: SCORER_ENGINE_TYPES, + validate: (d: string): boolean | string => + (SCORER_ENGINE_TYPES as readonly string[]).includes(d) || 'Invalid engine type', + required: true, + }, + status: { + message: messages.getMessage('flags.status.summary'), + promptMessage: 'Initial status', + options: SCORER_STATUSES, + validate: (d: string): boolean | string => (SCORER_STATUSES as readonly string[]).includes(d) || 'Invalid status', + default: 'Draft', + }, +} satisfies Record; + +type OutputEnumValueInput = { + value: string; + outcomeType: string; + isFallback: boolean; + isSystemFallback: boolean; +}; + +async function promptForSingleEnumValue(index: number): Promise { + const value = await promptForFlag({ + message: 'Output value name', + promptMessage: `Output value #${index + 1} (e.g., "Good", "Bad", "N/A")`, + validate: (d: string): boolean | string => d.length > 0 || 'Value cannot be empty', + }); + + const outcomeType = await promptForFlag({ + message: 'Outcome type', + promptMessage: 'Outcome type for this value', + options: SCORER_OUTCOME_TYPES, + validate: (d: string): boolean | string => (SCORER_OUTCOME_TYPES as readonly string[]).includes(d) || 'Invalid', + }); + + const isFallback = await confirm({ + message: 'Is this the fallback value?', + default: index === 0, + theme, + }); + + const addMore = await confirm({ + message: 'Add another output value?', + default: index < 1, + theme, + }); + + return { value, outcomeType, isFallback, isSystemFallback: false, addMore }; +} + +async function promptForOutputEnumValues(): Promise { + const values: OutputEnumValueInput[] = []; + let addMore = true; + + while (addMore) { + // eslint-disable-next-line no-await-in-loop + const result = await promptForSingleEnumValue(values.length); + addMore = result.addMore; + values.push({ + value: result.value, + outcomeType: result.outcomeType, + isFallback: result.isFallback, + isSystemFallback: result.isSystemFallback, + }); + } + + return values; +} + +export default class AgentScorerGenerateMetadataFile extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly flags = { + 'target-org': Flags.requiredOrg(), + 'api-version': Flags.orgApiVersion(), + ...makeFlags(FLAGGABLE_PROMPTS), + 'agent-api-name': Flags.string({ + summary: messages.getMessage('flags.agent-api-name.summary'), + }), + spec: Flags.file({ + summary: messages.getMessage('flags.spec.summary'), + exists: true, + }), + 'spec-schema': Flags.boolean({ + summary: messages.getMessage('flags.spec-schema.summary'), + default: false, + }), + 'output-dir': Flags.directory({ + summary: messages.getMessage('flags.output-dir.summary'), + default: join('force-app', 'main', 'default'), + }), + preview: Flags.boolean({ + summary: messages.getMessage('flags.preview.summary'), + }), + }; + + // eslint-disable-next-line complexity + public async run(): Promise { + const { flags } = await this.parse(AgentScorerGenerateMetadataFile); + + if (flags['spec-schema']) { + const schema = scorerSpecJsonSchema(); + // Under --json, styledJSON is a no-op (output is suppressed), so return the schema as the command result + // — it then rides in the standard {status, result} envelope for machine consumers. Otherwise pretty-print + // it to the terminal as before. + if (this.jsonEnabled()) { + return schema; + } + this.styledJSON(schema as unknown as import('@salesforce/ts-types').AnyJson); + return { path: '', apiName: '', contents: '' }; + } + + const outputDir = resolve(flags['output-dir']); + + const connection = flags['target-org'].getConnection(flags['api-version']); + + const spec: ScorerSpec = flags.spec + ? this.parseSpec(readFileSync(resolve(flags.spec), 'utf8')) + : await this.runInteractiveInterview(flags, connection); + + const scorerFileName = `${spec.apiName}.aiAgentScorerDefinition-meta.xml`; + const scorerPath = join(outputDir, 'aiAgentScorerDefinitions', scorerFileName); + const exists = existsSync(scorerPath); + + // `generate-metadata-file` only scaffolds a brand-new scorer — it never overwrites an existing one. Once the metadata XML + // exists it has lost its connection to the spec, so every further change (new versions, status, activation, + // rubric edits) is authored directly in the XML. Error out and point the user there rather than re-scaffold. + if (exists) { + throw messages.createError('error.scorerExists', [spec.apiName, scorerPath]); + } + + if (flags.preview) { + const result = await createScorerDefinition(spec, { outputDir, write: false }); + this.log('\n--- Scorer Definition (preview) ---\n'); + this.log(result.contents); + if (result.promptTemplateContents) { + this.log('\n--- Prompt Template (preview) ---\n'); + this.log(result.promptTemplateContents); + } + return { + path: result.path, + apiName: result.apiName, + contents: result.contents, + promptTemplatePath: result.promptTemplatePath, + }; + } + + const result = await createScorerDefinition(spec, { outputDir }); + this.log(`\nScorer definition written to: ${result.path}`); + if (result.promptTemplatePath) { + this.log(`Prompt template written to: ${result.promptTemplatePath}`); + } + // Make the scaffold-once model explicit: from here on the XML is the source of truth, not the spec. `this.log` + // is suppressed under --json, so the same guidance also rides along in the returned result for agent callers. + const guidance = messages.getMessage('info.editXmlDirectly'); + this.log(`\n${guidance}`); + + return { + path: result.path, + apiName: result.apiName, + contents: result.contents, + promptTemplatePath: result.promptTemplatePath, + guidance, + }; + } + + // eslint-disable-next-line class-methods-use-this + private parseSpec(raw: string): ScorerSpec { + let parsed: unknown; + try { + parsed = YAML.parse(raw); + } catch (e) { + throw messages.createError('error.invalidSpecYaml', [(e as Error).message]); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw messages.createError('error.invalidSpecShape', [typeof parsed]); + } + return parsed as ScorerSpec; + } + + private async runInteractiveInterview( + flags: Record, + connection: ReturnType + ): Promise { + if (this.jsonEnabled()) { + const missing = Object.entries(FLAGGABLE_PROMPTS) + .filter(([key, p]) => 'required' in p && p.required && !flags[key]) + .map(([key]) => key); + if (!flags['agent-api-name']) missing.push('agent-api-name'); + if (missing.length) { + throw messages.createError('error.missingRequiredFlags', [missing.join(', ')]); + } + } + + // Set expectations up front for the human doing the interview: this only scaffolds the XML, which becomes + // the source of truth. (Suppressed under --json, which drives the flow from flags rather than prompts.) + if (!this.jsonEnabled()) { + this.log(`\n${messages.getMessage('info.scaffoldIntro')}`); + } + + this.log(); + this.styledHeader('Scorer Definition'); + + const label = (flags.label as string) ?? (await promptForFlag(FLAGGABLE_PROMPTS.label)); + + const defaultApiName = labelToApiName(label); + const apiName = + (flags['api-name'] as string) ?? + (await inquirerInput({ + message: 'Scorer API name', + default: defaultApiName, + validate: FLAGGABLE_PROMPTS['api-name'].validate, + theme, + })); + + const description = + (flags.description as string) ?? + (this.jsonEnabled() ? undefined : await promptForFlag(FLAGGABLE_PROMPTS.description)); + const status = + (flags.status as string) ?? + (this.jsonEnabled() ? FLAGGABLE_PROMPTS.status.default : await promptForFlag(FLAGGABLE_PROMPTS.status)); + const lightningType = ((flags['lightning-type'] as SupportedLightningType) ?? + (await promptForFlag(FLAGGABLE_PROMPTS['lightning-type']))) as SupportedLightningType; + + this.log(); + this.styledHeader('Output Labels'); + const addLabels = this.jsonEnabled() + ? false + : await confirm({ + message: 'Add predefined output labels? (leave off for fully open-ended output)', + default: false, + theme, + }); + const outputEnumValues = addLabels ? await promptForOutputEnumValues() : undefined; + + const engineType = (flags['engine-type'] as string) ?? (await promptForFlag(FLAGGABLE_PROMPTS['engine-type'])); + const engineConfig = await this.promptForEngineConfig(engineType); + const agentAssociation = await this.promptForAgentAssociationDetails( + connection, + engineType, + flags['agent-api-name'] as string | undefined + ); + + return { + apiName, + lightningType, + inputScope: 'Session', + label, + description: description || undefined, + engineType: engineType as ScorerSpec['engineType'], + promptContent: engineConfig.promptContent, + promptTemplateName: engineConfig.promptTemplateName, + status: status as ScorerSpec['status'], + outputEnumValues: outputEnumValues as ScorerSpec['outputEnumValues'], + agentAssociation, + }; + } + + private async promptForEngineConfig( + engineType: string + ): Promise<{ promptContent?: string; promptTemplateName?: string }> { + if (engineType !== 'PromptTemplate') return {}; + // No flag exists yet for referencing an existing prompt template by name, so in --json/ + // non-interactive mode we always generate a new default prompt template. + if (this.jsonEnabled()) return {}; + + this.log(); + this.styledHeader('Prompt Template'); + + const promptChoice = await select({ + message: 'Prompt template source', + choices: [ + { name: 'Generate a new default prompt template', value: 'generate' }, + { name: 'Use an existing prompt template', value: 'existing' }, + ], + theme, + }); + + if (promptChoice === 'existing') { + const promptTemplateName = await inquirerInput({ + message: 'Existing prompt template API name', + validate: (d: string): boolean | string => d.length > 0 || 'Name cannot be empty', + theme, + }); + return { promptTemplateName }; + } + + return {}; + } + + private async promptForAgentAssociationDetails( + connection: ReturnType, + engineType: string, + agentApiNameFlag?: string + ): Promise { + let agentAssociation: ScorerSpec['agentAssociation']; + if (agentApiNameFlag) { + agentAssociation = { agentApiName: agentApiNameFlag, isActive: false }; + } else { + const agentsInOrg = await Agent.listRemote(connection); + if (!agentsInOrg.length) { + throw messages.createError('error.noAgentsInOrg'); + } + const agentApiName = await select({ + message: 'Select the agent to associate with this scorer', + choices: agentsInOrg + .filter((a) => !a.IsDeleted) + .sort((a, b) => a.DeveloperName.localeCompare(b.DeveloperName)) + .map((a) => ({ name: a.DeveloperName, value: a.DeveloperName })), + theme, + }); + agentAssociation = { agentApiName, isActive: false }; + } + + const associationInputScope = this.jsonEnabled() + ? 'Session' + : await select({ + message: 'Input scope for this agent association', + choices: SCORER_INPUT_SCOPES.map((s) => ({ name: s, value: s })), + default: 'Session', + theme, + }); + agentAssociation.inputScope = associationInputScope as 'Session' | 'Intent'; + + if (engineType === 'PromptTemplate') { + const isActive = this.jsonEnabled() + ? false + : await confirm({ + message: 'Activate scoring for this agent?', + default: false, + theme, + }); + agentAssociation.isActive = isActive; + + if (isActive) { + const samplingRateStr = await promptForFlag({ + message: 'Sampling rate (0.0 - 1.0)', + promptMessage: 'Sampling rate (0.0 to 1.0, where 1.0 = score every session)', + validate: (d: string): boolean | string => { + const n = parseFloat(d); + if (isNaN(n) || n < 0 || n > 1) return 'Must be between 0.0 and 1.0'; + return true; + }, + default: '1.0', + }); + agentAssociation.samplingRate = parseFloat(samplingRateStr); + } + } + + return agentAssociation; + } +} diff --git a/src/commands/agent/scorer/run.ts b/src/commands/agent/scorer/run.ts new file mode 100644 index 00000000..dd82d616 --- /dev/null +++ b/src/commands/agent/scorer/run.ts @@ -0,0 +1,131 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { resolve } from 'node:path'; +import { readFileSync } from 'node:fs'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, EnvironmentVariable } from '@salesforce/core'; +import { type SessionView, type ScorerResult, runScorer, loadScorerSpec, sessionViewJsonSchema } from '@salesforce/agents'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.scorer.run'); + +// The JSON Schema for the STDM session object, surfaced in the --data flag's help so authors (and agents) +// can hand-construct a valid session. Derived from the SessionView type in @salesforce/agents. +// +// oclif renders flag descriptions through wrap-ansi, which mangles pretty-printed JSON in two ways: it +// tokenizes each line on the regular space (U+0020), and it trimStart()s every wrapped row. Because U+00A0 +// (no-break space) is still matched by JS's \s, a plain NBSP indent gets trimmed away on any line with more +// than one whitespace-separated token. So we indent with no-break spaces (never split on) AND prefix each +// indented line with a zero-width space (U+200B) — a non-whitespace char that halts trimStart() before it +// can reach the indentation, so the nesting survives the formatter. Both chars render as no ink / blank +// columns, keeping the schema readable. +const NBSP = '\u00A0'; +const ZWSP = '\u200B'; +const prettySessionSchema = JSON.stringify(sessionViewJsonSchema(), null, 2).replace( + /^ +/gm, + (spaces) => `${ZWSP}${NBSP.repeat(spaces.length)}` +); +const DATA_SCHEMA_HELP = `${messages.getMessage('flags.data.description')}\n\n${prettySessionSchema}`; + +export type AgentScorerRunResult = ScorerResult & { + scorerApiName: string; + /** The version number that was actually run (highest Available by default, or the requested --scorer-version). */ + scorerVersion?: number; +}; + +export default class AgentScorerRun extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + public static readonly requiresProject = true; + + public static readonly envVariablesSection = toHelpSection('ENVIRONMENT VARIABLES', EnvironmentVariable.SF_TARGET_ORG); + + public static readonly flags = { + 'target-org': Flags.requiredOrg(), + 'api-version': Flags.orgApiVersion(), + 'api-name': Flags.string({ + summary: messages.getMessage('flags.api-name.summary'), + required: true, + }), + // eslint-disable-next-line sf-plugin/flag-min-max-default + 'scorer-version': Flags.integer({ + summary: messages.getMessage('flags.scorer-version.summary'), + min: 1, + }), + data: Flags.string({ + summary: messages.getMessage('flags.data.summary'), + description: DATA_SCHEMA_HELP, + exactlyOne: ['data', 'file'], + }), + file: Flags.file({ + summary: messages.getMessage('flags.file.summary'), + exists: true, + exactlyOne: ['data', 'file'], + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentScorerRun); + + const apiName = flags['api-name']; + + // Resolve the scorer from local project metadata by API name — the business logic throws a clear error + // if no scorer with this API name is authored in the project. + const directories = this.project!.getUniquePackageDirectories().map((pkgDir) => pkgDir.fullPath); + const spec = await loadScorerSpec({ apiName, directories, scorerVersion: flags['scorer-version'] }); + + const session = this.parseSession(flags.file ? readFileSync(resolve(flags.file), 'utf8') : flags.data!); + + const connection = flags['target-org'].getConnection(flags['api-version']); + const result = await runScorer(spec, session, connection); + + if (!this.jsonEnabled()) { + this.styledHeader(`Scorer: ${spec.apiName}`); + if (spec.scorerVersion) this.log(`Version: ${spec.scorerVersion}`); + this.log(`Outcome: ${result.ok ? 'ok' : 'error'}`); + if (result.output !== undefined) { + this.log(`Output: ${Array.isArray(result.output) ? result.output.join(', ') : String(result.output)}`); + } + if (result.explanation) this.log(`Explanation: ${result.explanation}`); + } + + // A scorer that didn't produce a valid score (engine failure, or a session the platform rejected) comes back + // with ok:false. Surface it as a command failure — non-zero exit, standard SfError envelope in --json — so a + // scripted loop can't mistake a failed evaluation for a passing one. The full result is attached as error data. + if (!result.ok) { + const error = messages.createError('error.scorerRunFailed', [spec.apiName, result.error ?? 'unknown error']); + error.data = { scorerApiName: spec.apiName, scorerVersion: spec.scorerVersion, ...result }; + throw error; + } + + return { scorerApiName: spec.apiName, scorerVersion: spec.scorerVersion, ...result }; + } + + // eslint-disable-next-line class-methods-use-this + private parseSession(raw: string): SessionView { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (e) { + throw messages.createError('error.invalidSessionJson', [(e as Error).message]); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw messages.createError('error.invalidSessionShape', [typeof parsed]); + } + return parsed as SessionView; + } +} diff --git a/test/commands/agent/scorer/generate-metadata-file.test.ts b/test/commands/agent/scorer/generate-metadata-file.test.ts new file mode 100644 index 00000000..00c53ced --- /dev/null +++ b/test/commands/agent/scorer/generate-metadata-file.test.ts @@ -0,0 +1,1179 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any */ + +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { createRequire } from 'node:module'; +import { mkdirSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { expect } from 'chai'; +import esmock from 'esmock'; +import sinon from 'sinon'; + +// The command reads specs via node:fs, but createScorerDefinition (in @salesforce/agents) writes +// output via node:fs/promises (writeFile/mkdir). node: core modules are singletons, so stubbing +// this shared instance captures the library's real writes without touching disk. Obtained through +// createRequire because ESM namespace objects are frozen and cannot be stubbed. +const fsPromises = createRequire(import.meta.url)('node:fs/promises') as { + writeFile: (...args: any[]) => Promise; + mkdir: (...args: any[]) => Promise; + readFile: (...args: any[]) => Promise; +}; +import YAML from 'yaml'; +import { TestContext, MockTestOrgData } from '@salesforce/core/testSetup'; +import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import * as agentsModule from '@salesforce/agents'; +import type { ScorerSpec } from '@salesforce/agents'; + +function makeLabeledSpec(overrides: Partial = {}): ScorerSpec { + return { + apiName: 'Test_Scorer', + lightningType: 'lightning__textType', + inputScope: 'Session', + label: 'Test Scorer', + description: 'A test scorer', + engineType: 'Manual', + status: 'Draft', + agentAssociation: { + agentApiName: 'My_Agent', + isActive: false, + }, + outputEnumValues: [ + { value: 'Positive', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Negative', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + { value: 'Neutral', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, + ], + ...overrides, + }; +} + +function makeOpenSpec(overrides: Partial = {}): ScorerSpec { + return { + apiName: 'Open_Scorer', + lightningType: 'lightning__textType', + inputScope: 'Session', + label: 'Open Scorer', + engineType: 'PromptTemplate', + status: 'Draft', + agentAssociation: { + agentApiName: 'My_Agent', + isActive: true, + samplingRate: 0.5, + inputScope: 'Intent', + }, + ...overrides, + }; +} + +function makePromptTemplateSpec(overrides: Partial = {}): ScorerSpec { + return { + apiName: 'Prompt_Scorer', + lightningType: 'lightning__textType', + inputScope: 'Session', + label: 'Prompt Scorer', + engineType: 'PromptTemplate', + status: 'Draft', + promptContent: 'Evaluate this session.\n\n{!$Input:Session}', + agentAssociation: { + agentApiName: 'My_Agent', + isActive: true, + samplingRate: 1.0, + }, + outputEnumValues: [ + { value: 'Pass', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Fail', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, + ], + ...overrides, + }; +} + +type WrittenFile = { path: string; content: string }; + +async function loadMockedCommand( + yamlSpec: ScorerSpec, + opts?: { + existsSync?: () => boolean; + confirmResult?: boolean; + } +): Promise<{ Command: any; writtenFiles: WrittenFile[]; createdDirs: string[] }> { + const yamlContent = YAML.stringify(yamlSpec); + const writtenFiles: WrittenFile[] = []; + const createdDirs: string[] = []; + const fileExists = opts?.existsSync ?? (() => false); + + const fsMock: Record = { + readFileSync: () => yamlContent, + existsSync: fileExists, + }; + + const mocks: Record = { 'node:fs': fsMock }; + + // Capture the scorer/prompt-template files written by createScorerDefinition (via + // node:fs/promises) without hitting disk; delegate any unrelated writes to the real fns. + const isScorerOutput = (p: unknown): boolean => + typeof p === 'string' && (p.includes('aiAgentScorerDefinitions') || p.includes('genAiPromptTemplates')); + const origWriteFile = fsPromises.writeFile; + const origMkdir = fsPromises.mkdir; + + sinon.stub(fsPromises, 'writeFile').callsFake((path: unknown, content: unknown, options: unknown) => { + if (isScorerOutput(path)) { + writtenFiles.push({ path: String(path), content: String(content) }); + return Promise.resolve(); + } + return origWriteFile(path, content, options); + }); + sinon.stub(fsPromises, 'mkdir').callsFake((path: unknown, options: unknown) => { + if (isScorerOutput(path)) { + createdDirs.push(String(path)); + return Promise.resolve(undefined); + } + return origMkdir(path, options); + }); + + if (opts?.confirmResult !== undefined) { + mocks['@inquirer/prompts'] = { + confirm: sinon.stub().resolves(opts.confirmResult), + select: sinon.stub().resolves('lightning__textType'), + input: sinon.stub().resolves(''), + }; + } + + const mod = await esmock('../../../../src/commands/agent/scorer/generate-metadata-file.js', mocks); + return { Command: mod.default, writtenFiles, createdDirs }; +} + +// Bare spec filenames used across the --spec tests. `spec: Flags.file({ exists: true })` makes +// oclif stat the path at parse time (via a CJS require of node:fs/promises deep inside +// @oclif/core, which esmock cannot intercept), so these must physically exist on disk. We create +// them in a temp dir and chdir there; the command's readFileSync is still mocked, so file +// contents are irrelevant — only their existence matters. +const SPEC_FILENAMES = [ + 'test.yaml', + 'test-scorer.yaml', + 'open-scorer.yaml', + 'prompt-scorer.yaml', + 'manual-scorer.yaml', +]; + +describe('agent scorer generate-metadata-file', () => { + const $$ = new TestContext(); + let testOrg: MockTestOrgData; + let originalCwd: string; + let specDir: string; + let sfCommandStubs: ReturnType; + + before(async function () { + // Warm up esmock to check it can load the module + try { + await esmock('../../../../src/commands/agent/scorer/generate-metadata-file.js', { + 'node:fs': { + readFileSync: () => '', + writeFileSync: () => {}, + mkdirSync: () => {}, + existsSync: () => false, + }, + }); + } catch (e: any) { + // eslint-disable-next-line no-console + console.error('esmock warmup failed:', e.message); + this.skip(); + } + + originalCwd = process.cwd(); + specDir = mkdtempSync(join(tmpdir(), 'scorer-specs-')); + for (const name of SPEC_FILENAMES) writeFileSync(join(specDir, name), ''); + process.chdir(specDir); + }); + + after(() => { + if (originalCwd) process.chdir(originalCwd); + if (specDir) rmSync(specDir, { recursive: true, force: true }); + }); + + beforeEach(async () => { + sfCommandStubs = stubSfCommandUx($$.SANDBOX); + testOrg = new MockTestOrgData(); + await $$.stubAuths(testOrg); + }); + + afterEach(() => { + $$.restore(); + sinon.restore(); + }); + + describe('--spec flag (YAML-driven) with --preview', () => { + it('should create a labeled scorer from a YAML spec', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.apiName).to.equal('Test_Scorer'); + expect(result.contents).to.include('AiAgentScorerDefinition'); + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('lightning__textType'); + expect(result.contents).to.include('OpenEnded'); + expect(result.contents).to.include('Session'); + expect(result.contents).to.include('Manual'); + expect(result.contents).to.include('Draft'); + expect(result.contents).to.include('My_Agent'); + expect(result.contents).to.include('Positive'); + expect(result.contents).to.include('Negative'); + expect(result.contents).to.include('Neutral'); + }); + + it('should create an OpenEnded (LightningType) scorer', async () => { + const { Command } = await loadMockedCommand(makeOpenSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'open-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.apiName).to.equal('Open_Scorer'); + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('lightning__textType'); + expect(result.contents).to.include('OpenEnded'); + expect(result.contents).to.include('Session'); + }); + + it('should include inputScope in agent association when specified', async () => { + const { Command } = await loadMockedCommand(makeOpenSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'open-scorer.yaml', + '--preview', + '--json', + ]); + + const agentAssocBlock = result.contents.substring( + result.contents.indexOf(''), + result.contents.indexOf('') + ''.length + ); + expect(agentAssocBlock).to.include('Intent'); + }); + + it('should include outputEnumValues when provided', async () => { + const spec = makeOpenSpec({ + outputEnumValues: [ + { value: 'GOOD', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'BAD', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + { value: 'N/A', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, + ], + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'open-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('OpenEnded'); + expect(result.contents).to.include('GOOD'); + expect(result.contents).to.include('BAD'); + expect(result.contents).to.include('N/A'); + expect(result.contents).to.include('Pass'); + expect(result.contents).to.include('Fail'); + expect(result.contents).to.include('NotApplicable'); + expect(result.contents).to.include('true'); + }); + + it('should not include outputEnumValue when none provided', async () => { + const { Command } = await loadMockedCommand(makeOpenSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'open-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).not.to.include(''); + expect(result.contents).not.to.include(''); + }); + + it('should generate prompt template path for PromptTemplate engine', async () => { + const { Command } = await loadMockedCommand(makePromptTemplateSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'prompt-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.promptTemplatePath).to.be.a('string'); + expect(result.promptTemplatePath).to.include('genAiPromptTemplates'); + expect(result.promptTemplatePath).to.include('Prompt_Scorer.genAiPromptTemplate-meta.xml'); + expect(result.contents).to.include('Prompt_Scorer'); + expect(result.contents).to.include('PromptTemplate'); + }); + + it('should not generate prompt template for Manual engine', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec({ engineType: 'Manual' })); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'manual-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.promptTemplatePath).to.be.undefined; + expect(result.contents).not.to.include(''); + expect(result.contents).to.include('Manual'); + }); + + it('should use promptTemplateName as engineRef and skip prompt template file generation', async () => { + const spec = makePromptTemplateSpec({ promptTemplateName: 'My_Existing_Template' }); + const { Command, writtenFiles } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + expect(result.contents).to.include('My_Existing_Template'); + expect(result.contents).to.include('PromptTemplate'); + expect(result.promptTemplatePath).to.be.undefined; + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile).to.be.undefined; + expect(writtenFiles).to.have.length(1); + }); + + it('should omit inputScope from agent association XML when not specified', async () => { + const spec = makeLabeledSpec(); + spec.agentAssociation.inputScope = undefined; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + const agentAssocBlock = result.contents.substring( + result.contents.indexOf(''), + result.contents.indexOf('') + ''.length + ); + expect(agentAssocBlock).to.include('My_Agent'); + expect(agentAssocBlock).not.to.include(''); + }); + + it('should include description when provided', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec({ description: 'Evaluates politeness' })); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('Evaluates politeness'); + }); + + it('should omit description when not provided', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec({ description: undefined })); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).not.to.include(''); + }); + + it('should default samplingRate to 1.0', async () => { + const spec = makeLabeledSpec(); + spec.agentAssociation.samplingRate = undefined; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('1'); + }); + + it('should use custom samplingRate', async () => { + const spec = makeLabeledSpec(); + spec.agentAssociation.samplingRate = 0.25; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('0.25'); + }); + + it('should set versionNumber to 1', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('1'); + }); + }); + + describe('prompt template type', () => { + it('should always use scorerOpenEnded type', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeOpenSpec()); + + await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('agentforce_session_tracing__scorerOpenEnded'); + }); + + it('should use scorerOpenEnded type even when labels are defined', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('agentforce_session_tracing__scorerOpenEnded'); + expect(promptFile!.content).to.include('AllowedLabels'); + expect(promptFile!.content).to.include('FallbackLabel'); + }); + }); + + describe('XML structure', () => { + it('should include XML declaration and namespace', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include(''); + expect(result.contents).to.include('xmlns="http://soap.sforce.com/2006/04/metadata"'); + }); + + it('should include isActive in agent association', async () => { + const spec = makeLabeledSpec(); + spec.agentAssociation.isActive = true; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('true'); + }); + + it('should include isFallback and isSystemFallback', async () => { + const spec = makeLabeledSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, + ], + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('false'); + expect(result.contents).to.include('true'); + expect(result.contents).to.include('false'); + }); + + it('should include label in scorerVersion', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec({ label: 'My Custom Label' })); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include(''); + }); + }); + + describe('file writing', () => { + it('should write scorer XML to correct path', async () => { + const { Command, writtenFiles, createdDirs } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + expect(result.path).to.include('/tmp/out'); + expect(result.path).to.include('aiAgentScorerDefinitions'); + expect(result.path).to.include('Test_Scorer.aiAgentScorerDefinition-meta.xml'); + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('AiAgentScorerDefinition'); + expect(createdDirs.some((d) => d.includes('aiAgentScorerDefinitions'))).to.be.true; + }); + + it('should write both scorer and prompt template for PromptTemplate', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + expect(writtenFiles).to.have.length(2); + const scorerFile = writtenFiles.find((f) => f.path.includes('aiAgentScorerDefinitions')); + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(scorerFile).to.not.be.undefined; + expect(promptFile).to.not.be.undefined; + expect(promptFile!.path).to.include('Prompt_Scorer.genAiPromptTemplate-meta.xml'); + expect(promptFile!.content).to.include('GenAiPromptTemplate'); + expect(result.promptTemplatePath).to.equal(promptFile!.path); + }); + + it('should not write files with --preview', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec()); + + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml', '--preview', '--json']); + + expect(writtenFiles).to.have.length(0); + }); + + it('should use default prompt content when promptContent not in spec', async () => { + const spec = makePromptTemplateSpec(); + delete (spec as any).promptContent; + const { Command, writtenFiles } = await loadMockedCommand(spec); + + await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('{!$Input:Session}'); + expect(promptFile!.content).to.include('{!$Input:AllowedLabels}'); + expect(promptFile!.content).to.include('{!$Input:FallbackLabel}'); + }); + + it('should omit label guidance from default prompt when no labels are defined', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeOpenSpec()); + + await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('{!$Input:Session}'); + expect(promptFile!.content).not.to.include('{!$Input:AllowedLabels}'); + }); + }); + + describe('output directory', () => { + it('should default to force-app/main/default', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.path).to.include('force-app/main/default/aiAgentScorerDefinitions'); + }); + + it('should use custom --output-dir', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/custom/path', + '--preview', + '--json', + ]); + + expect(result.path).to.include('/custom/path/aiAgentScorerDefinitions'); + }); + }); + + describe('existing scorer behavior', () => { + // `create` is a one-shot scaffolder: once the XML exists it is the source of truth, so a re-run must not + // overwrite it or silently mutate it. It errors and points the user at editing the metadata XML directly. + it('errors and writes nothing when the scorer already exists', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec(), { + existsSync: () => true, + }); + + try { + await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('already exists'); + // Directs the user to hand-edit the metadata XML rather than re-scaffolding via the CLI. + expect((err as Error).message).to.include('metadata XML'); + } + expect(writtenFiles).to.have.length(0); + }); + + // `this.log` is suppressed under --json, so an agent caller only sees the returned payload. The + // scaffold-once guidance rides along in `result.guidance` so it still reaches that caller. + it('returns scaffold-once guidance in the --json result on a fresh create', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + expect(result.guidance).to.be.a('string'); + expect(result.guidance).to.include('source of truth'); + }); + }); + + describe('prompt template XML details', () => { + it('should include developerName and masterLabel matching apiName', async () => { + const { Command, writtenFiles } = await loadMockedCommand( + makePromptTemplateSpec({ apiName: 'My_Prompt_Scorer' }) + ); + + await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('My_Prompt_Scorer'); + expect(promptFile!.content).to.include('My_Prompt_Scorer'); + }); + + it('should set overridable to false and visibility to Global', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('false'); + expect(promptFile!.content).to.include('Global'); + }); + + it('should set primaryModel and status Published', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('sfdc_ai__DefaultOpenAIGPT4OmniMini'); + expect(promptFile!.content).to.include('Published'); + }); + + it('should include activeVersionIdentifier and versionIdentifier', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include(''); + expect(promptFile!.content).to.include(''); + // Both should have the same value + const activeMatch = promptFile!.content.match(/(.+?)<\/activeVersionIdentifier>/); + const versionMatch = promptFile!.content.match(/(.+?)<\/versionIdentifier>/); + expect(activeMatch![1]).to.equal(versionMatch![1]); + expect(activeMatch![1]).to.match(/.+=_1$/); + }); + + it('should include Session input with correct definition', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include( + 'lightningtype://propertyType/agentforce_session_tracing__stdmDetailViewType' + ); + expect(promptFile!.content).to.include('Input:Session'); + }); + }); + + describe('--json mode error handling', () => { + it('should throw when required flags are missing', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + try { + await Command.run(['--target-org', testOrg.username, '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string }; + expect(error.message).to.include('Missing required flags'); + } + }); + + it('should list all missing required flags', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + try { + await Command.run(['--target-org', testOrg.username, '--label', 'Foo', '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string }; + expect(error.message).to.include('api-name'); + expect(error.message).to.include('lightning-type'); + expect(error.message).to.include('engine-type'); + expect(error.message).to.include('agent-api-name'); + } + }); + }); + + describe('output label validation', () => { + let tmpDir: string; + let specFile: string; + + beforeEach(() => { + tmpDir = join(process.cwd(), 'tmp-test-fallback-' + Date.now()); + mkdirSync(tmpDir, { recursive: true }); + specFile = join(tmpDir, 'scorer.yaml'); + }); + + afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('should throw when more than one output value is the fallback', async () => { + const spec = makeLabeledSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: true, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, + ], + }); + writeFileSync(specFile, YAML.stringify(spec)); + const { Command } = await loadMockedCommand(spec); + + try { + await Command.run(['--target-org', testOrg.username, '--spec', specFile, '--preview', '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string }; + expect(error.message).to.include('At most one outputEnumValue can be the fallback'); + expect(error.message).to.include('found 2'); + } + }); + + it('should pass with zero fallback values', async () => { + const spec = makeLabeledSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + ], + }); + writeFileSync(specFile, YAML.stringify(spec)); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run(['--target-org', testOrg.username, '--spec', specFile, '--preview', '--json']); + + expect(result.apiName).to.equal('Test_Scorer'); + expect(result.contents).to.include('Good'); + }); + + it('should pass with exactly one fallback value', async () => { + const spec = makeLabeledSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + { value: 'N/A', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, + ], + }); + writeFileSync(specFile, YAML.stringify(spec)); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run(['--target-org', testOrg.username, '--spec', specFile, '--preview', '--json']); + + expect(result.apiName).to.equal('Test_Scorer'); + expect(result.contents).to.include('N/A'); + }); + }); + + describe('edge cases', () => { + it('should handle LightningType with no outputEnumValues', async () => { + const spec: ScorerSpec = { + apiName: 'Lightning_Scorer', + lightningType: 'lightning__numberType', + inputScope: 'Session', + label: 'Lightning Scorer', + engineType: 'Manual', + agentAssociation: { agentApiName: 'Agent_X', isActive: false }, + }; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('lightning__numberType'); + }); + + it('should handle single output enum value', async () => { + const spec = makeLabeledSpec({ + outputEnumValues: [{ value: 'Only', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }], + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('Only'); + expect(result.contents).to.include('NotApplicable'); + expect(result.contents).to.include('true'); + }); + }); + + describe('--json without --spec (flag-only path)', () => { + it('completes without prompting when all required flags are supplied', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', + testOrg.username, + '--label', + 'My Scorer', + '--api-name', + 'My_Scorer', + '--lightning-type', + 'lightning__textType', + '--engine-type', + 'Manual', + '--agent-api-name', + 'My_Agent', + '--output-dir', + '/tmp/out-json-flags', + '--json', + ]); + + expect(result.apiName).to.equal('My_Scorer'); + // The status flag's default ('Draft') must be honored without prompting. + expect(result.contents).to.include('Draft'); + expect(writtenFiles).to.have.length(1); + }); + }); + + describe('--spec YAML parsing', () => { + it('throws a clear error when the spec file is not valid YAML', async () => { + const mod = await esmock('../../../../src/commands/agent/scorer/generate-metadata-file.js', { + 'node:fs': { readFileSync: () => 'foo: [1, 2', existsSync: () => false }, + }); + const Command = mod.default; + + try { + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml']); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('Could not parse the --spec file as YAML'); + } + }); + + it('throws a clear error when the spec file is not a YAML object', async () => { + const mod = await esmock('../../../../src/commands/agent/scorer/generate-metadata-file.js', { + 'node:fs': { readFileSync: () => '- 1\n- 2\n', existsSync: () => false }, + }); + const Command = mod.default; + + try { + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml']); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('must define a YAML object'); + } + }); + }); + + describe('--spec validation errors', () => { + // The --spec path hands the parsed YAML to createScorerDefinition → validateScorerSpec. These assert the + // command surfaces a clear, actionable message rather than a raw TypeError / silently-broken metadata. + it('surfaces a clear error (not a TypeError) when agentAssociation is missing', async () => { + const spec = makeLabeledSpec(); + delete (spec as any).agentAssociation; + const { Command, writtenFiles } = await loadMockedCommand(spec); + + try { + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml', '--preview', '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + const message = (err as Error).message; + expect(message).to.include('agentAssociation is required.'); + expect(message).to.not.match(/cannot read properties of undefined/i); + } + expect(writtenFiles).to.have.length(0); + }); + + it('surfaces a clear error when engineType is not a supported engine', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec({ engineType: 'prompttemplate' as any })); + + try { + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml', '--preview', '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include("Unsupported engineType 'prompttemplate'."); + } + }); + }); + + describe('--spec-schema', () => { + it('prints the schema JSON and skips org/spec resolution', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + // Deliberately omit --spec and every other required flag: --spec-schema must short-circuit + // before the required-flags gate or spec/org resolution ever runs. + const result = await Command.run(['--target-org', testOrg.username, '--spec-schema']); + + expect(result).to.deep.equal({ path: '', apiName: '', contents: '' }); + expect(sfCommandStubs.styledJSON.calledOnce).to.be.true; + const printed = sfCommandStubs.styledJSON.firstCall.args[0] as Record; + expect(printed).to.have.property('$ref', '#/definitions/ScorerSpec'); + expect(printed).to.have.property('definitions'); + }); + + it('returns the schema as the --json result instead of the empty payload', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + // Under --json, styledJSON is suppressed, so the schema must be the command result — it then rides in + // the standard {status, result} envelope rather than being dropped for the machine consumer --json serves. + const result = await Command.run(['--target-org', testOrg.username, '--spec-schema', '--json']); + + expect(result).to.have.property('$ref', '#/definitions/ScorerSpec'); + expect(result).to.have.property('definitions'); + expect(sfCommandStubs.styledJSON.called).to.be.false; + }); + }); + + describe('interactive interview (no --spec, no --json)', () => { + it('throws a clear error when the org has no agents to associate', async () => { + const mocks: Record = { + 'node:fs': { readFileSync: () => '', existsSync: () => false }, + '@inquirer/prompts': { + input: sinon.stub().resolves(''), + confirm: sinon.stub().resolves(false), + select: sinon.stub().resolves(''), + }, + '@salesforce/agents': { + ...agentsModule, + Agent: { listRemote: sinon.stub().resolves([]) }, + }, + }; + const mod = await esmock('../../../../src/commands/agent/scorer/generate-metadata-file.js', mocks); + const Command = mod.default; + + try { + // Supply every FLAGGABLE_PROMPTS-backed flag (including --description and --status, which + // are otherwise resolved via promptForFlag() in ../../../flags.js — a module esmock does not + // remock here, so any prompt routed through it would hit the real @inquirer/prompts and hang). + await Command.run([ + '--target-org', + testOrg.username, + '--label', + 'My Scorer', + '--api-name', + 'My_Scorer', + '--lightning-type', + 'lightning__textType', + '--engine-type', + 'Manual', + '--description', + 'A test description', + '--status', + 'Draft', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('No agents found in the org'); + } + }); + }); +}); diff --git a/test/commands/agent/scorer/run.test.ts b/test/commands/agent/scorer/run.test.ts new file mode 100644 index 00000000..86db38ca --- /dev/null +++ b/test/commands/agent/scorer/run.test.ts @@ -0,0 +1,471 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any */ + +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { expect } from 'chai'; +import esmock from 'esmock'; +import sinon from 'sinon'; +import { TestContext, MockTestOrgData } from '@salesforce/core/testSetup'; +import { SfProject } from '@salesforce/core'; +import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; + +const SPEC = { + apiName: 'Sentiment_Scorer', + lightningType: 'lightning__textType', + label: 'Sentiment Scorer', + engineType: 'PromptTemplate', + agentAssociation: { agentApiName: 'My_Agent', isActive: true }, +}; + +const SESSION = { + sessionState: { sessionId: 'S1', startTimestamp: '2026-01-01T00:00:00Z', channel: 'web' }, + actors: [], + metrics: { durationMs: 0, turns: 0 }, + runs: [], +}; + +const SESSION_FILE = 'session.json'; + +async function loadMockedCommand(opts?: { + runScorerResult?: any; + runScorerError?: Error; + loadScorerSpecError?: Error; + specOverride?: any; + schema?: unknown; +}): Promise<{ Command: any; runScorer: sinon.SinonStub; loadScorerSpec: sinon.SinonStub }> { + const runScorer = sinon.stub(); + if (opts?.runScorerError) runScorer.rejects(opts.runScorerError); + else runScorer.resolves(opts?.runScorerResult ?? { ok: true, output: 'Positive', explanation: 'Looks good.' }); + + const loadScorerSpec = sinon.stub(); + if (opts?.loadScorerSpecError) loadScorerSpec.rejects(opts.loadScorerSpecError); + else loadScorerSpec.resolves(opts?.specOverride ?? SPEC); + + const readFileSync = (path: unknown): string => { + const p = String(path); + if (p.endsWith('.json')) return JSON.stringify(SESSION); + return ''; + }; + + const mocks: Record = { + 'node:fs': { readFileSync }, + '@salesforce/agents': { + runScorer, + loadScorerSpec, + sessionViewJsonSchema: () => opts?.schema ?? { $schema: 'http://json-schema.org/draft-07/schema#' }, + }, + }; + + const mod = await esmock('../../../../src/commands/agent/scorer/run.js', mocks); + return { Command: mod.default, runScorer, loadScorerSpec }; +} + +describe('agent scorer run', () => { + const $$ = new TestContext(); + let testOrg: MockTestOrgData; + let originalCwd: string; + let workDir: string; + let sfCommandStubs: ReturnType; + + before(async function () { + try { + await esmock('../../../../src/commands/agent/scorer/run.js', { + 'node:fs': { readFileSync: () => '' }, + '@salesforce/agents': { + runScorer: () => Promise.resolve({ ok: true }), + loadScorerSpec: () => Promise.resolve(SPEC), + sessionViewJsonSchema: () => ({}), + }, + }); + } catch (e: any) { + // eslint-disable-next-line no-console + console.error('esmock warmup failed:', e.message); + this.skip(); + } + + originalCwd = process.cwd(); + workDir = mkdtempSync(join(tmpdir(), 'scorer-run-')); + // Flags.file({ exists: true }) stats this at parse time, so it must exist on disk. + writeFileSync(join(workDir, SESSION_FILE), ''); + process.chdir(workDir); + }); + + after(() => { + if (originalCwd) process.chdir(originalCwd); + if (workDir) rmSync(workDir, { recursive: true, force: true }); + }); + + beforeEach(async () => { + sfCommandStubs = stubSfCommandUx($$.SANDBOX); + testOrg = new MockTestOrgData(); + await $$.stubAuths(testOrg); + + // requiresProject: stub the project so package directories resolve without a real sfdx-project.json. + $$.inProject(true); + const mockProject = { + getPath: () => workDir, + getUniquePackageDirectories: () => [{ fullPath: join(workDir, 'force-app') }], + } as unknown as SfProject; + $$.SANDBOX.stub(SfProject, 'resolve').resolves(mockProject); + $$.SANDBOX.stub(SfProject, 'getInstance').returns(mockProject); + }); + + afterEach(() => { + $$.restore(); + sinon.restore(); + }); + + it('resolves the scorer by API name and runs it against a session file', async () => { + const { Command, runScorer, loadScorerSpec } = await loadMockedCommand(); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.scorerApiName).to.equal('Sentiment_Scorer'); + expect(result.ok).to.equal(true); + expect(result.output).to.equal('Positive'); + expect(result.explanation).to.equal('Looks good.'); + + expect(loadScorerSpec.calledOnce).to.be.true; + expect(loadScorerSpec.firstCall.firstArg.apiName).to.equal('Sentiment_Scorer'); + expect(loadScorerSpec.firstCall.firstArg.directories).to.be.an('array').that.is.not.empty; + + expect(runScorer.calledOnce).to.be.true; + const [passedSpec, passedSession] = runScorer.firstCall.args; + expect(passedSpec.apiName).to.equal('Sentiment_Scorer'); + expect(passedSession.sessionState.sessionId).to.equal('S1'); + }); + + it('plumbs --scorer-version through to loadScorerSpec', async () => { + const { Command, loadScorerSpec } = await loadMockedCommand(); + + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--scorer-version', '3', + '--file', SESSION_FILE, + '--json', + ]); + + expect(loadScorerSpec.calledOnce).to.be.true; + expect(loadScorerSpec.firstCall.firstArg.scorerVersion).to.equal(3); + }); + + it('leaves scorerVersion undefined when --scorer-version is omitted', async () => { + const { Command, loadScorerSpec } = await loadMockedCommand(); + + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(loadScorerSpec.firstCall.firstArg.scorerVersion).to.be.undefined; + }); + + describe('resolved-version reporting', () => { + it('reports the resolved scorerVersion in the JSON result', async () => { + const { Command } = await loadMockedCommand({ specOverride: { ...SPEC, scorerVersion: 2 } }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.scorerVersion).to.equal(2); + }); + + it('prints the resolved version in human-readable output', async () => { + const { Command } = await loadMockedCommand({ specOverride: { ...SPEC, scorerVersion: 2 } }); + + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Version: 2'); + }); + + it('omits scorerVersion (and the Version line) when the spec has no resolved version', async () => { + const { Command } = await loadMockedCommand(); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.scorerVersion).to.be.undefined; + }); + + it('includes the resolved scorerVersion in error.data on a failed result', async () => { + const { Command } = await loadMockedCommand({ + specOverride: { ...SPEC, scorerVersion: 2 }, + runScorerResult: { ok: false, error: 'no engine for Manual' }, + }); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { data?: any }; + expect(error.data?.scorerVersion).to.equal(2); + } + }); + }); + + it('runs a scorer against inline session JSON', async () => { + const { Command, runScorer } = await loadMockedCommand(); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', JSON.stringify(SESSION), + '--json', + ]); + + expect(result.scorerApiName).to.equal('Sentiment_Scorer'); + expect(result.ok).to.equal(true); + const [, passedSession] = runScorer.firstCall.args; + expect(passedSession.sessionState.sessionId).to.equal('S1'); + }); + + it('passes the connection from the target org to runScorer', async () => { + const { Command, runScorer } = await loadMockedCommand(); + + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + const [, , connection] = runScorer.firstCall.args; + expect(connection).to.not.be.undefined; + }); + + it('exits non-zero (throws) on an engine error result, with the result attached as error data', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: false, error: 'no engine for Manual' }, + }); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string; data?: any }; + expect(error.message).to.include('did not produce a valid score'); + expect(error.message).to.include('no engine for Manual'); + // The full result is preserved for scripted consumers on the error's data field. + expect(error.data?.ok).to.equal(false); + expect(error.data?.scorerApiName).to.equal('Sentiment_Scorer'); + } + }); + + it('surfaces the error when the scorer is not found in the project', async () => { + const { Command } = await loadMockedCommand({ + loadScorerSpecError: new Error("No scorer named 'Missing_Scorer' was found in this project."), + }); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Missing_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('was found in this project'); + } + }); + + it('throws when neither --data nor --file is provided', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message.toLowerCase()).to.match(/data|file/); + } + }); + + it('throws when both --data and --file are provided', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', JSON.stringify(SESSION), + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message.toLowerCase()).to.match(/data|file/); + } + }); + + it('throws a clear error when the session is not valid JSON', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', '{not valid json', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('Could not parse the STDM session as JSON'); + } + }); + + it('throws a clear error when the session JSON is not an object', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', '["not", "an", "object"]', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('must be a JSON object'); + } + }); + + describe('human-readable output (non --json)', () => { + it('renders an array output joined by commas', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: ['A', 'B'] }, + }); + + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Output: A, B'); + }); + + it('renders a numeric output', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: 42 }, + }); + + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Output: 42'); + }); + + it('renders the outcome/output/explanation lines before throwing on a failed result', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: false, output: 'Negative', explanation: 'Tone was hostile.', error: 'no engine for Manual' }, + }); + + try { + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('no engine for Manual'); + } + + // The human-readable summary is still printed before the command fails. + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Outcome: error'); + expect(logLines).to.include('Output: Negative'); + expect(logLines).to.include('Explanation: Tone was hostile.'); + }); + }); + + describe('--json with array/number output', () => { + it('returns array output as-is in the JSON result', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: ['A', 'B'] }, + }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.output).to.deep.equal(['A', 'B']); + }); + + it('returns numeric output as-is in the JSON result', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: 7 }, + }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.output).to.equal(7); + }); + }); + + describe('--data schema help formatting', () => { + it('preserves JSON nesting through the ZWSP/NBSP indent markers', async () => { + const schema = { type: 'object', properties: { foo: { type: 'string' }, bar: { type: 'number' } } }; + const { Command } = await loadMockedCommand({ schema }); + + const description = Command.flags.data.description as string; + const separatorIndex = description.indexOf('\n\n'); + expect(separatorIndex).to.be.greaterThan(-1); + const jsonPart = description.slice(separatorIndex + 2); + + // Strip the zero-width-space + no-break-space indent markers (see run.ts) before parsing. + const stripped = jsonPart.replace(/\u200B/g, '').replace(/\u00A0/g, ' '); + const parsed = JSON.parse(stripped); + expect(parsed).to.deep.equal(schema); + }); + }); +}); diff --git a/test/nuts/agent.scorer.generate-metadata-file.nut.ts b/test/nuts/agent.scorer.generate-metadata-file.nut.ts new file mode 100644 index 00000000..539cb563 --- /dev/null +++ b/test/nuts/agent.scorer.generate-metadata-file.nut.ts @@ -0,0 +1,105 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { join } from 'node:path'; +import { writeFileSync, readFileSync, existsSync } from 'node:fs'; +import { expect } from 'chai'; +import { TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; +import { parseScorerVersions } from '@salesforce/agents'; +import type { AgentScorerGenerateMetadataFileResult } from '../../src/commands/agent/scorer/generate-metadata-file.js'; + +// `agent scorer generate-metadata-file` writes local metadata only, but its `--target-org` is a required flag that resolves at +// parse time, so this NUT needs a default org. It uses a lightweight scratch org (devhub only — no Einstein +// provisioning or metadata deploy, which create doesn't need) rather than the shared heavyweight session. +describe('agent scorer generate-metadata-file NUTs', function () { + this.timeout(15 * 60 * 1000); + + const API_NAME = 'Nut_Create_Scorer'; + // A `Manual`-engine scorer needs no prompt template, so create writes a single self-contained definition file. + const makeSpec = (apiName: string, label: string): Record => ({ + apiName, + lightningType: 'lightning__textType', + inputScope: 'Session', + label, + engineType: 'Manual', + status: 'Draft', + agentAssociation: { agentApiName: 'My_Agent', isActive: false }, + }); + + let session: TestSession; + let outputDir: string; + let specPath: string; + let scorerPath: string; + + before(async () => { + session = await TestSession.create({ + project: { name: 'scorerGenerateMetadataFileNut' }, + devhubAuthStrategy: 'AUTO', + scratchOrgs: [{ setDefault: true, config: join('config', 'project-scratch-def.json') }], + }); + outputDir = join(session.project.dir, 'scorer-out'); + specPath = join(session.project.dir, 'nut-create-spec.json'); + scorerPath = join(outputDir, 'aiAgentScorerDefinitions', `${API_NAME}.aiAgentScorerDefinition-meta.xml`); + writeFileSync(specPath, JSON.stringify(makeSpec(API_NAME, 'NUT Create Scorer'))); + }); + + after(async () => { + await session?.clean(); + }); + + it('prints the spec JSON Schema with --spec-schema', () => { + const { stdout } = execCmd('agent scorer generate-metadata-file --spec-schema', { ensureExitCode: 0 }).shellOutput; + expect(stdout).to.include('ScorerSpec'); + expect(stdout).to.include('apiName'); + }); + + it('authors a scorer definition from a --spec file', () => { + const result = execCmd( + `agent scorer generate-metadata-file --spec "${specPath}" --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ).jsonOutput?.result; + + expect(result?.apiName).to.equal(API_NAME); + expect(result?.path).to.equal(scorerPath); + expect(existsSync(scorerPath)).to.equal(true); + expect(parseScorerVersions(readFileSync(scorerPath, 'utf8'))).to.have.length(1); + }); + + it('refuses to overwrite an existing scorer (scaffold-once; edit the XML directly instead)', () => { + const output = execCmd( + `agent scorer generate-metadata-file --spec "${specPath}" --output-dir "${outputDir}" --json`, + { ensureExitCode: 1 } + ).jsonOutput; + + expect(output?.message).to.match(new RegExp(API_NAME)); + // the existing file is left untouched (still the single scaffolded version) + expect(parseScorerVersions(readFileSync(scorerPath, 'utf8'))).to.have.length(1); + }); + + it('writes nothing with --preview', () => { + const previewName = 'Nut_Preview_Scorer'; + const previewSpec = join(session.project.dir, 'nut-preview-spec.json'); + writeFileSync(previewSpec, JSON.stringify(makeSpec(previewName, 'NUT Preview Scorer'))); + const previewPath = join(outputDir, 'aiAgentScorerDefinitions', `${previewName}.aiAgentScorerDefinition-meta.xml`); + + const result = execCmd( + `agent scorer generate-metadata-file --spec "${previewSpec}" --output-dir "${outputDir}" --preview --json`, + { ensureExitCode: 0 } + ).jsonOutput?.result; + + expect(result?.contents).to.include('AiAgentScorerDefinition'); + expect(existsSync(previewPath)).to.equal(false); + }); +});