From bba0a029a1866435184320e0e80024a79e8b4c6d Mon Sep 17 00:00:00 2001 From: nabilnaffar-sf Date: Wed, 1 Jul 2026 19:33:43 +0300 Subject: [PATCH 01/19] feat: add `sf agent scorer create` command for interactive scorer definition authoring --- messages/agent.scorer.create.md | 79 ++ package.json | 4 + src/commands/agent/scorer/create.ts | 744 +++++++++++++++++ test/commands/agent/scorer/create.test.ts | 973 ++++++++++++++++++++++ 4 files changed, 1800 insertions(+) create mode 100644 messages/agent.scorer.create.md create mode 100644 src/commands/agent/scorer/create.ts create mode 100644 test/commands/agent/scorer/create.test.ts diff --git a/messages/agent.scorer.create.md b/messages/agent.scorer.create.md new file mode 100644 index 00000000..c97fe685 --- /dev/null +++ b/messages/agent.scorer.create.md @@ -0,0 +1,79 @@ +# summary + +Create an agent scorer definition using an interactive interview or a spec file. + +# description + +Creates an AiAgentScorerDefinition metadata XML file either interactively (prompting for each field) or from a YAML spec file. + +Run with no flags to start the interactive interview. The command prompts you for the scorer's data type, input scope, engine type, output values, and agent associations. + +Alternatively, provide a --spec flag pointing to a YAML file that defines the scorer. This is useful for repeatable automation or when the scorer has many output values. + +Use --preview to see the generated XML without writing it to disk. + +# flags.api-name.summary + +API name of the scorer definition. + +# flags.agent-api-name.summary + +API name of the agent to associate with this scorer. + +# flags.data-type.summary + +Data type produced by the scorer (Text, Number, or OpenEnded). + +# flags.label.summary + +Display label for the scorer version. + +# flags.description.summary + +Description of what this scorer evaluates. + +# flags.engine-type.summary + +Engine type for scoring (Manual or PromptTemplate). + +# flags.status.summary + +Initial status of the scorer version (Available or Draft). + +# flags.spec.summary + +Path to a scorer spec YAML file. Bypasses interactive prompts. + +# flags.output-dir.summary + +Output directory for the generated metadata XML files (scorer definition and prompt template). + +# flags.preview.summary + +Preview the generated XML without writing to disk. + +# examples + +- Create a scorer interactively: + + <%= config.bin %> <%= command.id %> + +- Create a scorer from a spec file: + + <%= config.bin %> <%= command.id %> --spec specs/expert-analysis-scorer.yaml + +- Preview the XML that would be generated: + + <%= config.bin %> <%= command.id %> --spec specs/expert-analysis-scorer.yaml --preview + +- Create a manual scorer with flags (non-interactive): + + <%= config.bin %> <%= command.id %> --api-name Expert_Analysis --data-type Text --engine-type Manual --label Expert_Analysis --agent-api-name My_Agent --status Available + +- Create a prompt-based scorer (generates both scorer definition and prompt template): + + <%= config.bin %> <%= command.id %> --api-name sentiment_analysis --data-type Text --engine-type PromptTemplate --label sentiment_analysis --agent-api-name My_Agent + +# error.missingRequiredFlags + +Missing required flags: %s. When using --json, all required flags must be provided. diff --git a/package.json b/package.json index 17160dc7..9ec907eb 100644 --- a/package.json +++ b/package.json @@ -95,6 +95,10 @@ "description": "Command to validate an Agent Script file.", "external": true }, + "scorer": { + "description": "Commands to create and manage agent scorers.", + "external": true + }, "adl": { "description": "Commands to manage Agentforce Data Libraries.", "external": true, diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts new file mode 100644 index 00000000..c7079282 --- /dev/null +++ b/src/commands/agent/scorer/create.ts @@ -0,0 +1,744 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { join, resolve } from 'node:path'; +import { mkdirSync, readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, EnvironmentVariable } from '@salesforce/core'; +import { Agent } from '@salesforce/agents'; +import { XMLBuilder } from 'fast-xml-parser'; +import { confirm, select, input as inquirerInput } from '@inquirer/prompts'; +import YAML from 'yaml'; +import { FlaggablePrompt, makeFlags, promptForFlag } from '../../../flags.js'; +import { theme } from '../../../inquirer-theme.js'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.scorer.create'); + +export type AgentScorerCreateResult = { + path: string; + apiName: string; + contents: string; + promptTemplatePath?: string; +}; + +export type ScorerSpecFile = { + apiName: string; + dataType: 'Text' | 'Number' | 'LightningType'; + scorerType?: 'Predefined' | 'OpenEnded'; + lightningType?: string; + semanticType?: 'Dimension' | 'Measurement'; + inputScope?: 'Session' | 'Intent'; + label: string; + description?: string; + engineType: 'Manual' | 'PromptTemplate'; + promptContent?: string; + promptTemplateName?: string; + status?: 'Available' | 'Draft'; + agentAssociation: { + agentApiName: string; + isActive: boolean; + samplingRate?: number; + inputScope?: 'Session' | 'Intent'; + }; + outputEnumValues?: Array<{ + value: string; + outcomeType: 'Pass' | 'Fail' | 'NotApplicable'; + isFallback?: boolean; + isSystemFallback?: boolean; + }>; + specification?: { + valueSpecification: { + min: number; + max: number; + step: number; + threshold?: number; + }; + }; +}; + +const MAX_ENUM_VALUES = 101; + +const SUPPORTED_LIGHTNING_TYPES = [ + 'lightning__textType', + 'lightning__multilineTextType', + 'lightning__richTextType', + 'lightning__numberType', + 'lightning__integerType', + 'lightning__booleanType', + 'lightning__dateType', + 'lightning__dateTimeType', + 'lightning__dateTimeStringType', + 'lightning__urlType', + 'lightning__objectType', + 'lightning__listType', +]; + +const FLAGGABLE_PROMPTS = { + label: { + message: messages.getMessage('flags.label.summary'), + promptMessage: 'Scorer label (display name)', + validate: (d: string): boolean | string => d.length > 0 || 'Label cannot be empty', + required: true, + }, + 'api-name': { + message: messages.getMessage('flags.api-name.summary'), + promptMessage: 'Scorer API name', + validate: (d: string): boolean | string => { + if (!d.length) return 'API name cannot be empty'; + if (d.length > 35) return 'API name cannot exceed 35 characters'; + if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(d)) return 'Must start with letter, only alphanumerics and underscores'; + return true; + }, + required: true, + }, + 'data-type': { + message: messages.getMessage('flags.data-type.summary'), + promptMessage: 'What data type does this scorer produce?', + options: ['Text', 'Number', 'OpenEnded'], + validate: (d: string): boolean | string => ['Text', 'Number', 'OpenEnded'].includes(d) || 'Invalid data type', + required: true, + }, + description: { + message: messages.getMessage('flags.description.summary'), + promptMessage: 'Description (optional, press Enter to skip)', + validate: (): boolean | string => true, + }, + 'engine-type': { + message: messages.getMessage('flags.engine-type.summary'), + promptMessage: 'Scoring engine type', + options: ['Manual', 'PromptTemplate'], + validate: (d: string): boolean | string => + ['Manual', 'PromptTemplate'].includes(d) || 'Invalid engine type', + required: true, + }, + status: { + message: messages.getMessage('flags.status.summary'), + promptMessage: 'Initial status', + options: ['Draft', 'Available'], + validate: (d: string): boolean | string => ['Available', 'Draft'].includes(d) || 'Invalid status', + default: 'Draft', + }, +} satisfies Record; + +type OutputEnumValue = { + value: string; + outcomeType: string; + isFallback: boolean; + isSystemFallback: boolean; +}; + +async function promptForOutputEnumValues(): Promise { + const values: OutputEnumValue[] = []; + let addMore = true; + + while (addMore) { + const value = await promptForFlag({ + message: 'Output value name', + promptMessage: `Output value #${values.length + 1} (e.g., "Good", "Bad", "N/A")`, + validate: (d: string): boolean | string => d.length > 0 || 'Value cannot be empty', + }); + + const outcomeType = await promptForFlag({ + message: 'Outcome type', + promptMessage: 'Outcome type for this value', + options: ['Pass', 'Fail', 'NotApplicable'], + validate: (d: string): boolean | string => + ['Pass', 'Fail', 'NotApplicable'].includes(d) || 'Invalid', + }); + + const isFallback = await confirm({ + message: 'Is this the fallback value?', + default: values.length === 0, + theme, + }); + + values.push({ + value, + outcomeType, + isFallback, + isSystemFallback: false, + }); + + addMore = await confirm({ + message: 'Add another output value?', + default: values.length < 2, + theme, + }); + } + + return values; +} + +type NumberSpecification = { + min: number; + max: number; + step: number; + threshold?: number; +}; + +async function promptForNumberSpecification(): Promise { + const minStr = await inquirerInput({ + message: 'Minimum value', + default: '0', + validate: (d: string): boolean | string => !isNaN(parseFloat(d)) || 'Must be a number', + theme, + }); + + const maxStr = await inquirerInput({ + message: 'Maximum value', + default: '5', + validate: (d: string): boolean | string => !isNaN(parseFloat(d)) || 'Must be a number', + theme, + }); + + const min = parseFloat(minStr); + const max = parseFloat(maxStr); + + if (min >= max) { + throw new Error(`Minimum value (${min}) must be less than maximum value (${max})`); + } + + const stepStr = await inquirerInput({ + message: 'Step size', + default: '1', + validate: (d: string): boolean | string => { + const n = parseFloat(d); + if (isNaN(n) || n <= 0) return 'Step must be a positive number'; + const numValues = Math.floor((max - min) / n) + 1; + if (numValues > MAX_ENUM_VALUES) return `Step too small: would generate ${numValues} values (max ${MAX_ENUM_VALUES})`; + return true; + }, + theme, + }); + + const step = parseFloat(stepStr); + const numValues = Math.floor((max - min) / step) + 1; + + const addThreshold = await confirm({ + message: `Add a threshold value? (${numValues} output values will be generated from ${min} to ${max})`, + default: false, + theme, + }); + + let threshold: number | undefined; + if (addThreshold) { + const thresholdStr = await inquirerInput({ + message: `Threshold (must be between ${min} and ${max})`, + validate: (d: string): boolean | string => { + const n = parseFloat(d); + if (isNaN(n)) return 'Must be a number'; + if (n < min || n > max) return `Must be between ${min} and ${max}`; + return true; + }, + theme, + }); + threshold = parseFloat(thresholdStr); + } + + return { min, max, step, threshold }; +} + +function generateNumberEnumValues(spec: NumberSpecification): OutputEnumValue[] { + const values: OutputEnumValue[] = []; + const epsilon = 1e-9; + let current = spec.min; + + while (current <= spec.max + epsilon) { + const rounded = Math.round(current * 1e9) / 1e9; + values.push({ + value: String(rounded), + outcomeType: 'NotApplicable', + isFallback: false, + isSystemFallback: false, + }); + current += spec.step; + } + + return values; +} + +type AgentAssociation = { + agentApiName: string; + isActive: boolean; + samplingRate?: number; + inputScope?: 'Session' | 'Intent'; +}; + +function buildScorerXml(spec: ScorerSpecFile): string { + const engine: Record = {}; + if (spec.engineType === 'PromptTemplate') { + engine.engineRef = spec.promptTemplateName ?? spec.apiName; + } + engine.engineType = spec.engineType; + + const agentAssociationXml: Record = { + agentApiName: spec.agentAssociation.agentApiName, + ...(spec.agentAssociation.inputScope ? { inputScope: spec.agentAssociation.inputScope } : {}), + isActive: spec.agentAssociation.isActive, + samplingRate: spec.agentAssociation.samplingRate ?? 1.0, + }; + + const scorerVersion: Record = { + agentAssociation: agentAssociationXml, + ...(spec.description ? { description: spec.description } : {}), + engine, + label: spec.label, + }; + + // For Number type with specification, generate enum values from spec + if (spec.dataType === 'Number' && spec.specification) { + const numSpec = spec.specification.valueSpecification; + const enumValues = generateNumberEnumValues(numSpec); + scorerVersion.outputEnumValue = enumValues.map((v) => ({ + isFallback: false, + isSystemFallback: false, + outcomeType: v.outcomeType, + value: v.value, + })); + scorerVersion.specification = { + valueSpecification: { + min: numSpec.min, + max: numSpec.max, + step: numSpec.step, + ...(numSpec.threshold != null ? { threshold: numSpec.threshold } : {}), + }, + }; + } else if (spec.outputEnumValues) { + scorerVersion.outputEnumValue = spec.outputEnumValues.map((v) => ({ + isFallback: v.isFallback ?? false, + isSystemFallback: v.isSystemFallback ?? false, + outcomeType: v.outcomeType, + value: v.value, + })); + } + + scorerVersion.status = spec.status ?? 'Draft'; + scorerVersion.versionNumber = 1; + + const definition: Record = { + '@_xmlns': 'http://soap.sforce.com/2006/04/metadata', + dataType: spec.dataType, + inputScope: spec.inputScope ?? 'Session', + }; + + if (spec.lightningType) { + definition.lightningType = spec.lightningType; + } + if (spec.scorerType) { + definition.scorerType = spec.scorerType; + } + if (spec.semanticType) { + definition.semanticType = spec.semanticType; + } + + definition.scorerVersion = scorerVersion; + + const xmlObj = { + '?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' }, + AiAgentScorerDefinition: definition, + }; + + const builder = new XMLBuilder({ + format: true, + ignoreAttributes: false, + indentBy: ' ', + suppressBooleanAttributes: false, + }); + + return builder.build(xmlObj) as string; +} + +function getPromptTemplateType(spec: ScorerSpecFile): string { + if (spec.scorerType === 'OpenEnded') { + return 'agentforce_session_tracing__scorerOpenEnded'; + } + if (spec.semanticType === 'Measurement') { + return 'agentforce_session_tracing__scorerMeasurement'; + } + return 'agentforce_session_tracing__scorerMultilabel'; +} + +function buildPromptTemplateXml(apiName: string, promptContent: string, spec: ScorerSpecFile): string { + const templateType = getPromptTemplateType(spec); + + const isOpenEnded = spec.scorerType === 'OpenEnded'; + + const inputs = [ + { + apiName: 'Session', + definition: 'lightningtype://propertyType/agentforce_session_tracing__stdmDetailViewType', + referenceName: 'Input:Session', + required: true, + }, + { + apiName: 'AllowedLabels', + definition: 'primitive://String', + referenceName: 'Input:AllowedLabels', + required: !isOpenEnded, + }, + { + apiName: 'FallbackLabel', + definition: 'primitive://String', + referenceName: 'Input:FallbackLabel', + required: !isOpenEnded, + }, + ]; + + const xmlObj = { + '?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' }, + GenAiPromptTemplate: { + '@_xmlns': 'http://soap.sforce.com/2006/04/metadata', + developerName: apiName, + masterLabel: apiName, + overridable: false, + templateVersions: { + content: promptContent, + inputs, + primaryModel: 'sfdc_ai__DefaultOpenAIGPT4OmniMini', + status: 'Published', + }, + type: templateType, + visibility: 'Global', + }, + }; + + const builder = new XMLBuilder({ + format: true, + ignoreAttributes: false, + indentBy: ' ', + suppressBooleanAttributes: false, + }); + + return builder.build(xmlObj) as string; +} + +function labelToApiName(label: string): string { + return label.replace(/\s+/g, '_').replace(/[^A-Za-z0-9_]/g, ''); +} + +export default class AgentScorerCreate extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + + public static readonly envVariablesSection = toHelpSection( + 'ENVIRONMENT VARIABLES', + EnvironmentVariable.SF_TARGET_ORG + ); + + public static readonly flags = { + 'target-org': Flags.requiredOrg(), + 'api-version': Flags.orgApiVersion(), + ...makeFlags(FLAGGABLE_PROMPTS), + 'agent-api-name': Flags.string({ + summary: messages.getMessage('flags.agent-api-name.summary'), + }), + spec: Flags.file({ + summary: messages.getMessage('flags.spec.summary'), + exists: true, + }), + 'output-dir': Flags.directory({ + summary: messages.getMessage('flags.output-dir.summary'), + default: join('force-app', 'main', 'default'), + }), + preview: Flags.boolean({ + summary: messages.getMessage('flags.preview.summary'), + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentScorerCreate); + const connection = flags['target-org'].getConnection(flags['api-version']); + + let spec: ScorerSpecFile; + + if (flags.spec) { + spec = YAML.parse(readFileSync(resolve(flags.spec), 'utf8')) as ScorerSpecFile; + this.log(`Reading scorer spec from ${flags.spec}`); + } else { + if (this.jsonEnabled()) { + const missing = Object.entries(FLAGGABLE_PROMPTS) + .filter(([key, p]) => 'required' in p && p.required && !flags[key as keyof typeof flags]) + .map(([key]) => key); + if (!flags['agent-api-name']) missing.push('agent-api-name'); + if (missing.length) { + throw messages.createError('error.missingRequiredFlags', [missing.join(', ')]); + } + } + + this.log(); + this.styledHeader('Scorer Definition'); + + // 1. Label + const label = flags.label ?? (await promptForFlag(FLAGGABLE_PROMPTS.label)); + + // 2. API name (default derived from label) + const defaultApiName = labelToApiName(label); + let apiName: string; + if (flags['api-name']) { + apiName = flags['api-name']; + } else { + apiName = await inquirerInput({ + message: 'Scorer API name', + default: defaultApiName, + validate: FLAGGABLE_PROMPTS['api-name'].validate, + theme, + }); + } + + // 3. Description + const description = flags.description ?? (await promptForFlag(FLAGGABLE_PROMPTS.description)); + + // 4. Status (default Draft) + const status = flags.status ?? (await promptForFlag(FLAGGABLE_PROMPTS.status)); + + // 5. Data type (output scope) + const dataType = flags['data-type'] ?? (await promptForFlag(FLAGGABLE_PROMPTS['data-type'])); + + // 7. Type-specific collection + let outputEnumValues: OutputEnumValue[] | undefined; + let specification: ScorerSpecFile['specification'] | undefined; + let lightningType: string | undefined; + let scorerType: ScorerSpecFile['scorerType'] | undefined; + + if (dataType === 'Number') { + this.log(); + this.styledHeader('Number Scale'); + const numSpec = await promptForNumberSpecification(); + specification = { valueSpecification: numSpec }; + } else if (dataType === 'OpenEnded') { + this.log(); + this.styledHeader('Open Scorer Configuration'); + scorerType = 'OpenEnded'; + + lightningType = await select({ + message: 'Select the lightning type for open-ended values', + choices: SUPPORTED_LIGHTNING_TYPES.map((t) => ({ name: t, value: t })), + theme, + }); + + const addEnumValues = await confirm({ + message: 'Add output enum values?', + default: false, + theme, + }); + if (addEnumValues) { + outputEnumValues = await promptForOutputEnumValues(); + } + } else { + // Text + this.log(); + this.styledHeader('Output Values'); + outputEnumValues = await promptForOutputEnumValues(); + } + + // 7b. Semantic type (optional, for any data type) + const semanticType = await select({ + message: 'Semantic type (how this scorer is used in analytics)', + choices: [ + { name: 'None', value: '' }, + { name: 'Dimension (categorical grouping)', value: 'Dimension' }, + { name: 'Measurement (numeric aggregation)', value: 'Measurement' }, + ], + theme, + }); + + // 7. Engine type + const engineType = flags['engine-type'] ?? (await promptForFlag(FLAGGABLE_PROMPTS['engine-type'])); + + // Prompt template source (only for PromptTemplate engines, before agent selection) + let promptContent: string | undefined; + let existingPromptTemplateName: string | undefined; + if (engineType === 'PromptTemplate') { + this.log(); + this.styledHeader('Prompt Template'); + + const promptChoice = await select({ + message: 'Prompt template source', + choices: [ + { name: 'Generate a new default prompt template', value: 'generate' }, + { name: 'Use an existing prompt template', value: 'existing' }, + ], + theme, + }); + + if (promptChoice === 'existing') { + existingPromptTemplateName = await inquirerInput({ + message: 'Existing prompt template API name', + validate: (d: string): boolean | string => d.length > 0 || 'Name cannot be empty', + theme, + }); + } + } + + // 8. Select agent + let agentAssociation: AgentAssociation; + if (flags['agent-api-name']) { + agentAssociation = { agentApiName: flags['agent-api-name'], isActive: false }; + } else { + const agentsInOrg = await Agent.listRemote(connection); + if (!agentsInOrg.length) { + throw new Error('No agents found in the org.'); + } + const agentApiName = await select({ + message: 'Select the agent to associate with this scorer', + choices: agentsInOrg + .filter((a) => !a.IsDeleted) + .sort((a, b) => a.DeveloperName.localeCompare(b.DeveloperName)) + .map((a) => ({ name: a.DeveloperName, value: a.DeveloperName })), + theme, + }); + agentAssociation = { agentApiName, isActive: false }; + } + + // 9. Input scope for the agent association + const associationInputScope = await select({ + message: 'Input scope for this agent association', + choices: [ + { name: 'Session', value: 'Session' }, + { name: 'Intent', value: 'Intent' }, + ], + default: 'Session', + theme, + }); + agentAssociation.inputScope = associationInputScope as 'Session' | 'Intent'; + + // 10. Activation (only for PromptTemplate engines) + if (engineType === 'PromptTemplate') { + const isActive = await confirm({ + message: 'Activate scoring for this agent?', + default: false, + theme, + }); + agentAssociation.isActive = isActive; + + // 11. Sampling rate (only if active) + if (isActive) { + const samplingRateStr = await promptForFlag({ + message: 'Sampling rate (0.0 - 1.0)', + promptMessage: 'Sampling rate (0.0 to 1.0, where 1.0 = score every session)', + validate: (d: string): boolean | string => { + const n = parseFloat(d); + if (isNaN(n) || n < 0 || n > 1) return 'Must be between 0.0 and 1.0'; + return true; + }, + default: '1.0', + }); + agentAssociation.samplingRate = parseFloat(samplingRateStr); + } + } + + // Map 'OpenEnded' user-facing name to 'LightningType' metadata value + const resolvedDataType = dataType === 'OpenEnded' ? 'LightningType' : dataType; + + spec = { + apiName, + dataType: resolvedDataType as ScorerSpecFile['dataType'], + scorerType, + lightningType, + semanticType: (semanticType || undefined) as ScorerSpecFile['semanticType'], + inputScope: 'Session', + label, + description: description || undefined, + engineType: engineType as ScorerSpecFile['engineType'], + promptContent, + promptTemplateName: existingPromptTemplateName, + status: status as ScorerSpecFile['status'], + outputEnumValues: outputEnumValues as ScorerSpecFile['outputEnumValues'], + specification, + agentAssociation, + }; + } + + // ─── Generate scorer XML ────────────────────────────────────────────── + const scorerXml = buildScorerXml(spec); + const outputDir = resolve(flags['output-dir']); + const scorerDir = join(outputDir, 'aiAgentScorerDefinitions'); + const scorerFileName = `${spec.apiName}.aiAgentScorerDefinition-meta.xml`; + const scorerPath = join(scorerDir, scorerFileName); + + // ─── Generate prompt template XML (if PromptTemplate engine and not using existing) ── + let promptTemplatePath: string | undefined; + let promptTemplateXml: string | undefined; + if (spec.engineType === 'PromptTemplate' && !spec.promptTemplateName) { + const promptContent = spec.promptContent ?? buildDefaultPromptContent(spec); + promptTemplateXml = buildPromptTemplateXml(spec.apiName, promptContent, spec); + const promptDir = join(outputDir, 'genAiPromptTemplates'); + const promptFileName = `${spec.apiName}.genAiPromptTemplate-meta.xml`; + promptTemplatePath = join(promptDir, promptFileName); + } + + if (flags.preview) { + this.log('\n--- Scorer Definition (preview) ---\n'); + this.log(scorerXml); + if (promptTemplateXml) { + this.log('\n--- Prompt Template (preview) ---\n'); + this.log(promptTemplateXml); + } + return { path: scorerPath, apiName: spec.apiName, contents: scorerXml, promptTemplatePath }; + } + + // ─── Write scorer ───────────────────────────────────────────────────── + mkdirSync(scorerDir, { recursive: true }); + if (existsSync(scorerPath) && !this.jsonEnabled()) { + const overwrite = await confirm({ + message: `${scorerFileName} already exists. Overwrite?`, + default: false, + theme, + }); + if (!overwrite) { + this.log('Operation canceled.'); + return { path: '', apiName: spec.apiName, contents: '' }; + } + } + writeFileSync(scorerPath, scorerXml); + this.log(`\nScorer definition written to: ${scorerPath}`); + + // ─── Write prompt template ──────────────────────────────────────────── + if (promptTemplateXml && promptTemplatePath) { + const promptDir = join(outputDir, 'genAiPromptTemplates'); + mkdirSync(promptDir, { recursive: true }); + writeFileSync(promptTemplatePath, promptTemplateXml); + this.log(`Prompt template written to: ${promptTemplatePath}`); + } + + return { path: scorerPath, apiName: spec.apiName, contents: scorerXml, promptTemplatePath }; + } + +} + +function buildDefaultPromptContent(spec: Partial): string { + if (spec.scorerType === 'OpenEnded') { + return [ + `Analyze the following agent-user conversation and provide your evaluation.`, + ``, + `Your response must conform to the expected data type.`, + ``, + `session audit data:`, + `{!$Input:Session}`, + ].join('\n'); + } + + return [ + `Analyze the following agent-user conversation and evaluate it based on your scoring criteria.`, + ``, + `Respond with ONLY one of the allowed values: {!$Input:AllowedLabels}`, + `or fallback to: {!$Input:FallbackLabel}`, + ``, + `session audit data:`, + `{!$Input:Session}`, + ].join('\n'); +} diff --git a/test/commands/agent/scorer/create.test.ts b/test/commands/agent/scorer/create.test.ts new file mode 100644 index 00000000..34414289 --- /dev/null +++ b/test/commands/agent/scorer/create.test.ts @@ -0,0 +1,973 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any */ + +import { expect } from 'chai'; +import esmock from 'esmock'; +import sinon from 'sinon'; +import YAML from 'yaml'; +import { TestContext, MockTestOrgData } from '@salesforce/core/testSetup'; +import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import type { ScorerSpecFile } from '../../../../src/commands/agent/scorer/create.js'; + +function makeTextSpec(overrides: Partial = {}): ScorerSpecFile { + return { + apiName: 'Test_Scorer', + dataType: 'Text', + inputScope: 'Session', + label: 'Test Scorer', + description: 'A test scorer', + engineType: 'Manual', + status: 'Draft', + agentAssociation: { + agentApiName: 'My_Agent', + isActive: false, + }, + outputEnumValues: [ + { value: 'Positive', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Negative', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + { value: 'Neutral', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, + ], + ...overrides, + }; +} + +function makeNumberSpec(overrides: Partial = {}): ScorerSpecFile { + return { + apiName: 'Numeric_Scorer', + dataType: 'Number', + inputScope: 'Session', + label: 'Numeric Scorer', + engineType: 'Manual', + status: 'Available', + agentAssociation: { + agentApiName: 'My_Agent', + isActive: false, + }, + specification: { + valueSpecification: { + min: 0, + max: 5, + step: 1, + }, + }, + ...overrides, + }; +} + +function makeOpenSpec(overrides: Partial = {}): ScorerSpecFile { + return { + apiName: 'Open_Scorer', + dataType: 'LightningType', + scorerType: 'OpenEnded', + lightningType: 'lightning__textType', + inputScope: 'Session', + label: 'Open Scorer', + engineType: 'PromptTemplate', + status: 'Draft', + agentAssociation: { + agentApiName: 'My_Agent', + isActive: true, + samplingRate: 0.5, + inputScope: 'Intent', + }, + ...overrides, + }; +} + +function makePromptTemplateSpec(overrides: Partial = {}): ScorerSpecFile { + return { + apiName: 'Prompt_Scorer', + dataType: 'Text', + inputScope: 'Session', + label: 'Prompt Scorer', + engineType: 'PromptTemplate', + status: 'Draft', + promptContent: 'Evaluate this session.\n\n{!$Input:Session}', + agentAssociation: { + agentApiName: 'My_Agent', + isActive: true, + samplingRate: 1.0, + }, + outputEnumValues: [ + { value: 'Pass', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Fail', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, + ], + ...overrides, + }; +} + +type WrittenFile = { path: string; content: string }; + +async function loadMockedCommand( + yamlSpec: ScorerSpecFile, + opts?: { existsSync?: () => boolean; confirmResult?: boolean } +): Promise<{ Command: any; writtenFiles: WrittenFile[]; createdDirs: string[] }> { + const yamlContent = YAML.stringify(yamlSpec); + const writtenFiles: WrittenFile[] = []; + const createdDirs: string[] = []; + const fileExists = opts?.existsSync ?? (() => false); + + const fsMock: Record = { + readFileSync: () => yamlContent, + writeFileSync: (path: string, content: string) => { + writtenFiles.push({ path, content }); + }, + mkdirSync: (path: string) => { + createdDirs.push(path); + }, + existsSync: fileExists, + }; + + const mocks: Record = { 'node:fs': fsMock }; + + if (opts?.confirmResult !== undefined) { + mocks['@inquirer/prompts'] = { + confirm: sinon.stub().resolves(opts.confirmResult), + select: sinon.stub().resolves('Text'), + input: sinon.stub().resolves(''), + }; + } + + const mod = await esmock('../../../../src/commands/agent/scorer/create.js', mocks); + return { Command: mod.default, writtenFiles, createdDirs }; +} + +describe('agent scorer create', () => { + const $$ = new TestContext(); + let testOrg: MockTestOrgData; + + before(async function () { + // Warm up esmock to check it can load the module + try { + await esmock('../../../../src/commands/agent/scorer/create.js', { + 'node:fs': { + readFileSync: () => '', + writeFileSync: () => {}, + mkdirSync: () => {}, + existsSync: () => false, + }, + }); + } catch (e: any) { + console.error('esmock warmup failed:', e.message); + this.skip(); + } + }); + + beforeEach(async () => { + stubSfCommandUx($$.SANDBOX); + testOrg = new MockTestOrgData(); + await $$.stubAuths(testOrg); + }); + + afterEach(() => { + $$.restore(); + }); + + describe('--spec flag (YAML-driven) with --preview', () => { + it('should create a Text scorer from a YAML spec', async () => { + const { Command } = await loadMockedCommand(makeTextSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.apiName).to.equal('Test_Scorer'); + expect(result.contents).to.include('AiAgentScorerDefinition'); + expect(result.contents).to.include('Text'); + expect(result.contents).to.include('Session'); + expect(result.contents).to.include('Manual'); + expect(result.contents).to.include('Draft'); + expect(result.contents).to.include('My_Agent'); + expect(result.contents).to.include('Positive'); + expect(result.contents).to.include('Negative'); + expect(result.contents).to.include('Neutral'); + }); + + it('should create a Number scorer with specification', async () => { + const { Command } = await loadMockedCommand(makeNumberSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'numeric-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.apiName).to.equal('Numeric_Scorer'); + expect(result.contents).to.include('Number'); + expect(result.contents).to.include('0'); + expect(result.contents).to.include('5'); + expect(result.contents).to.include('1'); + expect(result.contents).to.include('0'); + expect(result.contents).to.include('5'); + expect(result.contents).to.include('Available'); + }); + + it('should create a Number scorer with threshold', async () => { + const spec = makeNumberSpec({ + specification: { valueSpecification: { min: 1, max: 10, step: 1, threshold: 7 } }, + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'threshold-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('7'); + expect(result.contents).to.include('1'); + expect(result.contents).to.include('10'); + }); + + it('should create an OpenEnded (LightningType) scorer', async () => { + const { Command } = await loadMockedCommand(makeOpenSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'open-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.apiName).to.equal('Open_Scorer'); + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('lightning__textType'); + expect(result.contents).to.include('OpenEnded'); + expect(result.contents).to.include('Session'); + }); + + it('should include inputScope in agent association when specified', async () => { + const { Command } = await loadMockedCommand(makeOpenSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'open-scorer.yaml', + '--preview', + '--json', + ]); + + const agentAssocBlock = result.contents.substring( + result.contents.indexOf(''), + result.contents.indexOf('') + ''.length + ); + expect(agentAssocBlock).to.include('Intent'); + }); + + it('should include outputEnumValues for OpenEnded scorer when provided', async () => { + const spec = makeOpenSpec({ + outputEnumValues: [ + { value: 'GOOD', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'BAD', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + { value: 'N/A', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, + ], + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'open-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('OpenEnded'); + expect(result.contents).to.include('GOOD'); + expect(result.contents).to.include('BAD'); + expect(result.contents).to.include('N/A'); + expect(result.contents).to.include('Pass'); + expect(result.contents).to.include('Fail'); + expect(result.contents).to.include('NotApplicable'); + expect(result.contents).to.include('true'); + }); + + it('should not include outputEnumValue for OpenEnded scorer when none provided', async () => { + const { Command } = await loadMockedCommand(makeOpenSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'open-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).not.to.include(''); + expect(result.contents).not.to.include(''); + }); + + it('should generate prompt template path for PromptTemplate engine', async () => { + const { Command } = await loadMockedCommand(makePromptTemplateSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'prompt-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.promptTemplatePath).to.be.a('string'); + expect(result.promptTemplatePath).to.include('genAiPromptTemplates'); + expect(result.promptTemplatePath).to.include('Prompt_Scorer.genAiPromptTemplate-meta.xml'); + expect(result.contents).to.include('Prompt_Scorer'); + expect(result.contents).to.include('PromptTemplate'); + }); + + it('should not generate prompt template for Manual engine', async () => { + const { Command } = await loadMockedCommand(makeTextSpec({ engineType: 'Manual' })); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'manual-scorer.yaml', + '--preview', + '--json', + ]); + + expect(result.promptTemplatePath).to.be.undefined; + expect(result.contents).not.to.include(''); + expect(result.contents).to.include('Manual'); + }); + + it('should use promptTemplateName as engineRef and skip prompt template file generation', async () => { + const spec = makePromptTemplateSpec({ promptTemplateName: 'My_Existing_Template' }); + const { Command, writtenFiles } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(result.contents).to.include('My_Existing_Template'); + expect(result.contents).to.include('PromptTemplate'); + expect(result.promptTemplatePath).to.be.undefined; + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile).to.be.undefined; + expect(writtenFiles).to.have.length(1); + }); + + it('should omit inputScope from agent association XML when not specified', async () => { + const spec = makeTextSpec(); + spec.agentAssociation.inputScope = undefined; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + const agentAssocBlock = result.contents.substring( + result.contents.indexOf(''), + result.contents.indexOf('') + ''.length + ); + expect(agentAssocBlock).to.include('My_Agent'); + expect(agentAssocBlock).not.to.include(''); + }); + + it('should include semanticType when set', async () => { + const { Command } = await loadMockedCommand(makeTextSpec({ semanticType: 'Dimension' })); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('Dimension'); + }); + + it('should include description when provided', async () => { + const { Command } = await loadMockedCommand(makeTextSpec({ description: 'Evaluates politeness' })); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('Evaluates politeness'); + }); + + it('should omit description when not provided', async () => { + const { Command } = await loadMockedCommand(makeTextSpec({ description: undefined })); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).not.to.include(''); + }); + + it('should default samplingRate to 1.0', async () => { + const spec = makeTextSpec(); + spec.agentAssociation.samplingRate = undefined; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('1'); + }); + + it('should use custom samplingRate', async () => { + const spec = makeTextSpec(); + spec.agentAssociation.samplingRate = 0.25; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('0.25'); + }); + + it('should set versionNumber to 1', async () => { + const { Command } = await loadMockedCommand(makeTextSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('1'); + }); + }); + + describe('prompt template type selection', () => { + it('should use scorerOpenEnded type for OpenEnded scorerType', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeOpenSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('agentforce_session_tracing__scorerOpenEnded'); + }); + + it('should use scorerMeasurement type for Measurement semanticType', async () => { + const { Command, writtenFiles } = await loadMockedCommand( + makePromptTemplateSpec({ semanticType: 'Measurement' }) + ); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('agentforce_session_tracing__scorerMeasurement'); + }); + + it('should use scorerMultilabel type for default Text scorers', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('agentforce_session_tracing__scorerMultilabel'); + }); + }); + + describe('number enum value generation', () => { + it('should generate correct values for integer steps', async () => { + const spec = makeNumberSpec({ + specification: { valueSpecification: { min: 0, max: 3, step: 1 } }, + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('0'); + expect(result.contents).to.include('1'); + expect(result.contents).to.include('2'); + expect(result.contents).to.include('3'); + }); + + it('should generate correct values for decimal steps', async () => { + const spec = makeNumberSpec({ + specification: { valueSpecification: { min: 0, max: 1, step: 0.5 } }, + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('0'); + expect(result.contents).to.include('0.5'); + expect(result.contents).to.include('1'); + }); + + it('should set outcomeType to NotApplicable for number values', async () => { + const spec = makeNumberSpec({ + specification: { valueSpecification: { min: 1, max: 2, step: 1 } }, + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + const matches = result.contents.match(/NotApplicable<\/outcomeType>/g); + expect(matches).to.have.length(2); + }); + + it('should handle large step generating few values', async () => { + const spec = makeNumberSpec({ + specification: { valueSpecification: { min: 0, max: 100, step: 50 } }, + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('0'); + expect(result.contents).to.include('50'); + expect(result.contents).to.include('100'); + }); + }); + + describe('XML structure', () => { + it('should include XML declaration and namespace', async () => { + const { Command } = await loadMockedCommand(makeTextSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include(''); + expect(result.contents).to.include('xmlns="http://soap.sforce.com/2006/04/metadata"'); + }); + + it('should include isActive in agent association', async () => { + const spec = makeTextSpec(); + spec.agentAssociation.isActive = true; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('true'); + }); + + it('should include isFallback and isSystemFallback', async () => { + const spec = makeTextSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, + ], + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('false'); + expect(result.contents).to.include('true'); + expect(result.contents).to.include('false'); + }); + + it('should include label in scorerVersion', async () => { + const { Command } = await loadMockedCommand(makeTextSpec({ label: 'My Custom Label' })); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include(''); + }); + }); + + describe('file writing', () => { + it('should write scorer XML to correct path', async () => { + const { Command, writtenFiles, createdDirs } = await loadMockedCommand(makeTextSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(result.path).to.include('/tmp/out'); + expect(result.path).to.include('aiAgentScorerDefinitions'); + expect(result.path).to.include('Test_Scorer.aiAgentScorerDefinition-meta.xml'); + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('AiAgentScorerDefinition'); + expect(createdDirs.some((d) => d.includes('aiAgentScorerDefinitions'))).to.be.true; + }); + + it('should write both scorer and prompt template for PromptTemplate', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(writtenFiles).to.have.length(2); + const scorerFile = writtenFiles.find((f) => f.path.includes('aiAgentScorerDefinitions')); + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(scorerFile).to.not.be.undefined; + expect(promptFile).to.not.be.undefined; + expect(promptFile!.path).to.include('Prompt_Scorer.genAiPromptTemplate-meta.xml'); + expect(promptFile!.content).to.include('GenAiPromptTemplate'); + expect(result.promptTemplatePath).to.equal(promptFile!.path); + }); + + it('should not write files with --preview', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeTextSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(writtenFiles).to.have.length(0); + }); + + it('should use default prompt content when promptContent not in spec', async () => { + const spec = makePromptTemplateSpec(); + delete (spec as any).promptContent; + const { Command, writtenFiles } = await loadMockedCommand(spec); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('{!$Input:Session}'); + expect(promptFile!.content).to.include('{!$Input:AllowedLabels}'); + expect(promptFile!.content).to.include('{!$Input:FallbackLabel}'); + }); + + it('should use OpenEnded default prompt for OpenEnded type', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeOpenSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('{!$Input:Session}'); + expect(promptFile!.content).not.to.include('{!$Input:AllowedLabels}'); + }); + }); + + describe('output directory', () => { + it('should default to force-app/main/default', async () => { + const { Command } = await loadMockedCommand(makeTextSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.path).to.include('force-app/main/default/aiAgentScorerDefinitions'); + }); + + it('should use custom --output-dir', async () => { + const { Command } = await loadMockedCommand(makeTextSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/custom/path', + '--preview', + '--json', + ]); + + expect(result.path).to.include('/custom/path/aiAgentScorerDefinitions'); + }); + }); + + describe('overwrite behavior', () => { + it('should cancel when user declines overwrite', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeTextSpec(), { + existsSync: () => true, + confirmResult: false, + }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + ]); + + expect(result.path).to.equal(''); + expect(result.contents).to.equal(''); + expect(writtenFiles).to.have.length(0); + }); + + it('should skip overwrite prompt in --json mode', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeTextSpec(), { + existsSync: () => true, + }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(result.path).to.not.equal(''); + expect(writtenFiles).to.have.length(1); + }); + }); + + describe('prompt template XML details', () => { + it('should include developerName and masterLabel matching apiName', async () => { + const { Command, writtenFiles } = await loadMockedCommand( + makePromptTemplateSpec({ apiName: 'My_Prompt_Scorer' }) + ); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('My_Prompt_Scorer'); + expect(promptFile!.content).to.include('My_Prompt_Scorer'); + }); + + it('should set overridable to false and visibility to Global', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('false'); + expect(promptFile!.content).to.include('Global'); + }); + + it('should set primaryModel and status Published', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('sfdc_ai__DefaultOpenAIGPT4OmniMini'); + expect(promptFile!.content).to.include('Published'); + }); + + it('should include Session input with correct definition', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include( + 'lightningtype://propertyType/agentforce_session_tracing__stdmDetailViewType' + ); + expect(promptFile!.content).to.include('Input:Session'); + }); + }); + + describe('--json mode error handling', () => { + it('should throw when required flags are missing', async () => { + const { Command } = await loadMockedCommand(makeTextSpec()); + + try { + await Command.run(['--target-org', testOrg.username, '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string }; + expect(error.message).to.include('Missing required flags'); + } + }); + + it('should list all missing required flags', async () => { + const { Command } = await loadMockedCommand(makeTextSpec()); + + try { + await Command.run(['--target-org', testOrg.username, '--label', 'Foo', '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string }; + expect(error.message).to.include('api-name'); + expect(error.message).to.include('data-type'); + expect(error.message).to.include('engine-type'); + expect(error.message).to.include('agent-api-name'); + } + }); + }); + + describe('edge cases', () => { + it('should handle LightningType with no outputEnumValues', async () => { + const spec: ScorerSpecFile = { + apiName: 'Lightning_Scorer', + dataType: 'LightningType', + scorerType: 'OpenEnded', + lightningType: 'lightning__numberType', + inputScope: 'Session', + label: 'Lightning Scorer', + engineType: 'Manual', + agentAssociation: { agentApiName: 'Agent_X', isActive: false }, + }; + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('lightning__numberType'); + }); + + it('should handle single output enum value', async () => { + const spec = makeTextSpec({ + outputEnumValues: [ + { value: 'Only', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, + ], + }); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('Only'); + expect(result.contents).to.include('NotApplicable'); + expect(result.contents).to.include('true'); + }); + + it('should include scorerType Predefined when set', async () => { + const { Command } = await loadMockedCommand(makeTextSpec({ scorerType: 'Predefined' })); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('Predefined'); + }); + + it('should include Measurement semanticType in XML', async () => { + const { Command } = await loadMockedCommand(makeNumberSpec({ semanticType: 'Measurement' })); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--preview', + '--json', + ]); + + expect(result.contents).to.include('Measurement'); + }); + }); +}); From 48517a8626db585349919ae2770106da6e42e2a3 Mon Sep 17 00:00:00 2001 From: nabilnaffar-sf Date: Wed, 1 Jul 2026 19:45:09 +0300 Subject: [PATCH 02/19] feat: add `sf agent scorer create` command for interactive scorer definition authoring --- src/commands/agent/scorer/create.ts | 502 ++++++++++++++-------------- 1 file changed, 259 insertions(+), 243 deletions(-) diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index c7079282..2eec060b 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -140,43 +140,45 @@ type OutputEnumValue = { isSystemFallback: boolean; }; -async function promptForOutputEnumValues(): Promise { - const values: OutputEnumValue[] = []; - let addMore = true; +async function promptForSingleEnumValue(index: number): Promise { + const value = await promptForFlag({ + message: 'Output value name', + promptMessage: `Output value #${index + 1} (e.g., "Good", "Bad", "N/A")`, + validate: (d: string): boolean | string => d.length > 0 || 'Value cannot be empty', + }); - while (addMore) { - const value = await promptForFlag({ - message: 'Output value name', - promptMessage: `Output value #${values.length + 1} (e.g., "Good", "Bad", "N/A")`, - validate: (d: string): boolean | string => d.length > 0 || 'Value cannot be empty', - }); + const outcomeType = await promptForFlag({ + message: 'Outcome type', + promptMessage: 'Outcome type for this value', + options: ['Pass', 'Fail', 'NotApplicable'], + validate: (d: string): boolean | string => + ['Pass', 'Fail', 'NotApplicable'].includes(d) || 'Invalid', + }); - const outcomeType = await promptForFlag({ - message: 'Outcome type', - promptMessage: 'Outcome type for this value', - options: ['Pass', 'Fail', 'NotApplicable'], - validate: (d: string): boolean | string => - ['Pass', 'Fail', 'NotApplicable'].includes(d) || 'Invalid', - }); + const isFallback = await confirm({ + message: 'Is this the fallback value?', + default: index === 0, + theme, + }); - const isFallback = await confirm({ - message: 'Is this the fallback value?', - default: values.length === 0, - theme, - }); + const addMore = await confirm({ + message: 'Add another output value?', + default: index < 1, + theme, + }); - values.push({ - value, - outcomeType, - isFallback, - isSystemFallback: false, - }); + return { value, outcomeType, isFallback, isSystemFallback: false, addMore }; +} - addMore = await confirm({ - message: 'Add another output value?', - default: values.length < 2, - theme, - }); +async function promptForOutputEnumValues(): Promise { + const values: OutputEnumValue[] = []; + let addMore = true; + + while (addMore) { + // eslint-disable-next-line no-await-in-loop + const result = await promptForSingleEnumValue(values.length); + addMore = result.addMore; + values.push({ value: result.value, outcomeType: result.outcomeType, isFallback: result.isFallback, isSystemFallback: result.isSystemFallback }); } return values; @@ -358,7 +360,8 @@ function buildScorerXml(spec: ScorerSpecFile): string { suppressBooleanAttributes: false, }); - return builder.build(xmlObj) as string; + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return builder.build(xmlObj); } function getPromptTemplateType(spec: ScorerSpecFile): string { @@ -422,7 +425,8 @@ function buildPromptTemplateXml(apiName: string, promptContent: string, spec: Sc suppressBooleanAttributes: false, }); - return builder.build(xmlObj) as string; + // eslint-disable-next-line @typescript-eslint/no-unsafe-return + return builder.build(xmlObj); } function labelToApiName(label: string): string { @@ -459,223 +463,109 @@ export default class AgentScorerCreate extends SfCommand { const { flags } = await this.parse(AgentScorerCreate); const connection = flags['target-org'].getConnection(flags['api-version']); - let spec: ScorerSpecFile; - - if (flags.spec) { - spec = YAML.parse(readFileSync(resolve(flags.spec), 'utf8')) as ScorerSpecFile; - this.log(`Reading scorer spec from ${flags.spec}`); - } else { - if (this.jsonEnabled()) { - const missing = Object.entries(FLAGGABLE_PROMPTS) - .filter(([key, p]) => 'required' in p && p.required && !flags[key as keyof typeof flags]) - .map(([key]) => key); - if (!flags['agent-api-name']) missing.push('agent-api-name'); - if (missing.length) { - throw messages.createError('error.missingRequiredFlags', [missing.join(', ')]); - } - } + const spec = flags.spec + ? this.loadSpecFromFile(flags.spec) + : await this.runInteractiveInterview(flags, connection); - this.log(); - this.styledHeader('Scorer Definition'); - - // 1. Label - const label = flags.label ?? (await promptForFlag(FLAGGABLE_PROMPTS.label)); - - // 2. API name (default derived from label) - const defaultApiName = labelToApiName(label); - let apiName: string; - if (flags['api-name']) { - apiName = flags['api-name']; - } else { - apiName = await inquirerInput({ - message: 'Scorer API name', - default: defaultApiName, - validate: FLAGGABLE_PROMPTS['api-name'].validate, - theme, - }); - } + return this.generateOutput(spec, flags); + } - // 3. Description - const description = flags.description ?? (await promptForFlag(FLAGGABLE_PROMPTS.description)); - - // 4. Status (default Draft) - const status = flags.status ?? (await promptForFlag(FLAGGABLE_PROMPTS.status)); - - // 5. Data type (output scope) - const dataType = flags['data-type'] ?? (await promptForFlag(FLAGGABLE_PROMPTS['data-type'])); - - // 7. Type-specific collection - let outputEnumValues: OutputEnumValue[] | undefined; - let specification: ScorerSpecFile['specification'] | undefined; - let lightningType: string | undefined; - let scorerType: ScorerSpecFile['scorerType'] | undefined; - - if (dataType === 'Number') { - this.log(); - this.styledHeader('Number Scale'); - const numSpec = await promptForNumberSpecification(); - specification = { valueSpecification: numSpec }; - } else if (dataType === 'OpenEnded') { - this.log(); - this.styledHeader('Open Scorer Configuration'); - scorerType = 'OpenEnded'; - - lightningType = await select({ - message: 'Select the lightning type for open-ended values', - choices: SUPPORTED_LIGHTNING_TYPES.map((t) => ({ name: t, value: t })), - theme, - }); + private loadSpecFromFile(specPath: string): ScorerSpecFile { + const spec = YAML.parse(readFileSync(resolve(specPath), 'utf8')) as ScorerSpecFile; + this.log(`Reading scorer spec from ${specPath}`); + return spec; + } - const addEnumValues = await confirm({ - message: 'Add output enum values?', - default: false, - theme, - }); - if (addEnumValues) { - outputEnumValues = await promptForOutputEnumValues(); - } - } else { - // Text - this.log(); - this.styledHeader('Output Values'); - outputEnumValues = await promptForOutputEnumValues(); + private async runInteractiveInterview( + flags: Record, + connection: ReturnType + ): Promise { + if (this.jsonEnabled()) { + const missing = Object.entries(FLAGGABLE_PROMPTS) + .filter(([key, p]) => 'required' in p && p.required && !flags[key]) + .map(([key]) => key); + if (!flags['agent-api-name']) missing.push('agent-api-name'); + if (missing.length) { + throw messages.createError('error.missingRequiredFlags', [missing.join(', ')]); } + } - // 7b. Semantic type (optional, for any data type) - const semanticType = await select({ - message: 'Semantic type (how this scorer is used in analytics)', - choices: [ - { name: 'None', value: '' }, - { name: 'Dimension (categorical grouping)', value: 'Dimension' }, - { name: 'Measurement (numeric aggregation)', value: 'Measurement' }, - ], - theme, - }); + this.log(); + this.styledHeader('Scorer Definition'); - // 7. Engine type - const engineType = flags['engine-type'] ?? (await promptForFlag(FLAGGABLE_PROMPTS['engine-type'])); - - // Prompt template source (only for PromptTemplate engines, before agent selection) - let promptContent: string | undefined; - let existingPromptTemplateName: string | undefined; - if (engineType === 'PromptTemplate') { - this.log(); - this.styledHeader('Prompt Template'); - - const promptChoice = await select({ - message: 'Prompt template source', - choices: [ - { name: 'Generate a new default prompt template', value: 'generate' }, - { name: 'Use an existing prompt template', value: 'existing' }, - ], - theme, - }); + const label = (flags.label as string) ?? (await promptForFlag(FLAGGABLE_PROMPTS.label)); - if (promptChoice === 'existing') { - existingPromptTemplateName = await inquirerInput({ - message: 'Existing prompt template API name', - validate: (d: string): boolean | string => d.length > 0 || 'Name cannot be empty', - theme, - }); - } - } + const defaultApiName = labelToApiName(label); + const apiName = (flags['api-name'] as string) ?? (await inquirerInput({ + message: 'Scorer API name', + default: defaultApiName, + validate: FLAGGABLE_PROMPTS['api-name'].validate, + theme, + })); - // 8. Select agent - let agentAssociation: AgentAssociation; - if (flags['agent-api-name']) { - agentAssociation = { agentApiName: flags['agent-api-name'], isActive: false }; - } else { - const agentsInOrg = await Agent.listRemote(connection); - if (!agentsInOrg.length) { - throw new Error('No agents found in the org.'); - } - const agentApiName = await select({ - message: 'Select the agent to associate with this scorer', - choices: agentsInOrg - .filter((a) => !a.IsDeleted) - .sort((a, b) => a.DeveloperName.localeCompare(b.DeveloperName)) - .map((a) => ({ name: a.DeveloperName, value: a.DeveloperName })), - theme, - }); - agentAssociation = { agentApiName, isActive: false }; - } + const description = (flags.description as string) ?? (await promptForFlag(FLAGGABLE_PROMPTS.description)); + const status = (flags.status as string) ?? (await promptForFlag(FLAGGABLE_PROMPTS.status)); + const dataType = (flags['data-type'] as string) ?? (await promptForFlag(FLAGGABLE_PROMPTS['data-type'])); - // 9. Input scope for the agent association - const associationInputScope = await select({ - message: 'Input scope for this agent association', - choices: [ - { name: 'Session', value: 'Session' }, - { name: 'Intent', value: 'Intent' }, - ], - default: 'Session', - theme, - }); - agentAssociation.inputScope = associationInputScope as 'Session' | 'Intent'; - - // 10. Activation (only for PromptTemplate engines) - if (engineType === 'PromptTemplate') { - const isActive = await confirm({ - message: 'Activate scoring for this agent?', - default: false, - theme, - }); - agentAssociation.isActive = isActive; - - // 11. Sampling rate (only if active) - if (isActive) { - const samplingRateStr = await promptForFlag({ - message: 'Sampling rate (0.0 - 1.0)', - promptMessage: 'Sampling rate (0.0 to 1.0, where 1.0 = score every session)', - validate: (d: string): boolean | string => { - const n = parseFloat(d); - if (isNaN(n) || n < 0 || n > 1) return 'Must be between 0.0 and 1.0'; - return true; - }, - default: '1.0', - }); - agentAssociation.samplingRate = parseFloat(samplingRateStr); - } - } + const dataTypeDetails = await this.promptForDataTypeDetails(dataType); - // Map 'OpenEnded' user-facing name to 'LightningType' metadata value - const resolvedDataType = dataType === 'OpenEnded' ? 'LightningType' : dataType; - - spec = { - apiName, - dataType: resolvedDataType as ScorerSpecFile['dataType'], - scorerType, - lightningType, - semanticType: (semanticType || undefined) as ScorerSpecFile['semanticType'], - inputScope: 'Session', - label, - description: description || undefined, - engineType: engineType as ScorerSpecFile['engineType'], - promptContent, - promptTemplateName: existingPromptTemplateName, - status: status as ScorerSpecFile['status'], - outputEnumValues: outputEnumValues as ScorerSpecFile['outputEnumValues'], - specification, - agentAssociation, - }; - } + const semanticType = await select({ + message: 'Semantic type (how this scorer is used in analytics)', + choices: [ + { name: 'None', value: '' }, + { name: 'Dimension (categorical grouping)', value: 'Dimension' }, + { name: 'Measurement (numeric aggregation)', value: 'Measurement' }, + ], + theme, + }); - // ─── Generate scorer XML ────────────────────────────────────────────── + const engineType = (flags['engine-type'] as string) ?? (await promptForFlag(FLAGGABLE_PROMPTS['engine-type'])); + const engineConfig = await this.promptForEngineConfig(engineType); + const agentAssociation = await this.promptForAgentAssociationDetails( + connection, engineType, flags['agent-api-name'] as string | undefined + ); + + const resolvedDataType = dataType === 'OpenEnded' ? 'LightningType' : dataType; + + return { + apiName, + dataType: resolvedDataType as ScorerSpecFile['dataType'], + scorerType: dataTypeDetails.scorerType, + lightningType: dataTypeDetails.lightningType, + semanticType: (semanticType || undefined) as ScorerSpecFile['semanticType'], + inputScope: 'Session', + label, + description: description || undefined, + engineType: engineType as ScorerSpecFile['engineType'], + promptContent: engineConfig.promptContent, + promptTemplateName: engineConfig.promptTemplateName, + status: status as ScorerSpecFile['status'], + outputEnumValues: dataTypeDetails.outputEnumValues as ScorerSpecFile['outputEnumValues'], + specification: dataTypeDetails.specification, + agentAssociation, + }; + } + + private async generateOutput( + spec: ScorerSpecFile, + flags: Record + ): Promise { const scorerXml = buildScorerXml(spec); - const outputDir = resolve(flags['output-dir']); + const outputDir = resolve(flags['output-dir'] as string); const scorerDir = join(outputDir, 'aiAgentScorerDefinitions'); const scorerFileName = `${spec.apiName}.aiAgentScorerDefinition-meta.xml`; const scorerPath = join(scorerDir, scorerFileName); - // ─── Generate prompt template XML (if PromptTemplate engine and not using existing) ── let promptTemplatePath: string | undefined; let promptTemplateXml: string | undefined; if (spec.engineType === 'PromptTemplate' && !spec.promptTemplateName) { - const promptContent = spec.promptContent ?? buildDefaultPromptContent(spec); - promptTemplateXml = buildPromptTemplateXml(spec.apiName, promptContent, spec); + const content = spec.promptContent ?? buildDefaultPromptContent(spec); + promptTemplateXml = buildPromptTemplateXml(spec.apiName, content, spec); const promptDir = join(outputDir, 'genAiPromptTemplates'); const promptFileName = `${spec.apiName}.genAiPromptTemplate-meta.xml`; promptTemplatePath = join(promptDir, promptFileName); @@ -691,7 +581,6 @@ export default class AgentScorerCreate extends SfCommand { + if (dataType === 'Number') { + this.log(); + this.styledHeader('Number Scale'); + const numSpec = await promptForNumberSpecification(); + return { specification: { valueSpecification: numSpec } }; + } + + if (dataType === 'OpenEnded') { + this.log(); + this.styledHeader('Open Scorer Configuration'); + + const lightningType = await select({ + message: 'Select the lightning type for open-ended values', + choices: SUPPORTED_LIGHTNING_TYPES.map((t) => ({ name: t, value: t })), + theme, + }); + + const addEnumValues = await confirm({ + message: 'Add output enum values?', + default: false, + theme, + }); + const outputEnumValues = addEnumValues ? await promptForOutputEnumValues() : undefined; + return { scorerType: 'OpenEnded', lightningType, outputEnumValues }; + } + + // Text + this.log(); + this.styledHeader('Output Values'); + const outputEnumValues = await promptForOutputEnumValues(); + return { outputEnumValues }; + } + + private async promptForEngineConfig(engineType: string): Promise<{ promptContent?: string; promptTemplateName?: string }> { + if (engineType !== 'PromptTemplate') return {}; + + this.log(); + this.styledHeader('Prompt Template'); + + const promptChoice = await select({ + message: 'Prompt template source', + choices: [ + { name: 'Generate a new default prompt template', value: 'generate' }, + { name: 'Use an existing prompt template', value: 'existing' }, + ], + theme, + }); + + if (promptChoice === 'existing') { + const promptTemplateName = await inquirerInput({ + message: 'Existing prompt template API name', + validate: (d: string): boolean | string => d.length > 0 || 'Name cannot be empty', + theme, + }); + return { promptTemplateName }; + } + + return {}; + } + + // eslint-disable-next-line class-methods-use-this + private async promptForAgentAssociationDetails( + connection: ReturnType, + engineType: string, + agentApiNameFlag?: string + ): Promise { + let agentAssociation: AgentAssociation; + if (agentApiNameFlag) { + agentAssociation = { agentApiName: agentApiNameFlag, isActive: false }; + } else { + const agentsInOrg = await Agent.listRemote(connection); + if (!agentsInOrg.length) { + throw new Error('No agents found in the org.'); + } + const agentApiName = await select({ + message: 'Select the agent to associate with this scorer', + choices: agentsInOrg + .filter((a) => !a.IsDeleted) + .sort((a, b) => a.DeveloperName.localeCompare(b.DeveloperName)) + .map((a) => ({ name: a.DeveloperName, value: a.DeveloperName })), + theme, + }); + agentAssociation = { agentApiName, isActive: false }; + } + + const associationInputScope = await select({ + message: 'Input scope for this agent association', + choices: [ + { name: 'Session', value: 'Session' }, + { name: 'Intent', value: 'Intent' }, + ], + default: 'Session', + theme, + }); + agentAssociation.inputScope = associationInputScope as 'Session' | 'Intent'; + + if (engineType === 'PromptTemplate') { + const isActive = await confirm({ + message: 'Activate scoring for this agent?', + default: false, + theme, + }); + agentAssociation.isActive = isActive; + + if (isActive) { + const samplingRateStr = await promptForFlag({ + message: 'Sampling rate (0.0 - 1.0)', + promptMessage: 'Sampling rate (0.0 to 1.0, where 1.0 = score every session)', + validate: (d: string): boolean | string => { + const n = parseFloat(d); + if (isNaN(n) || n < 0 || n > 1) return 'Must be between 0.0 and 1.0'; + return true; + }, + default: '1.0', + }); + agentAssociation.samplingRate = parseFloat(samplingRateStr); + } + } + + return agentAssociation; + } + } function buildDefaultPromptContent(spec: Partial): string { if (spec.scorerType === 'OpenEnded') { return [ - `Analyze the following agent-user conversation and provide your evaluation.`, - ``, - `Your response must conform to the expected data type.`, - ``, - `session audit data:`, - `{!$Input:Session}`, + 'Analyze the following agent-user conversation and provide your evaluation.', + '', + 'Your response must conform to the expected data type.', + '', + 'session audit data:', + '{!$Input:Session}', ].join('\n'); } return [ - `Analyze the following agent-user conversation and evaluate it based on your scoring criteria.`, - ``, - `Respond with ONLY one of the allowed values: {!$Input:AllowedLabels}`, - `or fallback to: {!$Input:FallbackLabel}`, - ``, - `session audit data:`, - `{!$Input:Session}`, + 'Analyze the following agent-user conversation and evaluate it based on your scoring criteria.', + '', + 'Respond with ONLY one of the allowed values: {!$Input:AllowedLabels}', + 'or fallback to: {!$Input:FallbackLabel}', + '', + 'session audit data:', + '{!$Input:Session}', ].join('\n'); } From 88fa53d358420ddee0d020733f18989ff756f26f Mon Sep 17 00:00:00 2001 From: nabilnaffar-sf Date: Wed, 1 Jul 2026 19:51:44 +0300 Subject: [PATCH 03/19] feat: add `sf agent scorer create` command for interactive scorer definition authoring --- src/commands/agent/scorer/create.ts | 9 +++ test/commands/agent/scorer/create.test.ts | 89 +++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index 2eec060b..15b51ec4 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -555,6 +555,15 @@ export default class AgentScorerCreate extends SfCommand ): Promise { + if (spec.dataType === 'Text' && spec.outputEnumValues) { + const fallbackCount = spec.outputEnumValues.filter((v) => v.isFallback).length; + if (fallbackCount !== 1) { + throw new Error( + `Text scorers must have exactly 1 fallback value, but found ${fallbackCount}.` + ); + } + } + const scorerXml = buildScorerXml(spec); const outputDir = resolve(flags['output-dir'] as string); const scorerDir = join(outputDir, 'aiAgentScorerDefinitions'); diff --git a/test/commands/agent/scorer/create.test.ts b/test/commands/agent/scorer/create.test.ts index 34414289..357659b3 100644 --- a/test/commands/agent/scorer/create.test.ts +++ b/test/commands/agent/scorer/create.test.ts @@ -16,6 +16,8 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any */ +import { join } from 'node:path'; +import { mkdirSync, writeFileSync, rmSync } from 'node:fs'; import { expect } from 'chai'; import esmock from 'esmock'; import sinon from 'sinon'; @@ -899,6 +901,93 @@ describe('agent scorer create', () => { }); }); + describe('Text scorer fallback validation', () => { + let tmpDir: string; + let specFile: string; + + beforeEach(() => { + tmpDir = join(process.cwd(), 'tmp-test-fallback-' + Date.now()); + mkdirSync(tmpDir, { recursive: true }); + specFile = join(tmpDir, 'scorer.yaml'); + }); + + afterEach(() => { + rmSync(tmpDir, { recursive: true, force: true }); + }); + + it('should throw when Text scorer has no fallback value', async () => { + const spec = makeTextSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + ], + }); + writeFileSync(specFile, YAML.stringify(spec)); + const { Command } = await loadMockedCommand(spec); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--spec', specFile, + '--preview', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string }; + expect(error.message).to.include('exactly 1 fallback value'); + expect(error.message).to.include('found 0'); + } + }); + + it('should throw when Text scorer has multiple fallback values', async () => { + const spec = makeTextSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: true, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, + ], + }); + writeFileSync(specFile, YAML.stringify(spec)); + const { Command } = await loadMockedCommand(spec); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--spec', specFile, + '--preview', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string }; + expect(error.message).to.include('exactly 1 fallback value'); + expect(error.message).to.include('found 2'); + } + }); + + it('should pass when Text scorer has exactly 1 fallback value', async () => { + const spec = makeTextSpec({ + outputEnumValues: [ + { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + { value: 'N/A', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, + ], + }); + writeFileSync(specFile, YAML.stringify(spec)); + const { Command } = await loadMockedCommand(spec); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', specFile, + '--preview', + '--json', + ]); + + expect(result.apiName).to.equal('Test_Scorer'); + expect(result.contents).to.include('N/A'); + }); + }); + describe('edge cases', () => { it('should handle LightningType with no outputEnumValues', async () => { const spec: ScorerSpecFile = { From d250545772ba26305431cbccf9f25158187283f5 Mon Sep 17 00:00:00 2001 From: nnaffar Date: Thu, 2 Jul 2026 13:44:10 +0300 Subject: [PATCH 04/19] adding --spec-schema for agents to discover the right yaml schema for authoring --- messages/agent.scorer.create.md | 8 ++ schemas/agent-scorer-create__spec.json | 189 +++++++++++++++++++++++++ src/commands/agent/scorer/create.ts | 18 ++- 3 files changed, 214 insertions(+), 1 deletion(-) create mode 100644 schemas/agent-scorer-create__spec.json diff --git a/messages/agent.scorer.create.md b/messages/agent.scorer.create.md index c97fe685..3144194e 100644 --- a/messages/agent.scorer.create.md +++ b/messages/agent.scorer.create.md @@ -44,6 +44,10 @@ Initial status of the scorer version (Available or Draft). Path to a scorer spec YAML file. Bypasses interactive prompts. +# flags.spec-schema.summary + +Output the JSON Schema for the --spec YAML file and exit. + # flags.output-dir.summary Output directory for the generated metadata XML files (scorer definition and prompt template). @@ -54,6 +58,10 @@ Preview the generated XML without writing to disk. # examples +- Show the JSON Schema for the spec YAML file: + + <%= config.bin %> <%= command.id %> --spec-schema + - Create a scorer interactively: <%= config.bin %> <%= command.id %> diff --git a/schemas/agent-scorer-create__spec.json b/schemas/agent-scorer-create__spec.json new file mode 100644 index 00000000..37ae4d5e --- /dev/null +++ b/schemas/agent-scorer-create__spec.json @@ -0,0 +1,189 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/ScorerSpecFile", + "definitions": { + "ScorerSpecFile": { + "type": "object", + "description": "YAML spec file for creating an agent scorer definition via `sf agent scorer create --spec `.", + "properties": { + "apiName": { + "type": "string", + "description": "API name of the scorer definition. Max 35 characters, must start with a letter, only alphanumerics and underscores.", + "pattern": "^[A-Za-z][A-Za-z0-9_]{0,34}$", + "maxLength": 35 + }, + "dataType": { + "type": "string", + "enum": ["Text", "Number", "LightningType"], + "description": "Data type produced by the scorer. Use 'Text' for categorical labels, 'Number' for numeric scales, 'LightningType' for open-ended evaluations." + }, + "scorerType": { + "type": "string", + "enum": ["Predefined", "OpenEnded"], + "description": "Set to 'OpenEnded' when dataType is 'LightningType' for free-form evaluation." + }, + "lightningType": { + "type": "string", + "enum": [ + "lightning__textType", + "lightning__multilineTextType", + "lightning__richTextType", + "lightning__numberType", + "lightning__integerType", + "lightning__booleanType", + "lightning__dateType", + "lightning__dateTimeType", + "lightning__dateTimeStringType", + "lightning__urlType", + "lightning__objectType", + "lightning__listType" + ], + "description": "Required when dataType is 'LightningType'. Specifies the lightning type for open-ended values." + }, + "semanticType": { + "type": "string", + "enum": ["Dimension", "Measurement"], + "description": "How this scorer is used in analytics. 'Dimension' for categorical grouping, 'Measurement' for numeric aggregation." + }, + "inputScope": { + "type": "string", + "enum": ["Session", "Intent"], + "default": "Session", + "description": "Whether the scorer evaluates an entire session or a single intent within a session." + }, + "label": { + "type": "string", + "description": "Display label for the scorer version.", + "minLength": 1 + }, + "description": { + "type": "string", + "description": "Human-readable description of what this scorer evaluates." + }, + "engineType": { + "type": "string", + "enum": ["Manual", "PromptTemplate"], + "description": "'Manual' for human-evaluated scoring, 'PromptTemplate' for LLM-evaluated scoring." + }, + "promptContent": { + "type": "string", + "description": "Prompt text for PromptTemplate engine type. Use {!$Input:Session} to reference the session data, {!$Input:AllowedLabels} for allowed output values, and {!$Input:FallbackLabel} for the fallback value. Ignored when engineType is 'Manual'." + }, + "promptTemplateName": { + "type": "string", + "description": "API name of an existing prompt template to use instead of generating a new one. Mutually exclusive with promptContent." + }, + "status": { + "type": "string", + "enum": ["Available", "Draft"], + "default": "Draft", + "description": "Initial status of the scorer version." + }, + "agentAssociation": { + "$ref": "#/definitions/AgentAssociation" + }, + "outputEnumValues": { + "type": "array", + "description": "Output value definitions. Required for 'Text' dataType. For 'Text' scorers, exactly one value must have isFallback: true.", + "items": { + "$ref": "#/definitions/OutputEnumValue" + } + }, + "specification": { + "$ref": "#/definitions/NumberSpecification", + "description": "Required when dataType is 'Number'. Defines the numeric scale." + } + }, + "required": ["apiName", "dataType", "label", "engineType", "agentAssociation"], + "additionalProperties": false + }, + "AgentAssociation": { + "type": "object", + "description": "Associates the scorer with an agent in the org.", + "properties": { + "agentApiName": { + "type": "string", + "description": "API name of the agent to associate with this scorer." + }, + "isActive": { + "type": "boolean", + "description": "Whether scoring is active for this agent association." + }, + "samplingRate": { + "type": "number", + "minimum": 0, + "maximum": 1, + "default": 1.0, + "description": "Fraction of sessions to score (0.0 to 1.0). Only relevant when isActive is true." + }, + "inputScope": { + "type": "string", + "enum": ["Session", "Intent"], + "description": "Override input scope for this specific agent association." + } + }, + "required": ["agentApiName", "isActive"], + "additionalProperties": false + }, + "OutputEnumValue": { + "type": "object", + "description": "A possible output value for the scorer.", + "properties": { + "value": { + "type": "string", + "description": "The output label (e.g., 'Good', 'Bad', 'N/A').", + "minLength": 1 + }, + "outcomeType": { + "type": "string", + "enum": ["Pass", "Fail", "NotApplicable"], + "description": "Maps this value to a pass/fail outcome for reporting." + }, + "isFallback": { + "type": "boolean", + "default": false, + "description": "Whether this is the fallback value. Exactly one value must be the fallback for Text scorers." + }, + "isSystemFallback": { + "type": "boolean", + "default": false, + "description": "Whether this is a system-generated fallback. Typically false for user-defined scorers." + } + }, + "required": ["value", "outcomeType"], + "additionalProperties": false + }, + "NumberSpecification": { + "type": "object", + "properties": { + "valueSpecification": { + "type": "object", + "description": "Defines the numeric scale. The number of generated values ((max - min) / step + 1) must not exceed 101.", + "properties": { + "min": { + "type": "number", + "description": "Minimum value of the scale." + }, + "max": { + "type": "number", + "description": "Maximum value of the scale. Must be greater than min." + }, + "step": { + "type": "number", + "exclusiveMinimum": 0, + "description": "Step size between values." + }, + "threshold": { + "type": "number", + "description": "Optional threshold value (must be between min and max)." + } + }, + "required": ["min", "max", "step"], + "additionalProperties": false + } + }, + "required": ["valueSpecification"], + "additionalProperties": false + } + } +} diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index 15b51ec4..70812e94 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -13,8 +13,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { join, resolve } from 'node:path'; +import { join, resolve, dirname } from 'node:path'; import { mkdirSync, readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { fileURLToPath } from 'node:url'; import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; import { Messages, EnvironmentVariable } from '@salesforce/core'; import { Agent } from '@salesforce/agents'; @@ -454,6 +455,10 @@ export default class AgentScorerCreate extends SfCommand { const { flags } = await this.parse(AgentScorerCreate); + + if (flags['spec-schema']) { + const schemaPath = resolve( + dirname(fileURLToPath(import.meta.url)), + '..', '..', '..', '..', 'schemas', 'agent-scorer-create__spec.json' + ); + const schema = readFileSync(schemaPath, 'utf8'); + this.styledJSON(JSON.parse(schema) as unknown as import('@salesforce/ts-types').AnyJson); + return { path: '', apiName: '', contents: '' }; + } + const connection = flags['target-org'].getConnection(flags['api-version']); const spec = flags.spec From af783e8368dd6518b4ea17acd2de4a8533b218a2 Mon Sep 17 00:00:00 2001 From: nnaffar Date: Sun, 5 Jul 2026 16:44:32 +0300 Subject: [PATCH 05/19] add support for measurements --- src/commands/agent/scorer/create.ts | 55 ++++++++++++++++----- test/commands/agent/scorer/create.test.ts | 58 +++++++++++++++++++++++ 2 files changed, 100 insertions(+), 13 deletions(-) diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index 70812e94..856dac13 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -14,6 +14,7 @@ * limitations under the License. */ import { join, resolve, dirname } from 'node:path'; +import { createHash } from 'node:crypto'; import { mkdirSync, readFileSync, writeFileSync, existsSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; @@ -379,32 +380,48 @@ function buildPromptTemplateXml(apiName: string, promptContent: string, spec: Sc const templateType = getPromptTemplateType(spec); const isOpenEnded = spec.scorerType === 'OpenEnded'; + const isMeasurement = templateType === 'agentforce_session_tracing__scorerMeasurement'; - const inputs = [ + const inputs: Array<{ apiName: string; definition: string; referenceName: string; required: boolean }> = [ { apiName: 'Session', definition: 'lightningtype://propertyType/agentforce_session_tracing__stdmDetailViewType', referenceName: 'Input:Session', required: true, }, - { - apiName: 'AllowedLabels', - definition: 'primitive://String', - referenceName: 'Input:AllowedLabels', - required: !isOpenEnded, - }, - { - apiName: 'FallbackLabel', - definition: 'primitive://String', - referenceName: 'Input:FallbackLabel', - required: !isOpenEnded, - }, ]; + if (isMeasurement) { + inputs.push({ + apiName: 'AllowedRange', + definition: 'primitive://String', + referenceName: 'Input:AllowedRange', + required: true, + }); + } else { + inputs.push( + { + apiName: 'AllowedLabels', + definition: 'primitive://String', + referenceName: 'Input:AllowedLabels', + required: !isOpenEnded, + }, + { + apiName: 'FallbackLabel', + definition: 'primitive://String', + referenceName: 'Input:FallbackLabel', + required: !isOpenEnded, + } + ); + } + + const versionIdentifier = createHash('sha256').update(promptContent).digest('base64') + '_1'; + const xmlObj = { '?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' }, GenAiPromptTemplate: { '@_xmlns': 'http://soap.sforce.com/2006/04/metadata', + activeVersionIdentifier: versionIdentifier, developerName: apiName, masterLabel: apiName, overridable: false, @@ -413,6 +430,7 @@ function buildPromptTemplateXml(apiName: string, promptContent: string, spec: Sc inputs, primaryModel: 'sfdc_ai__DefaultOpenAIGPT4OmniMini', status: 'Published', + versionIdentifier, }, type: templateType, visibility: 'Global', @@ -773,6 +791,17 @@ function buildDefaultPromptContent(spec: Partial): string { ].join('\n'); } + if (spec.semanticType === 'Measurement') { + return [ + 'Analyze the following agent-user conversation and evaluate it based on your scoring criteria.', + '', + 'Respond with ONLY a number within the allowed range: {!$Input:AllowedRange}', + '', + 'session audit data:', + '{!$Input:Session}', + ].join('\n'); + } + return [ 'Analyze the following agent-user conversation and evaluate it based on your scoring criteria.', '', diff --git a/test/commands/agent/scorer/create.test.ts b/test/commands/agent/scorer/create.test.ts index 357659b3..914f4ee6 100644 --- a/test/commands/agent/scorer/create.test.ts +++ b/test/commands/agent/scorer/create.test.ts @@ -502,6 +502,25 @@ describe('agent scorer create', () => { expect(promptFile!.content).to.include('agentforce_session_tracing__scorerMeasurement'); }); + it('should use AllowedRange input for scorerMeasurement type', async () => { + const { Command, writtenFiles } = await loadMockedCommand( + makePromptTemplateSpec({ semanticType: 'Measurement' }) + ); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('AllowedRange'); + expect(promptFile!.content).to.include('Input:AllowedRange'); + expect(promptFile!.content).not.to.include('AllowedLabels'); + expect(promptFile!.content).not.to.include('FallbackLabel'); + }); + it('should use scorerMultilabel type for default Text scorers', async () => { const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); @@ -740,6 +759,25 @@ describe('agent scorer create', () => { expect(promptFile!.content).to.include('{!$Input:Session}'); expect(promptFile!.content).not.to.include('{!$Input:AllowedLabels}'); }); + + it('should use Measurement default prompt with AllowedRange', async () => { + const spec = makePromptTemplateSpec({ semanticType: 'Measurement' }); + delete (spec as any).promptContent; + const { Command, writtenFiles } = await loadMockedCommand(spec); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include('{!$Input:Session}'); + expect(promptFile!.content).to.include('{!$Input:AllowedRange}'); + expect(promptFile!.content).not.to.include('{!$Input:AllowedLabels}'); + expect(promptFile!.content).not.to.include('{!$Input:FallbackLabel}'); + }); }); describe('output directory', () => { @@ -854,6 +892,26 @@ describe('agent scorer create', () => { expect(promptFile!.content).to.include('Published'); }); + it('should include activeVersionIdentifier and versionIdentifier', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); + + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + + const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); + expect(promptFile!.content).to.include(''); + expect(promptFile!.content).to.include(''); + // Both should have the same value + const activeMatch = promptFile!.content.match(/(.+?)<\/activeVersionIdentifier>/); + const versionMatch = promptFile!.content.match(/(.+?)<\/versionIdentifier>/); + expect(activeMatch![1]).to.equal(versionMatch![1]); + expect(activeMatch![1]).to.match(/.+=_1$/); + }); + it('should include Session input with correct definition', async () => { const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); From 1021df9e4a7d21f8ce6d9a26a9f6fec43a3dc805 Mon Sep 17 00:00:00 2001 From: nnaffar Date: Sun, 5 Jul 2026 16:47:48 +0300 Subject: [PATCH 06/19] adding --spec-schema for agents to discover the right yaml schema for authoring --- src/commands/agent/scorer/create.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index 856dac13..02ce1e89 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -109,7 +109,7 @@ const FLAGGABLE_PROMPTS = { 'data-type': { message: messages.getMessage('flags.data-type.summary'), promptMessage: 'What data type does this scorer produce?', - options: ['Text', 'Number', 'OpenEnded'], + options: ['OpenEnded', 'Text', 'Number'], validate: (d: string): boolean | string => ['Text', 'Number', 'OpenEnded'].includes(d) || 'Invalid data type', required: true, }, From 410cd4a9892f5c0d46908bd037286a2191dfaa1f Mon Sep 17 00:00:00 2001 From: nnaffar Date: Mon, 6 Jul 2026 20:31:04 +0300 Subject: [PATCH 07/19] offload logic to agents package --- src/commands/agent/scorer/create.ts | 447 ++++------------------------ 1 file changed, 64 insertions(+), 383 deletions(-) diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index 02ce1e89..f4f6b692 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -14,13 +14,18 @@ * limitations under the License. */ import { join, resolve, dirname } from 'node:path'; -import { createHash } from 'node:crypto'; -import { mkdirSync, readFileSync, writeFileSync, existsSync } from 'node:fs'; +import { readFileSync, existsSync } from 'node:fs'; import { fileURLToPath } from 'node:url'; import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; import { Messages, EnvironmentVariable } from '@salesforce/core'; -import { Agent } from '@salesforce/agents'; -import { XMLBuilder } from 'fast-xml-parser'; +import { + Agent, + type ScorerSpec, + createScorerDefinition, + labelToApiName, + SUPPORTED_LIGHTNING_TYPES, + MAX_ENUM_VALUES, +} from '@salesforce/agents'; import { confirm, select, input as inquirerInput } from '@inquirer/prompts'; import YAML from 'yaml'; import { FlaggablePrompt, makeFlags, promptForFlag } from '../../../flags.js'; @@ -36,57 +41,8 @@ export type AgentScorerCreateResult = { promptTemplatePath?: string; }; -export type ScorerSpecFile = { - apiName: string; - dataType: 'Text' | 'Number' | 'LightningType'; - scorerType?: 'Predefined' | 'OpenEnded'; - lightningType?: string; - semanticType?: 'Dimension' | 'Measurement'; - inputScope?: 'Session' | 'Intent'; - label: string; - description?: string; - engineType: 'Manual' | 'PromptTemplate'; - promptContent?: string; - promptTemplateName?: string; - status?: 'Available' | 'Draft'; - agentAssociation: { - agentApiName: string; - isActive: boolean; - samplingRate?: number; - inputScope?: 'Session' | 'Intent'; - }; - outputEnumValues?: Array<{ - value: string; - outcomeType: 'Pass' | 'Fail' | 'NotApplicable'; - isFallback?: boolean; - isSystemFallback?: boolean; - }>; - specification?: { - valueSpecification: { - min: number; - max: number; - step: number; - threshold?: number; - }; - }; -}; - -const MAX_ENUM_VALUES = 101; - -const SUPPORTED_LIGHTNING_TYPES = [ - 'lightning__textType', - 'lightning__multilineTextType', - 'lightning__richTextType', - 'lightning__numberType', - 'lightning__integerType', - 'lightning__booleanType', - 'lightning__dateType', - 'lightning__dateTimeType', - 'lightning__dateTimeStringType', - 'lightning__urlType', - 'lightning__objectType', - 'lightning__listType', -]; +/** @deprecated Use ScorerSpec from @salesforce/agents directly. */ +export type ScorerSpecFile = ScorerSpec; const FLAGGABLE_PROMPTS = { label: { @@ -135,14 +91,14 @@ const FLAGGABLE_PROMPTS = { }, } satisfies Record; -type OutputEnumValue = { +type OutputEnumValueInput = { value: string; outcomeType: string; isFallback: boolean; isSystemFallback: boolean; }; -async function promptForSingleEnumValue(index: number): Promise { +async function promptForSingleEnumValue(index: number): Promise { const value = await promptForFlag({ message: 'Output value name', promptMessage: `Output value #${index + 1} (e.g., "Good", "Bad", "N/A")`, @@ -172,8 +128,8 @@ async function promptForSingleEnumValue(index: number): Promise { - const values: OutputEnumValue[] = []; +async function promptForOutputEnumValues(): Promise { + const values: OutputEnumValueInput[] = []; let addMore = true; while (addMore) { @@ -186,14 +142,7 @@ async function promptForOutputEnumValues(): Promise { return values; } -type NumberSpecification = { - min: number; - max: number; - step: number; - threshold?: number; -}; - -async function promptForNumberSpecification(): Promise { +async function promptForNumberSpecification(): Promise<{ min: number; max: number; step: number; threshold?: number }> { const minStr = await inquirerInput({ message: 'Minimum value', default: '0', @@ -229,10 +178,9 @@ async function promptForNumberSpecification(): Promise { }); const step = parseFloat(stepStr); - const numValues = Math.floor((max - min) / step) + 1; const addThreshold = await confirm({ - message: `Add a threshold value? (${numValues} output values will be generated from ${min} to ${max})`, + message: `Add a threshold value? (${Math.floor((max - min) / step) + 1} output values will be generated from ${min} to ${max})`, default: false, theme, }); @@ -255,203 +203,6 @@ async function promptForNumberSpecification(): Promise { return { min, max, step, threshold }; } -function generateNumberEnumValues(spec: NumberSpecification): OutputEnumValue[] { - const values: OutputEnumValue[] = []; - const epsilon = 1e-9; - let current = spec.min; - - while (current <= spec.max + epsilon) { - const rounded = Math.round(current * 1e9) / 1e9; - values.push({ - value: String(rounded), - outcomeType: 'NotApplicable', - isFallback: false, - isSystemFallback: false, - }); - current += spec.step; - } - - return values; -} - -type AgentAssociation = { - agentApiName: string; - isActive: boolean; - samplingRate?: number; - inputScope?: 'Session' | 'Intent'; -}; - -function buildScorerXml(spec: ScorerSpecFile): string { - const engine: Record = {}; - if (spec.engineType === 'PromptTemplate') { - engine.engineRef = spec.promptTemplateName ?? spec.apiName; - } - engine.engineType = spec.engineType; - - const agentAssociationXml: Record = { - agentApiName: spec.agentAssociation.agentApiName, - ...(spec.agentAssociation.inputScope ? { inputScope: spec.agentAssociation.inputScope } : {}), - isActive: spec.agentAssociation.isActive, - samplingRate: spec.agentAssociation.samplingRate ?? 1.0, - }; - - const scorerVersion: Record = { - agentAssociation: agentAssociationXml, - ...(spec.description ? { description: spec.description } : {}), - engine, - label: spec.label, - }; - - // For Number type with specification, generate enum values from spec - if (spec.dataType === 'Number' && spec.specification) { - const numSpec = spec.specification.valueSpecification; - const enumValues = generateNumberEnumValues(numSpec); - scorerVersion.outputEnumValue = enumValues.map((v) => ({ - isFallback: false, - isSystemFallback: false, - outcomeType: v.outcomeType, - value: v.value, - })); - scorerVersion.specification = { - valueSpecification: { - min: numSpec.min, - max: numSpec.max, - step: numSpec.step, - ...(numSpec.threshold != null ? { threshold: numSpec.threshold } : {}), - }, - }; - } else if (spec.outputEnumValues) { - scorerVersion.outputEnumValue = spec.outputEnumValues.map((v) => ({ - isFallback: v.isFallback ?? false, - isSystemFallback: v.isSystemFallback ?? false, - outcomeType: v.outcomeType, - value: v.value, - })); - } - - scorerVersion.status = spec.status ?? 'Draft'; - scorerVersion.versionNumber = 1; - - const definition: Record = { - '@_xmlns': 'http://soap.sforce.com/2006/04/metadata', - dataType: spec.dataType, - inputScope: spec.inputScope ?? 'Session', - }; - - if (spec.lightningType) { - definition.lightningType = spec.lightningType; - } - if (spec.scorerType) { - definition.scorerType = spec.scorerType; - } - if (spec.semanticType) { - definition.semanticType = spec.semanticType; - } - - definition.scorerVersion = scorerVersion; - - const xmlObj = { - '?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' }, - AiAgentScorerDefinition: definition, - }; - - const builder = new XMLBuilder({ - format: true, - ignoreAttributes: false, - indentBy: ' ', - suppressBooleanAttributes: false, - }); - - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return builder.build(xmlObj); -} - -function getPromptTemplateType(spec: ScorerSpecFile): string { - if (spec.scorerType === 'OpenEnded') { - return 'agentforce_session_tracing__scorerOpenEnded'; - } - if (spec.semanticType === 'Measurement') { - return 'agentforce_session_tracing__scorerMeasurement'; - } - return 'agentforce_session_tracing__scorerMultilabel'; -} - -function buildPromptTemplateXml(apiName: string, promptContent: string, spec: ScorerSpecFile): string { - const templateType = getPromptTemplateType(spec); - - const isOpenEnded = spec.scorerType === 'OpenEnded'; - const isMeasurement = templateType === 'agentforce_session_tracing__scorerMeasurement'; - - const inputs: Array<{ apiName: string; definition: string; referenceName: string; required: boolean }> = [ - { - apiName: 'Session', - definition: 'lightningtype://propertyType/agentforce_session_tracing__stdmDetailViewType', - referenceName: 'Input:Session', - required: true, - }, - ]; - - if (isMeasurement) { - inputs.push({ - apiName: 'AllowedRange', - definition: 'primitive://String', - referenceName: 'Input:AllowedRange', - required: true, - }); - } else { - inputs.push( - { - apiName: 'AllowedLabels', - definition: 'primitive://String', - referenceName: 'Input:AllowedLabels', - required: !isOpenEnded, - }, - { - apiName: 'FallbackLabel', - definition: 'primitive://String', - referenceName: 'Input:FallbackLabel', - required: !isOpenEnded, - } - ); - } - - const versionIdentifier = createHash('sha256').update(promptContent).digest('base64') + '_1'; - - const xmlObj = { - '?xml': { '@_version': '1.0', '@_encoding': 'UTF-8' }, - GenAiPromptTemplate: { - '@_xmlns': 'http://soap.sforce.com/2006/04/metadata', - activeVersionIdentifier: versionIdentifier, - developerName: apiName, - masterLabel: apiName, - overridable: false, - templateVersions: { - content: promptContent, - inputs, - primaryModel: 'sfdc_ai__DefaultOpenAIGPT4OmniMini', - status: 'Published', - versionIdentifier, - }, - type: templateType, - visibility: 'Global', - }, - }; - - const builder = new XMLBuilder({ - format: true, - ignoreAttributes: false, - indentBy: ' ', - suppressBooleanAttributes: false, - }); - - // eslint-disable-next-line @typescript-eslint/no-unsafe-return - return builder.build(xmlObj); -} - -function labelToApiName(label: string): string { - return label.replace(/\s+/g, '_').replace(/[^A-Za-z0-9_]/g, ''); -} - export default class AgentScorerCreate extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); @@ -502,23 +253,51 @@ export default class AgentScorerCreate extends SfCommand, connection: ReturnType - ): Promise { + ): Promise { if (this.jsonEnabled()) { const missing = Object.entries(FLAGGABLE_PROMPTS) .filter(([key, p]) => 'required' in p && p.required && !flags[key]) @@ -568,92 +347,28 @@ export default class AgentScorerCreate extends SfCommand - ): Promise { - if (spec.dataType === 'Text' && spec.outputEnumValues) { - const fallbackCount = spec.outputEnumValues.filter((v) => v.isFallback).length; - if (fallbackCount !== 1) { - throw new Error( - `Text scorers must have exactly 1 fallback value, but found ${fallbackCount}.` - ); - } - } - - const scorerXml = buildScorerXml(spec); - const outputDir = resolve(flags['output-dir'] as string); - const scorerDir = join(outputDir, 'aiAgentScorerDefinitions'); - const scorerFileName = `${spec.apiName}.aiAgentScorerDefinition-meta.xml`; - const scorerPath = join(scorerDir, scorerFileName); - - let promptTemplatePath: string | undefined; - let promptTemplateXml: string | undefined; - if (spec.engineType === 'PromptTemplate' && !spec.promptTemplateName) { - const content = spec.promptContent ?? buildDefaultPromptContent(spec); - promptTemplateXml = buildPromptTemplateXml(spec.apiName, content, spec); - const promptDir = join(outputDir, 'genAiPromptTemplates'); - const promptFileName = `${spec.apiName}.genAiPromptTemplate-meta.xml`; - promptTemplatePath = join(promptDir, promptFileName); - } - - if (flags.preview) { - this.log('\n--- Scorer Definition (preview) ---\n'); - this.log(scorerXml); - if (promptTemplateXml) { - this.log('\n--- Prompt Template (preview) ---\n'); - this.log(promptTemplateXml); - } - return { path: scorerPath, apiName: spec.apiName, contents: scorerXml, promptTemplatePath }; - } - - mkdirSync(scorerDir, { recursive: true }); - if (existsSync(scorerPath) && !this.jsonEnabled()) { - const overwrite = await confirm({ - message: `${scorerFileName} already exists. Overwrite?`, - default: false, - theme, - }); - if (!overwrite) { - this.log('Operation canceled.'); - return { path: '', apiName: spec.apiName, contents: '' }; - } - } - writeFileSync(scorerPath, scorerXml); - this.log(`\nScorer definition written to: ${scorerPath}`); - - if (promptTemplateXml && promptTemplatePath) { - const promptDir = join(outputDir, 'genAiPromptTemplates'); - mkdirSync(promptDir, { recursive: true }); - writeFileSync(promptTemplatePath, promptTemplateXml); - this.log(`Prompt template written to: ${promptTemplatePath}`); - } - - return { path: scorerPath, apiName: spec.apiName, contents: scorerXml, promptTemplatePath }; - } - private async promptForDataTypeDetails(dataType: string): Promise<{ - outputEnumValues?: OutputEnumValue[]; - specification?: ScorerSpecFile['specification']; + outputEnumValues?: OutputEnumValueInput[]; + specification?: ScorerSpec['specification']; lightningType?: string; - scorerType?: ScorerSpecFile['scorerType']; + scorerType?: ScorerSpec['scorerType']; }> { if (dataType === 'Number') { this.log(); @@ -720,8 +435,8 @@ export default class AgentScorerCreate extends SfCommand, engineType: string, agentApiNameFlag?: string - ): Promise { - let agentAssociation: AgentAssociation; + ): Promise { + let agentAssociation: ScorerSpec['agentAssociation']; if (agentApiNameFlag) { agentAssociation = { agentApiName: agentApiNameFlag, isActive: false }; } else { @@ -778,37 +493,3 @@ export default class AgentScorerCreate extends SfCommand): string { - if (spec.scorerType === 'OpenEnded') { - return [ - 'Analyze the following agent-user conversation and provide your evaluation.', - '', - 'Your response must conform to the expected data type.', - '', - 'session audit data:', - '{!$Input:Session}', - ].join('\n'); - } - - if (spec.semanticType === 'Measurement') { - return [ - 'Analyze the following agent-user conversation and evaluate it based on your scoring criteria.', - '', - 'Respond with ONLY a number within the allowed range: {!$Input:AllowedRange}', - '', - 'session audit data:', - '{!$Input:Session}', - ].join('\n'); - } - - return [ - 'Analyze the following agent-user conversation and evaluate it based on your scoring criteria.', - '', - 'Respond with ONLY one of the allowed values: {!$Input:AllowedLabels}', - 'or fallback to: {!$Input:FallbackLabel}', - '', - 'session audit data:', - '{!$Input:Session}', - ].join('\n'); -} From e10c17dab5355993d0f89c08634d405ba502882a Mon Sep 17 00:00:00 2001 From: nnaffar Date: Sun, 6 Sep 2026 12:29:19 +0300 Subject: [PATCH 08/19] reuse consts from agents lib --- src/commands/agent/scorer/create.ts | 42 ++++++++++++++++++----------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index f4f6b692..51f1d336 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -23,8 +23,15 @@ import { type ScorerSpec, createScorerDefinition, labelToApiName, + scorerEnumValueCount, SUPPORTED_LIGHTNING_TYPES, MAX_ENUM_VALUES, + SCORER_API_NAME_MAX_LENGTH, + SCORER_API_NAME_PATTERN, + SCORER_ENGINE_TYPES, + SCORER_STATUSES, + SCORER_OUTCOME_TYPES, + SCORER_INPUT_SCOPES, } from '@salesforce/agents'; import { confirm, select, input as inquirerInput } from '@inquirer/prompts'; import YAML from 'yaml'; @@ -44,6 +51,11 @@ export type AgentScorerCreateResult = { /** @deprecated Use ScorerSpec from @salesforce/agents directly. */ export type ScorerSpecFile = ScorerSpec; +// The CLI presents "OpenEnded" as a data-type choice; it maps to core dataType 'LightningType' + +// scorerType 'OpenEnded' (see runInteractiveInterview). This is a UI-level set, distinct from the +// core SCORER_DATA_TYPES union, so it is defined locally. +const UI_SCORER_DATA_TYPES = ['OpenEnded', 'Text', 'Number'] as const; + const FLAGGABLE_PROMPTS = { label: { message: messages.getMessage('flags.label.summary'), @@ -56,8 +68,8 @@ const FLAGGABLE_PROMPTS = { promptMessage: 'Scorer API name', validate: (d: string): boolean | string => { if (!d.length) return 'API name cannot be empty'; - if (d.length > 35) return 'API name cannot exceed 35 characters'; - if (!/^[A-Za-z][A-Za-z0-9_]*$/.test(d)) return 'Must start with letter, only alphanumerics and underscores'; + if (d.length > SCORER_API_NAME_MAX_LENGTH) return `API name cannot exceed ${SCORER_API_NAME_MAX_LENGTH} characters`; + if (!SCORER_API_NAME_PATTERN.test(d)) return 'Must start with letter, only alphanumerics and underscores'; return true; }, required: true, @@ -65,8 +77,9 @@ const FLAGGABLE_PROMPTS = { 'data-type': { message: messages.getMessage('flags.data-type.summary'), promptMessage: 'What data type does this scorer produce?', - options: ['OpenEnded', 'Text', 'Number'], - validate: (d: string): boolean | string => ['Text', 'Number', 'OpenEnded'].includes(d) || 'Invalid data type', + options: UI_SCORER_DATA_TYPES, + validate: (d: string): boolean | string => + (UI_SCORER_DATA_TYPES as readonly string[]).includes(d) || 'Invalid data type', required: true, }, description: { @@ -77,16 +90,16 @@ const FLAGGABLE_PROMPTS = { 'engine-type': { message: messages.getMessage('flags.engine-type.summary'), promptMessage: 'Scoring engine type', - options: ['Manual', 'PromptTemplate'], + options: SCORER_ENGINE_TYPES, validate: (d: string): boolean | string => - ['Manual', 'PromptTemplate'].includes(d) || 'Invalid engine type', + (SCORER_ENGINE_TYPES as readonly string[]).includes(d) || 'Invalid engine type', required: true, }, status: { message: messages.getMessage('flags.status.summary'), promptMessage: 'Initial status', - options: ['Draft', 'Available'], - validate: (d: string): boolean | string => ['Available', 'Draft'].includes(d) || 'Invalid status', + options: SCORER_STATUSES, + validate: (d: string): boolean | string => (SCORER_STATUSES as readonly string[]).includes(d) || 'Invalid status', default: 'Draft', }, } satisfies Record; @@ -108,9 +121,9 @@ async function promptForSingleEnumValue(index: number): Promise - ['Pass', 'Fail', 'NotApplicable'].includes(d) || 'Invalid', + (SCORER_OUTCOME_TYPES as readonly string[]).includes(d) || 'Invalid', }); const isFallback = await confirm({ @@ -170,7 +183,7 @@ async function promptForNumberSpecification(): Promise<{ min: number; max: numbe validate: (d: string): boolean | string => { const n = parseFloat(d); if (isNaN(n) || n <= 0) return 'Step must be a positive number'; - const numValues = Math.floor((max - min) / n) + 1; + const numValues = scorerEnumValueCount(min, max, n); if (numValues > MAX_ENUM_VALUES) return `Step too small: would generate ${numValues} values (max ${MAX_ENUM_VALUES})`; return true; }, @@ -180,7 +193,7 @@ async function promptForNumberSpecification(): Promise<{ min: number; max: numbe const step = parseFloat(stepStr); const addThreshold = await confirm({ - message: `Add a threshold value? (${Math.floor((max - min) / step) + 1} output values will be generated from ${min} to ${max})`, + message: `Add a threshold value? (${scorerEnumValueCount(min, max, step)} output values will be generated from ${min} to ${max})`, default: false, theme, }); @@ -457,10 +470,7 @@ export default class AgentScorerCreate extends SfCommand({ message: 'Input scope for this agent association', - choices: [ - { name: 'Session', value: 'Session' }, - { name: 'Intent', value: 'Intent' }, - ], + choices: SCORER_INPUT_SCOPES.map((s) => ({ name: s, value: s })), default: 'Session', theme, }); From 9d3a58b584e199bd447eef164bf6f4eec5f219df Mon Sep 17 00:00:00 2001 From: nnaffar Date: Sun, 6 Sep 2026 22:34:52 +0300 Subject: [PATCH 09/19] fix(scorer): consume the generated spec schema and repair command tests Wire the create command to the single source of truth in @salesforce/agents and get its unit tests running again. - --spec-schema now emits scorerSpecJsonSchema() from the agents lib instead of a hand-maintained copy, so schemas/agent-scorer-create__spec.json is deleted; the tracked schema is now the command's generated result schema (schemas/agent-scorer-create.json). - Repair the command tests, which had silently rotted: the esmock warmup skipped the whole suite, so nobody noticed they were written against an older command. Make the --spec files exist on disk (oclif's Flags.file({ exists: true }) stats them at parse time, which esmock can't intercept), capture the lib's real fs/promises writes (it no longer writes via the mocked node:fs), point the measurement-template tests at Number specs (the template type is driven by dataType, not semanticType), and drop the obsolete number-enum-value tests (generateNumberEnumValues was removed; numbers now emit a compact valueSpecification). --- schemas/agent-scorer-create.json | 29 ++++ schemas/agent-scorer-create__spec.json | 189 ---------------------- src/commands/agent/scorer/create.ts | 18 +-- test/commands/agent/scorer/create.test.ts | 157 ++++++++---------- 4 files changed, 106 insertions(+), 287 deletions(-) create mode 100644 schemas/agent-scorer-create.json delete mode 100644 schemas/agent-scorer-create__spec.json diff --git a/schemas/agent-scorer-create.json b/schemas/agent-scorer-create.json new file mode 100644 index 00000000..65b33290 --- /dev/null +++ b/schemas/agent-scorer-create.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentScorerCreateResult", + "definitions": { + "AgentScorerCreateResult": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "apiName": { + "type": "string" + }, + "contents": { + "type": "string" + }, + "promptTemplatePath": { + "type": "string" + } + }, + "required": [ + "path", + "apiName", + "contents" + ], + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/schemas/agent-scorer-create__spec.json b/schemas/agent-scorer-create__spec.json deleted file mode 100644 index 37ae4d5e..00000000 --- a/schemas/agent-scorer-create__spec.json +++ /dev/null @@ -1,189 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$ref": "#/definitions/ScorerSpecFile", - "definitions": { - "ScorerSpecFile": { - "type": "object", - "description": "YAML spec file for creating an agent scorer definition via `sf agent scorer create --spec `.", - "properties": { - "apiName": { - "type": "string", - "description": "API name of the scorer definition. Max 35 characters, must start with a letter, only alphanumerics and underscores.", - "pattern": "^[A-Za-z][A-Za-z0-9_]{0,34}$", - "maxLength": 35 - }, - "dataType": { - "type": "string", - "enum": ["Text", "Number", "LightningType"], - "description": "Data type produced by the scorer. Use 'Text' for categorical labels, 'Number' for numeric scales, 'LightningType' for open-ended evaluations." - }, - "scorerType": { - "type": "string", - "enum": ["Predefined", "OpenEnded"], - "description": "Set to 'OpenEnded' when dataType is 'LightningType' for free-form evaluation." - }, - "lightningType": { - "type": "string", - "enum": [ - "lightning__textType", - "lightning__multilineTextType", - "lightning__richTextType", - "lightning__numberType", - "lightning__integerType", - "lightning__booleanType", - "lightning__dateType", - "lightning__dateTimeType", - "lightning__dateTimeStringType", - "lightning__urlType", - "lightning__objectType", - "lightning__listType" - ], - "description": "Required when dataType is 'LightningType'. Specifies the lightning type for open-ended values." - }, - "semanticType": { - "type": "string", - "enum": ["Dimension", "Measurement"], - "description": "How this scorer is used in analytics. 'Dimension' for categorical grouping, 'Measurement' for numeric aggregation." - }, - "inputScope": { - "type": "string", - "enum": ["Session", "Intent"], - "default": "Session", - "description": "Whether the scorer evaluates an entire session or a single intent within a session." - }, - "label": { - "type": "string", - "description": "Display label for the scorer version.", - "minLength": 1 - }, - "description": { - "type": "string", - "description": "Human-readable description of what this scorer evaluates." - }, - "engineType": { - "type": "string", - "enum": ["Manual", "PromptTemplate"], - "description": "'Manual' for human-evaluated scoring, 'PromptTemplate' for LLM-evaluated scoring." - }, - "promptContent": { - "type": "string", - "description": "Prompt text for PromptTemplate engine type. Use {!$Input:Session} to reference the session data, {!$Input:AllowedLabels} for allowed output values, and {!$Input:FallbackLabel} for the fallback value. Ignored when engineType is 'Manual'." - }, - "promptTemplateName": { - "type": "string", - "description": "API name of an existing prompt template to use instead of generating a new one. Mutually exclusive with promptContent." - }, - "status": { - "type": "string", - "enum": ["Available", "Draft"], - "default": "Draft", - "description": "Initial status of the scorer version." - }, - "agentAssociation": { - "$ref": "#/definitions/AgentAssociation" - }, - "outputEnumValues": { - "type": "array", - "description": "Output value definitions. Required for 'Text' dataType. For 'Text' scorers, exactly one value must have isFallback: true.", - "items": { - "$ref": "#/definitions/OutputEnumValue" - } - }, - "specification": { - "$ref": "#/definitions/NumberSpecification", - "description": "Required when dataType is 'Number'. Defines the numeric scale." - } - }, - "required": ["apiName", "dataType", "label", "engineType", "agentAssociation"], - "additionalProperties": false - }, - "AgentAssociation": { - "type": "object", - "description": "Associates the scorer with an agent in the org.", - "properties": { - "agentApiName": { - "type": "string", - "description": "API name of the agent to associate with this scorer." - }, - "isActive": { - "type": "boolean", - "description": "Whether scoring is active for this agent association." - }, - "samplingRate": { - "type": "number", - "minimum": 0, - "maximum": 1, - "default": 1.0, - "description": "Fraction of sessions to score (0.0 to 1.0). Only relevant when isActive is true." - }, - "inputScope": { - "type": "string", - "enum": ["Session", "Intent"], - "description": "Override input scope for this specific agent association." - } - }, - "required": ["agentApiName", "isActive"], - "additionalProperties": false - }, - "OutputEnumValue": { - "type": "object", - "description": "A possible output value for the scorer.", - "properties": { - "value": { - "type": "string", - "description": "The output label (e.g., 'Good', 'Bad', 'N/A').", - "minLength": 1 - }, - "outcomeType": { - "type": "string", - "enum": ["Pass", "Fail", "NotApplicable"], - "description": "Maps this value to a pass/fail outcome for reporting." - }, - "isFallback": { - "type": "boolean", - "default": false, - "description": "Whether this is the fallback value. Exactly one value must be the fallback for Text scorers." - }, - "isSystemFallback": { - "type": "boolean", - "default": false, - "description": "Whether this is a system-generated fallback. Typically false for user-defined scorers." - } - }, - "required": ["value", "outcomeType"], - "additionalProperties": false - }, - "NumberSpecification": { - "type": "object", - "properties": { - "valueSpecification": { - "type": "object", - "description": "Defines the numeric scale. The number of generated values ((max - min) / step + 1) must not exceed 101.", - "properties": { - "min": { - "type": "number", - "description": "Minimum value of the scale." - }, - "max": { - "type": "number", - "description": "Maximum value of the scale. Must be greater than min." - }, - "step": { - "type": "number", - "exclusiveMinimum": 0, - "description": "Step size between values." - }, - "threshold": { - "type": "number", - "description": "Optional threshold value (must be between min and max)." - } - }, - "required": ["min", "max", "step"], - "additionalProperties": false - } - }, - "required": ["valueSpecification"], - "additionalProperties": false - } - } -} diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index 51f1d336..ab57a8ee 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -13,9 +13,8 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import { join, resolve, dirname } from 'node:path'; +import { join, resolve } from 'node:path'; import { readFileSync, existsSync } from 'node:fs'; -import { fileURLToPath } from 'node:url'; import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; import { Messages, EnvironmentVariable } from '@salesforce/core'; import { @@ -24,6 +23,8 @@ import { createScorerDefinition, labelToApiName, scorerEnumValueCount, + scorerSpecJsonSchema, + type SupportedLightningType, SUPPORTED_LIGHTNING_TYPES, MAX_ENUM_VALUES, SCORER_API_NAME_MAX_LENGTH, @@ -255,12 +256,7 @@ export default class AgentScorerCreate extends SfCommand { if (dataType === 'Number') { @@ -394,7 +390,7 @@ export default class AgentScorerCreate extends SfCommand({ + const lightningType = await select({ message: 'Select the lightning type for open-ended values', choices: SUPPORTED_LIGHTNING_TYPES.map((t) => ({ name: t, value: t })), theme, diff --git a/test/commands/agent/scorer/create.test.ts b/test/commands/agent/scorer/create.test.ts index 914f4ee6..109a8721 100644 --- a/test/commands/agent/scorer/create.test.ts +++ b/test/commands/agent/scorer/create.test.ts @@ -17,10 +17,21 @@ /* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any */ import { join } from 'node:path'; -import { mkdirSync, writeFileSync, rmSync } from 'node:fs'; +import { tmpdir } from 'node:os'; +import { createRequire } from 'node:module'; +import { mkdirSync, mkdtempSync, writeFileSync, rmSync } from 'node:fs'; import { expect } from 'chai'; import esmock from 'esmock'; import sinon from 'sinon'; + +// The command reads specs via node:fs, but createScorerDefinition (in @salesforce/agents) writes +// output via node:fs/promises (writeFile/mkdir). node: core modules are singletons, so stubbing +// this shared instance captures the library's real writes without touching disk. Obtained through +// createRequire because ESM namespace objects are frozen and cannot be stubbed. +const fsPromises = createRequire(import.meta.url)('node:fs/promises') as { + writeFile: (...args: any[]) => Promise; + mkdir: (...args: any[]) => Promise; +}; import YAML from 'yaml'; import { TestContext, MockTestOrgData } from '@salesforce/core/testSetup'; import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; @@ -126,17 +137,32 @@ async function loadMockedCommand( const fsMock: Record = { readFileSync: () => yamlContent, - writeFileSync: (path: string, content: string) => { - writtenFiles.push({ path, content }); - }, - mkdirSync: (path: string) => { - createdDirs.push(path); - }, existsSync: fileExists, }; const mocks: Record = { 'node:fs': fsMock }; + // Capture the scorer/prompt-template files written by createScorerDefinition (via + // node:fs/promises) without hitting disk; delegate any unrelated writes to the real fns. + const isScorerOutput = (p: unknown): boolean => + typeof p === 'string' && (p.includes('aiAgentScorerDefinitions') || p.includes('genAiPromptTemplates')); + const origWriteFile = fsPromises.writeFile; + const origMkdir = fsPromises.mkdir; + sinon.stub(fsPromises, 'writeFile').callsFake((path: unknown, content: unknown, options: unknown) => { + if (isScorerOutput(path)) { + writtenFiles.push({ path: String(path), content: String(content) }); + return Promise.resolve(); + } + return origWriteFile(path, content, options); + }); + sinon.stub(fsPromises, 'mkdir').callsFake((path: unknown, options: unknown) => { + if (isScorerOutput(path)) { + createdDirs.push(String(path)); + return Promise.resolve(undefined); + } + return origMkdir(path, options); + }); + if (opts?.confirmResult !== undefined) { mocks['@inquirer/prompts'] = { confirm: sinon.stub().resolves(opts.confirmResult), @@ -149,9 +175,26 @@ async function loadMockedCommand( return { Command: mod.default, writtenFiles, createdDirs }; } +// Bare spec filenames used across the --spec tests. `spec: Flags.file({ exists: true })` makes +// oclif stat the path at parse time (via a CJS require of node:fs/promises deep inside +// @oclif/core, which esmock cannot intercept), so these must physically exist on disk. We create +// them in a temp dir and chdir there; the command's readFileSync is still mocked, so file +// contents are irrelevant — only their existence matters. +const SPEC_FILENAMES = [ + 'test.yaml', + 'test-scorer.yaml', + 'numeric-scorer.yaml', + 'threshold-scorer.yaml', + 'open-scorer.yaml', + 'prompt-scorer.yaml', + 'manual-scorer.yaml', +]; + describe('agent scorer create', () => { const $$ = new TestContext(); let testOrg: MockTestOrgData; + let originalCwd: string; + let specDir: string; before(async function () { // Warm up esmock to check it can load the module @@ -165,9 +208,20 @@ describe('agent scorer create', () => { }, }); } catch (e: any) { + // eslint-disable-next-line no-console console.error('esmock warmup failed:', e.message); this.skip(); } + + originalCwd = process.cwd(); + specDir = mkdtempSync(join(tmpdir(), 'scorer-specs-')); + for (const name of SPEC_FILENAMES) writeFileSync(join(specDir, name), ''); + process.chdir(specDir); + }); + + after(() => { + if (originalCwd) process.chdir(originalCwd); + if (specDir) rmSync(specDir, { recursive: true, force: true }); }); beforeEach(async () => { @@ -178,6 +232,7 @@ describe('agent scorer create', () => { afterEach(() => { $$.restore(); + sinon.restore(); }); describe('--spec flag (YAML-driven) with --preview', () => { @@ -218,8 +273,6 @@ describe('agent scorer create', () => { expect(result.contents).to.include('0'); expect(result.contents).to.include('5'); expect(result.contents).to.include('1'); - expect(result.contents).to.include('0'); - expect(result.contents).to.include('5'); expect(result.contents).to.include('Available'); }); @@ -486,9 +539,9 @@ describe('agent scorer create', () => { expect(promptFile!.content).to.include('agentforce_session_tracing__scorerOpenEnded'); }); - it('should use scorerMeasurement type for Measurement semanticType', async () => { + it('should use scorerMeasurement type for Number scorers', async () => { const { Command, writtenFiles } = await loadMockedCommand( - makePromptTemplateSpec({ semanticType: 'Measurement' }) + makeNumberSpec({ engineType: 'PromptTemplate' }) ); await Command.run([ @@ -504,7 +557,7 @@ describe('agent scorer create', () => { it('should use AllowedRange input for scorerMeasurement type', async () => { const { Command, writtenFiles } = await loadMockedCommand( - makePromptTemplateSpec({ semanticType: 'Measurement' }) + makeNumberSpec({ engineType: 'PromptTemplate' }) ); await Command.run([ @@ -536,79 +589,10 @@ describe('agent scorer create', () => { }); }); - describe('number enum value generation', () => { - it('should generate correct values for integer steps', async () => { - const spec = makeNumberSpec({ - specification: { valueSpecification: { min: 0, max: 3, step: 1 } }, - }); - const { Command } = await loadMockedCommand(spec); - - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--preview', - '--json', - ]); - - expect(result.contents).to.include('0'); - expect(result.contents).to.include('1'); - expect(result.contents).to.include('2'); - expect(result.contents).to.include('3'); - }); - - it('should generate correct values for decimal steps', async () => { - const spec = makeNumberSpec({ - specification: { valueSpecification: { min: 0, max: 1, step: 0.5 } }, - }); - const { Command } = await loadMockedCommand(spec); - - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--preview', - '--json', - ]); - - expect(result.contents).to.include('0'); - expect(result.contents).to.include('0.5'); - expect(result.contents).to.include('1'); - }); - - it('should set outcomeType to NotApplicable for number values', async () => { - const spec = makeNumberSpec({ - specification: { valueSpecification: { min: 1, max: 2, step: 1 } }, - }); - const { Command } = await loadMockedCommand(spec); - - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--preview', - '--json', - ]); - - const matches = result.contents.match(/NotApplicable<\/outcomeType>/g); - expect(matches).to.have.length(2); - }); - - it('should handle large step generating few values', async () => { - const spec = makeNumberSpec({ - specification: { valueSpecification: { min: 0, max: 100, step: 50 } }, - }); - const { Command } = await loadMockedCommand(spec); - - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--preview', - '--json', - ]); - - expect(result.contents).to.include('0'); - expect(result.contents).to.include('50'); - expect(result.contents).to.include('100'); - }); - }); + // NOTE: number scorers no longer expand min/max/step into enumerated entries; the + // generateNumberEnumValues helper was removed (they now emit a compact , + // covered by 'should create a Number scorer with specification'). The former + // 'number enum value generation' suite tested that removed behavior and was deleted. describe('XML structure', () => { it('should include XML declaration and namespace', async () => { @@ -761,8 +745,7 @@ describe('agent scorer create', () => { }); it('should use Measurement default prompt with AllowedRange', async () => { - const spec = makePromptTemplateSpec({ semanticType: 'Measurement' }); - delete (spec as any).promptContent; + const spec = makeNumberSpec({ engineType: 'PromptTemplate' }); const { Command, writtenFiles } = await loadMockedCommand(spec); await Command.run([ From b1f6721f65f33dd8cef60386f6ee079a2ad64f5f Mon Sep 17 00:00:00 2001 From: nnaffar Date: Mon, 7 Sep 2026 12:36:59 +0300 Subject: [PATCH 10/19] add support for scorer run. deprecate old dataTypes, keep lightning support as default --- command-snapshot.json | 1377 +++++++++++++-------- messages/agent.scorer.create.md | 10 +- messages/agent.scorer.run.md | 51 + schemas/agent-scorer-run.json | 49 + src/commands/agent/scorer/create.ts | 145 +-- src/commands/agent/scorer/run.ts | 115 ++ test/commands/agent/scorer/create.test.ts | 284 +---- test/commands/agent/scorer/run.test.ts | 284 +++++ 8 files changed, 1446 insertions(+), 869 deletions(-) create mode 100644 messages/agent.scorer.run.md create mode 100644 schemas/agent-scorer-run.json create mode 100644 src/commands/agent/scorer/run.ts create mode 100644 test/commands/agent/scorer/run.test.ts diff --git a/command-snapshot.json b/command-snapshot.json index 39455470..ef4d8a47 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -1,512 +1,867 @@ [ - { - "alias": [], - "command": "agent:activate", - "flagAliases": [], - "flagChars": ["n", "o"], - "flags": ["api-name", "api-version", "flags-dir", "json", "target-org", "version"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:create", - "flagAliases": [], - "flagChars": ["n", "o", "w"], - "flags": [ - "api-version", - "content-fields", - "data-category-ids", - "data-category-names", - "description", - "developer-name", - "flags-dir", - "index-mode", - "json", - "name", - "primary-index-field1", - "primary-index-field2", - "retriever-id", - "source-type", - "target-org", - "wait" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:delete", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "library-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:file:add", - "flagAliases": [], - "flagChars": ["f", "i", "o"], - "flags": ["api-version", "flags-dir", "json", "library-id", "path", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:file:delete", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "file-id", "flags-dir", "json", "library-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:file:list", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "library-id", "offset", "page-size", "status", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:get", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "library-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:list", - "flagAliases": [], - "flagChars": ["o"], - "flags": ["api-version", "flags-dir", "json", "source-type", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:status", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "include-artifacts", "json", "library-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:update", - "flagAliases": [], - "flagChars": ["i", "n", "o"], - "flags": [ - "api-version", - "content-fields", - "data-category-rule", - "description", - "flags-dir", - "json", - "library-id", - "name", - "restrict-to-public-articles", - "retriever-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:upload", - "flagAliases": [], - "flagChars": ["f", "i", "o", "w"], - "flags": ["api-version", "file", "flags-dir", "json", "library-id", "target-org", "wait"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:create", - "flagAliases": [], - "flagChars": ["o"], - "flags": ["api-name", "api-version", "flags-dir", "json", "name", "planner-id", "preview", "spec", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:deactivate", - "flagAliases": [], - "flagChars": ["n", "o"], - "flags": ["api-name", "api-version", "flags-dir", "json", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:agent-spec", - "flagAliases": [], - "flagChars": ["o"], - "flags": [ - "agent-user", - "api-version", - "company-description", - "company-name", - "company-website", - "enrich-logs", - "flags-dir", - "force-overwrite", - "full-interview", - "grounding-context", - "json", - "max-topics", - "output-file", - "prompt-template", - "role", - "spec", - "target-org", - "tone", - "type" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:authoring-bundle", - "flagAliases": [], - "flagChars": ["d", "f", "n", "o"], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "force-overwrite", - "json", - "name", - "no-spec", - "output-dir", - "spec", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:template", - "flagAliases": [], - "flagChars": ["f", "r", "s"], - "flags": ["agent-file", "agent-version", "api-version", "flags-dir", "json", "output-dir", "source-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:test-spec", - "flagAliases": [], - "flagChars": ["d", "f"], - "flags": ["flags-dir", "force-overwrite", "from-definition", "output-file", "test-runner"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:asset:list", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:asset:replace", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "assets", "assets-file", "flags-dir", "json", "mcp-server-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:create", - "flagAliases": [], - "flagChars": ["n", "o"], - "flags": [ - "api-version", - "auth-type", - "client-id", - "client-secret", - "description", - "flags-dir", - "identity-provider", - "json", - "label", - "name", - "scope", - "server-url", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:delete", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "no-prompt", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:fetch", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:get", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:list", - "flagAliases": [], - "flagChars": ["o"], - "flags": ["api-version", "flags-dir", "json", "label", "status", "target-org", "type"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:update", - "flagAliases": [], - "flagChars": ["i", "o"], - "flags": [ - "api-version", - "auth-type", - "client-id", - "client-secret", - "description", - "flags-dir", - "identity-provider", - "json", - "label", - "mcp-server-id", - "scope", - "server-url", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview", - "flagAliases": [], - "flagChars": ["d", "n", "o", "x"], - "flags": [ - "agent-json", - "apex-debug", - "api-name", - "api-version", - "authoring-bundle", - "context-variables", - "flags-dir", - "output-dir", - "target-org", - "use-live-actions" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:end", - "flagAliases": [], - "flagChars": ["n", "o", "p"], - "flags": [ - "all", - "api-name", - "api-version", - "authoring-bundle", - "flags-dir", - "json", - "no-prompt", - "session-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:send", - "flagAliases": [], - "flagChars": ["n", "o", "u"], - "flags": [ - "api-name", - "api-version", - "authoring-bundle", - "flags-dir", - "json", - "session-id", - "target-org", - "utterance" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:sessions", - "flagAliases": [], - "flagChars": [], - "flags": ["flags-dir", "json"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:start", - "flagAliases": [], - "flagChars": ["n", "o"], - "flags": [ - "agent-json", - "api-name", - "api-version", - "authoring-bundle", - "context-variables", - "flags-dir", - "json", - "simulate-actions", - "target-org", - "use-live-actions" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:publish:authoring-bundle", - "flagAliases": [], - "flagChars": ["n", "o", "v"], - "flags": ["api-name", "api-version", "concise", "flags-dir", "json", "skip-retrieve", "target-org", "verbose"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:create", - "flagAliases": [], - "flagChars": ["o"], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "force-overwrite", - "json", - "preview", - "spec", - "target-org", - "test-runner" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:list", - "flagAliases": [], - "flagChars": ["o"], - "flags": ["api-version", "flags-dir", "json", "target-org"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:results", - "flagAliases": [], - "flagChars": ["d", "i", "o"], - "flags": [ - "api-version", - "flags-dir", - "job-id", - "json", - "output-dir", - "result-format", - "target-org", - "test-runner", - "verbose" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:resume", - "flagAliases": [], - "flagChars": ["d", "i", "o", "r", "w"], - "flags": [ - "api-version", - "flags-dir", - "job-id", - "json", - "output-dir", - "result-format", - "target-org", - "test-runner", - "use-most-recent", - "verbose", - "wait" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:run", - "flagAliases": [], - "flagChars": ["d", "n", "o", "w"], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "json", - "output-dir", - "result-format", - "target-org", - "test-runner", - "verbose", - "wait" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:run-eval", - "flagAliases": [], - "flagChars": ["n", "o", "s"], - "flags": [ - "api-name", - "api-version", - "batch-size", - "flags-dir", - "json", - "no-normalize", - "result-format", - "spec", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:trace:delete", - "flagAliases": [], - "flagChars": ["a"], - "flags": ["agent", "flags-dir", "json", "no-prompt", "older-than", "session-id"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:trace:list", - "flagAliases": [], - "flagChars": ["a"], - "flags": ["agent", "flags-dir", "json", "session-id", "since"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:trace:read", - "flagAliases": [], - "flagChars": ["d", "f", "s", "t"], - "flags": ["dimension", "flags-dir", "format", "json", "session-id", "turn"], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:validate:authoring-bundle", - "flagAliases": [], - "flagChars": ["n", "o"], - "flags": ["api-name", "api-version", "flags-dir", "json", "target-org"], - "plugin": "@salesforce/plugin-agent" - } -] + { + "alias": [], + "command": "agent:activate", + "flagAliases": [], + "flagChars": [ + "n", + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "json", + "target-org", + "version" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:create", + "flagAliases": [], + "flagChars": [ + "n", + "o", + "w" + ], + "flags": [ + "api-version", + "content-fields", + "data-category-ids", + "data-category-names", + "description", + "developer-name", + "flags-dir", + "index-mode", + "json", + "name", + "primary-index-field1", + "primary-index-field2", + "retriever-id", + "source-type", + "target-org", + "wait" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:delete", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "library-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:file:add", + "flagAliases": [], + "flagChars": [ + "f", + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "library-id", + "path", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:file:delete", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "file-id", + "flags-dir", + "json", + "library-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:file:list", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "library-id", + "offset", + "page-size", + "status", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:get", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "library-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:list", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "source-type", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:status", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "include-artifacts", + "json", + "library-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:update", + "flagAliases": [], + "flagChars": [ + "i", + "n", + "o" + ], + "flags": [ + "api-version", + "content-fields", + "data-category-rule", + "description", + "flags-dir", + "json", + "library-id", + "name", + "restrict-to-public-articles", + "retriever-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:upload", + "flagAliases": [], + "flagChars": [ + "f", + "i", + "o", + "w" + ], + "flags": [ + "api-version", + "file", + "flags-dir", + "json", + "library-id", + "target-org", + "wait" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:create", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "json", + "name", + "planner-id", + "preview", + "spec", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:deactivate", + "flagAliases": [], + "flagChars": [ + "n", + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "json", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:agent-spec", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "agent-user", + "api-version", + "company-description", + "company-name", + "company-website", + "enrich-logs", + "flags-dir", + "force-overwrite", + "full-interview", + "grounding-context", + "json", + "max-topics", + "output-file", + "prompt-template", + "role", + "spec", + "target-org", + "tone", + "type" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:authoring-bundle", + "flagAliases": [], + "flagChars": [ + "d", + "f", + "n", + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "force-overwrite", + "json", + "name", + "no-spec", + "output-dir", + "spec", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:template", + "flagAliases": [], + "flagChars": [ + "f", + "r", + "s" + ], + "flags": [ + "agent-file", + "agent-version", + "api-version", + "flags-dir", + "json", + "output-dir", + "source-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:test-spec", + "flagAliases": [], + "flagChars": [ + "d", + "f" + ], + "flags": [ + "flags-dir", + "force-overwrite", + "from-definition", + "output-file", + "test-runner" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:asset:list", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "mcp-server-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:asset:replace", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "assets", + "assets-file", + "flags-dir", + "json", + "mcp-server-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:create", + "flagAliases": [], + "flagChars": [ + "n", + "o" + ], + "flags": [ + "api-version", + "auth-type", + "client-id", + "client-secret", + "description", + "flags-dir", + "identity-provider", + "json", + "label", + "name", + "scope", + "server-url", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:delete", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "mcp-server-id", + "no-prompt", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:fetch", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "mcp-server-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:get", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "mcp-server-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:list", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "label", + "status", + "target-org", + "type" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:update", + "flagAliases": [], + "flagChars": [ + "i", + "o" + ], + "flags": [ + "api-version", + "auth-type", + "client-id", + "client-secret", + "description", + "flags-dir", + "identity-provider", + "json", + "label", + "mcp-server-id", + "scope", + "server-url", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview", + "flagAliases": [], + "flagChars": [ + "d", + "n", + "o", + "x" + ], + "flags": [ + "agent-json", + "apex-debug", + "api-name", + "api-version", + "authoring-bundle", + "context-variables", + "flags-dir", + "output-dir", + "target-org", + "use-live-actions" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:end", + "flagAliases": [], + "flagChars": [ + "n", + "o", + "p" + ], + "flags": [ + "all", + "api-name", + "api-version", + "authoring-bundle", + "flags-dir", + "json", + "no-prompt", + "session-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:send", + "flagAliases": [], + "flagChars": [ + "n", + "o", + "u" + ], + "flags": [ + "api-name", + "api-version", + "authoring-bundle", + "flags-dir", + "json", + "session-id", + "target-org", + "utterance" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:sessions", + "flagAliases": [], + "flagChars": [], + "flags": [ + "flags-dir", + "json" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:start", + "flagAliases": [], + "flagChars": [ + "n", + "o" + ], + "flags": [ + "agent-json", + "api-name", + "api-version", + "authoring-bundle", + "context-variables", + "flags-dir", + "json", + "simulate-actions", + "target-org", + "use-live-actions" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:publish:authoring-bundle", + "flagAliases": [], + "flagChars": [ + "n", + "o", + "v" + ], + "flags": [ + "api-name", + "api-version", + "concise", + "flags-dir", + "json", + "skip-retrieve", + "target-org", + "verbose" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:scorer:create", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "agent-api-name", + "api-name", + "api-version", + "description", + "engine-type", + "flags-dir", + "json", + "label", + "lightning-type", + "output-dir", + "preview", + "spec", + "spec-schema", + "status", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:scorer:run", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-name", + "api-version", + "data", + "file", + "flags-dir", + "json", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:create", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "force-overwrite", + "json", + "preview", + "spec", + "target-org", + "test-runner" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:list", + "flagAliases": [], + "flagChars": [ + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "json", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:results", + "flagAliases": [], + "flagChars": [ + "d", + "i", + "o" + ], + "flags": [ + "api-version", + "flags-dir", + "job-id", + "json", + "output-dir", + "result-format", + "target-org", + "test-runner", + "verbose" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:resume", + "flagAliases": [], + "flagChars": [ + "d", + "i", + "o", + "r", + "w" + ], + "flags": [ + "api-version", + "flags-dir", + "job-id", + "json", + "output-dir", + "result-format", + "target-org", + "test-runner", + "use-most-recent", + "verbose", + "wait" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:run", + "flagAliases": [], + "flagChars": [ + "d", + "n", + "o", + "w" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "json", + "output-dir", + "result-format", + "target-org", + "test-runner", + "verbose", + "wait" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:run-eval", + "flagAliases": [], + "flagChars": [ + "n", + "o", + "s" + ], + "flags": [ + "api-name", + "api-version", + "batch-size", + "flags-dir", + "json", + "no-normalize", + "result-format", + "spec", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:trace:delete", + "flagAliases": [], + "flagChars": [ + "a" + ], + "flags": [ + "agent", + "flags-dir", + "json", + "no-prompt", + "older-than", + "session-id" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:trace:list", + "flagAliases": [], + "flagChars": [ + "a" + ], + "flags": [ + "agent", + "flags-dir", + "json", + "session-id", + "since" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:trace:read", + "flagAliases": [], + "flagChars": [ + "d", + "f", + "s", + "t" + ], + "flags": [ + "dimension", + "flags-dir", + "format", + "json", + "session-id", + "turn" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:validate:authoring-bundle", + "flagAliases": [], + "flagChars": [ + "n", + "o" + ], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "json", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + } +] \ No newline at end of file diff --git a/messages/agent.scorer.create.md b/messages/agent.scorer.create.md index 3144194e..91914423 100644 --- a/messages/agent.scorer.create.md +++ b/messages/agent.scorer.create.md @@ -6,7 +6,7 @@ Create an agent scorer definition using an interactive interview or a spec file. Creates an AiAgentScorerDefinition metadata XML file either interactively (prompting for each field) or from a YAML spec file. -Run with no flags to start the interactive interview. The command prompts you for the scorer's data type, input scope, engine type, output values, and agent associations. +Run with no flags to start the interactive interview. The command prompts you for the scorer's lightning type, optional output labels, engine type, and agent associations. Alternatively, provide a --spec flag pointing to a YAML file that defines the scorer. This is useful for repeatable automation or when the scorer has many output values. @@ -20,9 +20,9 @@ API name of the scorer definition. API name of the agent to associate with this scorer. -# flags.data-type.summary +# flags.lightning-type.summary -Data type produced by the scorer (Text, Number, or OpenEnded). +Lightning type the scorer's value conforms to (for example, lightning__textType or lightning__numberType). # flags.label.summary @@ -76,11 +76,11 @@ Preview the generated XML without writing to disk. - Create a manual scorer with flags (non-interactive): - <%= config.bin %> <%= command.id %> --api-name Expert_Analysis --data-type Text --engine-type Manual --label Expert_Analysis --agent-api-name My_Agent --status Available + <%= config.bin %> <%= command.id %> --api-name Expert_Analysis --lightning-type lightning__textType --engine-type Manual --label Expert_Analysis --agent-api-name My_Agent --status Available - Create a prompt-based scorer (generates both scorer definition and prompt template): - <%= config.bin %> <%= command.id %> --api-name sentiment_analysis --data-type Text --engine-type PromptTemplate --label sentiment_analysis --agent-api-name My_Agent + <%= config.bin %> <%= command.id %> --api-name sentiment_analysis --lightning-type lightning__textType --engine-type PromptTemplate --label sentiment_analysis --agent-api-name My_Agent # error.missingRequiredFlags diff --git a/messages/agent.scorer.run.md b/messages/agent.scorer.run.md new file mode 100644 index 00000000..260e77f1 --- /dev/null +++ b/messages/agent.scorer.run.md @@ -0,0 +1,51 @@ +# summary + +Run an agent scorer against an STDM session and print its score. + +# description + +Runs a scorer that is already authored in your project metadata, referenced by its API name, against a single STDM (Session Trace Data Model) session, then prints the resulting score, outcome labels, and explanation. + +The scorer is resolved from your project's package directories by API name. If no scorer with that API name exists locally, the command errors — author it first with `sf agent scorer create`. + +Provide the session either inline as a JSON string with --data, or as a path to a local JSON file with --file. Exactly one of the two is required. + +To help you hand-construct a valid session, run this command with --help: the full JSON Schema for the session object is printed under the --data flag. + +# flags.api-name.summary + +API name of the scorer to run. Must match a scorer authored in this project's metadata. + +# flags.data.summary + +Inline STDM session JSON to score. + +# flags.data.description + +Inline STDM session JSON to score. The value must be a JSON object matching the following JSON Schema (the shape of the scorer's Input:Session value): + +# flags.file.summary + +Path to a local JSON file containing the STDM session to score. + +# examples + +- Run a scorer against a session stored in a local file: + + <%= config.bin %> <%= command.id %> --api-name Sentiment_Scorer --file ./session.json + +- Run a scorer against an inline session JSON string: + + <%= config.bin %> <%= command.id %> --api-name Sentiment_Scorer --data '{"sessionState":{"sessionId":"1","startTimestamp":"2026-01-01T00:00:00Z","channel":"web"},"actors":[],"metrics":{"durationMs":0,"turns":0},"runs":[]}' + +- Show the session JSON Schema in the help output: + + <%= config.bin %> <%= command.id %> --help + +# error.invalidSessionJson + +Could not parse the STDM session as JSON: %s + +# error.invalidSessionShape + +The STDM session must be a JSON object matching the session schema (see --help). Received: %s diff --git a/schemas/agent-scorer-run.json b/schemas/agent-scorer-run.json new file mode 100644 index 00000000..9b5e9836 --- /dev/null +++ b/schemas/agent-scorer-run.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentScorerRunResult", + "definitions": { + "AgentScorerRunResult": { + "type": "object", + "additionalProperties": false, + "properties": { + "scorerApiName": { + "type": "string" + }, + "ok": { + "type": "boolean" + }, + "output": { + "anyOf": [ + { + "type": "number" + }, + { + "type": "string" + }, + { + "type": "array", + "items": { + "type": "string" + } + } + ], + "description": "The score: one or more labels, or a typed value matching the scorer's lightning type." + }, + "explanation": { + "type": "string" + }, + "raw": { + "type": "string", + "description": "Raw engine output (for debugging / loose formats)." + }, + "error": { + "type": "string" + } + }, + "required": [ + "ok", + "scorerApiName" + ] + } + } +} \ No newline at end of file diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index ab57a8ee..80398223 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -22,11 +22,9 @@ import { type ScorerSpec, createScorerDefinition, labelToApiName, - scorerEnumValueCount, scorerSpecJsonSchema, type SupportedLightningType, SUPPORTED_LIGHTNING_TYPES, - MAX_ENUM_VALUES, SCORER_API_NAME_MAX_LENGTH, SCORER_API_NAME_PATTERN, SCORER_ENGINE_TYPES, @@ -52,11 +50,6 @@ export type AgentScorerCreateResult = { /** @deprecated Use ScorerSpec from @salesforce/agents directly. */ export type ScorerSpecFile = ScorerSpec; -// The CLI presents "OpenEnded" as a data-type choice; it maps to core dataType 'LightningType' + -// scorerType 'OpenEnded' (see runInteractiveInterview). This is a UI-level set, distinct from the -// core SCORER_DATA_TYPES union, so it is defined locally. -const UI_SCORER_DATA_TYPES = ['OpenEnded', 'Text', 'Number'] as const; - const FLAGGABLE_PROMPTS = { label: { message: messages.getMessage('flags.label.summary'), @@ -75,12 +68,12 @@ const FLAGGABLE_PROMPTS = { }, required: true, }, - 'data-type': { - message: messages.getMessage('flags.data-type.summary'), - promptMessage: 'What data type does this scorer produce?', - options: UI_SCORER_DATA_TYPES, + 'lightning-type': { + message: messages.getMessage('flags.lightning-type.summary'), + promptMessage: 'Select the lightning type this scorer produces', + options: SUPPORTED_LIGHTNING_TYPES, validate: (d: string): boolean | string => - (UI_SCORER_DATA_TYPES as readonly string[]).includes(d) || 'Invalid data type', + (SUPPORTED_LIGHTNING_TYPES as readonly string[]).includes(d) || 'Invalid lightning type', required: true, }, description: { @@ -156,67 +149,6 @@ async function promptForOutputEnumValues(): Promise { return values; } -async function promptForNumberSpecification(): Promise<{ min: number; max: number; step: number; threshold?: number }> { - const minStr = await inquirerInput({ - message: 'Minimum value', - default: '0', - validate: (d: string): boolean | string => !isNaN(parseFloat(d)) || 'Must be a number', - theme, - }); - - const maxStr = await inquirerInput({ - message: 'Maximum value', - default: '5', - validate: (d: string): boolean | string => !isNaN(parseFloat(d)) || 'Must be a number', - theme, - }); - - const min = parseFloat(minStr); - const max = parseFloat(maxStr); - - if (min >= max) { - throw new Error(`Minimum value (${min}) must be less than maximum value (${max})`); - } - - const stepStr = await inquirerInput({ - message: 'Step size', - default: '1', - validate: (d: string): boolean | string => { - const n = parseFloat(d); - if (isNaN(n) || n <= 0) return 'Step must be a positive number'; - const numValues = scorerEnumValueCount(min, max, n); - if (numValues > MAX_ENUM_VALUES) return `Step too small: would generate ${numValues} values (max ${MAX_ENUM_VALUES})`; - return true; - }, - theme, - }); - - const step = parseFloat(stepStr); - - const addThreshold = await confirm({ - message: `Add a threshold value? (${scorerEnumValueCount(min, max, step)} output values will be generated from ${min} to ${max})`, - default: false, - theme, - }); - - let threshold: number | undefined; - if (addThreshold) { - const thresholdStr = await inquirerInput({ - message: `Threshold (must be between ${min} and ${max})`, - validate: (d: string): boolean | string => { - const n = parseFloat(d); - if (isNaN(n)) return 'Must be a number'; - if (n < min || n > max) return `Must be between ${min} and ${max}`; - return true; - }, - theme, - }); - threshold = parseFloat(thresholdStr); - } - - return { min, max, step, threshold }; -} - export default class AgentScorerCreate extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); @@ -332,19 +264,17 @@ export default class AgentScorerCreate extends SfCommand({ - message: 'Semantic type (how this scorer is used in analytics)', - choices: [ - { name: 'None', value: '' }, - { name: 'Dimension (categorical grouping)', value: 'Dimension' }, - { name: 'Measurement (numeric aggregation)', value: 'Measurement' }, - ], + this.log(); + this.styledHeader('Output Labels'); + const addLabels = await confirm({ + message: 'Add predefined output labels? (leave off for fully open-ended output)', + default: false, theme, }); + const outputEnumValues = addLabels ? await promptForOutputEnumValues() : undefined; const engineType = (flags['engine-type'] as string) ?? (await promptForFlag(FLAGGABLE_PROMPTS['engine-type'])); const engineConfig = await this.promptForEngineConfig(engineType); @@ -352,14 +282,9 @@ export default class AgentScorerCreate extends SfCommand { - if (dataType === 'Number') { - this.log(); - this.styledHeader('Number Scale'); - const numSpec = await promptForNumberSpecification(); - return { specification: { valueSpecification: numSpec } }; - } - - if (dataType === 'OpenEnded') { - this.log(); - this.styledHeader('Open Scorer Configuration'); - - const lightningType = await select({ - message: 'Select the lightning type for open-ended values', - choices: SUPPORTED_LIGHTNING_TYPES.map((t) => ({ name: t, value: t })), - theme, - }); - - const addEnumValues = await confirm({ - message: 'Add output enum values?', - default: false, - theme, - }); - const outputEnumValues = addEnumValues ? await promptForOutputEnumValues() : undefined; - return { scorerType: 'OpenEnded', lightningType, outputEnumValues }; - } - - // Text - this.log(); - this.styledHeader('Output Values'); - const outputEnumValues = await promptForOutputEnumValues(); - return { outputEnumValues }; - } - private async promptForEngineConfig(engineType: string): Promise<{ promptContent?: string; promptTemplateName?: string }> { if (engineType !== 'PromptTemplate') return {}; diff --git a/src/commands/agent/scorer/run.ts b/src/commands/agent/scorer/run.ts new file mode 100644 index 00000000..0623fe67 --- /dev/null +++ b/src/commands/agent/scorer/run.ts @@ -0,0 +1,115 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { resolve } from 'node:path'; +import { readFileSync } from 'node:fs'; +import { SfCommand, Flags, toHelpSection } from '@salesforce/sf-plugins-core'; +import { Messages, EnvironmentVariable } from '@salesforce/core'; +import { type SessionView, type ScorerResult, runScorer, loadScorerSpec, sessionViewJsonSchema } from '@salesforce/agents'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.scorer.run'); + +// The JSON Schema for the STDM session object, surfaced in the --data flag's help so authors (and agents) +// can hand-construct a valid session. Derived from the SessionView type in @salesforce/agents. +// +// oclif renders flag descriptions through wrap-ansi, which mangles pretty-printed JSON in two ways: it +// tokenizes each line on the regular space (U+0020), and it trimStart()s every wrapped row. Because U+00A0 +// (no-break space) is still matched by JS's \s, a plain NBSP indent gets trimmed away on any line with more +// than one whitespace-separated token. So we indent with no-break spaces (never split on) AND prefix each +// indented line with a zero-width space (U+200B) — a non-whitespace char that halts trimStart() before it +// can reach the indentation, so the nesting survives the formatter. Both chars render as no ink / blank +// columns, keeping the schema readable. +const NBSP = '\u00A0'; +const ZWSP = '\u200B'; +const prettySessionSchema = JSON.stringify(sessionViewJsonSchema(), null, 2).replace( + /^ +/gm, + (spaces) => `${ZWSP}${NBSP.repeat(spaces.length)}` +); +const DATA_SCHEMA_HELP = `${messages.getMessage('flags.data.description')}\n\n${prettySessionSchema}`; + +export type AgentScorerRunResult = ScorerResult & { + scorerApiName: string; +}; + +export default class AgentScorerRun extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + public static readonly requiresProject = true; + + public static readonly envVariablesSection = toHelpSection('ENVIRONMENT VARIABLES', EnvironmentVariable.SF_TARGET_ORG); + + public static readonly flags = { + 'target-org': Flags.requiredOrg(), + 'api-version': Flags.orgApiVersion(), + 'api-name': Flags.string({ + summary: messages.getMessage('flags.api-name.summary'), + required: true, + }), + data: Flags.string({ + summary: messages.getMessage('flags.data.summary'), + description: DATA_SCHEMA_HELP, + exactlyOne: ['data', 'file'], + }), + file: Flags.file({ + summary: messages.getMessage('flags.file.summary'), + exists: true, + exactlyOne: ['data', 'file'], + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentScorerRun); + + const apiName = flags['api-name']; + + // Resolve the scorer from local project metadata by API name — the business logic throws a clear error + // if no scorer with this API name is authored in the project. + const directories = this.project!.getUniquePackageDirectories().map((pkgDir) => pkgDir.fullPath); + const spec = await loadScorerSpec({ apiName, directories }); + + const session = this.parseSession(flags.file ? readFileSync(resolve(flags.file), 'utf8') : flags.data!); + + const connection = flags['target-org'].getConnection(flags['api-version']); + const result = await runScorer(spec, session, connection); + + if (!this.jsonEnabled()) { + this.styledHeader(`Scorer: ${spec.apiName}`); + this.log(`Outcome: ${result.ok ? 'ok' : 'error'}`); + if (result.output !== undefined) { + this.log(`Output: ${Array.isArray(result.output) ? result.output.join(', ') : String(result.output)}`); + } + if (result.explanation) this.log(`Explanation: ${result.explanation}`); + if (result.error) this.log(`Error: ${result.error}`); + } + + return { scorerApiName: spec.apiName, ...result }; + } + + // eslint-disable-next-line class-methods-use-this + private parseSession(raw: string): SessionView { + let parsed: unknown; + try { + parsed = JSON.parse(raw); + } catch (e) { + throw messages.createError('error.invalidSessionJson', [(e as Error).message]); + } + if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) { + throw messages.createError('error.invalidSessionShape', [typeof parsed]); + } + return parsed as SessionView; + } +} diff --git a/test/commands/agent/scorer/create.test.ts b/test/commands/agent/scorer/create.test.ts index 109a8721..6c2557bb 100644 --- a/test/commands/agent/scorer/create.test.ts +++ b/test/commands/agent/scorer/create.test.ts @@ -37,10 +37,10 @@ import { TestContext, MockTestOrgData } from '@salesforce/core/testSetup'; import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; import type { ScorerSpecFile } from '../../../../src/commands/agent/scorer/create.js'; -function makeTextSpec(overrides: Partial = {}): ScorerSpecFile { +function makeLabeledSpec(overrides: Partial = {}): ScorerSpecFile { return { apiName: 'Test_Scorer', - dataType: 'Text', + lightningType: 'lightning__textType', inputScope: 'Session', label: 'Test Scorer', description: 'A test scorer', @@ -59,34 +59,9 @@ function makeTextSpec(overrides: Partial = {}): ScorerSpecFile { }; } -function makeNumberSpec(overrides: Partial = {}): ScorerSpecFile { - return { - apiName: 'Numeric_Scorer', - dataType: 'Number', - inputScope: 'Session', - label: 'Numeric Scorer', - engineType: 'Manual', - status: 'Available', - agentAssociation: { - agentApiName: 'My_Agent', - isActive: false, - }, - specification: { - valueSpecification: { - min: 0, - max: 5, - step: 1, - }, - }, - ...overrides, - }; -} - function makeOpenSpec(overrides: Partial = {}): ScorerSpecFile { return { apiName: 'Open_Scorer', - dataType: 'LightningType', - scorerType: 'OpenEnded', lightningType: 'lightning__textType', inputScope: 'Session', label: 'Open Scorer', @@ -105,7 +80,7 @@ function makeOpenSpec(overrides: Partial = {}): ScorerSpecFile { function makePromptTemplateSpec(overrides: Partial = {}): ScorerSpecFile { return { apiName: 'Prompt_Scorer', - dataType: 'Text', + lightningType: 'lightning__textType', inputScope: 'Session', label: 'Prompt Scorer', engineType: 'PromptTemplate', @@ -166,7 +141,7 @@ async function loadMockedCommand( if (opts?.confirmResult !== undefined) { mocks['@inquirer/prompts'] = { confirm: sinon.stub().resolves(opts.confirmResult), - select: sinon.stub().resolves('Text'), + select: sinon.stub().resolves('lightning__textType'), input: sinon.stub().resolves(''), }; } @@ -183,8 +158,6 @@ async function loadMockedCommand( const SPEC_FILENAMES = [ 'test.yaml', 'test-scorer.yaml', - 'numeric-scorer.yaml', - 'threshold-scorer.yaml', 'open-scorer.yaml', 'prompt-scorer.yaml', 'manual-scorer.yaml', @@ -236,8 +209,8 @@ describe('agent scorer create', () => { }); describe('--spec flag (YAML-driven) with --preview', () => { - it('should create a Text scorer from a YAML spec', async () => { - const { Command } = await loadMockedCommand(makeTextSpec()); + it('should create a labeled scorer from a YAML spec', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ '--target-org', testOrg.username, @@ -248,7 +221,9 @@ describe('agent scorer create', () => { expect(result.apiName).to.equal('Test_Scorer'); expect(result.contents).to.include('AiAgentScorerDefinition'); - expect(result.contents).to.include('Text'); + expect(result.contents).to.include('LightningType'); + expect(result.contents).to.include('lightning__textType'); + expect(result.contents).to.include('OpenEnded'); expect(result.contents).to.include('Session'); expect(result.contents).to.include('Manual'); expect(result.contents).to.include('Draft'); @@ -258,42 +233,6 @@ describe('agent scorer create', () => { expect(result.contents).to.include('Neutral'); }); - it('should create a Number scorer with specification', async () => { - const { Command } = await loadMockedCommand(makeNumberSpec()); - - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'numeric-scorer.yaml', - '--preview', - '--json', - ]); - - expect(result.apiName).to.equal('Numeric_Scorer'); - expect(result.contents).to.include('Number'); - expect(result.contents).to.include('0'); - expect(result.contents).to.include('5'); - expect(result.contents).to.include('1'); - expect(result.contents).to.include('Available'); - }); - - it('should create a Number scorer with threshold', async () => { - const spec = makeNumberSpec({ - specification: { valueSpecification: { min: 1, max: 10, step: 1, threshold: 7 } }, - }); - const { Command } = await loadMockedCommand(spec); - - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'threshold-scorer.yaml', - '--preview', - '--json', - ]); - - expect(result.contents).to.include('7'); - expect(result.contents).to.include('1'); - expect(result.contents).to.include('10'); - }); - it('should create an OpenEnded (LightningType) scorer', async () => { const { Command } = await loadMockedCommand(makeOpenSpec()); @@ -328,7 +267,7 @@ describe('agent scorer create', () => { expect(agentAssocBlock).to.include('Intent'); }); - it('should include outputEnumValues for OpenEnded scorer when provided', async () => { + it('should include outputEnumValues when provided', async () => { const spec = makeOpenSpec({ outputEnumValues: [ { value: 'GOOD', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, @@ -356,7 +295,7 @@ describe('agent scorer create', () => { expect(result.contents).to.include('true'); }); - it('should not include outputEnumValue for OpenEnded scorer when none provided', async () => { + it('should not include outputEnumValue when none provided', async () => { const { Command } = await loadMockedCommand(makeOpenSpec()); const result = await Command.run([ @@ -388,7 +327,7 @@ describe('agent scorer create', () => { }); it('should not generate prompt template for Manual engine', async () => { - const { Command } = await loadMockedCommand(makeTextSpec({ engineType: 'Manual' })); + const { Command } = await loadMockedCommand(makeLabeledSpec({ engineType: 'Manual' })); const result = await Command.run([ '--target-org', testOrg.username, @@ -422,7 +361,7 @@ describe('agent scorer create', () => { }); it('should omit inputScope from agent association XML when not specified', async () => { - const spec = makeTextSpec(); + const spec = makeLabeledSpec(); spec.agentAssociation.inputScope = undefined; const { Command } = await loadMockedCommand(spec); @@ -441,21 +380,8 @@ describe('agent scorer create', () => { expect(agentAssocBlock).not.to.include(''); }); - it('should include semanticType when set', async () => { - const { Command } = await loadMockedCommand(makeTextSpec({ semanticType: 'Dimension' })); - - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--preview', - '--json', - ]); - - expect(result.contents).to.include('Dimension'); - }); - it('should include description when provided', async () => { - const { Command } = await loadMockedCommand(makeTextSpec({ description: 'Evaluates politeness' })); + const { Command } = await loadMockedCommand(makeLabeledSpec({ description: 'Evaluates politeness' })); const result = await Command.run([ '--target-org', testOrg.username, @@ -468,7 +394,7 @@ describe('agent scorer create', () => { }); it('should omit description when not provided', async () => { - const { Command } = await loadMockedCommand(makeTextSpec({ description: undefined })); + const { Command } = await loadMockedCommand(makeLabeledSpec({ description: undefined })); const result = await Command.run([ '--target-org', testOrg.username, @@ -481,7 +407,7 @@ describe('agent scorer create', () => { }); it('should default samplingRate to 1.0', async () => { - const spec = makeTextSpec(); + const spec = makeLabeledSpec(); spec.agentAssociation.samplingRate = undefined; const { Command } = await loadMockedCommand(spec); @@ -496,7 +422,7 @@ describe('agent scorer create', () => { }); it('should use custom samplingRate', async () => { - const spec = makeTextSpec(); + const spec = makeLabeledSpec(); spec.agentAssociation.samplingRate = 0.25; const { Command } = await loadMockedCommand(spec); @@ -511,7 +437,7 @@ describe('agent scorer create', () => { }); it('should set versionNumber to 1', async () => { - const { Command } = await loadMockedCommand(makeTextSpec()); + const { Command } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ '--target-org', testOrg.username, @@ -524,8 +450,8 @@ describe('agent scorer create', () => { }); }); - describe('prompt template type selection', () => { - it('should use scorerOpenEnded type for OpenEnded scorerType', async () => { + describe('prompt template type', () => { + it('should always use scorerOpenEnded type', async () => { const { Command, writtenFiles } = await loadMockedCommand(makeOpenSpec()); await Command.run([ @@ -539,42 +465,7 @@ describe('agent scorer create', () => { expect(promptFile!.content).to.include('agentforce_session_tracing__scorerOpenEnded'); }); - it('should use scorerMeasurement type for Number scorers', async () => { - const { Command, writtenFiles } = await loadMockedCommand( - makeNumberSpec({ engineType: 'PromptTemplate' }) - ); - - await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', - '--json', - ]); - - const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); - expect(promptFile!.content).to.include('agentforce_session_tracing__scorerMeasurement'); - }); - - it('should use AllowedRange input for scorerMeasurement type', async () => { - const { Command, writtenFiles } = await loadMockedCommand( - makeNumberSpec({ engineType: 'PromptTemplate' }) - ); - - await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', - '--json', - ]); - - const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); - expect(promptFile!.content).to.include('AllowedRange'); - expect(promptFile!.content).to.include('Input:AllowedRange'); - expect(promptFile!.content).not.to.include('AllowedLabels'); - expect(promptFile!.content).not.to.include('FallbackLabel'); - }); - - it('should use scorerMultilabel type for default Text scorers', async () => { + it('should use scorerOpenEnded type even when labels are defined', async () => { const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); await Command.run([ @@ -585,18 +476,15 @@ describe('agent scorer create', () => { ]); const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); - expect(promptFile!.content).to.include('agentforce_session_tracing__scorerMultilabel'); + expect(promptFile!.content).to.include('agentforce_session_tracing__scorerOpenEnded'); + expect(promptFile!.content).to.include('AllowedLabels'); + expect(promptFile!.content).to.include('FallbackLabel'); }); }); - // NOTE: number scorers no longer expand min/max/step into enumerated entries; the - // generateNumberEnumValues helper was removed (they now emit a compact , - // covered by 'should create a Number scorer with specification'). The former - // 'number enum value generation' suite tested that removed behavior and was deleted. - describe('XML structure', () => { it('should include XML declaration and namespace', async () => { - const { Command } = await loadMockedCommand(makeTextSpec()); + const { Command } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ '--target-org', testOrg.username, @@ -610,7 +498,7 @@ describe('agent scorer create', () => { }); it('should include isActive in agent association', async () => { - const spec = makeTextSpec(); + const spec = makeLabeledSpec(); spec.agentAssociation.isActive = true; const { Command } = await loadMockedCommand(spec); @@ -625,7 +513,7 @@ describe('agent scorer create', () => { }); it('should include isFallback and isSystemFallback', async () => { - const spec = makeTextSpec({ + const spec = makeLabeledSpec({ outputEnumValues: [ { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, { value: 'Bad', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, @@ -646,7 +534,7 @@ describe('agent scorer create', () => { }); it('should include label in scorerVersion', async () => { - const { Command } = await loadMockedCommand(makeTextSpec({ label: 'My Custom Label' })); + const { Command } = await loadMockedCommand(makeLabeledSpec({ label: 'My Custom Label' })); const result = await Command.run([ '--target-org', testOrg.username, @@ -661,7 +549,7 @@ describe('agent scorer create', () => { describe('file writing', () => { it('should write scorer XML to correct path', async () => { - const { Command, writtenFiles, createdDirs } = await loadMockedCommand(makeTextSpec()); + const { Command, writtenFiles, createdDirs } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ '--target-org', testOrg.username, @@ -699,7 +587,7 @@ describe('agent scorer create', () => { }); it('should not write files with --preview', async () => { - const { Command, writtenFiles } = await loadMockedCommand(makeTextSpec()); + const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec()); await Command.run([ '--target-org', testOrg.username, @@ -729,7 +617,7 @@ describe('agent scorer create', () => { expect(promptFile!.content).to.include('{!$Input:FallbackLabel}'); }); - it('should use OpenEnded default prompt for OpenEnded type', async () => { + it('should omit label guidance from default prompt when no labels are defined', async () => { const { Command, writtenFiles } = await loadMockedCommand(makeOpenSpec()); await Command.run([ @@ -743,29 +631,11 @@ describe('agent scorer create', () => { expect(promptFile!.content).to.include('{!$Input:Session}'); expect(promptFile!.content).not.to.include('{!$Input:AllowedLabels}'); }); - - it('should use Measurement default prompt with AllowedRange', async () => { - const spec = makeNumberSpec({ engineType: 'PromptTemplate' }); - const { Command, writtenFiles } = await loadMockedCommand(spec); - - await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', - '--json', - ]); - - const promptFile = writtenFiles.find((f) => f.path.includes('genAiPromptTemplates')); - expect(promptFile!.content).to.include('{!$Input:Session}'); - expect(promptFile!.content).to.include('{!$Input:AllowedRange}'); - expect(promptFile!.content).not.to.include('{!$Input:AllowedLabels}'); - expect(promptFile!.content).not.to.include('{!$Input:FallbackLabel}'); - }); }); describe('output directory', () => { it('should default to force-app/main/default', async () => { - const { Command } = await loadMockedCommand(makeTextSpec()); + const { Command } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ '--target-org', testOrg.username, @@ -778,7 +648,7 @@ describe('agent scorer create', () => { }); it('should use custom --output-dir', async () => { - const { Command } = await loadMockedCommand(makeTextSpec()); + const { Command } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ '--target-org', testOrg.username, @@ -794,7 +664,7 @@ describe('agent scorer create', () => { describe('overwrite behavior', () => { it('should cancel when user declines overwrite', async () => { - const { Command, writtenFiles } = await loadMockedCommand(makeTextSpec(), { + const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec(), { existsSync: () => true, confirmResult: false, }); @@ -811,7 +681,7 @@ describe('agent scorer create', () => { }); it('should skip overwrite prompt in --json mode', async () => { - const { Command, writtenFiles } = await loadMockedCommand(makeTextSpec(), { + const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec(), { existsSync: () => true, }); @@ -915,7 +785,7 @@ describe('agent scorer create', () => { describe('--json mode error handling', () => { it('should throw when required flags are missing', async () => { - const { Command } = await loadMockedCommand(makeTextSpec()); + const { Command } = await loadMockedCommand(makeLabeledSpec()); try { await Command.run(['--target-org', testOrg.username, '--json']); @@ -927,7 +797,7 @@ describe('agent scorer create', () => { }); it('should list all missing required flags', async () => { - const { Command } = await loadMockedCommand(makeTextSpec()); + const { Command } = await loadMockedCommand(makeLabeledSpec()); try { await Command.run(['--target-org', testOrg.username, '--label', 'Foo', '--json']); @@ -935,14 +805,14 @@ describe('agent scorer create', () => { } catch (err: unknown) { const error = err as { message: string }; expect(error.message).to.include('api-name'); - expect(error.message).to.include('data-type'); + expect(error.message).to.include('lightning-type'); expect(error.message).to.include('engine-type'); expect(error.message).to.include('agent-api-name'); } }); }); - describe('Text scorer fallback validation', () => { + describe('output label validation', () => { let tmpDir: string; let specFile: string; @@ -956,11 +826,11 @@ describe('agent scorer create', () => { rmSync(tmpDir, { recursive: true, force: true }); }); - it('should throw when Text scorer has no fallback value', async () => { - const spec = makeTextSpec({ + it('should throw when more than one output value is the fallback', async () => { + const spec = makeLabeledSpec({ outputEnumValues: [ - { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, - { value: 'Bad', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, + { value: 'Good', outcomeType: 'Pass', isFallback: true, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, ], }); writeFileSync(specFile, YAML.stringify(spec)); @@ -976,38 +846,34 @@ describe('agent scorer create', () => { expect.fail('should have thrown'); } catch (err: unknown) { const error = err as { message: string }; - expect(error.message).to.include('exactly 1 fallback value'); - expect(error.message).to.include('found 0'); + expect(error.message).to.include('At most one outputEnumValue can be the fallback'); + expect(error.message).to.include('found 2'); } }); - it('should throw when Text scorer has multiple fallback values', async () => { - const spec = makeTextSpec({ + it('should pass with zero fallback values', async () => { + const spec = makeLabeledSpec({ outputEnumValues: [ - { value: 'Good', outcomeType: 'Pass', isFallback: true, isSystemFallback: false }, - { value: 'Bad', outcomeType: 'Fail', isFallback: true, isSystemFallback: false }, + { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, + { value: 'Bad', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, ], }); writeFileSync(specFile, YAML.stringify(spec)); const { Command } = await loadMockedCommand(spec); - try { - await Command.run([ - '--target-org', testOrg.username, - '--spec', specFile, - '--preview', - '--json', - ]); - expect.fail('should have thrown'); - } catch (err: unknown) { - const error = err as { message: string }; - expect(error.message).to.include('exactly 1 fallback value'); - expect(error.message).to.include('found 2'); - } + const result = await Command.run([ + '--target-org', testOrg.username, + '--spec', specFile, + '--preview', + '--json', + ]); + + expect(result.apiName).to.equal('Test_Scorer'); + expect(result.contents).to.include('Good'); }); - it('should pass when Text scorer has exactly 1 fallback value', async () => { - const spec = makeTextSpec({ + it('should pass with exactly one fallback value', async () => { + const spec = makeLabeledSpec({ outputEnumValues: [ { value: 'Good', outcomeType: 'Pass', isFallback: false, isSystemFallback: false }, { value: 'Bad', outcomeType: 'Fail', isFallback: false, isSystemFallback: false }, @@ -1033,8 +899,6 @@ describe('agent scorer create', () => { it('should handle LightningType with no outputEnumValues', async () => { const spec: ScorerSpecFile = { apiName: 'Lightning_Scorer', - dataType: 'LightningType', - scorerType: 'OpenEnded', lightningType: 'lightning__numberType', inputScope: 'Session', label: 'Lightning Scorer', @@ -1055,7 +919,7 @@ describe('agent scorer create', () => { }); it('should handle single output enum value', async () => { - const spec = makeTextSpec({ + const spec = makeLabeledSpec({ outputEnumValues: [ { value: 'Only', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, ], @@ -1073,31 +937,5 @@ describe('agent scorer create', () => { expect(result.contents).to.include('NotApplicable'); expect(result.contents).to.include('true'); }); - - it('should include scorerType Predefined when set', async () => { - const { Command } = await loadMockedCommand(makeTextSpec({ scorerType: 'Predefined' })); - - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--preview', - '--json', - ]); - - expect(result.contents).to.include('Predefined'); - }); - - it('should include Measurement semanticType in XML', async () => { - const { Command } = await loadMockedCommand(makeNumberSpec({ semanticType: 'Measurement' })); - - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--preview', - '--json', - ]); - - expect(result.contents).to.include('Measurement'); - }); }); }); diff --git a/test/commands/agent/scorer/run.test.ts b/test/commands/agent/scorer/run.test.ts new file mode 100644 index 00000000..f3c32842 --- /dev/null +++ b/test/commands/agent/scorer/run.test.ts @@ -0,0 +1,284 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any */ + +import { join } from 'node:path'; +import { tmpdir } from 'node:os'; +import { mkdtempSync, writeFileSync, rmSync } from 'node:fs'; +import { expect } from 'chai'; +import esmock from 'esmock'; +import sinon from 'sinon'; +import { TestContext, MockTestOrgData } from '@salesforce/core/testSetup'; +import { SfProject } from '@salesforce/core'; +import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; + +const SPEC = { + apiName: 'Sentiment_Scorer', + lightningType: 'lightning__textType', + label: 'Sentiment Scorer', + engineType: 'PromptTemplate', + agentAssociation: { agentApiName: 'My_Agent', isActive: true }, +}; + +const SESSION = { + sessionState: { sessionId: 'S1', startTimestamp: '2026-01-01T00:00:00Z', channel: 'web' }, + actors: [], + metrics: { durationMs: 0, turns: 0 }, + runs: [], +}; + +const SESSION_FILE = 'session.json'; + +async function loadMockedCommand(opts?: { + runScorerResult?: any; + runScorerError?: Error; + loadScorerSpecError?: Error; +}): Promise<{ Command: any; runScorer: sinon.SinonStub; loadScorerSpec: sinon.SinonStub }> { + const runScorer = sinon.stub(); + if (opts?.runScorerError) runScorer.rejects(opts.runScorerError); + else runScorer.resolves(opts?.runScorerResult ?? { ok: true, output: 'Positive', explanation: 'Looks good.' }); + + const loadScorerSpec = sinon.stub(); + if (opts?.loadScorerSpecError) loadScorerSpec.rejects(opts.loadScorerSpecError); + else loadScorerSpec.resolves(SPEC); + + const readFileSync = (path: unknown): string => { + const p = String(path); + if (p.endsWith('.json')) return JSON.stringify(SESSION); + return ''; + }; + + const mocks: Record = { + 'node:fs': { readFileSync }, + '@salesforce/agents': { + runScorer, + loadScorerSpec, + sessionViewJsonSchema: () => ({ $schema: 'http://json-schema.org/draft-07/schema#' }), + }, + }; + + const mod = await esmock('../../../../src/commands/agent/scorer/run.js', mocks); + return { Command: mod.default, runScorer, loadScorerSpec }; +} + +describe('agent scorer run', () => { + const $$ = new TestContext(); + let testOrg: MockTestOrgData; + let originalCwd: string; + let workDir: string; + + before(async function () { + try { + await esmock('../../../../src/commands/agent/scorer/run.js', { + 'node:fs': { readFileSync: () => '' }, + '@salesforce/agents': { + runScorer: () => Promise.resolve({ ok: true }), + loadScorerSpec: () => Promise.resolve(SPEC), + sessionViewJsonSchema: () => ({}), + }, + }); + } catch (e: any) { + // eslint-disable-next-line no-console + console.error('esmock warmup failed:', e.message); + this.skip(); + } + + originalCwd = process.cwd(); + workDir = mkdtempSync(join(tmpdir(), 'scorer-run-')); + // Flags.file({ exists: true }) stats this at parse time, so it must exist on disk. + writeFileSync(join(workDir, SESSION_FILE), ''); + process.chdir(workDir); + }); + + after(() => { + if (originalCwd) process.chdir(originalCwd); + if (workDir) rmSync(workDir, { recursive: true, force: true }); + }); + + beforeEach(async () => { + stubSfCommandUx($$.SANDBOX); + testOrg = new MockTestOrgData(); + await $$.stubAuths(testOrg); + + // requiresProject: stub the project so package directories resolve without a real sfdx-project.json. + $$.inProject(true); + const mockProject = { + getPath: () => workDir, + getUniquePackageDirectories: () => [{ fullPath: join(workDir, 'force-app') }], + } as unknown as SfProject; + $$.SANDBOX.stub(SfProject, 'resolve').resolves(mockProject); + $$.SANDBOX.stub(SfProject, 'getInstance').returns(mockProject); + }); + + afterEach(() => { + $$.restore(); + sinon.restore(); + }); + + it('resolves the scorer by API name and runs it against a session file', async () => { + const { Command, runScorer, loadScorerSpec } = await loadMockedCommand(); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.scorerApiName).to.equal('Sentiment_Scorer'); + expect(result.ok).to.equal(true); + expect(result.output).to.equal('Positive'); + expect(result.explanation).to.equal('Looks good.'); + + expect(loadScorerSpec.calledOnce).to.be.true; + expect(loadScorerSpec.firstCall.firstArg.apiName).to.equal('Sentiment_Scorer'); + expect(loadScorerSpec.firstCall.firstArg.directories).to.be.an('array').that.is.not.empty; + + expect(runScorer.calledOnce).to.be.true; + const [passedSpec, passedSession] = runScorer.firstCall.args; + expect(passedSpec.apiName).to.equal('Sentiment_Scorer'); + expect(passedSession.sessionState.sessionId).to.equal('S1'); + }); + + it('runs a scorer against inline session JSON', async () => { + const { Command, runScorer } = await loadMockedCommand(); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', JSON.stringify(SESSION), + '--json', + ]); + + expect(result.scorerApiName).to.equal('Sentiment_Scorer'); + expect(result.ok).to.equal(true); + const [, passedSession] = runScorer.firstCall.args; + expect(passedSession.sessionState.sessionId).to.equal('S1'); + }); + + it('passes the connection from the target org to runScorer', async () => { + const { Command, runScorer } = await loadMockedCommand(); + + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + const [, , connection] = runScorer.firstCall.args; + expect(connection).to.not.be.undefined; + }); + + it('surfaces an engine error result', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: false, error: 'no engine for Manual' }, + }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.ok).to.equal(false); + expect(result.error).to.equal('no engine for Manual'); + }); + + it('surfaces the error when the scorer is not found in the project', async () => { + const { Command } = await loadMockedCommand({ + loadScorerSpecError: new Error("No scorer named 'Missing_Scorer' was found in this project."), + }); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Missing_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('was found in this project'); + } + }); + + it('throws when neither --data nor --file is provided', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message.toLowerCase()).to.match(/data|file/); + } + }); + + it('throws when both --data and --file are provided', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', JSON.stringify(SESSION), + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message.toLowerCase()).to.match(/data|file/); + } + }); + + it('throws a clear error when the session is not valid JSON', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', '{not valid json', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('Could not parse the STDM session as JSON'); + } + }); + + it('throws a clear error when the session JSON is not an object', async () => { + const { Command } = await loadMockedCommand(); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--data', '["not", "an", "object"]', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('must be a JSON object'); + } + }); +}); From 882167cea70dc27be78e687cb3186572ba3e6abb Mon Sep 17 00:00:00 2001 From: nnaffar Date: Mon, 7 Sep 2026 19:09:04 +0300 Subject: [PATCH 11/19] add support for scorer run. deprecate old dataTypes, keep lightning support as default --- messages/agent.scorer.create.md | 32 +++ messages/agent.scorer.run.md | 4 + src/commands/agent/scorer/create.ts | 155 ++++++++++++--- src/commands/agent/scorer/run.ts | 7 +- test/commands/agent/scorer/create.test.ts | 227 ++++++++++++++++++++-- test/commands/agent/scorer/run.test.ts | 121 +++++++++++- 6 files changed, 497 insertions(+), 49 deletions(-) diff --git a/messages/agent.scorer.create.md b/messages/agent.scorer.create.md index 91914423..a1d26495 100644 --- a/messages/agent.scorer.create.md +++ b/messages/agent.scorer.create.md @@ -48,6 +48,18 @@ Path to a scorer spec YAML file. Bypasses interactive prompts. Output the JSON Schema for the --spec YAML file and exit. +# flags.new-version.summary + +Add a new version to an existing scorer instead of erroring. The new version is numbered one higher than the current highest; if a new prompt rubric is supplied, the prompt template's active version is updated too. + +# flags.promote-version.summary + +Promote the given version number of an existing scorer to Available (its rubric is served on the next run). Requires --api-name; authors no content. + +# flags.archive-version.summary + +Archive the given version number of an existing scorer so it can no longer be run. Requires --api-name; authors no content. + # flags.output-dir.summary Output directory for the generated metadata XML files (scorer definition and prompt template). @@ -85,3 +97,23 @@ Preview the generated XML without writing to disk. # error.missingRequiredFlags Missing required flags: %s. When using --json, all required flags must be provided. + +# error.invalidSpecYaml + +Could not parse the --spec file as YAML: %s + +# error.invalidSpecShape + +The --spec file must define a YAML object matching the scorer spec schema (see --spec-schema). Received: %s + +# error.noAgentsInOrg + +No agents found in the org. Deploy an agent first, or specify one with --agent-api-name. + +# error.scorerExists + +A scorer named '%s' already exists in this project. To refine it, add a new version with --new-version; to change a version's status use --promote-version or --archive-version; or use --preview to see the generated XML without writing. + +# error.transitionNeedsApiName + +--promote-version and --archive-version require --api-name to identify which scorer to update. diff --git a/messages/agent.scorer.run.md b/messages/agent.scorer.run.md index 260e77f1..a8edd3d5 100644 --- a/messages/agent.scorer.run.md +++ b/messages/agent.scorer.run.md @@ -16,6 +16,10 @@ To help you hand-construct a valid session, run this command with --help: the fu API name of the scorer to run. Must match a scorer authored in this project's metadata. +# flags.scorer-version.summary + +Version number of the scorer to run. Omit to run the highest-numbered Available version; the command errors if none is Available. A Draft version must be selected explicitly; an Archived version cannot be run. + # flags.data.summary Inline STDM session JSON to score. diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index 80398223..7e57459e 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -21,6 +21,8 @@ import { Agent, type ScorerSpec, createScorerDefinition, + addScorerVersion, + setScorerVersionStatus, labelToApiName, scorerSpecJsonSchema, type SupportedLightningType, @@ -174,6 +176,20 @@ export default class AgentScorerCreate extends SfCommand { + const apiName = flags['api-name']; + if (!apiName) { + throw messages.createError('error.transitionNeedsApiName'); + } + + let path = ''; + if (flags['promote-version'] != null) { + const result = await setScorerVersionStatus({ + apiName, + outputDir, + versionNumber: flags['promote-version'], + status: 'Available', + }); + this.log(`Promoted version ${result.versionNumber} of ${apiName} to Available: ${result.path}`); + path = result.path; + } + if (flags['archive-version'] != null) { + const result = await setScorerVersionStatus({ + apiName, + outputDir, + versionNumber: flags['archive-version'], + status: 'Archived', + }); + this.log(`Archived version ${result.versionNumber} of ${apiName}: ${result.path}`); + path = result.path; + } + + return { path, apiName, contents: '' }; + } + private async runInteractiveInterview( flags: Record, connection: ReturnType @@ -262,18 +341,24 @@ export default class AgentScorerCreate extends SfCommand { if (engineType !== 'PromptTemplate') return {}; + // No flag exists yet for referencing an existing prompt template by name, so in --json/ + // non-interactive mode we always generate a new default prompt template. + if (this.jsonEnabled()) return {}; this.log(); this.styledHeader('Prompt Template'); @@ -324,7 +412,6 @@ export default class AgentScorerCreate extends SfCommand, engineType: string, @@ -336,7 +423,7 @@ export default class AgentScorerCreate extends SfCommand({ message: 'Select the agent to associate with this scorer', @@ -349,20 +436,24 @@ export default class AgentScorerCreate extends SfCommand({ - message: 'Input scope for this agent association', - choices: SCORER_INPUT_SCOPES.map((s) => ({ name: s, value: s })), - default: 'Session', - theme, - }); + const associationInputScope = this.jsonEnabled() + ? 'Session' + : await select({ + message: 'Input scope for this agent association', + choices: SCORER_INPUT_SCOPES.map((s) => ({ name: s, value: s })), + default: 'Session', + theme, + }); agentAssociation.inputScope = associationInputScope as 'Session' | 'Intent'; if (engineType === 'PromptTemplate') { - const isActive = await confirm({ - message: 'Activate scoring for this agent?', - default: false, - theme, - }); + const isActive = this.jsonEnabled() + ? false + : await confirm({ + message: 'Activate scoring for this agent?', + default: false, + theme, + }); agentAssociation.isActive = isActive; if (isActive) { diff --git a/src/commands/agent/scorer/run.ts b/src/commands/agent/scorer/run.ts index 0623fe67..a1a8b1d9 100644 --- a/src/commands/agent/scorer/run.ts +++ b/src/commands/agent/scorer/run.ts @@ -59,6 +59,11 @@ export default class AgentScorerRun extends SfCommand { summary: messages.getMessage('flags.api-name.summary'), required: true, }), + // eslint-disable-next-line sf-plugin/flag-min-max-default + 'scorer-version': Flags.integer({ + summary: messages.getMessage('flags.scorer-version.summary'), + min: 1, + }), data: Flags.string({ summary: messages.getMessage('flags.data.summary'), description: DATA_SCHEMA_HELP, @@ -79,7 +84,7 @@ export default class AgentScorerRun extends SfCommand { // Resolve the scorer from local project metadata by API name — the business logic throws a clear error // if no scorer with this API name is authored in the project. const directories = this.project!.getUniquePackageDirectories().map((pkgDir) => pkgDir.fullPath); - const spec = await loadScorerSpec({ apiName, directories }); + const spec = await loadScorerSpec({ apiName, directories, scorerVersion: flags['scorer-version'] }); const session = this.parseSession(flags.file ? readFileSync(resolve(flags.file), 'utf8') : flags.data!); diff --git a/test/commands/agent/scorer/create.test.ts b/test/commands/agent/scorer/create.test.ts index 6c2557bb..bacafbc3 100644 --- a/test/commands/agent/scorer/create.test.ts +++ b/test/commands/agent/scorer/create.test.ts @@ -14,7 +14,7 @@ * limitations under the License. */ -/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-explicit-any */ +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any */ import { join } from 'node:path'; import { tmpdir } from 'node:os'; @@ -31,10 +31,12 @@ import sinon from 'sinon'; const fsPromises = createRequire(import.meta.url)('node:fs/promises') as { writeFile: (...args: any[]) => Promise; mkdir: (...args: any[]) => Promise; + readFile: (...args: any[]) => Promise; }; import YAML from 'yaml'; import { TestContext, MockTestOrgData } from '@salesforce/core/testSetup'; import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import * as agentsModule from '@salesforce/agents'; import type { ScorerSpecFile } from '../../../../src/commands/agent/scorer/create.js'; function makeLabeledSpec(overrides: Partial = {}): ScorerSpecFile { @@ -103,7 +105,12 @@ type WrittenFile = { path: string; content: string }; async function loadMockedCommand( yamlSpec: ScorerSpecFile, - opts?: { existsSync?: () => boolean; confirmResult?: boolean } + opts?: { + existsSync?: () => boolean; + confirmResult?: boolean; + existingScorerXml?: string; + existingTemplateXml?: string; + } ): Promise<{ Command: any; writtenFiles: WrittenFile[]; createdDirs: string[] }> { const yamlContent = YAML.stringify(yamlSpec); const writtenFiles: WrittenFile[] = []; @@ -123,6 +130,22 @@ async function loadMockedCommand( typeof p === 'string' && (p.includes('aiAgentScorerDefinitions') || p.includes('genAiPromptTemplates')); const origWriteFile = fsPromises.writeFile; const origMkdir = fsPromises.mkdir; + const origReadFile = fsPromises.readFile; + + // addScorerVersion / setScorerVersionStatus read existing metadata via node:fs/promises.readFile. + // Serve the supplied fixture XML for those paths so the version/transition logic runs without disk. + if (opts?.existingScorerXml !== undefined || opts?.existingTemplateXml !== undefined) { + sinon.stub(fsPromises, 'readFile').callsFake((path: unknown, ...rest: any[]) => { + const p = String(path); + if (p.includes('genAiPromptTemplates') && opts.existingTemplateXml !== undefined) { + return Promise.resolve(opts.existingTemplateXml); + } + if (p.includes('aiAgentScorerDefinitions') && opts.existingScorerXml !== undefined) { + return Promise.resolve(opts.existingScorerXml); + } + return origReadFile(path, ...rest); + }); + } sinon.stub(fsPromises, 'writeFile').callsFake((path: unknown, content: unknown, options: unknown) => { if (isScorerOutput(path)) { writtenFiles.push({ path: String(path), content: String(content) }); @@ -168,6 +191,7 @@ describe('agent scorer create', () => { let testOrg: MockTestOrgData; let originalCwd: string; let specDir: string; + let sfCommandStubs: ReturnType; before(async function () { // Warm up esmock to check it can load the module @@ -198,7 +222,7 @@ describe('agent scorer create', () => { }); beforeEach(async () => { - stubSfCommandUx($$.SANDBOX); + sfCommandStubs = stubSfCommandUx($$.SANDBOX); testOrg = new MockTestOrgData(); await $$.stubAuths(testOrg); }); @@ -662,38 +686,108 @@ describe('agent scorer create', () => { }); }); - describe('overwrite behavior', () => { - it('should cancel when user declines overwrite', async () => { + describe('existing scorer behavior', () => { + it('errors when the scorer already exists and --new-version is not passed', async () => { const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec(), { existsSync: () => true, - confirmResult: false, + }); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--spec', 'test.yaml', + '--output-dir', '/tmp/out', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('already exists'); + expect((err as Error).message).to.include('--new-version'); + } + expect(writtenFiles).to.have.length(0); + }); + + it('adds a new version when the scorer exists and --new-version is passed', async () => { + const spec = makeLabeledSpec(); + const existingScorerXml = (agentsModule as any).buildScorerXml(spec); + const { Command, writtenFiles } = await loadMockedCommand(spec, { + existsSync: () => true, + existingScorerXml, }); const result = await Command.run([ '--target-org', testOrg.username, '--spec', 'test.yaml', '--output-dir', '/tmp/out', + '--new-version', + '--json', ]); - expect(result.path).to.equal(''); - expect(result.contents).to.equal(''); - expect(writtenFiles).to.have.length(0); + expect(result.apiName).to.equal('Test_Scorer'); + // v1 is preserved and v2 is appended. + expect(result.contents).to.include('1'); + expect(result.contents).to.include('2'); + const scorerFile = writtenFiles.find((f) => f.path.includes('aiAgentScorerDefinitions')); + expect(scorerFile).to.not.be.undefined; }); + }); - it('should skip overwrite prompt in --json mode', async () => { - const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec(), { + describe('status transitions', () => { + it('promotes a version to Available with --promote-version', async () => { + const spec = makeLabeledSpec({ status: 'Draft' }); + const existingScorerXml = (agentsModule as any).buildScorerXml(spec); + const { Command, writtenFiles } = await loadMockedCommand(spec, { existsSync: () => true, + existingScorerXml, }); const result = await Command.run([ '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--api-name', 'Test_Scorer', + '--promote-version', '1', '--output-dir', '/tmp/out', '--json', ]); - expect(result.path).to.not.equal(''); - expect(writtenFiles).to.have.length(1); + expect(result.apiName).to.equal('Test_Scorer'); + const scorerFile = writtenFiles.find((f) => f.path.includes('aiAgentScorerDefinitions')); + expect(scorerFile!.content).to.include('Available'); + }); + + it('archives a version with --archive-version', async () => { + const spec = makeLabeledSpec({ status: 'Available' }); + const existingScorerXml = (agentsModule as any).buildScorerXml(spec); + const { Command, writtenFiles } = await loadMockedCommand(spec, { + existsSync: () => true, + existingScorerXml, + }); + + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Test_Scorer', + '--archive-version', '1', + '--output-dir', '/tmp/out', + '--json', + ]); + + const scorerFile = writtenFiles.find((f) => f.path.includes('aiAgentScorerDefinitions')); + expect(scorerFile!.content).to.include('Archived'); + }); + + it('errors when --promote-version is used without --api-name', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--promote-version', '1', + '--output-dir', '/tmp/out', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('--api-name'); + } }); }); @@ -938,4 +1032,109 @@ describe('agent scorer create', () => { expect(result.contents).to.include('true'); }); }); + + describe('--json without --spec (flag-only path)', () => { + it('completes without prompting when all required flags are supplied', async () => { + const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec()); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--label', 'My Scorer', + '--api-name', 'My_Scorer', + '--lightning-type', 'lightning__textType', + '--engine-type', 'Manual', + '--agent-api-name', 'My_Agent', + '--output-dir', '/tmp/out-json-flags', + '--json', + ]); + + expect(result.apiName).to.equal('My_Scorer'); + // The status flag's default ('Draft') must be honored without prompting. + expect(result.contents).to.include('Draft'); + expect(writtenFiles).to.have.length(1); + }); + }); + + describe('--spec YAML parsing', () => { + it('throws a clear error when the spec file is not valid YAML', async () => { + const mod = await esmock('../../../../src/commands/agent/scorer/create.js', { + 'node:fs': { readFileSync: () => 'foo: [1, 2', existsSync: () => false }, + }); + const Command = mod.default; + + try { + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml']); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('Could not parse the --spec file as YAML'); + } + }); + + it('throws a clear error when the spec file is not a YAML object', async () => { + const mod = await esmock('../../../../src/commands/agent/scorer/create.js', { + 'node:fs': { readFileSync: () => '- 1\n- 2\n', existsSync: () => false }, + }); + const Command = mod.default; + + try { + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml']); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('must define a YAML object'); + } + }); + }); + + describe('--spec-schema', () => { + it('prints the schema JSON and skips org/spec resolution', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + // Deliberately omit --spec and every other required flag: --spec-schema must short-circuit + // before the required-flags gate or spec/org resolution ever runs. + const result = await Command.run(['--target-org', testOrg.username, '--spec-schema']); + + expect(result).to.deep.equal({ path: '', apiName: '', contents: '' }); + expect(sfCommandStubs.styledJSON.calledOnce).to.be.true; + const printed = sfCommandStubs.styledJSON.firstCall.args[0] as Record; + expect(printed).to.have.property('$ref', '#/definitions/ScorerSpec'); + expect(printed).to.have.property('definitions'); + }); + }); + + describe('interactive interview (no --spec, no --json)', () => { + it('throws a clear error when the org has no agents to associate', async () => { + const mocks: Record = { + 'node:fs': { readFileSync: () => '', existsSync: () => false }, + '@inquirer/prompts': { + input: sinon.stub().resolves(''), + confirm: sinon.stub().resolves(false), + select: sinon.stub().resolves(''), + }, + '@salesforce/agents': { + ...agentsModule, + Agent: { listRemote: sinon.stub().resolves([]) }, + }, + }; + const mod = await esmock('../../../../src/commands/agent/scorer/create.js', mocks); + const Command = mod.default; + + try { + // Supply every FLAGGABLE_PROMPTS-backed flag (including --description and --status, which + // are otherwise resolved via promptForFlag() in ../../../flags.js — a module esmock does not + // remock here, so any prompt routed through it would hit the real @inquirer/prompts and hang). + await Command.run([ + '--target-org', testOrg.username, + '--label', 'My Scorer', + '--api-name', 'My_Scorer', + '--lightning-type', 'lightning__textType', + '--engine-type', 'Manual', + '--description', 'A test description', + '--status', 'Draft', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('No agents found in the org'); + } + }); + }); }); diff --git a/test/commands/agent/scorer/run.test.ts b/test/commands/agent/scorer/run.test.ts index f3c32842..f8935a7d 100644 --- a/test/commands/agent/scorer/run.test.ts +++ b/test/commands/agent/scorer/run.test.ts @@ -47,6 +47,7 @@ async function loadMockedCommand(opts?: { runScorerResult?: any; runScorerError?: Error; loadScorerSpecError?: Error; + schema?: unknown; }): Promise<{ Command: any; runScorer: sinon.SinonStub; loadScorerSpec: sinon.SinonStub }> { const runScorer = sinon.stub(); if (opts?.runScorerError) runScorer.rejects(opts.runScorerError); @@ -67,7 +68,7 @@ async function loadMockedCommand(opts?: { '@salesforce/agents': { runScorer, loadScorerSpec, - sessionViewJsonSchema: () => ({ $schema: 'http://json-schema.org/draft-07/schema#' }), + sessionViewJsonSchema: () => opts?.schema ?? { $schema: 'http://json-schema.org/draft-07/schema#' }, }, }; @@ -80,6 +81,7 @@ describe('agent scorer run', () => { let testOrg: MockTestOrgData; let originalCwd: string; let workDir: string; + let sfCommandStubs: ReturnType; before(async function () { try { @@ -110,7 +112,7 @@ describe('agent scorer run', () => { }); beforeEach(async () => { - stubSfCommandUx($$.SANDBOX); + sfCommandStubs = stubSfCommandUx($$.SANDBOX); testOrg = new MockTestOrgData(); await $$.stubAuths(testOrg); @@ -154,6 +156,34 @@ describe('agent scorer run', () => { expect(passedSession.sessionState.sessionId).to.equal('S1'); }); + it('plumbs --scorer-version through to loadScorerSpec', async () => { + const { Command, loadScorerSpec } = await loadMockedCommand(); + + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--scorer-version', '3', + '--file', SESSION_FILE, + '--json', + ]); + + expect(loadScorerSpec.calledOnce).to.be.true; + expect(loadScorerSpec.firstCall.firstArg.scorerVersion).to.equal(3); + }); + + it('leaves scorerVersion undefined when --scorer-version is omitted', async () => { + const { Command, loadScorerSpec } = await loadMockedCommand(); + + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(loadScorerSpec.firstCall.firstArg.scorerVersion).to.be.undefined; + }); + it('runs a scorer against inline session JSON', async () => { const { Command, runScorer } = await loadMockedCommand(); @@ -281,4 +311,91 @@ describe('agent scorer run', () => { expect((err as Error).message).to.include('must be a JSON object'); } }); + + describe('human-readable output (non --json)', () => { + it('renders an array output joined by commas', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: ['A', 'B'] }, + }); + + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Output: A, B'); + }); + + it('renders a numeric output', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: 42 }, + }); + + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Output: 42'); + }); + + it('renders explanation and error lines', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: false, output: 'Negative', explanation: 'Tone was hostile.', error: 'no engine for Manual' }, + }); + + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Outcome: error'); + expect(logLines).to.include('Output: Negative'); + expect(logLines).to.include('Explanation: Tone was hostile.'); + expect(logLines).to.include('Error: no engine for Manual'); + }); + }); + + describe('--json with array/number output', () => { + it('returns array output as-is in the JSON result', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: ['A', 'B'] }, + }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.output).to.deep.equal(['A', 'B']); + }); + + it('returns numeric output as-is in the JSON result', async () => { + const { Command } = await loadMockedCommand({ + runScorerResult: { ok: true, output: 7 }, + }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.output).to.equal(7); + }); + }); + + describe('--data schema help formatting', () => { + it('preserves JSON nesting through the ZWSP/NBSP indent markers', async () => { + const schema = { type: 'object', properties: { foo: { type: 'string' }, bar: { type: 'number' } } }; + const { Command } = await loadMockedCommand({ schema }); + + const description = Command.flags.data.description as string; + const separatorIndex = description.indexOf('\n\n'); + expect(separatorIndex).to.be.greaterThan(-1); + const jsonPart = description.slice(separatorIndex + 2); + + // Strip the zero-width-space + no-break-space indent markers (see run.ts) before parsing. + const stripped = jsonPart.replace(/\u200B/g, '').replace(/\u00A0/g, ' '); + const parsed = JSON.parse(stripped); + expect(parsed).to.deep.equal(schema); + }); + }); }); From 34b0e81537ecbab57d052784b252d91e0af6a993 Mon Sep 17 00:00:00 2001 From: nnaffar Date: Mon, 7 Sep 2026 19:17:55 +0300 Subject: [PATCH 12/19] add support for scorer run. deprecate old dataTypes, keep lightning support as default --- command-snapshot.json | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/command-snapshot.json b/command-snapshot.json index ef4d8a47..e97d2a7e 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -632,14 +632,17 @@ "agent-api-name", "api-name", "api-version", + "archive-version", "description", "engine-type", "flags-dir", "json", "label", "lightning-type", + "new-version", "output-dir", "preview", + "promote-version", "spec", "spec-schema", "status", @@ -661,6 +664,7 @@ "file", "flags-dir", "json", + "scorer-version", "target-org" ], "plugin": "@salesforce/plugin-agent" From 88a20649de3b34d1f31341323a6f89446cc357d4 Mon Sep 17 00:00:00 2001 From: nnaffar Date: Mon, 7 Sep 2026 20:26:20 +0300 Subject: [PATCH 13/19] allow editing the status and activation --- command-snapshot.json | 20 +- messages/agent.scorer.create.md | 14 +- messages/agent.scorer.edit.md | 69 +++++++ messages/agent.scorer.run.md | 4 + schemas/agent-scorer-edit.json | 26 +++ src/commands/agent/scorer/create.ts | 81 ++------ src/commands/agent/scorer/edit.ts | 129 ++++++++++++ src/commands/agent/scorer/run.ts | 10 +- test/commands/agent/scorer/create.test.ts | 54 +---- test/commands/agent/scorer/edit.test.ts | 240 ++++++++++++++++++++++ test/commands/agent/scorer/run.test.ts | 38 ++-- 11 files changed, 548 insertions(+), 137 deletions(-) create mode 100644 messages/agent.scorer.edit.md create mode 100644 schemas/agent-scorer-edit.json create mode 100644 src/commands/agent/scorer/edit.ts create mode 100644 test/commands/agent/scorer/edit.test.ts diff --git a/command-snapshot.json b/command-snapshot.json index e97d2a7e..d71e6c8b 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -632,7 +632,6 @@ "agent-api-name", "api-name", "api-version", - "archive-version", "description", "engine-type", "flags-dir", @@ -642,7 +641,6 @@ "new-version", "output-dir", "preview", - "promote-version", "spec", "spec-schema", "status", @@ -650,6 +648,24 @@ ], "plugin": "@salesforce/plugin-agent" }, + { + "alias": [], + "command": "agent:scorer:edit", + "flagAliases": [], + "flagChars": [], + "flags": [ + "activate", + "api-name", + "deactivate", + "flags-dir", + "json", + "output-dir", + "preview", + "status", + "version" + ], + "plugin": "@salesforce/plugin-agent" + }, { "alias": [], "command": "agent:scorer:run", diff --git a/messages/agent.scorer.create.md b/messages/agent.scorer.create.md index a1d26495..68844522 100644 --- a/messages/agent.scorer.create.md +++ b/messages/agent.scorer.create.md @@ -52,14 +52,6 @@ Output the JSON Schema for the --spec YAML file and exit. Add a new version to an existing scorer instead of erroring. The new version is numbered one higher than the current highest; if a new prompt rubric is supplied, the prompt template's active version is updated too. -# flags.promote-version.summary - -Promote the given version number of an existing scorer to Available (its rubric is served on the next run). Requires --api-name; authors no content. - -# flags.archive-version.summary - -Archive the given version number of an existing scorer so it can no longer be run. Requires --api-name; authors no content. - # flags.output-dir.summary Output directory for the generated metadata XML files (scorer definition and prompt template). @@ -112,8 +104,4 @@ No agents found in the org. Deploy an agent first, or specify one with --agent-a # error.scorerExists -A scorer named '%s' already exists in this project. To refine it, add a new version with --new-version; to change a version's status use --promote-version or --archive-version; or use --preview to see the generated XML without writing. - -# error.transitionNeedsApiName - ---promote-version and --archive-version require --api-name to identify which scorer to update. +A scorer named '%s' already exists in this project. To refine it, add a new version with --new-version; to change a version's status use `sf agent scorer edit`; or use --preview to see the generated XML without writing. diff --git a/messages/agent.scorer.edit.md b/messages/agent.scorer.edit.md new file mode 100644 index 00000000..a3afdb05 --- /dev/null +++ b/messages/agent.scorer.edit.md @@ -0,0 +1,69 @@ +# summary + +Change the status or agent-association activation of a version of an existing agent scorer. + +# description + +Edits one version of a scorer in place: promote it (`--status Available`, so its rubric is served by default on the next run), archive it (`--status Archived`, so it can no longer be run), and/or turn its agent association on or off (`--activate` / `--deactivate`). Editing a scorer never authors content: a version's rubric is immutable once it exists — status and activation are the only fields that change. To add a new version with a refined rubric, use `sf agent scorer create --new-version`. + +This command edits the scorer's local metadata XML only; it does not require an org connection. Deploy the updated scorer definition afterward for the org to reflect the change. + +Platform activation rules (enforced on deploy): an active association (`--activate`) requires the version's status to be `Available`, and at most one version of a scorer may hold an active association. + +# flags.api-name.summary + +API name of the scorer to edit. Must match a scorer authored in this project's metadata. + +# flags.version.summary + +Version number to edit. + +# flags.status.summary + +New status for the version: Draft, Available (promote — served by default on the next run), or Archived (can no longer be run). + +# flags.activate.summary + +Activate the version's agent association (start scoring the associated agent's sessions). Requires the version's status to be Available. + +# flags.deactivate.summary + +Deactivate the version's agent association (stop scoring the associated agent's sessions). + +# flags.output-dir.summary + +Directory containing the scorer's metadata XML (where the scorer definition was authored). + +# flags.preview.summary + +Preview the resulting XML without writing to disk. + +# examples + +- Promote version 2 of a scorer to Available: + + <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available + +- Archive version 1 so it can no longer be run: + + <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 1 --status Archived + +- Promote a version and activate its agent association in a single command: + + <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available --activate + +- Deactivate the agent association on a version: + + <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --deactivate + +- Preview a status change without writing to disk: + + <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available --preview + +# error.noChange + +Specify at least one change: --status, --activate, or --deactivate. + +# error.scorerNotFound + +No scorer '%s' was found at %s. Author it first with `sf agent scorer create`. diff --git a/messages/agent.scorer.run.md b/messages/agent.scorer.run.md index a8edd3d5..378f8111 100644 --- a/messages/agent.scorer.run.md +++ b/messages/agent.scorer.run.md @@ -53,3 +53,7 @@ Could not parse the STDM session as JSON: %s # error.invalidSessionShape The STDM session must be a JSON object matching the session schema (see --help). Received: %s + +# error.scorerRunFailed + +Scorer '%s' did not produce a valid score: %s. The command exits non-zero so a scripted loop won't treat a failed evaluation as a passing one; the full result (including any explanation) is in the error's "data" field when run with --json. diff --git a/schemas/agent-scorer-edit.json b/schemas/agent-scorer-edit.json new file mode 100644 index 00000000..de6a2aea --- /dev/null +++ b/schemas/agent-scorer-edit.json @@ -0,0 +1,26 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentScorerEditResult", + "definitions": { + "AgentScorerEditResult": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "apiName": { + "type": "string" + }, + "contents": { + "type": "string" + } + }, + "required": [ + "path", + "apiName", + "contents" + ], + "additionalProperties": false + } + } +} \ No newline at end of file diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index 7e57459e..dcd45d4b 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -22,7 +22,6 @@ import { type ScorerSpec, createScorerDefinition, addScorerVersion, - setScorerVersionStatus, labelToApiName, scorerSpecJsonSchema, type SupportedLightningType, @@ -180,16 +179,6 @@ export default class AgentScorerCreate extends SfCommand { - const apiName = flags['api-name']; - if (!apiName) { - throw messages.createError('error.transitionNeedsApiName'); - } - - let path = ''; - if (flags['promote-version'] != null) { - const result = await setScorerVersionStatus({ - apiName, - outputDir, - versionNumber: flags['promote-version'], - status: 'Available', - }); - this.log(`Promoted version ${result.versionNumber} of ${apiName} to Available: ${result.path}`); - path = result.path; - } - if (flags['archive-version'] != null) { - const result = await setScorerVersionStatus({ - apiName, - outputDir, - versionNumber: flags['archive-version'], - status: 'Archived', - }); - this.log(`Archived version ${result.versionNumber} of ${apiName}: ${result.path}`); - path = result.path; - } - - return { path, apiName, contents: '' }; - } - private async runInteractiveInterview( flags: Record, connection: ReturnType diff --git a/src/commands/agent/scorer/edit.ts b/src/commands/agent/scorer/edit.ts new file mode 100644 index 00000000..536d865a --- /dev/null +++ b/src/commands/agent/scorer/edit.ts @@ -0,0 +1,129 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { join, resolve } from 'node:path'; +import { readFile, writeFile } from 'node:fs/promises'; +import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; +import { Messages } from '@salesforce/core'; +import { + setVersionStatusInScorerXml, + setVersionAssociationActiveInScorerXml, + SCORER_VERSION_STATUSES, + type ScorerVersionStatus, +} from '@salesforce/agents'; + +Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.scorer.edit'); + +export type AgentScorerEditResult = { + path: string; + apiName: string; + contents: string; +}; + +/** + * Edit one version of an already-authored scorer, in place: change its status (`--status`) and/or turn its + * agent association on or off (`--activate` / `--deactivate`). Editing never authors content — a version's + * rubric is immutable once it exists; status and activation are the only fields that change. To add a new + * version (a refined rubric) use `sf agent scorer create --new-version`. + * + * This is a purely local metadata operation (it edits the scorer's XML on disk), so it needs no org connection; + * deploy the definition afterward for the org to reflect the change. + */ +export default class AgentScorerEdit extends SfCommand { + public static readonly summary = messages.getMessage('summary'); + public static readonly description = messages.getMessage('description'); + public static readonly examples = messages.getMessages('examples'); + + public static readonly flags = { + 'api-name': Flags.string({ + summary: messages.getMessage('flags.api-name.summary'), + required: true, + }), + // eslint-disable-next-line sf-plugin/flag-min-max-default + version: Flags.integer({ + summary: messages.getMessage('flags.version.summary'), + required: true, + min: 1, + }), + status: Flags.string({ + summary: messages.getMessage('flags.status.summary'), + options: SCORER_VERSION_STATUSES, + }), + activate: Flags.boolean({ + summary: messages.getMessage('flags.activate.summary'), + exclusive: ['deactivate'], + }), + deactivate: Flags.boolean({ + summary: messages.getMessage('flags.deactivate.summary'), + exclusive: ['activate'], + }), + 'output-dir': Flags.directory({ + summary: messages.getMessage('flags.output-dir.summary'), + default: join('force-app', 'main', 'default'), + }), + preview: Flags.boolean({ + summary: messages.getMessage('flags.preview.summary'), + }), + }; + + public async run(): Promise { + const { flags } = await this.parse(AgentScorerEdit); + const apiName = flags['api-name']; + const version = flags.version; + const outputDir = resolve(flags['output-dir']); + const status = flags.status as ScorerVersionStatus | undefined; + // --activate → true, --deactivate → false, neither → leave the association untouched. + const activate = flags.activate ? true : flags.deactivate ? false : undefined; + + if (!status && activate === undefined) { + throw messages.createError('error.noChange'); + } + + const scorerPath = join(outputDir, 'aiAgentScorerDefinitions', `${apiName}.aiAgentScorerDefinition-meta.xml`); + + let existingXml: string; + try { + existingXml = await readFile(scorerPath, 'utf8'); + } catch { + throw messages.createError('error.scorerNotFound', [apiName, scorerPath]); + } + + // Apply every requested change in memory against a single load, then write once, so a combined status + + // activation edit (and its --preview) reflects both changes together. + let contents = existingXml; + const changes: string[] = []; + if (status) { + contents = setVersionStatusInScorerXml(contents, apiName, version, status); + changes.push(`status → ${status}`); + } + if (activate !== undefined) { + contents = setVersionAssociationActiveInScorerXml(contents, apiName, version, activate); + changes.push(activate ? 'agent association activated' : 'agent association deactivated'); + } + + if (flags.preview) { + this.log(`\n--- ${apiName} v${version} (${changes.join(', ')}) — preview ---\n`); + this.log(contents); + return { path: scorerPath, apiName, contents }; + } + + await writeFile(scorerPath, contents); + this.log(`Updated ${apiName} v${version} (${changes.join(', ')}): ${scorerPath}`); + this.log('Deploy the scorer definition for the org to reflect this change.'); + + return { path: scorerPath, apiName, contents }; + } +} diff --git a/src/commands/agent/scorer/run.ts b/src/commands/agent/scorer/run.ts index a1a8b1d9..98584cc1 100644 --- a/src/commands/agent/scorer/run.ts +++ b/src/commands/agent/scorer/run.ts @@ -98,7 +98,15 @@ export default class AgentScorerRun extends SfCommand { this.log(`Output: ${Array.isArray(result.output) ? result.output.join(', ') : String(result.output)}`); } if (result.explanation) this.log(`Explanation: ${result.explanation}`); - if (result.error) this.log(`Error: ${result.error}`); + } + + // A scorer that didn't produce a valid score (engine failure, or a session the platform rejected) comes back + // with ok:false. Surface it as a command failure — non-zero exit, standard SfError envelope in --json — so a + // scripted loop can't mistake a failed evaluation for a passing one. The full result is attached as error data. + if (!result.ok) { + const error = messages.createError('error.scorerRunFailed', [spec.apiName, result.error ?? 'unknown error']); + error.data = { scorerApiName: spec.apiName, ...result }; + throw error; } return { scorerApiName: spec.apiName, ...result }; diff --git a/test/commands/agent/scorer/create.test.ts b/test/commands/agent/scorer/create.test.ts index bacafbc3..cff57291 100644 --- a/test/commands/agent/scorer/create.test.ts +++ b/test/commands/agent/scorer/create.test.ts @@ -730,11 +730,9 @@ describe('agent scorer create', () => { const scorerFile = writtenFiles.find((f) => f.path.includes('aiAgentScorerDefinitions')); expect(scorerFile).to.not.be.undefined; }); - }); - describe('status transitions', () => { - it('promotes a version to Available with --promote-version', async () => { - const spec = makeLabeledSpec({ status: 'Draft' }); + it('previews the appended version (not a fresh v1) with --new-version --preview, writing nothing', async () => { + const spec = makeLabeledSpec(); const existingScorerXml = (agentsModule as any).buildScorerXml(spec); const { Command, writtenFiles } = await loadMockedCommand(spec, { existsSync: () => true, @@ -743,51 +741,17 @@ describe('agent scorer create', () => { const result = await Command.run([ '--target-org', testOrg.username, - '--api-name', 'Test_Scorer', - '--promote-version', '1', - '--output-dir', '/tmp/out', - '--json', - ]); - - expect(result.apiName).to.equal('Test_Scorer'); - const scorerFile = writtenFiles.find((f) => f.path.includes('aiAgentScorerDefinitions')); - expect(scorerFile!.content).to.include('Available'); - }); - - it('archives a version with --archive-version', async () => { - const spec = makeLabeledSpec({ status: 'Available' }); - const existingScorerXml = (agentsModule as any).buildScorerXml(spec); - const { Command, writtenFiles } = await loadMockedCommand(spec, { - existsSync: () => true, - existingScorerXml, - }); - - await Command.run([ - '--target-org', testOrg.username, - '--api-name', 'Test_Scorer', - '--archive-version', '1', + '--spec', 'test.yaml', '--output-dir', '/tmp/out', + '--new-version', + '--preview', '--json', ]); - const scorerFile = writtenFiles.find((f) => f.path.includes('aiAgentScorerDefinitions')); - expect(scorerFile!.content).to.include('Archived'); - }); - - it('errors when --promote-version is used without --api-name', async () => { - const { Command } = await loadMockedCommand(makeLabeledSpec()); - - try { - await Command.run([ - '--target-org', testOrg.username, - '--promote-version', '1', - '--output-dir', '/tmp/out', - '--json', - ]); - expect.fail('should have thrown'); - } catch (err: unknown) { - expect((err as Error).message).to.include('--api-name'); - } + // Preview reflects the artifact --new-version would write: v1 preserved, v2 appended — not a fresh v1. + expect(result.contents).to.include('1'); + expect(result.contents).to.include('2'); + expect(writtenFiles).to.have.length(0); }); }); diff --git a/test/commands/agent/scorer/edit.test.ts b/test/commands/agent/scorer/edit.test.ts new file mode 100644 index 00000000..473d52ab --- /dev/null +++ b/test/commands/agent/scorer/edit.test.ts @@ -0,0 +1,240 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* eslint-disable @typescript-eslint/no-unsafe-assignment, @typescript-eslint/no-unsafe-member-access, @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-argument, @typescript-eslint/no-explicit-any */ + +import { expect } from 'chai'; +import esmock from 'esmock'; +import sinon from 'sinon'; +import { TestContext } from '@salesforce/core/testSetup'; +import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; +import * as agentsModule from '@salesforce/agents'; + +type WrittenFile = { path: string; content: string }; + +const SPEC: any = { + apiName: 'Test_Scorer', + lightningType: 'lightning__textType', + inputScope: 'Session', + label: 'Test Scorer', + engineType: 'Manual', + status: 'Draft', + agentAssociation: { agentApiName: 'My_Agent', isActive: false }, +}; + +const SPEC_ACTIVE: any = { ...SPEC, agentAssociation: { agentApiName: 'My_Agent', isActive: true } }; + +/** Build a scorer XML fixture with `versions` sequential versions (v1..vN) from `spec`. */ +function scorerXmlWithVersions(versions: number, spec: any = SPEC): string { + let xml = (agentsModule as any).buildScorerXml(spec); + for (let i = 2; i <= versions; i++) { + ({ xml } = (agentsModule as any).addVersionToScorerXml(xml, spec)); + } + return String(xml); +} + +// The edit command reads/writes the scorer XML via node:fs/promises. esmock swaps that module for the command +// only, so reads return a fixture and writes are captured — no disk I/O and no reliance on core-module stubbing. +async function loadMockedCommand( + existingScorerXml: string | null +): Promise<{ Command: any; writtenFiles: WrittenFile[] }> { + const writtenFiles: WrittenFile[] = []; + + const readFile = (): Promise => + // null models a missing scorer file (readFile rejects). + existingScorerXml == null + ? Promise.reject(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })) + : Promise.resolve(existingScorerXml); + + const writeFile = (path: unknown, content: unknown): Promise => { + writtenFiles.push({ path: String(path), content: String(content) }); + return Promise.resolve(); + }; + + const mod = await esmock('../../../../src/commands/agent/scorer/edit.js', { + 'node:fs/promises': { readFile, writeFile }, + }); + return { Command: mod.default, writtenFiles }; +} + +describe('agent scorer edit', () => { + const $$ = new TestContext(); + let sfCommandStubs: ReturnType; + + before(async function () { + try { + await esmock('../../../../src/commands/agent/scorer/edit.js', {}); + } catch (e: any) { + // eslint-disable-next-line no-console + console.error('esmock warmup failed:', e.message); + this.skip(); + } + }); + + beforeEach(() => { + sfCommandStubs = stubSfCommandUx($$.SANDBOX); + }); + + afterEach(() => { + $$.restore(); + sinon.restore(); + }); + + it('promotes a version to Available with --status Available', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + const result = await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--status', 'Available', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(result.apiName).to.equal('Test_Scorer'); + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('Available'); + }); + + it('archives a version with --status Archived', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--status', 'Archived', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('Archived'); + }); + + it('activates the agent association with --activate', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--activate', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('true'); + }); + + it('deactivates the agent association with --deactivate', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1, SPEC_ACTIVE)); + + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--deactivate', + '--output-dir', '/tmp/out', + '--json', + ]); + + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('false'); + }); + + it('changes status and activation together in a single write', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--status', 'Available', + '--activate', + '--output-dir', '/tmp/out', + '--json', + ]); + + // A single load → both mutations → one write, so the written XML carries both changes. + expect(writtenFiles).to.have.length(1); + expect(writtenFiles[0].content).to.include('Available'); + expect(writtenFiles[0].content).to.include('true'); + }); + + it('errors when neither --status nor --activate/--deactivate is provided', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + try { + await Command.run(['--api-name', 'Test_Scorer', '--version', '1', '--output-dir', '/tmp/out', '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/status|activate|deactivate/); + } + expect(writtenFiles).to.have.length(0); + }); + + it('rejects --activate together with --deactivate', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + try { + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--activate', + '--deactivate', + '--output-dir', '/tmp/out', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.match(/activate|deactivate/); + } + expect(writtenFiles).to.have.length(0); + }); + + it('errors when the scorer file is not found', async () => { + const { Command, writtenFiles } = await loadMockedCommand(null); + + try { + await Command.run([ + '--api-name', 'Missing_Scorer', + '--version', '1', + '--status', 'Available', + '--output-dir', '/tmp/out', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('was found at'); + } + expect(writtenFiles).to.have.length(0); + }); + + it('writes nothing with --preview but prints the resulting XML', async () => { + const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); + + const result = await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--status', 'Available', + '--output-dir', '/tmp/out', + '--preview', + ]); + + expect(writtenFiles).to.have.length(0); + expect(result.contents).to.include('Available'); + const logged = sfCommandStubs.log.args.map((a) => String(a[0])).join('\n'); + expect(logged).to.include('Available'); + }); +}); diff --git a/test/commands/agent/scorer/run.test.ts b/test/commands/agent/scorer/run.test.ts index f8935a7d..cc85740b 100644 --- a/test/commands/agent/scorer/run.test.ts +++ b/test/commands/agent/scorer/run.test.ts @@ -214,20 +214,27 @@ describe('agent scorer run', () => { expect(connection).to.not.be.undefined; }); - it('surfaces an engine error result', async () => { + it('exits non-zero (throws) on an engine error result, with the result attached as error data', async () => { const { Command } = await loadMockedCommand({ runScorerResult: { ok: false, error: 'no engine for Manual' }, }); - const result = await Command.run([ - '--target-org', testOrg.username, - '--api-name', 'Sentiment_Scorer', - '--file', SESSION_FILE, - '--json', - ]); - - expect(result.ok).to.equal(false); - expect(result.error).to.equal('no engine for Manual'); + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { message: string; data?: any }; + expect(error.message).to.include('did not produce a valid score'); + expect(error.message).to.include('no engine for Manual'); + // The full result is preserved for scripted consumers on the error's data field. + expect(error.data?.ok).to.equal(false); + expect(error.data?.scorerApiName).to.equal('Sentiment_Scorer'); + } }); it('surfaces the error when the scorer is not found in the project', async () => { @@ -335,18 +342,23 @@ describe('agent scorer run', () => { expect(logLines).to.include('Output: 42'); }); - it('renders explanation and error lines', async () => { + it('renders the outcome/output/explanation lines before throwing on a failed result', async () => { const { Command } = await loadMockedCommand({ runScorerResult: { ok: false, output: 'Negative', explanation: 'Tone was hostile.', error: 'no engine for Manual' }, }); - await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + try { + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include('no engine for Manual'); + } + // The human-readable summary is still printed before the command fails. const logLines = sfCommandStubs.log.args.map((a) => a[0]); expect(logLines).to.include('Outcome: error'); expect(logLines).to.include('Output: Negative'); expect(logLines).to.include('Explanation: Tone was hostile.'); - expect(logLines).to.include('Error: no engine for Manual'); }); }); From 58946024d4988494df89e11427edab381d79ae2e Mon Sep 17 00:00:00 2001 From: nnaffar Date: Mon, 7 Sep 2026 20:49:59 +0300 Subject: [PATCH 14/19] add NUTs --- src/commands/agent/scorer/create.ts | 3 - test/commands/agent/scorer/create.test.ts | 12 +- test/nuts/agent.scorer.create.nut.ts | 115 +++++++++++++++ test/nuts/agent.scorer.edit.nut.ts | 167 ++++++++++++++++++++++ 4 files changed, 288 insertions(+), 9 deletions(-) create mode 100644 test/nuts/agent.scorer.create.nut.ts create mode 100644 test/nuts/agent.scorer.edit.nut.ts diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/create.ts index dcd45d4b..60e25d22 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/create.ts @@ -48,9 +48,6 @@ export type AgentScorerCreateResult = { promptTemplatePath?: string; }; -/** @deprecated Use ScorerSpec from @salesforce/agents directly. */ -export type ScorerSpecFile = ScorerSpec; - const FLAGGABLE_PROMPTS = { label: { message: messages.getMessage('flags.label.summary'), diff --git a/test/commands/agent/scorer/create.test.ts b/test/commands/agent/scorer/create.test.ts index cff57291..d4597bd0 100644 --- a/test/commands/agent/scorer/create.test.ts +++ b/test/commands/agent/scorer/create.test.ts @@ -37,9 +37,9 @@ import YAML from 'yaml'; import { TestContext, MockTestOrgData } from '@salesforce/core/testSetup'; import { stubSfCommandUx } from '@salesforce/sf-plugins-core'; import * as agentsModule from '@salesforce/agents'; -import type { ScorerSpecFile } from '../../../../src/commands/agent/scorer/create.js'; +import type { ScorerSpec } from '@salesforce/agents'; -function makeLabeledSpec(overrides: Partial = {}): ScorerSpecFile { +function makeLabeledSpec(overrides: Partial = {}): ScorerSpec { return { apiName: 'Test_Scorer', lightningType: 'lightning__textType', @@ -61,7 +61,7 @@ function makeLabeledSpec(overrides: Partial = {}): ScorerSpecFil }; } -function makeOpenSpec(overrides: Partial = {}): ScorerSpecFile { +function makeOpenSpec(overrides: Partial = {}): ScorerSpec { return { apiName: 'Open_Scorer', lightningType: 'lightning__textType', @@ -79,7 +79,7 @@ function makeOpenSpec(overrides: Partial = {}): ScorerSpecFile { }; } -function makePromptTemplateSpec(overrides: Partial = {}): ScorerSpecFile { +function makePromptTemplateSpec(overrides: Partial = {}): ScorerSpec { return { apiName: 'Prompt_Scorer', lightningType: 'lightning__textType', @@ -104,7 +104,7 @@ function makePromptTemplateSpec(overrides: Partial = {}): Scorer type WrittenFile = { path: string; content: string }; async function loadMockedCommand( - yamlSpec: ScorerSpecFile, + yamlSpec: ScorerSpec, opts?: { existsSync?: () => boolean; confirmResult?: boolean; @@ -955,7 +955,7 @@ describe('agent scorer create', () => { describe('edge cases', () => { it('should handle LightningType with no outputEnumValues', async () => { - const spec: ScorerSpecFile = { + const spec: ScorerSpec = { apiName: 'Lightning_Scorer', lightningType: 'lightning__numberType', inputScope: 'Session', diff --git a/test/nuts/agent.scorer.create.nut.ts b/test/nuts/agent.scorer.create.nut.ts new file mode 100644 index 00000000..07b4a43e --- /dev/null +++ b/test/nuts/agent.scorer.create.nut.ts @@ -0,0 +1,115 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { join } from 'node:path'; +import { writeFileSync, readFileSync, existsSync } from 'node:fs'; +import { expect } from 'chai'; +import { TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; +import { parseScorerVersions } from '@salesforce/agents'; +import type { AgentScorerCreateResult } from '../../src/commands/agent/scorer/create.js'; + +// `agent scorer create` writes local metadata only, but its `--target-org` is a required flag that resolves at +// parse time, so this NUT needs a default org. It uses a lightweight scratch org (devhub only — no Einstein +// provisioning or metadata deploy, which create doesn't need) rather than the shared heavyweight session. +describe('agent scorer create NUTs', function () { + this.timeout(15 * 60 * 1000); + + const API_NAME = 'Nut_Create_Scorer'; + // A `Manual`-engine scorer needs no prompt template, so create writes a single self-contained definition file. + const makeSpec = (apiName: string, label: string): Record => ({ + apiName, + lightningType: 'lightning__textType', + inputScope: 'Session', + label, + engineType: 'Manual', + status: 'Draft', + agentAssociation: { agentApiName: 'My_Agent', isActive: false }, + }); + + let session: TestSession; + let outputDir: string; + let specPath: string; + let scorerPath: string; + + before(async () => { + session = await TestSession.create({ + project: { name: 'scorerCreateNut' }, + devhubAuthStrategy: 'AUTO', + scratchOrgs: [{ setDefault: true, config: join('config', 'project-scratch-def.json') }], + }); + outputDir = join(session.project.dir, 'scorer-out'); + specPath = join(session.project.dir, 'nut-create-spec.json'); + scorerPath = join(outputDir, 'aiAgentScorerDefinitions', `${API_NAME}.aiAgentScorerDefinition-meta.xml`); + writeFileSync(specPath, JSON.stringify(makeSpec(API_NAME, 'NUT Create Scorer'))); + }); + + after(async () => { + await session?.clean(); + }); + + it('prints the spec JSON Schema with --spec-schema', () => { + const { stdout } = execCmd('agent scorer create --spec-schema', { ensureExitCode: 0 }).shellOutput; + expect(stdout).to.include('ScorerSpec'); + expect(stdout).to.include('apiName'); + }); + + it('authors a scorer definition from a --spec file', () => { + const result = execCmd( + `agent scorer create --spec "${specPath}" --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ).jsonOutput?.result; + + expect(result?.apiName).to.equal(API_NAME); + expect(result?.path).to.equal(scorerPath); + expect(existsSync(scorerPath)).to.equal(true); + expect(parseScorerVersions(readFileSync(scorerPath, 'utf8'))).to.have.length(1); + }); + + it('appends a new version with --new-version', () => { + execCmd( + `agent scorer create --spec "${specPath}" --output-dir "${outputDir}" --new-version --json`, + { ensureExitCode: 0 } + ); + + const versions = parseScorerVersions(readFileSync(scorerPath, 'utf8')); + expect(versions.map((v) => v.versionNumber)).to.deep.equal([1, 2]); + }); + + it('refuses to overwrite an existing scorer without --new-version', () => { + const output = execCmd( + `agent scorer create --spec "${specPath}" --output-dir "${outputDir}" --json`, + { ensureExitCode: 1 } + ).jsonOutput; + + expect(output?.message).to.match(new RegExp(API_NAME)); + // the existing file is left untouched (still two versions from the prior test) + expect(parseScorerVersions(readFileSync(scorerPath, 'utf8'))).to.have.length(2); + }); + + it('writes nothing with --preview', () => { + const previewName = 'Nut_Preview_Scorer'; + const previewSpec = join(session.project.dir, 'nut-preview-spec.json'); + writeFileSync(previewSpec, JSON.stringify(makeSpec(previewName, 'NUT Preview Scorer'))); + const previewPath = join(outputDir, 'aiAgentScorerDefinitions', `${previewName}.aiAgentScorerDefinition-meta.xml`); + + const result = execCmd( + `agent scorer create --spec "${previewSpec}" --output-dir "${outputDir}" --preview --json`, + { ensureExitCode: 0 } + ).jsonOutput?.result; + + expect(result?.contents).to.include('AiAgentScorerDefinition'); + expect(existsSync(previewPath)).to.equal(false); + }); +}); diff --git a/test/nuts/agent.scorer.edit.nut.ts b/test/nuts/agent.scorer.edit.nut.ts new file mode 100644 index 00000000..06af70dd --- /dev/null +++ b/test/nuts/agent.scorer.edit.nut.ts @@ -0,0 +1,167 @@ +/* + * Copyright 2026, Salesforce, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +import { join } from 'node:path'; +import { mkdirSync, writeFileSync, readFileSync } from 'node:fs'; +import { expect } from 'chai'; +import { TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; +import { buildScorerXml, addVersionToScorerXml, parseScorerVersions, type ScorerSpec } from '@salesforce/agents'; +import type { AgentScorerEditResult } from '../../src/commands/agent/scorer/edit.js'; + +// `agent scorer edit` is a purely local XML edit — it needs NO org connection — so this NUT runs against a bare +// project with no scratch org. It authors a scorer fixture on disk (via the agents lib, the same way `create` +// would), then exercises the real command end to end: status changes, activation toggles, and the error paths. +describe('agent scorer edit NUTs', () => { + const API_NAME = 'Nut_Edit_Scorer'; + // A `Manual`-engine scorer needs no prompt template, keeping the fixture self-contained. + const spec: ScorerSpec = { + apiName: API_NAME, + lightningType: 'lightning__textType', + inputScope: 'Session', + label: 'NUT Edit Scorer', + engineType: 'Manual', + status: 'Draft', + agentAssociation: { agentApiName: 'My_Agent', isActive: false }, + }; + + let session: TestSession; + let outputDir: string; + let scorerPath: string; + + before(async () => { + session = await TestSession.create({ project: { name: 'scorerEditNut' } }); + outputDir = join(session.project.dir, 'force-app', 'main', 'default'); + scorerPath = join(outputDir, 'aiAgentScorerDefinitions', `${API_NAME}.aiAgentScorerDefinition-meta.xml`); + }); + + after(async () => { + await session?.clean(); + }); + + /** (Re)author a fresh two-version fixture (v1, v2 — both Draft, inactive) so each test starts from a known state. */ + function authorFixture(active = false): void { + const seed = active + ? { ...spec, agentAssociation: { agentApiName: 'My_Agent', isActive: true } } + : spec; + let xml = buildScorerXml(seed); + ({ xml } = addVersionToScorerXml(xml, seed)); + mkdirSync(join(outputDir, 'aiAgentScorerDefinitions'), { recursive: true }); + writeFileSync(scorerPath, xml); + } + + const versionsOnDisk = (): ReturnType => parseScorerVersions(readFileSync(scorerPath, 'utf8')); + + beforeEach(() => authorFixture()); + + it('promotes a version to Available with --status Available', () => { + const result = execCmd( + `agent scorer edit --api-name ${API_NAME} --version 2 --status Available --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ).jsonOutput?.result; + + expect(result?.apiName).to.equal(API_NAME); + const versions = versionsOnDisk(); + expect(versions.find((v) => v.versionNumber === 2)?.status).to.equal('Available'); + // untouched versions keep their status + expect(versions.find((v) => v.versionNumber === 1)?.status).to.equal('Draft'); + }); + + it('archives a version with --status Archived', () => { + execCmd( + `agent scorer edit --api-name ${API_NAME} --version 1 --status Archived --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ); + + expect(versionsOnDisk().find((v) => v.versionNumber === 1)?.status).to.equal('Archived'); + }); + + it('activates the agent association with --activate', () => { + execCmd( + `agent scorer edit --api-name ${API_NAME} --version 2 --activate --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ); + + expect(versionsOnDisk().find((v) => v.versionNumber === 2)?.isActive).to.equal(true); + }); + + it('deactivates the agent association with --deactivate', () => { + authorFixture(true); + execCmd( + `agent scorer edit --api-name ${API_NAME} --version 2 --deactivate --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ); + + expect(versionsOnDisk().find((v) => v.versionNumber === 2)?.isActive).to.equal(false); + }); + + it('changes status and activation together in a single invocation', () => { + execCmd( + `agent scorer edit --api-name ${API_NAME} --version 2 --status Available --activate --output-dir "${outputDir}" --json`, + { ensureExitCode: 0 } + ); + + const v2 = versionsOnDisk().find((v) => v.versionNumber === 2); + expect(v2?.status).to.equal('Available'); + expect(v2?.isActive).to.equal(true); + }); + + it('writes nothing with --preview but prints the resulting XML', () => { + const before = readFileSync(scorerPath, 'utf8'); + const output = execCmd( + `agent scorer edit --api-name ${API_NAME} --version 2 --status Available --output-dir "${outputDir}" --preview`, + { ensureExitCode: 0 } + ); + + // file untouched... + expect(readFileSync(scorerPath, 'utf8')).to.equal(before); + expect(versionsOnDisk().find((v) => v.versionNumber === 2)?.status).to.equal('Draft'); + // ...but the promoted XML was printed + expect(output.shellOutput.stdout).to.include('Available'); + }); + + it('errors when neither --status nor --activate/--deactivate is provided', () => { + const output = execCmd( + `agent scorer edit --api-name ${API_NAME} --version 1 --output-dir "${outputDir}" --json`, + { ensureExitCode: 1 } + ).jsonOutput; + + expect(output?.message).to.match(/status|activate|deactivate/); + }); + + it('rejects --activate together with --deactivate', () => { + execCmd( + `agent scorer edit --api-name ${API_NAME} --version 1 --activate --deactivate --output-dir "${outputDir}" --json`, + { ensureExitCode: 'nonZero' } + ); + }); + + it('errors on an unknown version', () => { + const output = execCmd( + `agent scorer edit --api-name ${API_NAME} --version 9 --status Available --output-dir "${outputDir}" --json`, + { ensureExitCode: 1 } + ).jsonOutput; + + expect(output?.message).to.match(/version 9/); + }); + + it('errors when the scorer file is not found', () => { + const output = execCmd( + `agent scorer edit --api-name Missing_Scorer --version 1 --status Available --output-dir "${outputDir}" --json`, + { ensureExitCode: 1 } + ).jsonOutput; + + expect(output?.message).to.include('was found at'); + }); +}); From 30a6869ad6a2fc4080c8c0da27c899dc9f9352f9 Mon Sep 17 00:00:00 2001 From: nnaffar Date: Mon, 7 Sep 2026 21:05:51 +0300 Subject: [PATCH 15/19] minor fixes for error modes, adding UTs --- src/commands/agent/scorer/edit.ts | 9 +++++-- test/commands/agent/scorer/edit.test.ts | 32 +++++++++++++++++++++---- 2 files changed, 35 insertions(+), 6 deletions(-) diff --git a/src/commands/agent/scorer/edit.ts b/src/commands/agent/scorer/edit.ts index 536d865a..7048afee 100644 --- a/src/commands/agent/scorer/edit.ts +++ b/src/commands/agent/scorer/edit.ts @@ -97,8 +97,13 @@ export default class AgentScorerEdit extends SfCommand { let existingXml: string; try { existingXml = await readFile(scorerPath, 'utf8'); - } catch { - throw messages.createError('error.scorerNotFound', [apiName, scorerPath]); + } catch (err) { + // Only a missing file means "not authored yet"; surface any other read failure (EACCES, EISDIR, …) + // as-is so the user isn't wrongly told to `create` a scorer that already exists. + if ((err as NodeJS.ErrnoException).code === 'ENOENT') { + throw messages.createError('error.scorerNotFound', [apiName, scorerPath]); + } + throw err; } // Apply every requested change in memory against a single load, then write once, so a combined status + diff --git a/test/commands/agent/scorer/edit.test.ts b/test/commands/agent/scorer/edit.test.ts index 473d52ab..432707ee 100644 --- a/test/commands/agent/scorer/edit.test.ts +++ b/test/commands/agent/scorer/edit.test.ts @@ -49,15 +49,18 @@ function scorerXmlWithVersions(versions: number, spec: any = SPEC): string { // The edit command reads/writes the scorer XML via node:fs/promises. esmock swaps that module for the command // only, so reads return a fixture and writes are captured — no disk I/O and no reliance on core-module stubbing. async function loadMockedCommand( - existingScorerXml: string | null + existingScorerXml: string | null, + readError?: NodeJS.ErrnoException ): Promise<{ Command: any; writtenFiles: WrittenFile[] }> { const writtenFiles: WrittenFile[] = []; - const readFile = (): Promise => - // null models a missing scorer file (readFile rejects). - existingScorerXml == null + const readFile = (): Promise => { + // readError models a non-ENOENT read failure (EACCES, EISDIR, …); null models a missing file (ENOENT). + if (readError) return Promise.reject(readError); + return existingScorerXml == null ? Promise.reject(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })) : Promise.resolve(existingScorerXml); + }; const writeFile = (path: unknown, content: unknown): Promise => { writtenFiles.push({ path: String(path), content: String(content) }); @@ -221,6 +224,27 @@ describe('agent scorer edit', () => { expect(writtenFiles).to.have.length(0); }); + it('rethrows a non-ENOENT read error instead of reporting "scorer not found"', async () => { + const eacces = Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' }); + const { Command, writtenFiles } = await loadMockedCommand('unused', eacces); + + try { + await Command.run([ + '--api-name', 'Test_Scorer', + '--version', '1', + '--status', 'Available', + '--output-dir', '/tmp/out', + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + // the raw fs error propagates; the user is NOT wrongly told the scorer was not found + expect((err as Error).message).to.include('EACCES'); + expect((err as Error).message).to.not.include('was found at'); + } + expect(writtenFiles).to.have.length(0); + }); + it('writes nothing with --preview but prints the resulting XML', async () => { const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); From d14a5bad27e4d373a2dd07e029771294d710586d Mon Sep 17 00:00:00 2001 From: nnaffar Date: Mon, 7 Sep 2026 21:25:18 +0300 Subject: [PATCH 16/19] return version indicator --- src/commands/agent/scorer/run.ts | 7 ++- test/commands/agent/scorer/run.test.ts | 60 +++++++++++++++++++++++++- 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/commands/agent/scorer/run.ts b/src/commands/agent/scorer/run.ts index 98584cc1..dd82d616 100644 --- a/src/commands/agent/scorer/run.ts +++ b/src/commands/agent/scorer/run.ts @@ -42,6 +42,8 @@ const DATA_SCHEMA_HELP = `${messages.getMessage('flags.data.description')}\n\n${ export type AgentScorerRunResult = ScorerResult & { scorerApiName: string; + /** The version number that was actually run (highest Available by default, or the requested --scorer-version). */ + scorerVersion?: number; }; export default class AgentScorerRun extends SfCommand { @@ -93,6 +95,7 @@ export default class AgentScorerRun extends SfCommand { if (!this.jsonEnabled()) { this.styledHeader(`Scorer: ${spec.apiName}`); + if (spec.scorerVersion) this.log(`Version: ${spec.scorerVersion}`); this.log(`Outcome: ${result.ok ? 'ok' : 'error'}`); if (result.output !== undefined) { this.log(`Output: ${Array.isArray(result.output) ? result.output.join(', ') : String(result.output)}`); @@ -105,11 +108,11 @@ export default class AgentScorerRun extends SfCommand { // scripted loop can't mistake a failed evaluation for a passing one. The full result is attached as error data. if (!result.ok) { const error = messages.createError('error.scorerRunFailed', [spec.apiName, result.error ?? 'unknown error']); - error.data = { scorerApiName: spec.apiName, ...result }; + error.data = { scorerApiName: spec.apiName, scorerVersion: spec.scorerVersion, ...result }; throw error; } - return { scorerApiName: spec.apiName, ...result }; + return { scorerApiName: spec.apiName, scorerVersion: spec.scorerVersion, ...result }; } // eslint-disable-next-line class-methods-use-this diff --git a/test/commands/agent/scorer/run.test.ts b/test/commands/agent/scorer/run.test.ts index cc85740b..86db38ca 100644 --- a/test/commands/agent/scorer/run.test.ts +++ b/test/commands/agent/scorer/run.test.ts @@ -47,6 +47,7 @@ async function loadMockedCommand(opts?: { runScorerResult?: any; runScorerError?: Error; loadScorerSpecError?: Error; + specOverride?: any; schema?: unknown; }): Promise<{ Command: any; runScorer: sinon.SinonStub; loadScorerSpec: sinon.SinonStub }> { const runScorer = sinon.stub(); @@ -55,7 +56,7 @@ async function loadMockedCommand(opts?: { const loadScorerSpec = sinon.stub(); if (opts?.loadScorerSpecError) loadScorerSpec.rejects(opts.loadScorerSpecError); - else loadScorerSpec.resolves(SPEC); + else loadScorerSpec.resolves(opts?.specOverride ?? SPEC); const readFileSync = (path: unknown): string => { const p = String(path); @@ -184,6 +185,63 @@ describe('agent scorer run', () => { expect(loadScorerSpec.firstCall.firstArg.scorerVersion).to.be.undefined; }); + describe('resolved-version reporting', () => { + it('reports the resolved scorerVersion in the JSON result', async () => { + const { Command } = await loadMockedCommand({ specOverride: { ...SPEC, scorerVersion: 2 } }); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.scorerVersion).to.equal(2); + }); + + it('prints the resolved version in human-readable output', async () => { + const { Command } = await loadMockedCommand({ specOverride: { ...SPEC, scorerVersion: 2 } }); + + await Command.run(['--target-org', testOrg.username, '--api-name', 'Sentiment_Scorer', '--file', SESSION_FILE]); + + const logLines = sfCommandStubs.log.args.map((a) => a[0]); + expect(logLines).to.include('Version: 2'); + }); + + it('omits scorerVersion (and the Version line) when the spec has no resolved version', async () => { + const { Command } = await loadMockedCommand(); + + const result = await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + + expect(result.scorerVersion).to.be.undefined; + }); + + it('includes the resolved scorerVersion in error.data on a failed result', async () => { + const { Command } = await loadMockedCommand({ + specOverride: { ...SPEC, scorerVersion: 2 }, + runScorerResult: { ok: false, error: 'no engine for Manual' }, + }); + + try { + await Command.run([ + '--target-org', testOrg.username, + '--api-name', 'Sentiment_Scorer', + '--file', SESSION_FILE, + '--json', + ]); + expect.fail('should have thrown'); + } catch (err: unknown) { + const error = err as { data?: any }; + expect(error.data?.scorerVersion).to.equal(2); + } + }); + }); + it('runs a scorer against inline session JSON', async () => { const { Command, runScorer } = await loadMockedCommand(); From 7087097851e7095f32edf88284abfa0f2feea7cc Mon Sep 17 00:00:00 2001 From: nnaffar Date: Wed, 9 Sep 2026 22:17:49 +0300 Subject: [PATCH 17/19] feat(scorer)!: rename create to generate-metadata-file; drop edit MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Remove the 'agent scorer edit' command and the create '--new-version' refine path — scorers are scaffold-once, so there is no in-CLI update surface. Rename the scaffolding command to 'generate-metadata-file' to make its one-shot role explicit. Make the command self-document: the generated AiAgentScorerDefinition metadata XML (created here or retrieved from an org) is the source of truth, not the command or the spec. Name the AiAgentScorerDefinition entity in the summary and interview intro, and surface the same guidance in the --json result payload (where this.log is suppressed). Regenerate command-snapshot.json for the renamed command id. BREAKING CHANGE: 'agent scorer create' is renamed to 'agent scorer generate-metadata-file' and 'agent scorer edit' is removed. --- command-snapshot.json | 1429 +++++++---------- messages/agent.scorer.edit.md | 69 - ...=> agent.scorer.generate-metadata-file.md} | 24 +- messages/agent.scorer.run.md | 2 +- package.json | 2 +- schemas/agent-scorer-create.json | 29 - schemas/agent-scorer-edit.json | 26 - ...agent-scorer-generate__metadata__file.json | 29 + src/commands/agent/scorer/edit.ts | 134 -- .../{create.ts => generate-metadata-file.ts} | 108 +- test/commands/agent/scorer/edit.test.ts | 264 --- ...test.ts => generate-metadata-file.test.ts} | 410 ++--- test/nuts/agent.scorer.edit.nut.ts | 167 -- ...gent.scorer.generate-metadata-file.nut.ts} | 38 +- 14 files changed, 884 insertions(+), 1847 deletions(-) delete mode 100644 messages/agent.scorer.edit.md rename messages/{agent.scorer.create.md => agent.scorer.generate-metadata-file.md} (53%) delete mode 100644 schemas/agent-scorer-create.json delete mode 100644 schemas/agent-scorer-edit.json create mode 100644 schemas/agent-scorer-generate__metadata__file.json delete mode 100644 src/commands/agent/scorer/edit.ts rename src/commands/agent/scorer/{create.ts => generate-metadata-file.ts} (82%) delete mode 100644 test/commands/agent/scorer/edit.test.ts rename test/commands/agent/scorer/{create.test.ts => generate-metadata-file.test.ts} (82%) delete mode 100644 test/nuts/agent.scorer.edit.nut.ts rename test/nuts/{agent.scorer.create.nut.ts => agent.scorer.generate-metadata-file.nut.ts} (71%) diff --git a/command-snapshot.json b/command-snapshot.json index d71e6c8b..c3080f7d 100644 --- a/command-snapshot.json +++ b/command-snapshot.json @@ -1,887 +1,544 @@ [ - { - "alias": [], - "command": "agent:activate", - "flagAliases": [], - "flagChars": [ - "n", - "o" - ], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "json", - "target-org", - "version" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:create", - "flagAliases": [], - "flagChars": [ - "n", - "o", - "w" - ], - "flags": [ - "api-version", - "content-fields", - "data-category-ids", - "data-category-names", - "description", - "developer-name", - "flags-dir", - "index-mode", - "json", - "name", - "primary-index-field1", - "primary-index-field2", - "retriever-id", - "source-type", - "target-org", - "wait" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:delete", - "flagAliases": [], - "flagChars": [ - "i", - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "json", - "library-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:file:add", - "flagAliases": [], - "flagChars": [ - "f", - "i", - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "json", - "library-id", - "path", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:file:delete", - "flagAliases": [], - "flagChars": [ - "i", - "o" - ], - "flags": [ - "api-version", - "file-id", - "flags-dir", - "json", - "library-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:file:list", - "flagAliases": [], - "flagChars": [ - "i", - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "json", - "library-id", - "offset", - "page-size", - "status", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:get", - "flagAliases": [], - "flagChars": [ - "i", - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "json", - "library-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:list", - "flagAliases": [], - "flagChars": [ - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "json", - "source-type", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:status", - "flagAliases": [], - "flagChars": [ - "i", - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "include-artifacts", - "json", - "library-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:update", - "flagAliases": [], - "flagChars": [ - "i", - "n", - "o" - ], - "flags": [ - "api-version", - "content-fields", - "data-category-rule", - "description", - "flags-dir", - "json", - "library-id", - "name", - "restrict-to-public-articles", - "retriever-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:adl:upload", - "flagAliases": [], - "flagChars": [ - "f", - "i", - "o", - "w" - ], - "flags": [ - "api-version", - "file", - "flags-dir", - "json", - "library-id", - "target-org", - "wait" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:create", - "flagAliases": [], - "flagChars": [ - "o" - ], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "json", - "name", - "planner-id", - "preview", - "spec", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:deactivate", - "flagAliases": [], - "flagChars": [ - "n", - "o" - ], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "json", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:agent-spec", - "flagAliases": [], - "flagChars": [ - "o" - ], - "flags": [ - "agent-user", - "api-version", - "company-description", - "company-name", - "company-website", - "enrich-logs", - "flags-dir", - "force-overwrite", - "full-interview", - "grounding-context", - "json", - "max-topics", - "output-file", - "prompt-template", - "role", - "spec", - "target-org", - "tone", - "type" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:authoring-bundle", - "flagAliases": [], - "flagChars": [ - "d", - "f", - "n", - "o" - ], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "force-overwrite", - "json", - "name", - "no-spec", - "output-dir", - "spec", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:template", - "flagAliases": [], - "flagChars": [ - "f", - "r", - "s" - ], - "flags": [ - "agent-file", - "agent-version", - "api-version", - "flags-dir", - "json", - "output-dir", - "source-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:generate:test-spec", - "flagAliases": [], - "flagChars": [ - "d", - "f" - ], - "flags": [ - "flags-dir", - "force-overwrite", - "from-definition", - "output-file", - "test-runner" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:asset:list", - "flagAliases": [], - "flagChars": [ - "i", - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "json", - "mcp-server-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:asset:replace", - "flagAliases": [], - "flagChars": [ - "i", - "o" - ], - "flags": [ - "api-version", - "assets", - "assets-file", - "flags-dir", - "json", - "mcp-server-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:create", - "flagAliases": [], - "flagChars": [ - "n", - "o" - ], - "flags": [ - "api-version", - "auth-type", - "client-id", - "client-secret", - "description", - "flags-dir", - "identity-provider", - "json", - "label", - "name", - "scope", - "server-url", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:delete", - "flagAliases": [], - "flagChars": [ - "i", - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "json", - "mcp-server-id", - "no-prompt", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:fetch", - "flagAliases": [], - "flagChars": [ - "i", - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "json", - "mcp-server-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:get", - "flagAliases": [], - "flagChars": [ - "i", - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "json", - "mcp-server-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:list", - "flagAliases": [], - "flagChars": [ - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "json", - "label", - "status", - "target-org", - "type" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:mcp:update", - "flagAliases": [], - "flagChars": [ - "i", - "o" - ], - "flags": [ - "api-version", - "auth-type", - "client-id", - "client-secret", - "description", - "flags-dir", - "identity-provider", - "json", - "label", - "mcp-server-id", - "scope", - "server-url", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview", - "flagAliases": [], - "flagChars": [ - "d", - "n", - "o", - "x" - ], - "flags": [ - "agent-json", - "apex-debug", - "api-name", - "api-version", - "authoring-bundle", - "context-variables", - "flags-dir", - "output-dir", - "target-org", - "use-live-actions" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:end", - "flagAliases": [], - "flagChars": [ - "n", - "o", - "p" - ], - "flags": [ - "all", - "api-name", - "api-version", - "authoring-bundle", - "flags-dir", - "json", - "no-prompt", - "session-id", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:send", - "flagAliases": [], - "flagChars": [ - "n", - "o", - "u" - ], - "flags": [ - "api-name", - "api-version", - "authoring-bundle", - "flags-dir", - "json", - "session-id", - "target-org", - "utterance" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:sessions", - "flagAliases": [], - "flagChars": [], - "flags": [ - "flags-dir", - "json" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:preview:start", - "flagAliases": [], - "flagChars": [ - "n", - "o" - ], - "flags": [ - "agent-json", - "api-name", - "api-version", - "authoring-bundle", - "context-variables", - "flags-dir", - "json", - "simulate-actions", - "target-org", - "use-live-actions" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:publish:authoring-bundle", - "flagAliases": [], - "flagChars": [ - "n", - "o", - "v" - ], - "flags": [ - "api-name", - "api-version", - "concise", - "flags-dir", - "json", - "skip-retrieve", - "target-org", - "verbose" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:scorer:create", - "flagAliases": [], - "flagChars": [ - "o" - ], - "flags": [ - "agent-api-name", - "api-name", - "api-version", - "description", - "engine-type", - "flags-dir", - "json", - "label", - "lightning-type", - "new-version", - "output-dir", - "preview", - "spec", - "spec-schema", - "status", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:scorer:edit", - "flagAliases": [], - "flagChars": [], - "flags": [ - "activate", - "api-name", - "deactivate", - "flags-dir", - "json", - "output-dir", - "preview", - "status", - "version" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:scorer:run", - "flagAliases": [], - "flagChars": [ - "o" - ], - "flags": [ - "api-name", - "api-version", - "data", - "file", - "flags-dir", - "json", - "scorer-version", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:create", - "flagAliases": [], - "flagChars": [ - "o" - ], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "force-overwrite", - "json", - "preview", - "spec", - "target-org", - "test-runner" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:list", - "flagAliases": [], - "flagChars": [ - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "json", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:results", - "flagAliases": [], - "flagChars": [ - "d", - "i", - "o" - ], - "flags": [ - "api-version", - "flags-dir", - "job-id", - "json", - "output-dir", - "result-format", - "target-org", - "test-runner", - "verbose" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:resume", - "flagAliases": [], - "flagChars": [ - "d", - "i", - "o", - "r", - "w" - ], - "flags": [ - "api-version", - "flags-dir", - "job-id", - "json", - "output-dir", - "result-format", - "target-org", - "test-runner", - "use-most-recent", - "verbose", - "wait" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:run", - "flagAliases": [], - "flagChars": [ - "d", - "n", - "o", - "w" - ], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "json", - "output-dir", - "result-format", - "target-org", - "test-runner", - "verbose", - "wait" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:test:run-eval", - "flagAliases": [], - "flagChars": [ - "n", - "o", - "s" - ], - "flags": [ - "api-name", - "api-version", - "batch-size", - "flags-dir", - "json", - "no-normalize", - "result-format", - "spec", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:trace:delete", - "flagAliases": [], - "flagChars": [ - "a" - ], - "flags": [ - "agent", - "flags-dir", - "json", - "no-prompt", - "older-than", - "session-id" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:trace:list", - "flagAliases": [], - "flagChars": [ - "a" - ], - "flags": [ - "agent", - "flags-dir", - "json", - "session-id", - "since" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:trace:read", - "flagAliases": [], - "flagChars": [ - "d", - "f", - "s", - "t" - ], - "flags": [ - "dimension", - "flags-dir", - "format", - "json", - "session-id", - "turn" - ], - "plugin": "@salesforce/plugin-agent" - }, - { - "alias": [], - "command": "agent:validate:authoring-bundle", - "flagAliases": [], - "flagChars": [ - "n", - "o" - ], - "flags": [ - "api-name", - "api-version", - "flags-dir", - "json", - "target-org" - ], - "plugin": "@salesforce/plugin-agent" - } -] \ No newline at end of file + { + "alias": [], + "command": "agent:activate", + "flagAliases": [], + "flagChars": ["n", "o"], + "flags": ["api-name", "api-version", "flags-dir", "json", "target-org", "version"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:create", + "flagAliases": [], + "flagChars": ["n", "o", "w"], + "flags": [ + "api-version", + "content-fields", + "data-category-ids", + "data-category-names", + "description", + "developer-name", + "flags-dir", + "index-mode", + "json", + "name", + "primary-index-field1", + "primary-index-field2", + "retriever-id", + "source-type", + "target-org", + "wait" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:delete", + "flagAliases": [], + "flagChars": ["i", "o"], + "flags": ["api-version", "flags-dir", "json", "library-id", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:file:add", + "flagAliases": [], + "flagChars": ["f", "i", "o"], + "flags": ["api-version", "flags-dir", "json", "library-id", "path", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:file:delete", + "flagAliases": [], + "flagChars": ["i", "o"], + "flags": ["api-version", "file-id", "flags-dir", "json", "library-id", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:file:list", + "flagAliases": [], + "flagChars": ["i", "o"], + "flags": ["api-version", "flags-dir", "json", "library-id", "offset", "page-size", "status", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:get", + "flagAliases": [], + "flagChars": ["i", "o"], + "flags": ["api-version", "flags-dir", "json", "library-id", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:list", + "flagAliases": [], + "flagChars": ["o"], + "flags": ["api-version", "flags-dir", "json", "source-type", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:status", + "flagAliases": [], + "flagChars": ["i", "o"], + "flags": ["api-version", "flags-dir", "include-artifacts", "json", "library-id", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:update", + "flagAliases": [], + "flagChars": ["i", "n", "o"], + "flags": [ + "api-version", + "content-fields", + "data-category-rule", + "description", + "flags-dir", + "json", + "library-id", + "name", + "restrict-to-public-articles", + "retriever-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:adl:upload", + "flagAliases": [], + "flagChars": ["f", "i", "o", "w"], + "flags": ["api-version", "file", "flags-dir", "json", "library-id", "target-org", "wait"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:create", + "flagAliases": [], + "flagChars": ["o"], + "flags": ["api-name", "api-version", "flags-dir", "json", "name", "planner-id", "preview", "spec", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:deactivate", + "flagAliases": [], + "flagChars": ["n", "o"], + "flags": ["api-name", "api-version", "flags-dir", "json", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:agent-spec", + "flagAliases": [], + "flagChars": ["o"], + "flags": [ + "agent-user", + "api-version", + "company-description", + "company-name", + "company-website", + "enrich-logs", + "flags-dir", + "force-overwrite", + "full-interview", + "grounding-context", + "json", + "max-topics", + "output-file", + "prompt-template", + "role", + "spec", + "target-org", + "tone", + "type" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:authoring-bundle", + "flagAliases": [], + "flagChars": ["d", "f", "n", "o"], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "force-overwrite", + "json", + "name", + "no-spec", + "output-dir", + "spec", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:template", + "flagAliases": [], + "flagChars": ["f", "r", "s"], + "flags": ["agent-file", "agent-version", "api-version", "flags-dir", "json", "output-dir", "source-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:generate:test-spec", + "flagAliases": [], + "flagChars": ["d", "f"], + "flags": ["flags-dir", "force-overwrite", "from-definition", "output-file", "test-runner"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:asset:list", + "flagAliases": [], + "flagChars": ["i", "o"], + "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:asset:replace", + "flagAliases": [], + "flagChars": ["i", "o"], + "flags": ["api-version", "assets", "assets-file", "flags-dir", "json", "mcp-server-id", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:create", + "flagAliases": [], + "flagChars": ["n", "o"], + "flags": [ + "api-version", + "auth-type", + "client-id", + "client-secret", + "description", + "flags-dir", + "identity-provider", + "json", + "label", + "name", + "scope", + "server-url", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:delete", + "flagAliases": [], + "flagChars": ["i", "o"], + "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "no-prompt", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:fetch", + "flagAliases": [], + "flagChars": ["i", "o"], + "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:get", + "flagAliases": [], + "flagChars": ["i", "o"], + "flags": ["api-version", "flags-dir", "json", "mcp-server-id", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:list", + "flagAliases": [], + "flagChars": ["o"], + "flags": ["api-version", "flags-dir", "json", "label", "status", "target-org", "type"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:mcp:update", + "flagAliases": [], + "flagChars": ["i", "o"], + "flags": [ + "api-version", + "auth-type", + "client-id", + "client-secret", + "description", + "flags-dir", + "identity-provider", + "json", + "label", + "mcp-server-id", + "scope", + "server-url", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview", + "flagAliases": [], + "flagChars": ["d", "n", "o", "x"], + "flags": [ + "agent-json", + "apex-debug", + "api-name", + "api-version", + "authoring-bundle", + "context-variables", + "flags-dir", + "output-dir", + "target-org", + "use-live-actions" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:end", + "flagAliases": [], + "flagChars": ["n", "o", "p"], + "flags": [ + "all", + "api-name", + "api-version", + "authoring-bundle", + "flags-dir", + "json", + "no-prompt", + "session-id", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:send", + "flagAliases": [], + "flagChars": ["n", "o", "u"], + "flags": [ + "api-name", + "api-version", + "authoring-bundle", + "flags-dir", + "json", + "session-id", + "target-org", + "utterance" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:sessions", + "flagAliases": [], + "flagChars": [], + "flags": ["flags-dir", "json"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:preview:start", + "flagAliases": [], + "flagChars": ["n", "o"], + "flags": [ + "agent-json", + "api-name", + "api-version", + "authoring-bundle", + "context-variables", + "flags-dir", + "json", + "simulate-actions", + "target-org", + "use-live-actions" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:publish:authoring-bundle", + "flagAliases": [], + "flagChars": ["n", "o", "v"], + "flags": ["api-name", "api-version", "concise", "flags-dir", "json", "skip-retrieve", "target-org", "verbose"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:scorer:generate-metadata-file", + "flagAliases": [], + "flagChars": ["o"], + "flags": [ + "agent-api-name", + "api-name", + "api-version", + "description", + "engine-type", + "flags-dir", + "json", + "label", + "lightning-type", + "output-dir", + "preview", + "spec", + "spec-schema", + "status", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:scorer:run", + "flagAliases": [], + "flagChars": ["o"], + "flags": ["api-name", "api-version", "data", "file", "flags-dir", "json", "scorer-version", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:create", + "flagAliases": [], + "flagChars": ["o"], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "force-overwrite", + "json", + "preview", + "spec", + "target-org", + "test-runner" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:list", + "flagAliases": [], + "flagChars": ["o"], + "flags": ["api-version", "flags-dir", "json", "target-org"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:results", + "flagAliases": [], + "flagChars": ["d", "i", "o"], + "flags": [ + "api-version", + "flags-dir", + "job-id", + "json", + "output-dir", + "result-format", + "target-org", + "test-runner", + "verbose" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:resume", + "flagAliases": [], + "flagChars": ["d", "i", "o", "r", "w"], + "flags": [ + "api-version", + "flags-dir", + "job-id", + "json", + "output-dir", + "result-format", + "target-org", + "test-runner", + "use-most-recent", + "verbose", + "wait" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:run", + "flagAliases": [], + "flagChars": ["d", "n", "o", "w"], + "flags": [ + "api-name", + "api-version", + "flags-dir", + "json", + "output-dir", + "result-format", + "target-org", + "test-runner", + "verbose", + "wait" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:test:run-eval", + "flagAliases": [], + "flagChars": ["n", "o", "s"], + "flags": [ + "api-name", + "api-version", + "batch-size", + "flags-dir", + "json", + "no-normalize", + "result-format", + "spec", + "target-org" + ], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:trace:delete", + "flagAliases": [], + "flagChars": ["a"], + "flags": ["agent", "flags-dir", "json", "no-prompt", "older-than", "session-id"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:trace:list", + "flagAliases": [], + "flagChars": ["a"], + "flags": ["agent", "flags-dir", "json", "session-id", "since"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:trace:read", + "flagAliases": [], + "flagChars": ["d", "f", "s", "t"], + "flags": ["dimension", "flags-dir", "format", "json", "session-id", "turn"], + "plugin": "@salesforce/plugin-agent" + }, + { + "alias": [], + "command": "agent:validate:authoring-bundle", + "flagAliases": [], + "flagChars": ["n", "o"], + "flags": ["api-name", "api-version", "flags-dir", "json", "target-org"], + "plugin": "@salesforce/plugin-agent" + } +] diff --git a/messages/agent.scorer.edit.md b/messages/agent.scorer.edit.md deleted file mode 100644 index a3afdb05..00000000 --- a/messages/agent.scorer.edit.md +++ /dev/null @@ -1,69 +0,0 @@ -# summary - -Change the status or agent-association activation of a version of an existing agent scorer. - -# description - -Edits one version of a scorer in place: promote it (`--status Available`, so its rubric is served by default on the next run), archive it (`--status Archived`, so it can no longer be run), and/or turn its agent association on or off (`--activate` / `--deactivate`). Editing a scorer never authors content: a version's rubric is immutable once it exists — status and activation are the only fields that change. To add a new version with a refined rubric, use `sf agent scorer create --new-version`. - -This command edits the scorer's local metadata XML only; it does not require an org connection. Deploy the updated scorer definition afterward for the org to reflect the change. - -Platform activation rules (enforced on deploy): an active association (`--activate`) requires the version's status to be `Available`, and at most one version of a scorer may hold an active association. - -# flags.api-name.summary - -API name of the scorer to edit. Must match a scorer authored in this project's metadata. - -# flags.version.summary - -Version number to edit. - -# flags.status.summary - -New status for the version: Draft, Available (promote — served by default on the next run), or Archived (can no longer be run). - -# flags.activate.summary - -Activate the version's agent association (start scoring the associated agent's sessions). Requires the version's status to be Available. - -# flags.deactivate.summary - -Deactivate the version's agent association (stop scoring the associated agent's sessions). - -# flags.output-dir.summary - -Directory containing the scorer's metadata XML (where the scorer definition was authored). - -# flags.preview.summary - -Preview the resulting XML without writing to disk. - -# examples - -- Promote version 2 of a scorer to Available: - - <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available - -- Archive version 1 so it can no longer be run: - - <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 1 --status Archived - -- Promote a version and activate its agent association in a single command: - - <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available --activate - -- Deactivate the agent association on a version: - - <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --deactivate - -- Preview a status change without writing to disk: - - <%= config.bin %> <%= command.id %> --api-name Resolution_Quality_Judge --version 2 --status Available --preview - -# error.noChange - -Specify at least one change: --status, --activate, or --deactivate. - -# error.scorerNotFound - -No scorer '%s' was found at %s. Author it first with `sf agent scorer create`. diff --git a/messages/agent.scorer.create.md b/messages/agent.scorer.generate-metadata-file.md similarity index 53% rename from messages/agent.scorer.create.md rename to messages/agent.scorer.generate-metadata-file.md index 68844522..93fa6477 100644 --- a/messages/agent.scorer.create.md +++ b/messages/agent.scorer.generate-metadata-file.md @@ -1,6 +1,6 @@ # summary -Create an agent scorer definition using an interactive interview or a spec file. +Scaffold an AiAgentScorerDefinition metadata XML file (an agent scorer), interactively or from a spec file. This is a one-time starter, not the source of truth: once the AiAgentScorerDefinition XML is created (or retrieved from an org), edit it directly. # description @@ -12,6 +12,8 @@ Alternatively, provide a --spec flag pointing to a YAML file that defines the sc Use --preview to see the generated XML without writing it to disk. +This command is a one-shot scaffolding helper: it generates the scorer definition (and, for PromptTemplate scorers, its prompt template) metadata XML to give you a fast start. The generated XML — not this command and not the spec — is the source of truth. Once a definition exists locally, whether you created it here or retrieved it from an org, it has no connection back to the spec: make every further change (add a version, promote or archive a version, toggle an agent association, or edit the prompt rubric) directly in the metadata XML. If you already know the XML structure you can author it by hand and skip this command entirely; it exists because the XML is intricate and encodes rules the spec cannot fully capture. + # flags.api-name.summary API name of the scorer definition. @@ -22,7 +24,7 @@ API name of the agent to associate with this scorer. # flags.lightning-type.summary -Lightning type the scorer's value conforms to (for example, lightning__textType or lightning__numberType). +Lightning type the scorer's value conforms to (for example, lightning**textType or lightning**numberType). # flags.label.summary @@ -48,10 +50,6 @@ Path to a scorer spec YAML file. Bypasses interactive prompts. Output the JSON Schema for the --spec YAML file and exit. -# flags.new-version.summary - -Add a new version to an existing scorer instead of erroring. The new version is numbered one higher than the current highest; if a new prompt rubric is supplied, the prompt template's active version is updated too. - # flags.output-dir.summary Output directory for the generated metadata XML files (scorer definition and prompt template). @@ -80,11 +78,11 @@ Preview the generated XML without writing to disk. - Create a manual scorer with flags (non-interactive): - <%= config.bin %> <%= command.id %> --api-name Expert_Analysis --lightning-type lightning__textType --engine-type Manual --label Expert_Analysis --agent-api-name My_Agent --status Available + <%= config.bin %> <%= command.id %> --api-name Expert_Analysis --lightning-type lightning\_\_textType --engine-type Manual --label Expert_Analysis --agent-api-name My_Agent --status Available - Create a prompt-based scorer (generates both scorer definition and prompt template): - <%= config.bin %> <%= command.id %> --api-name sentiment_analysis --lightning-type lightning__textType --engine-type PromptTemplate --label sentiment_analysis --agent-api-name My_Agent + <%= config.bin %> <%= command.id %> --api-name sentiment_analysis --lightning-type lightning\_\_textType --engine-type PromptTemplate --label sentiment_analysis --agent-api-name My_Agent # error.missingRequiredFlags @@ -104,4 +102,12 @@ No agents found in the org. Deploy an agent first, or specify one with --agent-a # error.scorerExists -A scorer named '%s' already exists in this project. To refine it, add a new version with --new-version; to change a version's status use `sf agent scorer edit`; or use --preview to see the generated XML without writing. +A scorer named '%s' already exists in this project. `generate-metadata-file` only scaffolds a new scorer and never overwrites an existing one. Edit its metadata XML directly instead (%s) — that is where you add a new version, promote or archive a version, toggle an agent association, or change the prompt rubric. To scaffold a different scorer, choose a new API name. + +# info.scaffoldIntro + +Heads up: this command only scaffolds the scorer's AiAgentScorerDefinition metadata XML to get you started — the generated file, not this interview, is the source of truth. After it's written, make any further changes directly in the AiAgentScorerDefinition XML. + +# info.editXmlDirectly + +Done. This scorer is now defined by its metadata XML, which is the source of truth from here on — the spec is no longer connected to it. Make any further change (add a version, promote or archive a version, toggle an agent association, or edit the prompt rubric) directly in the generated XML file. diff --git a/messages/agent.scorer.run.md b/messages/agent.scorer.run.md index 378f8111..da92a736 100644 --- a/messages/agent.scorer.run.md +++ b/messages/agent.scorer.run.md @@ -6,7 +6,7 @@ Run an agent scorer against an STDM session and print its score. Runs a scorer that is already authored in your project metadata, referenced by its API name, against a single STDM (Session Trace Data Model) session, then prints the resulting score, outcome labels, and explanation. -The scorer is resolved from your project's package directories by API name. If no scorer with that API name exists locally, the command errors — author it first with `sf agent scorer create`. +The scorer is resolved from your project's package directories by API name. If no scorer with that API name exists locally, the command errors — author it first with `sf agent scorer generate-metadata-file`. Provide the session either inline as a JSON string with --data, or as a path to a local JSON file with --file. Exactly one of the two is required. diff --git a/package.json b/package.json index 9ec907eb..7660a49e 100644 --- a/package.json +++ b/package.json @@ -96,7 +96,7 @@ "external": true }, "scorer": { - "description": "Commands to create and manage agent scorers.", + "description": "Commands to scaffold and run agent scorers.", "external": true }, "adl": { diff --git a/schemas/agent-scorer-create.json b/schemas/agent-scorer-create.json deleted file mode 100644 index 65b33290..00000000 --- a/schemas/agent-scorer-create.json +++ /dev/null @@ -1,29 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$ref": "#/definitions/AgentScorerCreateResult", - "definitions": { - "AgentScorerCreateResult": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "apiName": { - "type": "string" - }, - "contents": { - "type": "string" - }, - "promptTemplatePath": { - "type": "string" - } - }, - "required": [ - "path", - "apiName", - "contents" - ], - "additionalProperties": false - } - } -} \ No newline at end of file diff --git a/schemas/agent-scorer-edit.json b/schemas/agent-scorer-edit.json deleted file mode 100644 index de6a2aea..00000000 --- a/schemas/agent-scorer-edit.json +++ /dev/null @@ -1,26 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-07/schema#", - "$ref": "#/definitions/AgentScorerEditResult", - "definitions": { - "AgentScorerEditResult": { - "type": "object", - "properties": { - "path": { - "type": "string" - }, - "apiName": { - "type": "string" - }, - "contents": { - "type": "string" - } - }, - "required": [ - "path", - "apiName", - "contents" - ], - "additionalProperties": false - } - } -} \ No newline at end of file diff --git a/schemas/agent-scorer-generate__metadata__file.json b/schemas/agent-scorer-generate__metadata__file.json new file mode 100644 index 00000000..8fb99405 --- /dev/null +++ b/schemas/agent-scorer-generate__metadata__file.json @@ -0,0 +1,29 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "$ref": "#/definitions/AgentScorerGenerateMetadataFileResult", + "definitions": { + "AgentScorerGenerateMetadataFileResult": { + "type": "object", + "properties": { + "path": { + "type": "string" + }, + "apiName": { + "type": "string" + }, + "contents": { + "type": "string" + }, + "promptTemplatePath": { + "type": "string" + }, + "guidance": { + "type": "string", + "description": "Guidance surfaced to a caller running with --json (where `this.log` output is suppressed): a written scorer's metadata XML is the source of truth, so any further change is made directly in the XML." + } + }, + "required": ["path", "apiName", "contents"], + "additionalProperties": false + } + } +} diff --git a/src/commands/agent/scorer/edit.ts b/src/commands/agent/scorer/edit.ts deleted file mode 100644 index 7048afee..00000000 --- a/src/commands/agent/scorer/edit.ts +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Copyright 2026, Salesforce, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { join, resolve } from 'node:path'; -import { readFile, writeFile } from 'node:fs/promises'; -import { SfCommand, Flags } from '@salesforce/sf-plugins-core'; -import { Messages } from '@salesforce/core'; -import { - setVersionStatusInScorerXml, - setVersionAssociationActiveInScorerXml, - SCORER_VERSION_STATUSES, - type ScorerVersionStatus, -} from '@salesforce/agents'; - -Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); -const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.scorer.edit'); - -export type AgentScorerEditResult = { - path: string; - apiName: string; - contents: string; -}; - -/** - * Edit one version of an already-authored scorer, in place: change its status (`--status`) and/or turn its - * agent association on or off (`--activate` / `--deactivate`). Editing never authors content — a version's - * rubric is immutable once it exists; status and activation are the only fields that change. To add a new - * version (a refined rubric) use `sf agent scorer create --new-version`. - * - * This is a purely local metadata operation (it edits the scorer's XML on disk), so it needs no org connection; - * deploy the definition afterward for the org to reflect the change. - */ -export default class AgentScorerEdit extends SfCommand { - public static readonly summary = messages.getMessage('summary'); - public static readonly description = messages.getMessage('description'); - public static readonly examples = messages.getMessages('examples'); - - public static readonly flags = { - 'api-name': Flags.string({ - summary: messages.getMessage('flags.api-name.summary'), - required: true, - }), - // eslint-disable-next-line sf-plugin/flag-min-max-default - version: Flags.integer({ - summary: messages.getMessage('flags.version.summary'), - required: true, - min: 1, - }), - status: Flags.string({ - summary: messages.getMessage('flags.status.summary'), - options: SCORER_VERSION_STATUSES, - }), - activate: Flags.boolean({ - summary: messages.getMessage('flags.activate.summary'), - exclusive: ['deactivate'], - }), - deactivate: Flags.boolean({ - summary: messages.getMessage('flags.deactivate.summary'), - exclusive: ['activate'], - }), - 'output-dir': Flags.directory({ - summary: messages.getMessage('flags.output-dir.summary'), - default: join('force-app', 'main', 'default'), - }), - preview: Flags.boolean({ - summary: messages.getMessage('flags.preview.summary'), - }), - }; - - public async run(): Promise { - const { flags } = await this.parse(AgentScorerEdit); - const apiName = flags['api-name']; - const version = flags.version; - const outputDir = resolve(flags['output-dir']); - const status = flags.status as ScorerVersionStatus | undefined; - // --activate → true, --deactivate → false, neither → leave the association untouched. - const activate = flags.activate ? true : flags.deactivate ? false : undefined; - - if (!status && activate === undefined) { - throw messages.createError('error.noChange'); - } - - const scorerPath = join(outputDir, 'aiAgentScorerDefinitions', `${apiName}.aiAgentScorerDefinition-meta.xml`); - - let existingXml: string; - try { - existingXml = await readFile(scorerPath, 'utf8'); - } catch (err) { - // Only a missing file means "not authored yet"; surface any other read failure (EACCES, EISDIR, …) - // as-is so the user isn't wrongly told to `create` a scorer that already exists. - if ((err as NodeJS.ErrnoException).code === 'ENOENT') { - throw messages.createError('error.scorerNotFound', [apiName, scorerPath]); - } - throw err; - } - - // Apply every requested change in memory against a single load, then write once, so a combined status + - // activation edit (and its --preview) reflects both changes together. - let contents = existingXml; - const changes: string[] = []; - if (status) { - contents = setVersionStatusInScorerXml(contents, apiName, version, status); - changes.push(`status → ${status}`); - } - if (activate !== undefined) { - contents = setVersionAssociationActiveInScorerXml(contents, apiName, version, activate); - changes.push(activate ? 'agent association activated' : 'agent association deactivated'); - } - - if (flags.preview) { - this.log(`\n--- ${apiName} v${version} (${changes.join(', ')}) — preview ---\n`); - this.log(contents); - return { path: scorerPath, apiName, contents }; - } - - await writeFile(scorerPath, contents); - this.log(`Updated ${apiName} v${version} (${changes.join(', ')}): ${scorerPath}`); - this.log('Deploy the scorer definition for the org to reflect this change.'); - - return { path: scorerPath, apiName, contents }; - } -} diff --git a/src/commands/agent/scorer/create.ts b/src/commands/agent/scorer/generate-metadata-file.ts similarity index 82% rename from src/commands/agent/scorer/create.ts rename to src/commands/agent/scorer/generate-metadata-file.ts index 60e25d22..c4efc402 100644 --- a/src/commands/agent/scorer/create.ts +++ b/src/commands/agent/scorer/generate-metadata-file.ts @@ -21,7 +21,6 @@ import { Agent, type ScorerSpec, createScorerDefinition, - addScorerVersion, labelToApiName, scorerSpecJsonSchema, type SupportedLightningType, @@ -39,13 +38,18 @@ import { FlaggablePrompt, makeFlags, promptForFlag } from '../../../flags.js'; import { theme } from '../../../inquirer-theme.js'; Messages.importMessagesDirectoryFromMetaUrl(import.meta.url); -const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.scorer.create'); +const messages = Messages.loadMessages('@salesforce/plugin-agent', 'agent.scorer.generate-metadata-file'); -export type AgentScorerCreateResult = { +export type AgentScorerGenerateMetadataFileResult = { path: string; apiName: string; contents: string; promptTemplatePath?: string; + /** + * Guidance surfaced to a caller running with --json (where `this.log` output is suppressed): a written + * scorer's metadata XML is the source of truth, so any further change is made directly in the XML. + */ + guidance?: string; }; const FLAGGABLE_PROMPTS = { @@ -60,7 +64,8 @@ const FLAGGABLE_PROMPTS = { promptMessage: 'Scorer API name', validate: (d: string): boolean | string => { if (!d.length) return 'API name cannot be empty'; - if (d.length > SCORER_API_NAME_MAX_LENGTH) return `API name cannot exceed ${SCORER_API_NAME_MAX_LENGTH} characters`; + if (d.length > SCORER_API_NAME_MAX_LENGTH) + return `API name cannot exceed ${SCORER_API_NAME_MAX_LENGTH} characters`; if (!SCORER_API_NAME_PATTERN.test(d)) return 'Must start with letter, only alphanumerics and underscores'; return true; }, @@ -114,8 +119,7 @@ async function promptForSingleEnumValue(index: number): Promise - (SCORER_OUTCOME_TYPES as readonly string[]).includes(d) || 'Invalid', + validate: (d: string): boolean | string => (SCORER_OUTCOME_TYPES as readonly string[]).includes(d) || 'Invalid', }); const isFallback = await confirm({ @@ -141,13 +145,18 @@ async function promptForOutputEnumValues(): Promise { // eslint-disable-next-line no-await-in-loop const result = await promptForSingleEnumValue(values.length); addMore = result.addMore; - values.push({ value: result.value, outcomeType: result.outcomeType, isFallback: result.isFallback, isSystemFallback: result.isSystemFallback }); + values.push({ + value: result.value, + outcomeType: result.outcomeType, + isFallback: result.isFallback, + isSystemFallback: result.isSystemFallback, + }); } return values; } -export default class AgentScorerCreate extends SfCommand { +export default class AgentScorerGenerateMetadataFile extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); @@ -172,10 +181,6 @@ export default class AgentScorerCreate extends SfCommand { - const { flags } = await this.parse(AgentScorerCreate); + public async run(): Promise { + const { flags } = await this.parse(AgentScorerGenerateMetadataFile); if (flags['spec-schema']) { this.styledJSON(scorerSpecJsonSchema() as unknown as import('@salesforce/ts-types').AnyJson); @@ -206,40 +211,26 @@ export default class AgentScorerCreate extends SfCommand { + private async promptForEngineConfig( + engineType: string + ): Promise<{ promptContent?: string; promptTemplateName?: string }> { if (engineType !== 'PromptTemplate') return {}; // No flag exists yet for referencing an existing prompt template by name, so in --json/ // non-interactive mode we always generate a new default prompt template. @@ -425,5 +438,4 @@ export default class AgentScorerCreate extends SfCommand { - const writtenFiles: WrittenFile[] = []; - - const readFile = (): Promise => { - // readError models a non-ENOENT read failure (EACCES, EISDIR, …); null models a missing file (ENOENT). - if (readError) return Promise.reject(readError); - return existingScorerXml == null - ? Promise.reject(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })) - : Promise.resolve(existingScorerXml); - }; - - const writeFile = (path: unknown, content: unknown): Promise => { - writtenFiles.push({ path: String(path), content: String(content) }); - return Promise.resolve(); - }; - - const mod = await esmock('../../../../src/commands/agent/scorer/edit.js', { - 'node:fs/promises': { readFile, writeFile }, - }); - return { Command: mod.default, writtenFiles }; -} - -describe('agent scorer edit', () => { - const $$ = new TestContext(); - let sfCommandStubs: ReturnType; - - before(async function () { - try { - await esmock('../../../../src/commands/agent/scorer/edit.js', {}); - } catch (e: any) { - // eslint-disable-next-line no-console - console.error('esmock warmup failed:', e.message); - this.skip(); - } - }); - - beforeEach(() => { - sfCommandStubs = stubSfCommandUx($$.SANDBOX); - }); - - afterEach(() => { - $$.restore(); - sinon.restore(); - }); - - it('promotes a version to Available with --status Available', async () => { - const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); - - const result = await Command.run([ - '--api-name', 'Test_Scorer', - '--version', '1', - '--status', 'Available', - '--output-dir', '/tmp/out', - '--json', - ]); - - expect(result.apiName).to.equal('Test_Scorer'); - expect(writtenFiles).to.have.length(1); - expect(writtenFiles[0].content).to.include('Available'); - }); - - it('archives a version with --status Archived', async () => { - const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); - - await Command.run([ - '--api-name', 'Test_Scorer', - '--version', '1', - '--status', 'Archived', - '--output-dir', '/tmp/out', - '--json', - ]); - - expect(writtenFiles).to.have.length(1); - expect(writtenFiles[0].content).to.include('Archived'); - }); - - it('activates the agent association with --activate', async () => { - const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); - - await Command.run([ - '--api-name', 'Test_Scorer', - '--version', '1', - '--activate', - '--output-dir', '/tmp/out', - '--json', - ]); - - expect(writtenFiles).to.have.length(1); - expect(writtenFiles[0].content).to.include('true'); - }); - - it('deactivates the agent association with --deactivate', async () => { - const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1, SPEC_ACTIVE)); - - await Command.run([ - '--api-name', 'Test_Scorer', - '--version', '1', - '--deactivate', - '--output-dir', '/tmp/out', - '--json', - ]); - - expect(writtenFiles).to.have.length(1); - expect(writtenFiles[0].content).to.include('false'); - }); - - it('changes status and activation together in a single write', async () => { - const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); - - await Command.run([ - '--api-name', 'Test_Scorer', - '--version', '1', - '--status', 'Available', - '--activate', - '--output-dir', '/tmp/out', - '--json', - ]); - - // A single load → both mutations → one write, so the written XML carries both changes. - expect(writtenFiles).to.have.length(1); - expect(writtenFiles[0].content).to.include('Available'); - expect(writtenFiles[0].content).to.include('true'); - }); - - it('errors when neither --status nor --activate/--deactivate is provided', async () => { - const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); - - try { - await Command.run(['--api-name', 'Test_Scorer', '--version', '1', '--output-dir', '/tmp/out', '--json']); - expect.fail('should have thrown'); - } catch (err: unknown) { - expect((err as Error).message).to.match(/status|activate|deactivate/); - } - expect(writtenFiles).to.have.length(0); - }); - - it('rejects --activate together with --deactivate', async () => { - const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); - - try { - await Command.run([ - '--api-name', 'Test_Scorer', - '--version', '1', - '--activate', - '--deactivate', - '--output-dir', '/tmp/out', - '--json', - ]); - expect.fail('should have thrown'); - } catch (err: unknown) { - expect((err as Error).message).to.match(/activate|deactivate/); - } - expect(writtenFiles).to.have.length(0); - }); - - it('errors when the scorer file is not found', async () => { - const { Command, writtenFiles } = await loadMockedCommand(null); - - try { - await Command.run([ - '--api-name', 'Missing_Scorer', - '--version', '1', - '--status', 'Available', - '--output-dir', '/tmp/out', - '--json', - ]); - expect.fail('should have thrown'); - } catch (err: unknown) { - expect((err as Error).message).to.include('was found at'); - } - expect(writtenFiles).to.have.length(0); - }); - - it('rethrows a non-ENOENT read error instead of reporting "scorer not found"', async () => { - const eacces = Object.assign(new Error('EACCES: permission denied'), { code: 'EACCES' }); - const { Command, writtenFiles } = await loadMockedCommand('unused', eacces); - - try { - await Command.run([ - '--api-name', 'Test_Scorer', - '--version', '1', - '--status', 'Available', - '--output-dir', '/tmp/out', - '--json', - ]); - expect.fail('should have thrown'); - } catch (err: unknown) { - // the raw fs error propagates; the user is NOT wrongly told the scorer was not found - expect((err as Error).message).to.include('EACCES'); - expect((err as Error).message).to.not.include('was found at'); - } - expect(writtenFiles).to.have.length(0); - }); - - it('writes nothing with --preview but prints the resulting XML', async () => { - const { Command, writtenFiles } = await loadMockedCommand(scorerXmlWithVersions(1)); - - const result = await Command.run([ - '--api-name', 'Test_Scorer', - '--version', '1', - '--status', 'Available', - '--output-dir', '/tmp/out', - '--preview', - ]); - - expect(writtenFiles).to.have.length(0); - expect(result.contents).to.include('Available'); - const logged = sfCommandStubs.log.args.map((a) => String(a[0])).join('\n'); - expect(logged).to.include('Available'); - }); -}); diff --git a/test/commands/agent/scorer/create.test.ts b/test/commands/agent/scorer/generate-metadata-file.test.ts similarity index 82% rename from test/commands/agent/scorer/create.test.ts rename to test/commands/agent/scorer/generate-metadata-file.test.ts index d4597bd0..d7bacb26 100644 --- a/test/commands/agent/scorer/create.test.ts +++ b/test/commands/agent/scorer/generate-metadata-file.test.ts @@ -108,8 +108,6 @@ async function loadMockedCommand( opts?: { existsSync?: () => boolean; confirmResult?: boolean; - existingScorerXml?: string; - existingTemplateXml?: string; } ): Promise<{ Command: any; writtenFiles: WrittenFile[]; createdDirs: string[] }> { const yamlContent = YAML.stringify(yamlSpec); @@ -130,22 +128,7 @@ async function loadMockedCommand( typeof p === 'string' && (p.includes('aiAgentScorerDefinitions') || p.includes('genAiPromptTemplates')); const origWriteFile = fsPromises.writeFile; const origMkdir = fsPromises.mkdir; - const origReadFile = fsPromises.readFile; - - // addScorerVersion / setScorerVersionStatus read existing metadata via node:fs/promises.readFile. - // Serve the supplied fixture XML for those paths so the version/transition logic runs without disk. - if (opts?.existingScorerXml !== undefined || opts?.existingTemplateXml !== undefined) { - sinon.stub(fsPromises, 'readFile').callsFake((path: unknown, ...rest: any[]) => { - const p = String(path); - if (p.includes('genAiPromptTemplates') && opts.existingTemplateXml !== undefined) { - return Promise.resolve(opts.existingTemplateXml); - } - if (p.includes('aiAgentScorerDefinitions') && opts.existingScorerXml !== undefined) { - return Promise.resolve(opts.existingScorerXml); - } - return origReadFile(path, ...rest); - }); - } + sinon.stub(fsPromises, 'writeFile').callsFake((path: unknown, content: unknown, options: unknown) => { if (isScorerOutput(path)) { writtenFiles.push({ path: String(path), content: String(content) }); @@ -169,7 +152,7 @@ async function loadMockedCommand( }; } - const mod = await esmock('../../../../src/commands/agent/scorer/create.js', mocks); + const mod = await esmock('../../../../src/commands/agent/scorer/generate-metadata-file.js', mocks); return { Command: mod.default, writtenFiles, createdDirs }; } @@ -186,7 +169,7 @@ const SPEC_FILENAMES = [ 'manual-scorer.yaml', ]; -describe('agent scorer create', () => { +describe('agent scorer generate-metadata-file', () => { const $$ = new TestContext(); let testOrg: MockTestOrgData; let originalCwd: string; @@ -196,7 +179,7 @@ describe('agent scorer create', () => { before(async function () { // Warm up esmock to check it can load the module try { - await esmock('../../../../src/commands/agent/scorer/create.js', { + await esmock('../../../../src/commands/agent/scorer/generate-metadata-file.js', { 'node:fs': { readFileSync: () => '', writeFileSync: () => {}, @@ -237,8 +220,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test-scorer.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test-scorer.yaml', '--preview', '--json', ]); @@ -261,8 +246,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeOpenSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'open-scorer.yaml', + '--target-org', + testOrg.username, + '--spec', + 'open-scorer.yaml', '--preview', '--json', ]); @@ -278,8 +265,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeOpenSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'open-scorer.yaml', + '--target-org', + testOrg.username, + '--spec', + 'open-scorer.yaml', '--preview', '--json', ]); @@ -302,8 +291,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(spec); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'open-scorer.yaml', + '--target-org', + testOrg.username, + '--spec', + 'open-scorer.yaml', '--preview', '--json', ]); @@ -323,8 +314,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeOpenSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'open-scorer.yaml', + '--target-org', + testOrg.username, + '--spec', + 'open-scorer.yaml', '--preview', '--json', ]); @@ -337,8 +330,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makePromptTemplateSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'prompt-scorer.yaml', + '--target-org', + testOrg.username, + '--spec', + 'prompt-scorer.yaml', '--preview', '--json', ]); @@ -354,8 +349,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeLabeledSpec({ engineType: 'Manual' })); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'manual-scorer.yaml', + '--target-org', + testOrg.username, + '--spec', + 'manual-scorer.yaml', '--preview', '--json', ]); @@ -370,9 +367,12 @@ describe('agent scorer create', () => { const { Command, writtenFiles } = await loadMockedCommand(spec); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -390,8 +390,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(spec); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -408,8 +410,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeLabeledSpec({ description: 'Evaluates politeness' })); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -421,8 +425,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeLabeledSpec({ description: undefined })); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -436,8 +442,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(spec); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -451,8 +459,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(spec); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -464,8 +474,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -479,9 +491,12 @@ describe('agent scorer create', () => { const { Command, writtenFiles } = await loadMockedCommand(makeOpenSpec()); await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -493,9 +508,12 @@ describe('agent scorer create', () => { const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -511,8 +529,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -527,8 +547,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(spec); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -546,8 +568,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(spec); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -561,8 +585,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeLabeledSpec({ label: 'My Custom Label' })); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -576,9 +602,12 @@ describe('agent scorer create', () => { const { Command, writtenFiles, createdDirs } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -594,9 +623,12 @@ describe('agent scorer create', () => { const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -613,12 +645,7 @@ describe('agent scorer create', () => { it('should not write files with --preview', async () => { const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec()); - await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--preview', - '--json', - ]); + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml', '--preview', '--json']); expect(writtenFiles).to.have.length(0); }); @@ -629,9 +656,12 @@ describe('agent scorer create', () => { const { Command, writtenFiles } = await loadMockedCommand(spec); await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -645,9 +675,12 @@ describe('agent scorer create', () => { const { Command, writtenFiles } = await loadMockedCommand(makeOpenSpec()); await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -662,8 +695,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -675,9 +710,12 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/custom/path', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/custom/path', '--preview', '--json', ]); @@ -687,71 +725,49 @@ describe('agent scorer create', () => { }); describe('existing scorer behavior', () => { - it('errors when the scorer already exists and --new-version is not passed', async () => { + // `create` is a one-shot scaffolder: once the XML exists it is the source of truth, so a re-run must not + // overwrite it or silently mutate it. It errors and points the user at editing the metadata XML directly. + it('errors and writes nothing when the scorer already exists', async () => { const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec(), { existsSync: () => true, }); try { await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); expect.fail('should have thrown'); } catch (err: unknown) { expect((err as Error).message).to.include('already exists'); - expect((err as Error).message).to.include('--new-version'); + // Directs the user to hand-edit the metadata XML rather than re-scaffolding via the CLI. + expect((err as Error).message).to.include('metadata XML'); } expect(writtenFiles).to.have.length(0); }); - it('adds a new version when the scorer exists and --new-version is passed', async () => { - const spec = makeLabeledSpec(); - const existingScorerXml = (agentsModule as any).buildScorerXml(spec); - const { Command, writtenFiles } = await loadMockedCommand(spec, { - existsSync: () => true, - existingScorerXml, - }); - - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', - '--new-version', - '--json', - ]); - - expect(result.apiName).to.equal('Test_Scorer'); - // v1 is preserved and v2 is appended. - expect(result.contents).to.include('1'); - expect(result.contents).to.include('2'); - const scorerFile = writtenFiles.find((f) => f.path.includes('aiAgentScorerDefinitions')); - expect(scorerFile).to.not.be.undefined; - }); - - it('previews the appended version (not a fresh v1) with --new-version --preview, writing nothing', async () => { - const spec = makeLabeledSpec(); - const existingScorerXml = (agentsModule as any).buildScorerXml(spec); - const { Command, writtenFiles } = await loadMockedCommand(spec, { - existsSync: () => true, - existingScorerXml, - }); + // `this.log` is suppressed under --json, so an agent caller only sees the returned payload. The + // scaffold-once guidance rides along in `result.guidance` so it still reaches that caller. + it('returns scaffold-once guidance in the --json result on a fresh create', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', - '--new-version', - '--preview', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); - // Preview reflects the artifact --new-version would write: v1 preserved, v2 appended — not a fresh v1. - expect(result.contents).to.include('1'); - expect(result.contents).to.include('2'); - expect(writtenFiles).to.have.length(0); + expect(result.guidance).to.be.a('string'); + expect(result.guidance).to.include('source of truth'); }); }); @@ -762,9 +778,12 @@ describe('agent scorer create', () => { ); await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -777,9 +796,12 @@ describe('agent scorer create', () => { const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -792,9 +814,12 @@ describe('agent scorer create', () => { const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -807,9 +832,12 @@ describe('agent scorer create', () => { const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -827,9 +855,12 @@ describe('agent scorer create', () => { const { Command, writtenFiles } = await loadMockedCommand(makePromptTemplateSpec()); await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', - '--output-dir', '/tmp/out', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', + '--output-dir', + '/tmp/out', '--json', ]); @@ -895,12 +926,7 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(spec); try { - await Command.run([ - '--target-org', testOrg.username, - '--spec', specFile, - '--preview', - '--json', - ]); + await Command.run(['--target-org', testOrg.username, '--spec', specFile, '--preview', '--json']); expect.fail('should have thrown'); } catch (err: unknown) { const error = err as { message: string }; @@ -919,12 +945,7 @@ describe('agent scorer create', () => { writeFileSync(specFile, YAML.stringify(spec)); const { Command } = await loadMockedCommand(spec); - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', specFile, - '--preview', - '--json', - ]); + const result = await Command.run(['--target-org', testOrg.username, '--spec', specFile, '--preview', '--json']); expect(result.apiName).to.equal('Test_Scorer'); expect(result.contents).to.include('Good'); @@ -941,12 +962,7 @@ describe('agent scorer create', () => { writeFileSync(specFile, YAML.stringify(spec)); const { Command } = await loadMockedCommand(spec); - const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', specFile, - '--preview', - '--json', - ]); + const result = await Command.run(['--target-org', testOrg.username, '--spec', specFile, '--preview', '--json']); expect(result.apiName).to.equal('Test_Scorer'); expect(result.contents).to.include('N/A'); @@ -966,8 +982,10 @@ describe('agent scorer create', () => { const { Command } = await loadMockedCommand(spec); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -978,15 +996,15 @@ describe('agent scorer create', () => { it('should handle single output enum value', async () => { const spec = makeLabeledSpec({ - outputEnumValues: [ - { value: 'Only', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }, - ], + outputEnumValues: [{ value: 'Only', outcomeType: 'NotApplicable', isFallback: true, isSystemFallback: false }], }); const { Command } = await loadMockedCommand(spec); const result = await Command.run([ - '--target-org', testOrg.username, - '--spec', 'test.yaml', + '--target-org', + testOrg.username, + '--spec', + 'test.yaml', '--preview', '--json', ]); @@ -1002,13 +1020,20 @@ describe('agent scorer create', () => { const { Command, writtenFiles } = await loadMockedCommand(makeLabeledSpec()); const result = await Command.run([ - '--target-org', testOrg.username, - '--label', 'My Scorer', - '--api-name', 'My_Scorer', - '--lightning-type', 'lightning__textType', - '--engine-type', 'Manual', - '--agent-api-name', 'My_Agent', - '--output-dir', '/tmp/out-json-flags', + '--target-org', + testOrg.username, + '--label', + 'My Scorer', + '--api-name', + 'My_Scorer', + '--lightning-type', + 'lightning__textType', + '--engine-type', + 'Manual', + '--agent-api-name', + 'My_Agent', + '--output-dir', + '/tmp/out-json-flags', '--json', ]); @@ -1021,7 +1046,7 @@ describe('agent scorer create', () => { describe('--spec YAML parsing', () => { it('throws a clear error when the spec file is not valid YAML', async () => { - const mod = await esmock('../../../../src/commands/agent/scorer/create.js', { + const mod = await esmock('../../../../src/commands/agent/scorer/generate-metadata-file.js', { 'node:fs': { readFileSync: () => 'foo: [1, 2', existsSync: () => false }, }); const Command = mod.default; @@ -1035,7 +1060,7 @@ describe('agent scorer create', () => { }); it('throws a clear error when the spec file is not a YAML object', async () => { - const mod = await esmock('../../../../src/commands/agent/scorer/create.js', { + const mod = await esmock('../../../../src/commands/agent/scorer/generate-metadata-file.js', { 'node:fs': { readFileSync: () => '- 1\n- 2\n', existsSync: () => false }, }); const Command = mod.default; @@ -1079,7 +1104,7 @@ describe('agent scorer create', () => { Agent: { listRemote: sinon.stub().resolves([]) }, }, }; - const mod = await esmock('../../../../src/commands/agent/scorer/create.js', mocks); + const mod = await esmock('../../../../src/commands/agent/scorer/generate-metadata-file.js', mocks); const Command = mod.default; try { @@ -1087,13 +1112,20 @@ describe('agent scorer create', () => { // are otherwise resolved via promptForFlag() in ../../../flags.js — a module esmock does not // remock here, so any prompt routed through it would hit the real @inquirer/prompts and hang). await Command.run([ - '--target-org', testOrg.username, - '--label', 'My Scorer', - '--api-name', 'My_Scorer', - '--lightning-type', 'lightning__textType', - '--engine-type', 'Manual', - '--description', 'A test description', - '--status', 'Draft', + '--target-org', + testOrg.username, + '--label', + 'My Scorer', + '--api-name', + 'My_Scorer', + '--lightning-type', + 'lightning__textType', + '--engine-type', + 'Manual', + '--description', + 'A test description', + '--status', + 'Draft', ]); expect.fail('should have thrown'); } catch (err: unknown) { diff --git a/test/nuts/agent.scorer.edit.nut.ts b/test/nuts/agent.scorer.edit.nut.ts deleted file mode 100644 index 06af70dd..00000000 --- a/test/nuts/agent.scorer.edit.nut.ts +++ /dev/null @@ -1,167 +0,0 @@ -/* - * Copyright 2026, Salesforce, Inc. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -import { join } from 'node:path'; -import { mkdirSync, writeFileSync, readFileSync } from 'node:fs'; -import { expect } from 'chai'; -import { TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; -import { buildScorerXml, addVersionToScorerXml, parseScorerVersions, type ScorerSpec } from '@salesforce/agents'; -import type { AgentScorerEditResult } from '../../src/commands/agent/scorer/edit.js'; - -// `agent scorer edit` is a purely local XML edit — it needs NO org connection — so this NUT runs against a bare -// project with no scratch org. It authors a scorer fixture on disk (via the agents lib, the same way `create` -// would), then exercises the real command end to end: status changes, activation toggles, and the error paths. -describe('agent scorer edit NUTs', () => { - const API_NAME = 'Nut_Edit_Scorer'; - // A `Manual`-engine scorer needs no prompt template, keeping the fixture self-contained. - const spec: ScorerSpec = { - apiName: API_NAME, - lightningType: 'lightning__textType', - inputScope: 'Session', - label: 'NUT Edit Scorer', - engineType: 'Manual', - status: 'Draft', - agentAssociation: { agentApiName: 'My_Agent', isActive: false }, - }; - - let session: TestSession; - let outputDir: string; - let scorerPath: string; - - before(async () => { - session = await TestSession.create({ project: { name: 'scorerEditNut' } }); - outputDir = join(session.project.dir, 'force-app', 'main', 'default'); - scorerPath = join(outputDir, 'aiAgentScorerDefinitions', `${API_NAME}.aiAgentScorerDefinition-meta.xml`); - }); - - after(async () => { - await session?.clean(); - }); - - /** (Re)author a fresh two-version fixture (v1, v2 — both Draft, inactive) so each test starts from a known state. */ - function authorFixture(active = false): void { - const seed = active - ? { ...spec, agentAssociation: { agentApiName: 'My_Agent', isActive: true } } - : spec; - let xml = buildScorerXml(seed); - ({ xml } = addVersionToScorerXml(xml, seed)); - mkdirSync(join(outputDir, 'aiAgentScorerDefinitions'), { recursive: true }); - writeFileSync(scorerPath, xml); - } - - const versionsOnDisk = (): ReturnType => parseScorerVersions(readFileSync(scorerPath, 'utf8')); - - beforeEach(() => authorFixture()); - - it('promotes a version to Available with --status Available', () => { - const result = execCmd( - `agent scorer edit --api-name ${API_NAME} --version 2 --status Available --output-dir "${outputDir}" --json`, - { ensureExitCode: 0 } - ).jsonOutput?.result; - - expect(result?.apiName).to.equal(API_NAME); - const versions = versionsOnDisk(); - expect(versions.find((v) => v.versionNumber === 2)?.status).to.equal('Available'); - // untouched versions keep their status - expect(versions.find((v) => v.versionNumber === 1)?.status).to.equal('Draft'); - }); - - it('archives a version with --status Archived', () => { - execCmd( - `agent scorer edit --api-name ${API_NAME} --version 1 --status Archived --output-dir "${outputDir}" --json`, - { ensureExitCode: 0 } - ); - - expect(versionsOnDisk().find((v) => v.versionNumber === 1)?.status).to.equal('Archived'); - }); - - it('activates the agent association with --activate', () => { - execCmd( - `agent scorer edit --api-name ${API_NAME} --version 2 --activate --output-dir "${outputDir}" --json`, - { ensureExitCode: 0 } - ); - - expect(versionsOnDisk().find((v) => v.versionNumber === 2)?.isActive).to.equal(true); - }); - - it('deactivates the agent association with --deactivate', () => { - authorFixture(true); - execCmd( - `agent scorer edit --api-name ${API_NAME} --version 2 --deactivate --output-dir "${outputDir}" --json`, - { ensureExitCode: 0 } - ); - - expect(versionsOnDisk().find((v) => v.versionNumber === 2)?.isActive).to.equal(false); - }); - - it('changes status and activation together in a single invocation', () => { - execCmd( - `agent scorer edit --api-name ${API_NAME} --version 2 --status Available --activate --output-dir "${outputDir}" --json`, - { ensureExitCode: 0 } - ); - - const v2 = versionsOnDisk().find((v) => v.versionNumber === 2); - expect(v2?.status).to.equal('Available'); - expect(v2?.isActive).to.equal(true); - }); - - it('writes nothing with --preview but prints the resulting XML', () => { - const before = readFileSync(scorerPath, 'utf8'); - const output = execCmd( - `agent scorer edit --api-name ${API_NAME} --version 2 --status Available --output-dir "${outputDir}" --preview`, - { ensureExitCode: 0 } - ); - - // file untouched... - expect(readFileSync(scorerPath, 'utf8')).to.equal(before); - expect(versionsOnDisk().find((v) => v.versionNumber === 2)?.status).to.equal('Draft'); - // ...but the promoted XML was printed - expect(output.shellOutput.stdout).to.include('Available'); - }); - - it('errors when neither --status nor --activate/--deactivate is provided', () => { - const output = execCmd( - `agent scorer edit --api-name ${API_NAME} --version 1 --output-dir "${outputDir}" --json`, - { ensureExitCode: 1 } - ).jsonOutput; - - expect(output?.message).to.match(/status|activate|deactivate/); - }); - - it('rejects --activate together with --deactivate', () => { - execCmd( - `agent scorer edit --api-name ${API_NAME} --version 1 --activate --deactivate --output-dir "${outputDir}" --json`, - { ensureExitCode: 'nonZero' } - ); - }); - - it('errors on an unknown version', () => { - const output = execCmd( - `agent scorer edit --api-name ${API_NAME} --version 9 --status Available --output-dir "${outputDir}" --json`, - { ensureExitCode: 1 } - ).jsonOutput; - - expect(output?.message).to.match(/version 9/); - }); - - it('errors when the scorer file is not found', () => { - const output = execCmd( - `agent scorer edit --api-name Missing_Scorer --version 1 --status Available --output-dir "${outputDir}" --json`, - { ensureExitCode: 1 } - ).jsonOutput; - - expect(output?.message).to.include('was found at'); - }); -}); diff --git a/test/nuts/agent.scorer.create.nut.ts b/test/nuts/agent.scorer.generate-metadata-file.nut.ts similarity index 71% rename from test/nuts/agent.scorer.create.nut.ts rename to test/nuts/agent.scorer.generate-metadata-file.nut.ts index 07b4a43e..539cb563 100644 --- a/test/nuts/agent.scorer.create.nut.ts +++ b/test/nuts/agent.scorer.generate-metadata-file.nut.ts @@ -18,12 +18,12 @@ import { writeFileSync, readFileSync, existsSync } from 'node:fs'; import { expect } from 'chai'; import { TestSession, execCmd } from '@salesforce/cli-plugins-testkit'; import { parseScorerVersions } from '@salesforce/agents'; -import type { AgentScorerCreateResult } from '../../src/commands/agent/scorer/create.js'; +import type { AgentScorerGenerateMetadataFileResult } from '../../src/commands/agent/scorer/generate-metadata-file.js'; -// `agent scorer create` writes local metadata only, but its `--target-org` is a required flag that resolves at +// `agent scorer generate-metadata-file` writes local metadata only, but its `--target-org` is a required flag that resolves at // parse time, so this NUT needs a default org. It uses a lightweight scratch org (devhub only — no Einstein // provisioning or metadata deploy, which create doesn't need) rather than the shared heavyweight session. -describe('agent scorer create NUTs', function () { +describe('agent scorer generate-metadata-file NUTs', function () { this.timeout(15 * 60 * 1000); const API_NAME = 'Nut_Create_Scorer'; @@ -45,7 +45,7 @@ describe('agent scorer create NUTs', function () { before(async () => { session = await TestSession.create({ - project: { name: 'scorerCreateNut' }, + project: { name: 'scorerGenerateMetadataFileNut' }, devhubAuthStrategy: 'AUTO', scratchOrgs: [{ setDefault: true, config: join('config', 'project-scratch-def.json') }], }); @@ -60,14 +60,14 @@ describe('agent scorer create NUTs', function () { }); it('prints the spec JSON Schema with --spec-schema', () => { - const { stdout } = execCmd('agent scorer create --spec-schema', { ensureExitCode: 0 }).shellOutput; + const { stdout } = execCmd('agent scorer generate-metadata-file --spec-schema', { ensureExitCode: 0 }).shellOutput; expect(stdout).to.include('ScorerSpec'); expect(stdout).to.include('apiName'); }); it('authors a scorer definition from a --spec file', () => { - const result = execCmd( - `agent scorer create --spec "${specPath}" --output-dir "${outputDir}" --json`, + const result = execCmd( + `agent scorer generate-metadata-file --spec "${specPath}" --output-dir "${outputDir}" --json`, { ensureExitCode: 0 } ).jsonOutput?.result; @@ -77,25 +77,15 @@ describe('agent scorer create NUTs', function () { expect(parseScorerVersions(readFileSync(scorerPath, 'utf8'))).to.have.length(1); }); - it('appends a new version with --new-version', () => { - execCmd( - `agent scorer create --spec "${specPath}" --output-dir "${outputDir}" --new-version --json`, - { ensureExitCode: 0 } - ); - - const versions = parseScorerVersions(readFileSync(scorerPath, 'utf8')); - expect(versions.map((v) => v.versionNumber)).to.deep.equal([1, 2]); - }); - - it('refuses to overwrite an existing scorer without --new-version', () => { - const output = execCmd( - `agent scorer create --spec "${specPath}" --output-dir "${outputDir}" --json`, + it('refuses to overwrite an existing scorer (scaffold-once; edit the XML directly instead)', () => { + const output = execCmd( + `agent scorer generate-metadata-file --spec "${specPath}" --output-dir "${outputDir}" --json`, { ensureExitCode: 1 } ).jsonOutput; expect(output?.message).to.match(new RegExp(API_NAME)); - // the existing file is left untouched (still two versions from the prior test) - expect(parseScorerVersions(readFileSync(scorerPath, 'utf8'))).to.have.length(2); + // the existing file is left untouched (still the single scaffolded version) + expect(parseScorerVersions(readFileSync(scorerPath, 'utf8'))).to.have.length(1); }); it('writes nothing with --preview', () => { @@ -104,8 +94,8 @@ describe('agent scorer create NUTs', function () { writeFileSync(previewSpec, JSON.stringify(makeSpec(previewName, 'NUT Preview Scorer'))); const previewPath = join(outputDir, 'aiAgentScorerDefinitions', `${previewName}.aiAgentScorerDefinition-meta.xml`); - const result = execCmd( - `agent scorer create --spec "${previewSpec}" --output-dir "${outputDir}" --preview --json`, + const result = execCmd( + `agent scorer generate-metadata-file --spec "${previewSpec}" --output-dir "${outputDir}" --preview --json`, { ensureExitCode: 0 } ).jsonOutput?.result; From d749f93a7274f946c6a8fdd1b589fde8e3a0d8d5 Mon Sep 17 00:00:00 2001 From: nnaffar Date: Wed, 9 Sep 2026 22:34:45 +0300 Subject: [PATCH 18/19] chore(schema): sync drifted command schemas with code Regenerate scorer-run and mcp schemas that had drifted from the code. --- schemas/agent-mcp-create.json | 37 ++++++++++------------------------- schemas/agent-mcp-fetch.json | 20 +++++++------------ schemas/agent-scorer-run.json | 11 ++++++----- 3 files changed, 23 insertions(+), 45 deletions(-) diff --git a/schemas/agent-mcp-create.json b/schemas/agent-mcp-create.json index a4c5b095..a6ea809a 100644 --- a/schemas/agent-mcp-create.json +++ b/schemas/agent-mcp-create.json @@ -15,10 +15,7 @@ } } }, - "required": [ - "server", - "assets" - ], + "required": ["server", "assets"], "additionalProperties": false }, "McpServerOutput": { @@ -61,12 +58,7 @@ "type": "string" } }, - "required": [ - "id", - "name", - "type", - "status" - ], + "required": ["id", "name", "type", "status"], "additionalProperties": false }, "McpServerType": { @@ -86,17 +78,12 @@ "type": "string" } }, - "required": [ - "authType" - ], + "required": ["authType"], "additionalProperties": false }, "McpAuthType": { "type": "string", - "enum": [ - "OAUTH", - "NO_AUTH" - ] + "enum": ["OAUTH", "NO_AUTH"] }, "McpFetchedAsset": { "type": "object", @@ -124,21 +111,17 @@ }, "status": { "type": "string" + }, + "securityWarning": { + "type": "string" } }, - "required": [ - "name", - "kind" - ], + "required": ["name", "kind"], "additionalProperties": false }, "McpAssetKind": { "type": "string", - "enum": [ - "MCP_TOOL", - "MCP_PROMPT", - "MCP_RESOURCE" - ] + "enum": ["MCP_TOOL", "MCP_PROMPT", "MCP_RESOURCE"] } } -} \ No newline at end of file +} diff --git a/schemas/agent-mcp-fetch.json b/schemas/agent-mcp-fetch.json index b99f8e9c..b15d7242 100644 --- a/schemas/agent-mcp-fetch.json +++ b/schemas/agent-mcp-fetch.json @@ -15,9 +15,7 @@ } } }, - "required": [ - "assets" - ], + "required": ["assets"], "additionalProperties": false }, "McpFetchedAsset": { @@ -46,21 +44,17 @@ }, "status": { "type": "string" + }, + "securityWarning": { + "type": "string" } }, - "required": [ - "name", - "kind" - ], + "required": ["name", "kind"], "additionalProperties": false }, "McpAssetKind": { "type": "string", - "enum": [ - "MCP_TOOL", - "MCP_PROMPT", - "MCP_RESOURCE" - ] + "enum": ["MCP_TOOL", "MCP_PROMPT", "MCP_RESOURCE"] } } -} \ No newline at end of file +} diff --git a/schemas/agent-scorer-run.json b/schemas/agent-scorer-run.json index 9b5e9836..808b4ff7 100644 --- a/schemas/agent-scorer-run.json +++ b/schemas/agent-scorer-run.json @@ -9,6 +9,10 @@ "scorerApiName": { "type": "string" }, + "scorerVersion": { + "type": "number", + "description": "The version number that was actually run (highest Available by default, or the requested --scorer-version)." + }, "ok": { "type": "boolean" }, @@ -40,10 +44,7 @@ "type": "string" } }, - "required": [ - "ok", - "scorerApiName" - ] + "required": ["ok", "scorerApiName"] } } -} \ No newline at end of file +} From 65e02e8610d1bf3cd9d4f380427cddab3ad56454 Mon Sep 17 00:00:00 2001 From: nnaffar Date: Thu, 10 Sep 2026 12:20:38 +0300 Subject: [PATCH 19/19] feat(scorer): return spec schema as result under --spec-schema --json - generate-metadata-file: under `--spec-schema --json`, styledJSON is a no-op (output suppressed), so the schema was dropped for machine consumers. Return it as the command result instead so it rides in the standard {status, result} envelope; non-JSON runs still pretty-print as before. Widen the command result type to a union (AgentScorerGenerateMetadataFileCommandResult) and regenerate its schema. - Add tests: `--spec-schema --json` returns the schema as the result (styledJSON not called); `--spec` surfaces clear, actionable validation errors (missing agentAssociation, unsupported engineType) rather than raw TypeErrors. --- ...agent-scorer-generate__metadata__file.json | 18 +++++++- .../agent/scorer/generate-metadata-file.ts | 25 +++++++++-- .../scorer/generate-metadata-file.test.ts | 43 +++++++++++++++++++ 3 files changed, 82 insertions(+), 4 deletions(-) diff --git a/schemas/agent-scorer-generate__metadata__file.json b/schemas/agent-scorer-generate__metadata__file.json index 8fb99405..3ed1b068 100644 --- a/schemas/agent-scorer-generate__metadata__file.json +++ b/schemas/agent-scorer-generate__metadata__file.json @@ -1,7 +1,18 @@ { "$schema": "http://json-schema.org/draft-07/schema#", - "$ref": "#/definitions/AgentScorerGenerateMetadataFileResult", + "$ref": "#/definitions/AgentScorerGenerateMetadataFileCommandResult", "definitions": { + "AgentScorerGenerateMetadataFileCommandResult": { + "anyOf": [ + { + "$ref": "#/definitions/AgentScorerGenerateMetadataFileResult" + }, + { + "$ref": "#/definitions/ScorerSpecSchemaResult" + } + ], + "description": "Everything `run` can resolve to: a scaffolded/previewed scorer, or (under `--spec-schema --json`) the schema." + }, "AgentScorerGenerateMetadataFileResult": { "type": "object", "properties": { @@ -24,6 +35,11 @@ }, "required": ["path", "apiName", "contents"], "additionalProperties": false + }, + "ScorerSpecSchemaResult": { + "type": "object", + "additionalProperties": {}, + "description": "The scorer spec JSON Schema payload. Under `--spec-schema --json` (where `styledJSON` is suppressed) it is returned as the command result so the schema still rides in the standard {status, result} envelope for machine consumers, matching what `--spec-schema` prints without `--json`." } } } diff --git a/src/commands/agent/scorer/generate-metadata-file.ts b/src/commands/agent/scorer/generate-metadata-file.ts index c4efc402..37ddebcd 100644 --- a/src/commands/agent/scorer/generate-metadata-file.ts +++ b/src/commands/agent/scorer/generate-metadata-file.ts @@ -52,6 +52,18 @@ export type AgentScorerGenerateMetadataFileResult = { guidance?: string; }; +/** + * The scorer spec JSON Schema payload. Under `--spec-schema --json` (where `styledJSON` is suppressed) it is + * returned as the command result so the schema still rides in the standard {status, result} envelope for + * machine consumers, matching what `--spec-schema` prints without `--json`. + */ +export type ScorerSpecSchemaResult = Record; + +/** Everything `run` can resolve to: a scaffolded/previewed scorer, or (under `--spec-schema --json`) the schema. */ +export type AgentScorerGenerateMetadataFileCommandResult = + | AgentScorerGenerateMetadataFileResult + | ScorerSpecSchemaResult; + const FLAGGABLE_PROMPTS = { label: { message: messages.getMessage('flags.label.summary'), @@ -156,7 +168,7 @@ async function promptForOutputEnumValues(): Promise { return values; } -export default class AgentScorerGenerateMetadataFile extends SfCommand { +export default class AgentScorerGenerateMetadataFile extends SfCommand { public static readonly summary = messages.getMessage('summary'); public static readonly description = messages.getMessage('description'); public static readonly examples = messages.getMessages('examples'); @@ -191,11 +203,18 @@ export default class AgentScorerGenerateMetadataFile extends SfCommand { + public async run(): Promise { const { flags } = await this.parse(AgentScorerGenerateMetadataFile); if (flags['spec-schema']) { - this.styledJSON(scorerSpecJsonSchema() as unknown as import('@salesforce/ts-types').AnyJson); + const schema = scorerSpecJsonSchema(); + // Under --json, styledJSON is a no-op (output is suppressed), so return the schema as the command result + // — it then rides in the standard {status, result} envelope for machine consumers. Otherwise pretty-print + // it to the terminal as before. + if (this.jsonEnabled()) { + return schema; + } + this.styledJSON(schema as unknown as import('@salesforce/ts-types').AnyJson); return { path: '', apiName: '', contents: '' }; } diff --git a/test/commands/agent/scorer/generate-metadata-file.test.ts b/test/commands/agent/scorer/generate-metadata-file.test.ts index d7bacb26..00c53ced 100644 --- a/test/commands/agent/scorer/generate-metadata-file.test.ts +++ b/test/commands/agent/scorer/generate-metadata-file.test.ts @@ -1074,6 +1074,37 @@ describe('agent scorer generate-metadata-file', () => { }); }); + describe('--spec validation errors', () => { + // The --spec path hands the parsed YAML to createScorerDefinition → validateScorerSpec. These assert the + // command surfaces a clear, actionable message rather than a raw TypeError / silently-broken metadata. + it('surfaces a clear error (not a TypeError) when agentAssociation is missing', async () => { + const spec = makeLabeledSpec(); + delete (spec as any).agentAssociation; + const { Command, writtenFiles } = await loadMockedCommand(spec); + + try { + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml', '--preview', '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + const message = (err as Error).message; + expect(message).to.include('agentAssociation is required.'); + expect(message).to.not.match(/cannot read properties of undefined/i); + } + expect(writtenFiles).to.have.length(0); + }); + + it('surfaces a clear error when engineType is not a supported engine', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec({ engineType: 'prompttemplate' as any })); + + try { + await Command.run(['--target-org', testOrg.username, '--spec', 'test.yaml', '--preview', '--json']); + expect.fail('should have thrown'); + } catch (err: unknown) { + expect((err as Error).message).to.include("Unsupported engineType 'prompttemplate'."); + } + }); + }); + describe('--spec-schema', () => { it('prints the schema JSON and skips org/spec resolution', async () => { const { Command } = await loadMockedCommand(makeLabeledSpec()); @@ -1088,6 +1119,18 @@ describe('agent scorer generate-metadata-file', () => { expect(printed).to.have.property('$ref', '#/definitions/ScorerSpec'); expect(printed).to.have.property('definitions'); }); + + it('returns the schema as the --json result instead of the empty payload', async () => { + const { Command } = await loadMockedCommand(makeLabeledSpec()); + + // Under --json, styledJSON is suppressed, so the schema must be the command result — it then rides in + // the standard {status, result} envelope rather than being dropped for the machine consumer --json serves. + const result = await Command.run(['--target-org', testOrg.username, '--spec-schema', '--json']); + + expect(result).to.have.property('$ref', '#/definitions/ScorerSpec'); + expect(result).to.have.property('definitions'); + expect(sfCommandStubs.styledJSON.called).to.be.false; + }); }); describe('interactive interview (no --spec, no --json)', () => {