feat: Agentforce Scorers Authoring Command

command-snapshot.json

@@ -376,6 +376,38 @@
     "flags": ["api-name", "api-version", "concise", "flags-dir", "json", "skip-retrieve", "target-org", "verbose"],
     "plugin": "@salesforce/plugin-agent"
   },
+  {
+    "alias": [],
+    "command": "agent:scorer:generate-metadata-file",
+    "flagAliases": [],
+    "flagChars": ["o"],
+    "flags": [
+      "agent-api-name",
+      "api-name",
+      "api-version",
+      "description",
+      "engine-type",
+      "flags-dir",
+      "json",
+      "label",
+      "lightning-type",
+      "output-dir",
+      "preview",
+      "spec",
+      "spec-schema",
+      "status",
+      "target-org"
+    ],
+    "plugin": "@salesforce/plugin-agent"
+  },
+  {
+    "alias": [],
+    "command": "agent:scorer:run",
+    "flagAliases": [],
+    "flagChars": ["o"],
+    "flags": ["api-name", "api-version", "data", "file", "flags-dir", "json", "scorer-version", "target-org"],
+    "plugin": "@salesforce/plugin-agent"
+  },
   {
     "alias": [],
     "command": "agent:test:create",

messages/agent.scorer.generate-metadata-file.md

@@ -0,0 +1,113 @@
+# summary
+
+Scaffold an AiAgentScorerDefinition metadata XML file (an agent scorer), interactively or from a spec file. This is a one-time starter, not the source of truth: once the AiAgentScorerDefinition XML is created (or retrieved from an org), edit it directly.
+
+# description
+
+Creates an AiAgentScorerDefinition metadata XML file either interactively (prompting for each field) or from a YAML spec file.
+
+Run with no flags to start the interactive interview. The command prompts you for the scorer's lightning type, optional output labels, engine type, and agent associations.
+
+Alternatively, provide a --spec flag pointing to a YAML file that defines the scorer. This is useful for repeatable automation or when the scorer has many output values.
+
+Use --preview to see the generated XML without writing it to disk.
+
+This command is a one-shot scaffolding helper: it generates the scorer definition (and, for PromptTemplate scorers, its prompt template) metadata XML to give you a fast start. The generated XML — not this command and not the spec — is the source of truth. Once a definition exists locally, whether you created it here or retrieved it from an org, it has no connection back to the spec: make every further change (add a version, promote or archive a version, toggle an agent association, or edit the prompt rubric) directly in the metadata XML. If you already know the XML structure you can author it by hand and skip this command entirely; it exists because the XML is intricate and encodes rules the spec cannot fully capture.
+
+# flags.api-name.summary
+
+API name of the scorer definition.
+
+# flags.agent-api-name.summary
+
+API name of the agent to associate with this scorer.
+
+# flags.lightning-type.summary
+
+Lightning type the scorer's value conforms to (for example, lightning**textType or lightning**numberType).
+
+# flags.label.summary
+
+Display label for the scorer version.
+
+# flags.description.summary
+
+Description of what this scorer evaluates.
+
+# flags.engine-type.summary
+
+Engine type for scoring (Manual or PromptTemplate).
+
+# flags.status.summary
+
+Initial status of the scorer version (Available or Draft).
+
+# flags.spec.summary
+
+Path to a scorer spec YAML file. Bypasses interactive prompts.
+
+# flags.spec-schema.summary
+
+Output the JSON Schema for the --spec YAML file and exit.
+
+# flags.output-dir.summary
+
+Output directory for the generated metadata XML files (scorer definition and prompt template).
+
+# flags.preview.summary
+
+Preview the generated XML without writing to disk.
+
+# examples
+
+- Show the JSON Schema for the spec YAML file:
+
+  <%= config.bin %> <%= command.id %> --spec-schema
+
+- Create a scorer interactively:
+
+  <%= config.bin %> <%= command.id %>
+
+- Create a scorer from a spec file:
+
+  <%= config.bin %> <%= command.id %> --spec specs/expert-analysis-scorer.yaml
+
+- Preview the XML that would be generated:
+
+  <%= config.bin %> <%= command.id %> --spec specs/expert-analysis-scorer.yaml --preview
+
+- Create a manual scorer with flags (non-interactive):
+
+  <%= config.bin %> <%= command.id %> --api-name Expert_Analysis --lightning-type lightning\_\_textType --engine-type Manual --label Expert_Analysis --agent-api-name My_Agent --status Available
+
+- Create a prompt-based scorer (generates both scorer definition and prompt template):
+
+  <%= config.bin %> <%= command.id %> --api-name sentiment_analysis --lightning-type lightning\_\_textType --engine-type PromptTemplate --label sentiment_analysis --agent-api-name My_Agent
+
+# error.missingRequiredFlags
+
+Missing required flags: %s. When using --json, all required flags must be provided.
+
+# error.invalidSpecYaml
+
+Could not parse the --spec file as YAML: %s
+
+# error.invalidSpecShape
+
+The --spec file must define a YAML object matching the scorer spec schema (see --spec-schema). Received: %s
+
+# error.noAgentsInOrg
+
+No agents found in the org. Deploy an agent first, or specify one with --agent-api-name.
+
+# error.scorerExists
+
+A scorer named '%s' already exists in this project. `generate-metadata-file` only scaffolds a new scorer and never overwrites an existing one. Edit its metadata XML directly instead (%s) — that is where you add a new version, promote or archive a version, toggle an agent association, or change the prompt rubric. To scaffold a different scorer, choose a new API name.
+
+# info.scaffoldIntro
+
+Heads up: this command only scaffolds the scorer's AiAgentScorerDefinition metadata XML to get you started — the generated file, not this interview, is the source of truth. After it's written, make any further changes directly in the AiAgentScorerDefinition XML.
+
+# info.editXmlDirectly
+
+Done. This scorer is now defined by its metadata XML, which is the source of truth from here on — the spec is no longer connected to it. Make any further change (add a version, promote or archive a version, toggle an agent association, or edit the prompt rubric) directly in the generated XML file.

messages/agent.scorer.run.md

@@ -0,0 +1,59 @@
+# summary
+
+Run an agent scorer against an STDM session and print its score.
+
+# description
+
+Runs a scorer that is already authored in your project metadata, referenced by its API name, against a single STDM (Session Trace Data Model) session, then prints the resulting score, outcome labels, and explanation.
+
+The scorer is resolved from your project's package directories by API name. If no scorer with that API name exists locally, the command errors — author it first with `sf agent scorer generate-metadata-file`.
+
+Provide the session either inline as a JSON string with --data, or as a path to a local JSON file with --file. Exactly one of the two is required.
+
+To help you hand-construct a valid session, run this command with --help: the full JSON Schema for the session object is printed under the --data flag.
+
+# flags.api-name.summary
+
+API name of the scorer to run. Must match a scorer authored in this project's metadata.
+
+# flags.scorer-version.summary
+
+Version number of the scorer to run. Omit to run the highest-numbered Available version; the command errors if none is Available. A Draft version must be selected explicitly; an Archived version cannot be run.
+
+# flags.data.summary
+
+Inline STDM session JSON to score.
+
+# flags.data.description
+
+Inline STDM session JSON to score. The value must be a JSON object matching the following JSON Schema (the shape of the scorer's Input:Session value):
+
+# flags.file.summary
+
+Path to a local JSON file containing the STDM session to score.
+
+# examples
+
+- Run a scorer against a session stored in a local file:
+
+  <%= config.bin %> <%= command.id %> --api-name Sentiment_Scorer --file ./session.json
+
+- Run a scorer against an inline session JSON string:
+
+  <%= config.bin %> <%= command.id %> --api-name Sentiment_Scorer --data '{"sessionState":{"sessionId":"1","startTimestamp":"2026-01-01T00:00:00Z","channel":"web"},"actors":[],"metrics":{"durationMs":0,"turns":0},"runs":[]}'
+
+- Show the session JSON Schema in the help output:
+
+  <%= config.bin %> <%= command.id %> --help
+
+# error.invalidSessionJson
+
+Could not parse the STDM session as JSON: %s
+
+# error.invalidSessionShape
+
+The STDM session must be a JSON object matching the session schema (see --help). Received: %s
+
+# error.scorerRunFailed
+
+Scorer '%s' did not produce a valid score: %s. The command exits non-zero so a scripted loop won't treat a failed evaluation as a passing one; the full result (including any explanation) is in the error's "data" field when run with --json.

package.json

@@ -95,6 +95,10 @@
             "description": "Command to validate an Agent Script file.",
             "external": true
           },
+          "scorer": {
+            "description": "Commands to scaffold and run agent scorers.",
+            "external": true
+          },
           "adl": {
             "description": "Commands to manage Agentforce Data Libraries.",
             "external": true,

schemas/agent-mcp-create.json

@@ -15,10 +15,7 @@
           }
         }
       },
-      "required": [
-        "server",
-        "assets"
-      ],
+      "required": ["server", "assets"],
       "additionalProperties": false
     },
     "McpServerOutput": {
@@ -61,12 +58,7 @@
           "type": "string"
         }
       },
-      "required": [
-        "id",
-        "name",
-        "type",
-        "status"
-      ],
+      "required": ["id", "name", "type", "status"],
       "additionalProperties": false
     },
     "McpServerType": {
@@ -86,17 +78,12 @@
           "type": "string"
         }
       },
-      "required": [
-        "authType"
-      ],
+      "required": ["authType"],
       "additionalProperties": false
     },
     "McpAuthType": {
       "type": "string",
-      "enum": [
-        "OAUTH",
-        "NO_AUTH"
-      ]
+      "enum": ["OAUTH", "NO_AUTH"]
     },
     "McpFetchedAsset": {
       "type": "object",
@@ -124,21 +111,17 @@
         },
         "status": {
           "type": "string"
+        },
+        "securityWarning": {
+          "type": "string"
         }
       },
-      "required": [
-        "name",
-        "kind"
-      ],
+      "required": ["name", "kind"],
       "additionalProperties": false
     },
     "McpAssetKind": {
       "type": "string",
-      "enum": [
-        "MCP_TOOL",
-        "MCP_PROMPT",
-        "MCP_RESOURCE"
-      ]
+      "enum": ["MCP_TOOL", "MCP_PROMPT", "MCP_RESOURCE"]
     }
   }
-}
+}

schemas/agent-mcp-fetch.json

@@ -15,9 +15,7 @@
           }
         }
       },
-      "required": [
-        "assets"
-      ],
+      "required": ["assets"],
       "additionalProperties": false
     },
     "McpFetchedAsset": {
@@ -46,21 +44,17 @@
         },
         "status": {
           "type": "string"
+        },
+        "securityWarning": {
+          "type": "string"
         }
       },
-      "required": [
-        "name",
-        "kind"
-      ],
+      "required": ["name", "kind"],
       "additionalProperties": false
     },
     "McpAssetKind": {
       "type": "string",
-      "enum": [
-        "MCP_TOOL",
-        "MCP_PROMPT",
-        "MCP_RESOURCE"
-      ]
+      "enum": ["MCP_TOOL", "MCP_PROMPT", "MCP_RESOURCE"]
     }
   }
-}
+}

schemas/agent-scorer-generate__metadata__file.json

@@ -0,0 +1,45 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$ref": "#/definitions/AgentScorerGenerateMetadataFileCommandResult",
+  "definitions": {
+    "AgentScorerGenerateMetadataFileCommandResult": {
+      "anyOf": [
+        {
+          "$ref": "#/definitions/AgentScorerGenerateMetadataFileResult"
+        },
+        {
+          "$ref": "#/definitions/ScorerSpecSchemaResult"
+        }
+      ],
+      "description": "Everything `run` can resolve to: a scaffolded/previewed scorer, or (under `--spec-schema --json`) the schema."
+    },
+    "AgentScorerGenerateMetadataFileResult": {
+      "type": "object",
+      "properties": {
+        "path": {
+          "type": "string"
+        },
+        "apiName": {
+          "type": "string"
+        },
+        "contents": {
+          "type": "string"
+        },
+        "promptTemplatePath": {
+          "type": "string"
+        },
+        "guidance": {
+          "type": "string",
+          "description": "Guidance surfaced to a caller running with --json (where `this.log` output is suppressed): a written scorer's metadata XML is the source of truth, so any further change is made directly in the XML."
+        }
+      },
+      "required": ["path", "apiName", "contents"],
+      "additionalProperties": false
+    },
+    "ScorerSpecSchemaResult": {
+      "type": "object",
+      "additionalProperties": {},
+      "description": "The scorer spec JSON Schema payload. Under `--spec-schema --json` (where `styledJSON` is suppressed) it is returned as the command result so the schema still rides in the standard {status, result} envelope for machine consumers, matching what `--spec-schema` prints without `--json`."
+    }
+  }
+}