diff --git a/CHANGELOG.md b/CHANGELOG.md index a54c8a7c..62134542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,18 @@ This format follows [Keep a Changelog](https://keepachangelog.com/) and adheres ## [Unreleased] +### Security +- **Trusted PR evaluation (#499).** Generated GitHub PR gates execute a trusted + base checkout and import only immutable candidate dataset/prompt data. Azure + DevOps gates use a closed protected-ref `extends` template with PR-only + resources, fixed Git/API destinations and constrained logging commands. + Configuration/authentication changes are rejected in credentialed candidate + evaluation. Foundry detection now checks hostname boundaries, and credentialed + HTTP redirects cannot cross origins or downgrade HTTPS. Private/sovereign + hosts and explicit local configuration remain supported. Regenerate copied + workflows and configure the documented external resource/identity protections + and exclusive status provenance before relying on a secure merge gate. + ## [0.16.1] - 2026-11-06 ### Changed diff --git a/docs/ci-github-actions.md b/docs/ci-github-actions.md index 9deef55e..4967b22c 100644 --- a/docs/ci-github-actions.md +++ b/docs/ci-github-actions.md @@ -127,6 +127,131 @@ GitHub or Azure, and call out anything that needs owner/admin permission. ## Configuration walkthrough +### PR gate trust boundary and migration + +Regenerate existing copied PR workflows after installing a release containing +the #499 fix. Updating the package alone does not update copied YAML files. +The authenticated evaluator must execute only administrator-reviewed workflow, +toolkit, configuration, recipes and scripts. PR content is read as immutable +Git blobs, never checked out, installed or imported in the credentialed job. +Only JSONL rows at the configured dataset path and plain instructions at the +configured prompt path are copied into that trusted checkout. + +Changes to `agentops.yaml`, including destinations, authentication, thresholds, +recipes and prompt paths, fail the PR gate before evaluation. Review and land +such configuration through a separate controlled change, then rerun candidate +evaluation. Unrelated code changes do not run with evaluation credentials. +Code tests remain a separate, unprivileged workflow. Remote Foundry datasets +continue to come from trusted configuration. Local CLI configuration precedence, +the flat schema and exit codes are unchanged. + +HTTPS is required for credential-bearing PR destinations. Automatic Foundry +classification uses the hostname, not Azure-looking text in a path or query. +Private/custom hosts can use explicit `responses` or `invocations` protocols, +and supported sovereign Azure hostnames remain recognized. Credentialed HTTP +redirects may remain within the same origin, never across hosts, ports or an +HTTPS downgrade. Git input fetching does not follow redirects when using its +read token. + +#### GitHub Actions + +PR templates use `pull_request_target`, check out the exact trusted base SHA +and fetch candidate blobs without persisted checkout credentials. Do not add +candidate checkout, package installation, imports, actions, recipes or shell +execution to this workflow. The changed trigger deliberately permits evaluating +fork PR data with the same sandbox configuration. An optional environment +approval can remain in place, but a maintainer must not approve an environment +for arbitrary PR-authored code. + +Restrict the sandbox Azure identity and any status-writer key to the protected +workflow/ref. An environment name or default OIDC environment subject alone +does not identify a workflow. Protect allowed deployment branches and use a +workflow-bound custom OIDC subject where supported, or require an administrator +to review the exact workflow/revision before granting environment access. +Do not make the same credentials available to ordinary PR jobs or arbitrary +branch `workflow_dispatch` runs. + +The status is posted on the actual PR head SHA. **That SHA and the status name +alone are not trustworthy merge evidence.** Other writers and PR-authored +workflows can publish a status using the shared GitHub Actions identity. +For a required `agentops/pr-eval` gate, install a dedicated status-only GitHub +App, put its client ID in `AGENTOPS_STATUS_APP_CLIENT_ID`, and keep +`AGENTOPS_STATUS_APP_PRIVATE_KEY` in the protected evaluator environment. +Select that App as the required status's **expected source** in the branch +rule/ruleset. Do not select "any source" or the shared GitHub Actions App. +Restrict access to this key as carefully as Azure credentials. Without the App, +evaluation still runs and posts an informational status, but that shared status +must not be used as proof that the trusted evaluator approved a merge. +See [GitHub's expected-source rules](https://docs.github.com/en/repositories/configuring-branches-and-merges-in-your-repository/managing-rulesets/available-rules-for-rulesets#require-status-checks-to-pass-before-merging). + +#### Azure DevOps + +Generation writes two files: the PR entrypoint at +`.azuredevops/pipelines/agentops-pr.yml` and a closed `extends` template at +`.azuredevops/templates/agentops-pr-trusted.yml`. A separate repository is +**not required**. Store the template on an administrator-protected ref in the +same repository, or use an existing protected template repository. +For Azure Repos Git, configure an automatic build-validation branch policy +pointing at the entrypoint to trigger PR runs. Azure Repos does not use the +YAML `pr` trigger. Keep the exclusive required status policy below as the +authoritative proof, even when build validation also requires a passing build. +Replace `YOUR_PROJECT/YOUR_REPOSITORY` in both the wrapper resource and every +literal trusted checkout, and select the same protected ref throughout. +The template checks out that literal protected revision, not a repository +alias or target branch supplied by PR YAML. Its config and scripts govern +evaluation even if the PR targets another branch. + +Create PR-only resources rather than reusing deployment identities: the +`agentops-pr` variable group and the literal `agentops-pr-azure` service +connection. The group contains the usual Foundry sandbox values plus +`AGENTOPS_ADO_REPOSITORY_URL`, `AGENTOPS_ADO_COLLECTION_URI`, +`AGENTOPS_ADO_PROJECT_ID` and `AGENTOPS_ADO_REPOSITORY_ID`. These are the approved +HTTPS source Git URL, collection URL, project ID and repository ID. Git/API +tokens are sent only to these protected destinations, not caller-overridden +`Build.*` or `System.*` destinations. An optional +`AGENTOPS_AGENT_PROMPT_FILE` must likewise be administrator-controlled. + +On the service connection, add a Required template check binding the exact +repository, protected ref and `.azuredevops/templates/agentops-pr-trusted.yml` +path. Restrict updates to the variable group to administrators. Nonsecret +group values are not protected by access checks. If the group contains secrets, +its **own** checks must independently enforce the trusted template before +exposing them. A connection check alone cannot protect a secret group when a +malicious root YAML removes the connection. Verify that the group's available +checks reject that negative case, or keep its secrets unavailable to PR builds. +See [variable group security](https://learn.microsoft.com/en-us/azure/devops/pipelines/library/variable-groups?view=azure-devops). +Restrict pipeline +permissions, protect updates to the ref/resources/checks, limit queue-time +variables, and deny ordinary PR pipelines access to all other privileged +connections, variable groups, secure files and agent pools. Do not add custom +step/job/auth/config parameters to the closed template. It rejects free +execution variables, isolates package installation, and restricts logging +commands and variables that individual tasks may set. Use disposable hosted +agents rather than a shared credential-bearing worktree. + +Register the **exclusive service connection principal** in the ADO organization +and grant it the permission to publish PR statuses. Its ADO token publishes +`agentops/pr-eval` for the verified source SHA/current PR iteration. Configure +a required status policy with that principal as **Authorized identity**, apply +the policy immediately and reset it on source changes. Give the Build Service +only source-read and optional PR-comment permissions, not status authority for +this policy. A PR can remove protected resources and fake a successful build +without triggering resource checks, so a build-validation result or a shared +Build Service status is not sufficient proof. + +These are administrator-managed prerequisites, not settings provisioned by +the generator. Until enforced, do not grant credentials to PR execution or +treat its result as a secure merge gate. The same-repository design depends +on [Required template checks](https://learn.microsoft.com/en-us/azure/devops/pipelines/process/approvals?view=azure-devops#required-template), +[closed security templates](https://learn.microsoft.com/en-us/azure/devops/pipelines/security/templates?view=azure-devops) +and [identity-bound status policies](https://learn.microsoft.com/en-us/azure/devops/repos/git/pr-status-policy?view=azure-devops). + +Candidate prompts and dataset rows still influence the agent and its tools. +This separation prevents CI code/configuration from selecting credential +destinations, not general prompt injection. Keep the evaluation Foundry project +and tools sandboxed with least privilege, no production data and no production +deployment rights. + ### 1. Repository variables (OIDC) In Settings → Secrets and variables → Actions → **Variables**, add: diff --git a/src/agentops/core/agentops_config.py b/src/agentops/core/agentops_config.py index 062f5511..f6725e96 100644 --- a/src/agentops/core/agentops_config.py +++ b/src/agentops/core/agentops_config.py @@ -31,6 +31,7 @@ from dataclasses import dataclass from pathlib import Path from typing import Any, Dict, List, Literal, Mapping, Optional +from urllib.parse import urlsplit from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator @@ -1166,20 +1167,21 @@ class TargetResolution: def _looks_like_foundry_url(url: str) -> bool: """Return ``True`` when ``url`` matches a Foundry hosted endpoint pattern. - Heuristic - Foundry URLs include the segment ``/agents/`` and the host - ends in a Foundry-recognized domain. We err on the side of accepting more - URLs as Foundry hosted (the user can force ``http-json`` via ``protocol``). + This classifies a hostname, not a credential destination allowlist. + Explicit protocols support private/custom hosts without trusting text + in a URL's path, query, userinfo, or an attacker-controlled suffix. """ - lowered = url.lower() + hostname = (urlsplit(url).hostname or "").lower().rstrip(".") foundry_domains = ( ".azure.com", + ".azure.us", + ".azure.cn", ".azureml.ms", - ".cognitiveservices.azure.com", - ".services.ai.azure.com", - ".inference.ml.azure.com", ".azurewebsites.net", # rare; users can override + ".azurewebsites.us", + ".chinacloudsites.cn", ) - return any(domain in lowered for domain in foundry_domains) + return any(hostname.endswith(domain) for domain in foundry_domains) #: Matches the ``/agents//versions/`` segment pair that Foundry @@ -1358,7 +1360,9 @@ def classify_agent( lowered = raw.lower() if lowered.startswith(("http://", "https://")): - if _looks_like_foundry_url(raw): + if protocol in {"responses", "invocations"} or ( + protocol != "http-json" and _looks_like_foundry_url(raw) + ): resolved_protocol: Protocol = protocol or "responses" if resolved_protocol not in {"responses", "invocations"}: raise ValueError( diff --git a/src/agentops/pipeline/commit_info.py b/src/agentops/pipeline/commit_info.py index 395b9d35..c5358260 100644 --- a/src/agentops/pipeline/commit_info.py +++ b/src/agentops/pipeline/commit_info.py @@ -16,7 +16,7 @@ from agentops.core.results import CommitInfo -_CI_SHA_ENV_VARS = ("GITHUB_SHA", "BUILD_SOURCEVERSION") +_CI_SHA_ENV_VARS = ("AGENTOPS_PR_SHA", "GITHUB_SHA", "BUILD_SOURCEVERSION") _FIELD_SEP = "\x1f" _GIT_SHOW_FORMAT = f"%H{_FIELD_SEP}%h{_FIELD_SEP}%s{_FIELD_SEP}%an{_FIELD_SEP}%aI" _GIT_TIMEOUT_SECONDS = 5 @@ -52,7 +52,7 @@ def _pull_request_head_sha() -> Optional[str]: def _ci_git_sha(*, workspace: Optional[Path] = None) -> Optional[str]: - if os.environ.get("GITHUB_EVENT_NAME") == "pull_request": + if os.environ.get("GITHUB_EVENT_NAME") in {"pull_request", "pull_request_target"}: head_sha = _pull_request_head_sha() # The generated PR workflow's default `actions/checkout` has no # `ref`/`fetch-depth` override, so it checks out only the diff --git a/src/agentops/pipeline/invocations.py b/src/agentops/pipeline/invocations.py index c5497c29..2ca38ee1 100644 --- a/src/agentops/pipeline/invocations.py +++ b/src/agentops/pipeline/invocations.py @@ -24,7 +24,8 @@ import urllib.parse import urllib.request from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional +from http.client import HTTPMessage, HTTPResponse +from typing import Any, Dict, IO, List, Optional from agentops.core.agentops_config import AgentOpsConfig, TargetResolution from agentops.pipeline.diagnostics import with_tenant_mismatch_guidance @@ -201,6 +202,39 @@ def _row_input(row: Dict[str, Any]) -> str: return str(value) +def _http_origin(url: str) -> tuple[str, str, int]: + parts = urllib.parse.urlsplit(url) + port = parts.port if parts.port is not None else (443 if parts.scheme == "https" else 80) + return parts.scheme.lower(), (parts.hostname or "").lower(), port + + +class _SameOriginRedirectHandler(urllib.request.HTTPRedirectHandler): + def redirect_request( + self, req: urllib.request.Request, fp: IO[bytes], code: int, + msg: str, headers: HTTPMessage, newurl: str, + ) -> urllib.request.Request | None: + if _http_origin(req.full_url) != _http_origin(newurl): + raise RuntimeError( + "Refusing an authenticated HTTP redirect to another origin. " + "Configure the final trusted endpoint directly." + ) + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +def _open_http(request: urllib.request.Request, *, timeout: float) -> HTTPResponse: + # Custom headers can carry credentials under arbitrary names. + public_headers = {"content-type": "application/json"} + authenticated = any( + public_headers.get(name.lower()) != value + for name, value in request.header_items() + ) + if authenticated: + return urllib.request.build_opener(_SameOriginRedirectHandler()).open( + request, timeout=timeout + ) + return urllib.request.urlopen(request, timeout=timeout) # noqa: S310 + + def _http_request_json( *, method: str, @@ -216,7 +250,7 @@ def _http_request_json( last_exc: Optional[BaseException] = None for attempt in range(1, 4): try: - with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 + with _open_http(request, timeout=timeout) as response: payload = response.read().decode("utf-8") break except urllib.error.HTTPError as exc: @@ -263,7 +297,7 @@ def _http_request_stream( last_exc: Optional[BaseException] = None for attempt in range(1, 4): try: - with urllib.request.urlopen(request, timeout=timeout) as response: # noqa: S310 + with _open_http(request, timeout=timeout) as response: # HTTPResponse is iterable line-by-line; joining restores the # full body (including newlines) for both sse and text modes. chunks = [line.decode("utf-8", errors="replace") for line in response] diff --git a/src/agentops/pipeline/prompt_deploy.py b/src/agentops/pipeline/prompt_deploy.py index 96f0b237..26180da8 100644 --- a/src/agentops/pipeline/prompt_deploy.py +++ b/src/agentops/pipeline/prompt_deploy.py @@ -623,7 +623,9 @@ def _detect_pr_stage() -> Optional[str]: DevOps (``BUILD_REASON == 'PullRequest'``). """ - if os.environ.get("GITHUB_EVENT_NAME") == "pull_request": + if os.environ.get("AGENTOPS_PR_NUMBER", "").isdigit(): + return os.environ["AGENTOPS_PR_NUMBER"] + if os.environ.get("GITHUB_EVENT_NAME") in {"pull_request", "pull_request_target"}: ref = os.environ.get("GITHUB_REF", "") # ``refs/pull//merge`` or ``refs/pull//head``. if ref.startswith("refs/pull/"): @@ -648,7 +650,8 @@ def _detect_pr_stage() -> Optional[str]: def _git_sha() -> str: return ( - os.environ.get("GITHUB_SHA") + os.environ.get("AGENTOPS_PR_SHA") + or os.environ.get("GITHUB_SHA") or os.environ.get("BUILD_SOURCEVERSION") or os.environ.get("Build.SourceVersion") or "" @@ -687,7 +690,10 @@ def _write_candidate_eval_config( dataset_path = dataset_reference.local_path or Path(str(config.dataset)) if not dataset_path.is_absolute(): dataset_path = (source_config_path.parent / dataset_path).resolve() - data["dataset"] = str(dataset_path) + try: + data["dataset"] = os.path.relpath(dataset_path, destination.parent.resolve()) + except ValueError: # Paths on different Windows drives cannot be relative. + data["dataset"] = str(dataset_path) destination.parent.mkdir(parents=True, exist_ok=True) save_yaml(destination, data) diff --git a/src/agentops/services/cicd.py b/src/agentops/services/cicd.py index babade14..5999a104 100644 --- a/src/agentops/services/cicd.py +++ b/src/agentops/services/cicd.py @@ -2,6 +2,7 @@ from __future__ import annotations +import re from dataclasses import dataclass, field from importlib.resources import files from pathlib import Path @@ -26,7 +27,6 @@ recommended_eval_runner, ) - _TEMPLATE_PACKAGE = "agentops.templates" _CLOUD_EVAL_CONFIG_NAME = ".agentops.cloud.yaml" _CI_EVAL_OUTPUT = ".agentops/results/latest" @@ -110,7 +110,7 @@ def _agentops_install_spec(version: str | None = None) -> str: "azure-devops": { "pr": ( "pipelines/azuredevops/agentops-pr.yml", - ".azuredevops/pipelines/agentops-pr.yml", + ".azuredevops/templates/agentops-pr-trusted.yml", ), "dev": ( "pipelines/azuredevops/agentops-deploy-dev.yml", @@ -184,6 +184,8 @@ def _write_template( template_content = template_resource.read_text(encoding="utf-8") for key, value in (substitutions or {}).items(): template_content = template_content.replace(key, value) + if "__PR_INPUT_TASKS__" in (substitutions or {}): + template_content = _restrict_ado_pr_logging(template_content) existed_before = output_path.exists() @@ -283,7 +285,7 @@ def _branch_block_ado(*branches: str) -> str: "azure-devops": { "pr": ( "pipelines/azuredevops/agentops-pr-prompt-agent.yml", - ".azuredevops/pipelines/agentops-pr.yml", + ".azuredevops/templates/agentops-pr-trusted.yml", ), "dev": ( "pipelines/azuredevops/agentops-deploy-prompt-agent.yml", @@ -369,12 +371,134 @@ def _eval_substitutions( return _github_eval_substitutions(eval_runner, config_path, kind=kind) +def _pr_substitutions(platform: str, *, prompt_agent: bool) -> Mapping[str, str]: + prompt = str(prompt_agent).lower() + if platform == "azure-devops": + preparation = f""" - bash: python -I -m agentops.services.pr_gate + displayName: Prepare trusted data-only PR inputs + env: + AGENTOPS_PR_SHA: $(Build.SourceVersion) + AGENTOPS_PR_SOURCE_REF: $(Build.SourceBranch) + AGENTOPS_PR_GIT_URL: $(AGENTOPS_ADO_REPOSITORY_URL) + AGENTOPS_PR_GIT_TOKEN: $(System.AccessToken) + AGENTOPS_PR_PROMPT_CANDIDATE: "{prompt}" + AGENTOPS_AGENT_PROMPT_FILE: $(AGENTOPS_AGENT_PROMPT_FILE) + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: $(AZURE_AI_FOUNDRY_PROJECT_ENDPOINT) + AZURE_OPENAI_ENDPOINT: $(AZURE_OPENAI_ENDPOINT)""" + status = """ - task: AzureCLI@2 + displayName: Publish required AgentOps PR status + condition: always() + inputs: + azureSubscription: agentops-pr-azure + scriptType: bash + scriptLocation: inlineScript + inlineScript: python -I -m agentops.services.pr_gate status + env: + AGENTOPS_ADO_COLLECTION_URI: $(AGENTOPS_ADO_COLLECTION_URI) + AGENTOPS_ADO_PROJECT_ID: $(AGENTOPS_ADO_PROJECT_ID) + AGENTOPS_ADO_REPOSITORY_ID: $(AGENTOPS_ADO_REPOSITORY_ID)""" + return { + "__PR_INPUT_TASKS__": preparation, + "__PR_STATUS_TASK__": status, + "__PR_STAGE_FAILURE_STATUS__": status.replace( + "condition: always()", "condition: failed()" + ), + "__PR_VARIABLE_GUARD__": """ - ${{ each variable in variables }}: + - ${{ if not(or(startsWith(lower(variable.key), 'build.'), startsWith(lower(variable.key), 'system.'), startsWith(lower(variable.key), 'agent.'), startsWith(lower(variable.key), 'pipeline.'), startsWith(lower(variable.key), 'resources.'), startsWith(lower(variable.key), 'release.'))) }}: + '${{ variable.key }}': error +""", + } + preparation = f""" - name: Prepare trusted data-only PR inputs + id: pr_input + env: + AGENTOPS_PR_SHA: ${{{{ github.event.pull_request.head.sha }}}} + AGENTOPS_PR_NUMBER: ${{{{ github.event.pull_request.number }}}} + AGENTOPS_PR_GIT_TOKEN: ${{{{ github.token }}}} + AGENTOPS_PR_PROMPT_CANDIDATE: "{prompt}" + AGENTOPS_AGENT_PROMPT_FILE: ${{{{ vars.AGENTOPS_AGENT_PROMPT_FILE || '' }}}} + AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{{{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }}}} + AZURE_OPENAI_ENDPOINT: ${{{{ vars.AZURE_OPENAI_ENDPOINT }}}} + run: python -I -m agentops.services.pr_gate""" + status = """ - name: Create exclusive PR status token + id: status_token + if: always() && github.event_name == 'pull_request_target' && vars.AGENTOPS_STATUS_APP_CLIENT_ID != '' + uses: actions/create-github-app-token@64ab729a6294000f5aa4a4bc0b73d1f32bc45510 # v3 + with: + client-id: ${{ vars.AGENTOPS_STATUS_APP_CLIENT_ID }} + private-key: ${{ secrets.AGENTOPS_STATUS_APP_PRIVATE_KEY }} + permission-statuses: write + + - name: Publish required AgentOps PR status + if: always() && github.event_name == 'pull_request_target' + uses: actions/github-script@v8 + env: + AGENTOPS_JOB_STATUS: ${{ job.status }} + with: + # Without an exclusive App this is informational. Do not require + # a shared Actions status as proof of the trusted evaluator. + github-token: ${{ steps.status_token.outputs.token || github.token }} + script: | + await github.rest.repos.createCommitStatus({ + ...context.repo, + sha: context.payload.pull_request.head.sha, + context: 'agentops/pr-eval', + state: process.env.AGENTOPS_JOB_STATUS === 'success' ? 'success' : 'failure', + description: 'AgentOps trusted PR evaluation', + target_url: `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}` + }); +""" + return { + "__PR_INPUT_STEPS__": preparation, + "__PR_STATUS_STEP__": status, + "__PR_STAGE_FAILURE_STATUS__": status.replace( + "always() &&", "failure() &&" + ), + } + + +def _github_config_env(config_path: str) -> str: + if config_path == "$AGENTOPS_PR_CONFIG": + return "${{ env.AGENTOPS_PR_CONFIG }}" + return config_path + + +def _restrict_ado_pr_logging(content: str) -> str: + """Prevent candidate responses from changing subsequent task environments.""" + def replace(match: re.Match[str]) -> str: + indent, step = match.group(1), match.group(2) + end = re.search(rf"(?m)^{re.escape(indent)}- ", content[match.end():]) + body = content[match.start():match.end() + end.start()] if end else content[match.start():] + variables: tuple[str, ...] + if "Prepare trusted data-only PR inputs" in body: + variables = ("AGENTOPS_PR_PREPARED", "AGENTOPS_PR_SHA", "AGENTOPS_PR_NUMBER") + elif re.search(r"(?m)^[ \t]*displayName: Prepare AgentOps cloud eval config$", body): + variables = ("AGENTOPS_CI_CONFIG",) + elif "task.setvariable variable=AGENTOPS_EVAL_EXIT_CODE" in body: + variables = ("AGENTOPS_EVAL_EXIT_CODE",) + elif "officialPreflight" in body: + variables = ("official*",) + else: + variables = () + return ( + f"{indent}- target:\n{indent} commands: restricted\n" + f"{indent} settableVariables:{'' if variables else ' []'}\n" + + "".join(f"{indent} - {name}\n" for name in variables) + + f"{indent} {step}" + ) + + content = content.replace( + "azureSubscription: $(AZURE_SERVICE_CONNECTION)", + "azureSubscription: agentops-pr-azure", + ) + return re.sub(r"(?m)^([ \t]*)- (bash:|task: AzureCLI@2)", replace, content) + + def _github_governance_steps(config_path: str) -> str: return f""" - name: Detect AgentOps governance gates id: governance env: - AGENTOPS_CONFIG: "{config_path}" + AGENTOPS_CONFIG: "{_github_config_env(config_path)}" run: | python - <<'PY' import os @@ -487,7 +611,7 @@ def _github_eval_substitutions( return { "__EVAL_STEPS__": f""" - name: Prepare AgentOps cloud eval config env: - AGENTOPS_SOURCE_CONFIG: "{config_path}" + AGENTOPS_SOURCE_CONFIG: "{_github_config_env(config_path)}" run: | python - <<'PY' import os @@ -919,7 +1043,7 @@ def generate_cicd_workflows( "__AGENTOPS_INSTALL_SPEC__": _agentops_install_spec(), } eval_config = ( - "${{ inputs.config || 'agentops.yaml' }}" + "$AGENTOPS_PR_CONFIG" if platform == "github" and kind == "pr" else "$(AGENTOPS_CONFIG)" if platform == "azure-devops" @@ -937,6 +1061,19 @@ def generate_cicd_workflows( substitutions.update(prompt_values) else: template_path, output_rel = template_map[kind] + if kind == "pr": + substitutions.update( + _pr_substitutions( + platform, prompt_agent=effective_deploy_mode == "prompt-agent" + ) + ) + if platform == "azure-devops": + _write_template( + templates_root, + "pipelines/azuredevops/agentops-pr-entry.yml", + directory / ".azuredevops/pipelines/agentops-pr.yml", + force, result, + ) ado_indent = ( 10 if kind == "pr" diff --git a/src/agentops/services/pr_gate.py b/src/agentops/services/pr_gate.py new file mode 100644 index 00000000..4d4398bb --- /dev/null +++ b/src/agentops/services/pr_gate.py @@ -0,0 +1,308 @@ +"""Internal, data-only PR input preparation for trusted generated pipelines. + +Run the installed package with ``python -I -m agentops.services.pr_gate``. +The workflow definition and initial checkout must be trusted. A candidate is +read as Git blobs, never checked out, installed, imported, or executed. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import urllib.parse +import urllib.request +from pathlib import Path, PurePosixPath +from typing import Any + +from ruamel.yaml import YAML +from ruamel.yaml.error import YAMLError + +from agentops.core.agentops_config import AgentOpsConfig, is_unexpanded_ci_token +from agentops.core.dataset_source import parse_dataset_reference +from agentops.utils.yaml import load_yaml, save_yaml + +_SHA = re.compile(r"[0-9a-fA-F]{40}(?:[0-9a-fA-F]{24})?") +_MERGE_REF = re.compile(r"refs/pull/([1-9][0-9]*)/merge") +_INPUT_DIR = Path(".agentops/pr-input") +_MAX_CONFIG = 1024 * 1024 +_MAX_DATASET = 100 * 1024 * 1024 + + +def _git(root: Path, *args: str, token: str = "") -> bytes: + env = os.environ.copy() + if token: + # Do not put the read token in command arguments or persist it in Git. + remote = env.get("AGENTOPS_PR_GIT_URL") or _git( + root, "remote", "get-url", "origin" + ).decode().strip() + _validate_https(remote) + env.update( + GIT_CONFIG_COUNT="2", + GIT_CONFIG_KEY_0=f"http.{remote}.extraheader", + GIT_CONFIG_VALUE_0=f"AUTHORIZATION: bearer {token}", + GIT_CONFIG_KEY_1="http.followRedirects", + GIT_CONFIG_VALUE_1="false", + ) + result = subprocess.run( + ["git", *args], cwd=root, env=env, capture_output=True, check=False, + timeout=120, + ) + if result.returncode: + # Git/server errors may contain credential-bearing remote URLs. + raise ValueError(f"PR input Git operation failed: {args[0]}") + return result.stdout + + +def _repo_path(value: str) -> str: + path = PurePosixPath(value) + if ( + not value or path.is_absolute() or "\\" in value or ":" in value + or any(ord(char) < 32 for char in value) + or any(part in {"", ".", ".."} for part in value.split("/")) + ): + raise ValueError("PR input paths must be relative repository file paths") + return path.as_posix() + + +def _blob(root: Path, sha: str, path: str, limit: int) -> bytes: + path = _repo_path(path) + listing = _git(root, "ls-tree", "-z", sha, "--", path) + entries = [entry for entry in listing.split(b"\0") if entry] + if len(entries) != 1: + raise ValueError(f"PR input must identify one regular Git file: {path}") + metadata, actual_path = entries[0].split(b"\t", 1) + mode, kind, oid = metadata.decode("ascii").split() + if mode not in {"100644", "100755"} or kind != "blob" or actual_path.decode() != path: + raise ValueError(f"PR input is not a regular Git file: {path}") + object_id = oid.strip() + if int(_git(root, "cat-file", "-s", object_id)) > limit: + raise ValueError(f"PR input exceeds its size limit: {path}") + return _git(root, "cat-file", "blob", object_id) + + +def _workspace_file(root: Path, value: str) -> Path: + path = root / _repo_path(value) + if not path.resolve().is_relative_to(root.resolve()) or path.is_symlink(): + raise ValueError("PR input path escapes the trusted workspace") + return path + + +def _validate_https(value: str) -> None: + parts = urllib.parse.urlsplit(value) + if ( + parts.scheme != "https" or not parts.hostname + or parts.username is not None or parts.password is not None + ): + raise ValueError("Credential-bearing PR evaluation destinations require HTTPS") + + +def prepare_inputs( + root: Path, *, candidate_sha: str, config_path: str = "agentops.yaml", +) -> None: + """Overlay only validated dataset/prompt data onto the trusted workspace.""" + if not _SHA.fullmatch(candidate_sha): + raise ValueError("PR input requires an immutable candidate commit SHA") + root = root.resolve() + config_path = _repo_path(config_path) + source = _workspace_file(root, config_path) + trusted = load_yaml(source) + proposed = YAML(typ="safe").load( + _blob(root, candidate_sha, config_path, _MAX_CONFIG).decode("utf-8") + ) + if not isinstance(proposed, dict) or not all( + isinstance(key, str) and re.fullmatch(r"[a-z][a-z0-9_]*", key) + for key in proposed + ): + raise ValueError("PR agentops.yaml must be a mapping") + changed = sorted( + key for key in trusted.keys() | proposed.keys() + if trusted.get(key) != proposed.get(key) + ) + if changed: + raise ValueError( + "PR evaluation uses trusted-base configuration. Merge configuration " + "changes through review before using them in this gate. Changed fields: " + + ", ".join(str(key) for key in changed) + ) + config = AgentOpsConfig.model_validate(trusted) + for endpoint in ( + config.project_endpoint, + os.environ.get("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT"), + os.environ.get("AZURE_OPENAI_ENDPOINT"), + ): + if endpoint: + _validate_https(endpoint) + if ( + config.agent and config.agent.lower().startswith(("http://", "https://")) + and (config.auth_header_env or config.headers or config.resolved_target().kind == "foundry_hosted") + ): + _validate_https(config.agent) + assert_config = trusted.get("assert") + if isinstance(assert_config, dict): + extra_env = assert_config.get("env") or {} + if isinstance(extra_env, dict) and extra_env.get("AZURE_API_BASE"): + _validate_https(str(extra_env["AZURE_API_BASE"])) + + reference = parse_dataset_reference(config.dataset) + pending: list[tuple[Path, bytes]] = [] + if not reference.source_uri: + relative = PurePosixPath(config_path).parent / str(config.dataset).replace("\\", "/") + dataset = _workspace_file(root, relative.as_posix()) + if dataset.suffix.lower() != ".jsonl": + raise ValueError("PR candidate datasets must be JSONL data files") + payload = _blob(root, candidate_sha, relative.as_posix(), _MAX_DATASET) + for line in payload.decode("utf-8").splitlines(): + if line.strip() and not isinstance(json.loads(line), dict): + raise ValueError("Each PR dataset row must be a JSON object") + pending.append((dataset, payload)) + + prompt = os.environ.get("AGENTOPS_AGENT_PROMPT_FILE") or trusted.get("prompt_file") + if prompt and is_unexpanded_ci_token(str(prompt)): + prompt = trusted.get("prompt_file") + if not prompt and os.environ.get("AGENTOPS_PR_PROMPT_CANDIDATE") == "true": + prompt = ".agentops/prompts/agent-instructions.md" + if prompt: + relative = PurePosixPath(config_path).parent / str(prompt).replace("\\", "/") + payload = _blob(root, candidate_sha, relative.as_posix(), _MAX_CONFIG) + if not payload.decode("utf-8").strip(): + raise ValueError("PR prompt input must not be empty") + destination = _workspace_file(root, (_INPUT_DIR / "prompt.txt").as_posix()) + pending.append((destination, payload)) + trusted["prompt_file"] = os.path.relpath(destination, source.parent) + + # Validate everything before writing. Never overlay scripts or config from + # the candidate, including .env, recipes, ASSERT config, or bootstrap tools. + for path, payload in pending: + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(payload) + save_yaml(source, trusted) + + +def prepare_from_environment(root: Path) -> None: + """Fetch one immutable candidate using trusted CI event metadata.""" + env = os.environ + sha = env.get("AGENTOPS_PR_SHA", "") + if not sha: + if env.get("GITHUB_EVENT_NAME") == "workflow_dispatch": + # A dispatch evaluates its selected trusted revision, not a PR. + prepare_inputs( + root, candidate_sha=_git(root, "rev-parse", "HEAD").decode().strip(), + config_path=env.get("AGENTOPS_PR_CONFIG", "agentops.yaml"), + ) + return + raise ValueError("Missing immutable PR candidate metadata") + if not _SHA.fullmatch(sha): + raise ValueError("Invalid immutable PR candidate commit SHA") + source_ref = env.get("AGENTOPS_PR_SOURCE_REF", "") + match = None + if source_ref: + match = _MERGE_REF.fullmatch(source_ref) + if match is None: + raise ValueError("The ADO input pipeline must validate a PR merge ref") + token = env.get("AGENTOPS_PR_GIT_TOKEN", "") + remote = "origin" + if source_ref: + remote = env["AGENTOPS_PR_GIT_URL"] + _validate_https(remote) + _git(root, "fetch", "--no-tags", "--no-recurse-submodules", "--depth=2", + remote, source_ref or sha, token=token) + fetched = _git(root, "rev-parse", "FETCH_HEAD").decode().strip() + if fetched.lower() != sha.lower(): + raise ValueError("PR revision changed during preparation, queue a fresh evaluation") + source_sha = sha + if match is not None: + parents = _git(root, "rev-list", "--parents", "-n", "1", sha).decode().split() + if len(parents) != 3: + raise ValueError("ADO PR input must be the server-generated merge commit") + # Keep executing the explicitly protected checkout, not a target branch + # selected through PR-authored pipeline metadata. + record = {"source_sha": parents[2], "pr": int(match.group(1))} + source_sha = parents[2] + target = _workspace_file(root, (_INPUT_DIR / "revision.json").as_posix()) + target.parent.mkdir(parents=True, exist_ok=True) + target.write_text(json.dumps(record), encoding="utf-8") + prepare_inputs( + root, candidate_sha=sha, + config_path=env.get("AGENTOPS_PR_CONFIG", "agentops.yaml"), + ) + if match is not None: + print("##vso[task.setvariable variable=AGENTOPS_PR_PREPARED;isReadOnly=true]true") + print(f"##vso[task.setvariable variable=AGENTOPS_PR_SHA;isReadOnly=true]{source_sha}") + print(f"##vso[task.setvariable variable=AGENTOPS_PR_NUMBER;isReadOnly=true]{match.group(1)}") + elif env.get("GITHUB_ENV"): + with Path(env["GITHUB_ENV"]).open("a", encoding="utf-8") as handle: + handle.write(f"AGENTOPS_PR_SHA={sha}\n") + handle.write(f"AGENTOPS_PR_NUMBER={env.get('AGENTOPS_PR_NUMBER', '')}\n") + + +def publish_ado_status(root: Path) -> None: + """Publish a revision-bound required status from the protected evaluator.""" + env = os.environ + record_path = root / _INPUT_DIR / "revision.json" + if not record_path.exists(): + raise ValueError("No verified PR revision is available for status publication") + record = json.loads(record_path.read_text(encoding="utf-8")) + # These values are explicitly mapped from the protected PR variable group. + collection = env["AGENTOPS_ADO_COLLECTION_URI"].rstrip("/") + _validate_https(collection) + project = urllib.parse.quote(env["AGENTOPS_ADO_PROJECT_ID"], safe="") + repository = urllib.parse.quote(env["AGENTOPS_ADO_REPOSITORY_ID"], safe="") + base = f"{collection}/{project}/_apis/git/repositories/{repository}/pullRequests/{record['pr']}" + headers = {"Authorization": f"Bearer {_ado_status_token()}"} + + def request(url: str, payload: dict[str, Any] | None = None) -> Any: + data = json.dumps(payload).encode() if payload is not None else None + req = urllib.request.Request( + url, data=data, headers={**headers, "Content-Type": "application/json"}, + ) + # No redirects to another credential destination. + from agentops.pipeline.invocations import _SameOriginRedirectHandler + + with urllib.request.build_opener(_SameOriginRedirectHandler()).open(req, timeout=30) as response: + return json.loads(response.read()) + + iterations = request(f"{base}/iterations?api-version=7.1")["value"] + latest = max(iterations, key=lambda item: item["id"]) + if latest["sourceRefCommit"]["commitId"] != record["source_sha"]: + raise ValueError("PR changed since evaluation, refusing to approve a newer revision") + succeeded = env.get("AGENT_JOBSTATUS", "").lower() == "succeeded" + request( + f"{base}/statuses?api-version=7.1", + { + "state": "succeeded" if succeeded else "failed", + "description": "AgentOps trusted PR evaluation", + "context": {"name": "pr-eval", "genre": "agentops"}, + "iterationId": latest["id"], + "targetUrl": f"{collection}/{project}/_build/results?buildId={env['BUILD_BUILDID']}", + }, + ) + + +def _ado_status_token() -> str: + # Do not publish trusted evidence under the PR job's Build Service identity. + from azure.identity import AzureCliCredential + + return AzureCliCredential(process_timeout=30).get_token( + "499b84ac-1321-427f-aa17-267ca6975798/.default" + ).token + + +def main() -> None: + try: + if sys.argv[1:] == ["status"]: + publish_ado_status(Path.cwd()) + elif not sys.argv[1:]: + prepare_from_environment(Path.cwd()) + else: + raise ValueError("Unsupported internal PR gate operation") + except (ValueError, KeyError, OSError, YAMLError, subprocess.SubprocessError) as exc: + message = str(exc).replace("##vso[", "# #vso[").replace("\n", "\\n").replace("\r", "\\r") + print(f"AgentOps PR gate error: {message}", file=sys.stderr) + raise SystemExit(1) from exc + + +if __name__ == "__main__": + main() diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-pr-entry.yml b/src/agentops/templates/pipelines/azuredevops/agentops-pr-entry.yml new file mode 100644 index 00000000..ab429025 --- /dev/null +++ b/src/agentops/templates/pipelines/azuredevops/agentops-pr-entry.yml @@ -0,0 +1,23 @@ +# AgentOps - PR entrypoint using an enforced trusted template +# +# Configure the repository below and keep its template ref protected. +# The agentops-pr-azure connection MUST enforce this exact repo/ref/template. +# Keep group updates admin-only. Any group secrets need independent checks +# enforcing the same template, even if malicious YAML removes the connection. +# Require agentops/pr-eval from the connection's exclusive identity, not a +# Build Service status or this pipeline's build result alone. +trigger: none +pr: + branches: + include: + - develop + - release/* + - main +resources: + repositories: + - repository: agentops_trusted + type: git + name: YOUR_PROJECT/YOUR_REPOSITORY + ref: refs/heads/develop +extends: + template: /.azuredevops/templates/agentops-pr-trusted.yml@agentops_trusted diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-pr-prompt-agent.yml b/src/agentops/templates/pipelines/azuredevops/agentops-pr-prompt-agent.yml index a91f5c88..51c71d3a 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-pr-prompt-agent.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-pr-prompt-agent.yml @@ -19,69 +19,67 @@ # - Merge is what promotes the candidate via the deploy pipeline. # This PR pipeline does not record the candidate as deployed. # -# Setup (one-time): -# 1. Create a variable group named `agentops` in Pipelines -> Library -# with: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT, AZURE_OPENAI_ENDPOINT, -# AZURE_OPENAI_DEPLOYMENT, AGENTOPS_AGENT_PROMPT_FILE (optional). -# 2. Create a service connection to your Azure subscription named -# `agentops-azure`. -# 3. Grant the build service "Contribute to pull requests" permission -# on the repository so the PR-comment step can post. +# Set each literal checkout to the same protected project/repository/ref as +# the extends wrapper. Configure PR-only resources and exact Required template +# checks, plus the exclusive status policy in docs/ci-github-actions.md. # # Generated by `agentops workflow generate --platform azure-devops --deploy-mode prompt-agent`. -trigger: none - -pr: - branches: - include: - - develop - - release/* - - main - -pool: - vmImage: ubuntu-latest - -variables: - - group: agentops - - name: AZURE_SERVICE_CONNECTION - value: agentops-azure - - name: AGENTOPS_CONFIG - value: agentops.yaml - - name: TARGET_ENVIRONMENT - value: __ENV_NAME__ - +# Enforced, protected extends template, not a PR-authored inline pipeline. stages: +__PR_VARIABLE_GUARD__ - stage: stage_candidate displayName: Stage Foundry prompt candidate (PR) + pool: + vmImage: ubuntu-latest + variables: + - group: agentops-pr + - name: AGENTOPS_CONFIG + value: agentops.yaml + - name: TARGET_ENVIRONMENT + value: __ENV_NAME__ + - name: NODE_OPTIONS + value: "" + - name: BASH_ENV + value: /dev/null + - name: PYTHONPATH + value: "" + - name: PYTHONHOME + value: "" jobs: - job: stage_candidate steps: - - checkout: self + - checkout: git://YOUR_PROJECT/YOUR_REPOSITORY@refs/heads/develop fetchDepth: 1 + persistCredentials: false - task: UsePythonVersion@0 displayName: Set up Python 3.11 inputs: versionSpec: "3.11" + - bash: | + python -I -m pip --isolated install --upgrade pip + python -I -m pip --isolated install "agentops-accelerator__AGENTOPS_INSTALL_SPEC__" + displayName: Install AgentOps Toolkit + +__PR_INPUT_TASKS__ + - task: AzureCLI@2 displayName: Create candidate agent version from prompt_file inputs: - azureSubscription: $(AZURE_SERVICE_CONNECTION) + azureSubscription: agentops-pr-azure scriptType: bash scriptLocation: inlineScript inlineScript: | - python -m pip install --upgrade pip - python -m pip install "agentops-accelerator__AGENTOPS_INSTALL_SPEC__" - python -m agentops.pipeline.prompt_deploy stage \ + python -I -m agentops.pipeline.prompt_deploy stage \ --config "$(AGENTOPS_CONFIG)" \ --environment "$(TARGET_ENVIRONMENT)" \ --out ".agentops/deployments/foundry-agent.json" \ --eval-config ".agentops/deployments/agentops.candidate.yaml" env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: $(AZURE_AI_FOUNDRY_PROJECT_ENDPOINT) - AGENTOPS_AGENT_PROMPT_FILE: $(AGENTOPS_AGENT_PROMPT_FILE) + AGENTOPS_AGENT_PROMPT_FILE: .agentops/pr-input/prompt.txt - task: PublishPipelineArtifact@1 displayName: Upload candidate deployment record @@ -90,16 +88,35 @@ stages: artifact: foundry-agent-__ENV_KEY__-candidate publishLocation: pipeline +__PR_STAGE_FAILURE_STATUS__ + - stage: eval displayName: AgentOps eval (PR candidate gate) dependsOn: stage_candidate + pool: + vmImage: ubuntu-latest + variables: + - group: agentops-pr + - name: AGENTOPS_CONFIG + value: agentops.yaml + - name: TARGET_ENVIRONMENT + value: __ENV_NAME__ + - name: NODE_OPTIONS + value: "" + - name: BASH_ENV + value: /dev/null + - name: PYTHONPATH + value: "" + - name: PYTHONHOME + value: "" jobs: - job: run_eval displayName: Eval candidate for $(TARGET_ENVIRONMENT) timeoutInMinutes: 45 steps: - - checkout: self + - checkout: git://YOUR_PROJECT/YOUR_REPOSITORY@refs/heads/develop fetchDepth: 1 + persistCredentials: false - task: DownloadPipelineArtifact@2 displayName: Download candidate deployment record @@ -113,17 +130,19 @@ stages: versionSpec: "3.11" - bash: | - python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[cockpit]__AGENTOPS_INSTALL_SPEC__" + python -I -m pip --isolated install --upgrade pip + python -I -m pip --isolated install "agentops-accelerator[cockpit]__AGENTOPS_INSTALL_SPEC__" displayName: Install AgentOps Toolkit +__PR_INPUT_TASKS__ + __EVAL_TASKS__ - task: AzureCLI@2 displayName: Run AgentOps doctor - condition: always() + condition: and(always(), eq(variables.AGENTOPS_PR_PREPARED, 'true')) inputs: - azureSubscription: $(AZURE_SERVICE_CONNECTION) + azureSubscription: agentops-pr-azure scriptType: bash scriptLocation: inlineScript inlineScript: | @@ -209,3 +228,5 @@ __EVAL_TASKS__ condition: always() env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +__PR_STATUS_TASK__ diff --git a/src/agentops/templates/pipelines/azuredevops/agentops-pr.yml b/src/agentops/templates/pipelines/azuredevops/agentops-pr.yml index 9dd5cd2d..1c875003 100644 --- a/src/agentops/templates/pipelines/azuredevops/agentops-pr.yml +++ b/src/agentops/templates/pipelines/azuredevops/agentops-pr.yml @@ -1,50 +1,41 @@ # AgentOps — PR evaluation gate (Azure DevOps Pipelines) # -# Runs on pull requests targeting develop, release/**, or main and gates -# the merge on the AgentOps eval result. Posts the rendered Markdown -# report as an idempotent PR comment via the Azure DevOps REST API. -# -# Setup (one-time): -# 1. Create a variable group named `agentops` in Pipelines → Library with: -# AZURE_AI_FOUNDRY_PROJECT_ENDPOINT -# AZURE_OPENAI_ENDPOINT -# AZURE_OPENAI_DEPLOYMENT -# 2. Create a service connection to your Azure subscription named -# `agentops-azure` (Pipelines → Service connections → New). -# 3. Grant the build service "Contribute to pull requests" permission -# on the repository so the PR-comment step can post. +# Keep this template on a protected ref. Set the literal checkout below to +# that same protected project/repository/ref, not a caller-controlled alias. +# Configure agentops-pr resources and exact Required template checks, plus +# the exclusive status identity policy described in docs/ci-github-actions.md. # # Generated by `agentops workflow generate --platform azure-devops`. -trigger: none - -pr: - branches: - include: - - develop - - release/* - - main - -pool: - vmImage: ubuntu-latest - -variables: - - group: agentops - - name: AZURE_SERVICE_CONNECTION - value: agentops-azure - - name: AGENTOPS_CONFIG - value: agentops.yaml - +# Closed extends template. No steps/jobs/config/auth parameters are exposed. +# Require its exact protected ref on the PR-only connection. Independently +# enforce that trust on any secret group, and restrict group updates to admins. stages: +__PR_VARIABLE_GUARD__ - stage: eval displayName: AgentOps eval (PR gate) + pool: + vmImage: ubuntu-latest + variables: + - group: agentops-pr + - name: AGENTOPS_CONFIG + value: agentops.yaml + - name: NODE_OPTIONS + value: "" + - name: BASH_ENV + value: /dev/null + - name: PYTHONPATH + value: "" + - name: PYTHONHOME + value: "" jobs: - job: run_eval displayName: Run AgentOps eval timeoutInMinutes: 30 steps: - - checkout: self + - checkout: git://YOUR_PROJECT/YOUR_REPOSITORY@refs/heads/develop fetchDepth: 1 + persistCredentials: false - task: UsePythonVersion@0 displayName: Set up Python 3.11 @@ -52,17 +43,19 @@ stages: versionSpec: "3.11" - bash: | - python -m pip install --upgrade pip - python -m pip install "agentops-accelerator[cockpit]__AGENTOPS_INSTALL_SPEC__" + python -I -m pip --isolated install --upgrade pip + python -I -m pip --isolated install "agentops-accelerator[cockpit]__AGENTOPS_INSTALL_SPEC__" displayName: Install AgentOps Toolkit +__PR_INPUT_TASKS__ + __EVAL_TASKS__ - task: AzureCLI@2 displayName: Run AgentOps doctor - condition: always() + condition: and(always(), eq(variables.AGENTOPS_PR_PREPARED, 'true')) inputs: - azureSubscription: $(AZURE_SERVICE_CONNECTION) + azureSubscription: agentops-pr-azure scriptType: bash scriptLocation: inlineScript inlineScript: | @@ -152,3 +145,5 @@ __EVAL_TASKS__ condition: always() env: SYSTEM_ACCESSTOKEN: $(System.AccessToken) + +__PR_STATUS_TASK__ diff --git a/src/agentops/templates/workflows/agentops-pr-prompt-agent.yml b/src/agentops/templates/workflows/agentops-pr-prompt-agent.yml index 99809873..168ee7db 100644 --- a/src/agentops/templates/workflows/agentops-pr-prompt-agent.yml +++ b/src/agentops/templates/workflows/agentops-pr-prompt-agent.yml @@ -24,7 +24,7 @@ name: AgentOps PR on: - pull_request: + pull_request_target: branches: - develop - "release/**" @@ -39,12 +39,16 @@ on: permissions: contents: read pull-requests: write + statuses: write id-token: write concurrency: - group: agentops-pr-${{ github.workflow }}-${{ github.ref }} + group: agentops-pr-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + AGENTOPS_PR_CONFIG: ${{ inputs.config || 'agentops.yaml' }} + jobs: stage-candidate: name: Stage Foundry prompt candidate (PR) @@ -54,15 +58,11 @@ jobs: environment: __ENV_NAME__ timeout-minutes: 20 steps: - - name: Checkout + - name: Checkout trusted base uses: actions/checkout@v6 - - - name: Azure login (OIDC) - uses: azure/login@v3 with: - client-id: ${{ vars.AZURE_CLIENT_ID }} - tenant-id: ${{ vars.AZURE_TENANT_ID }} - subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + ref: ${{ github.event.pull_request.base.sha || github.sha }} + persist-credentials: false - name: Set up Python uses: actions/setup-python@v6 @@ -78,13 +78,22 @@ jobs: run: | uv pip install --system "agentops-accelerator[cockpit]__AGENTOPS_INSTALL_SPEC__" +__PR_INPUT_STEPS__ + + - name: Azure login (OIDC) + uses: azure/login@v3 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + - name: Create candidate agent version from prompt_file env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} - AGENTOPS_AGENT_PROMPT_FILE: ${{ vars.AGENTOPS_AGENT_PROMPT_FILE || '' }} + AGENTOPS_AGENT_PROMPT_FILE: .agentops/pr-input/prompt.txt run: | - python -m agentops.pipeline.prompt_deploy stage \ - --config "${{ inputs.config || 'agentops.yaml' }}" \ + python -I -m agentops.pipeline.prompt_deploy stage \ + --config "$AGENTOPS_PR_CONFIG" \ --environment "__ENV_NAME__" \ --out ".agentops/deployments/foundry-agent.json" \ --eval-config ".agentops/deployments/agentops.candidate.yaml" @@ -98,6 +107,8 @@ jobs: .agentops/deployments/agentops.candidate.yaml if-no-files-found: error +__PR_STAGE_FAILURE_STATUS__ + eval: name: __EVAL_JOB_NAME__ needs: stage-candidate @@ -105,8 +116,11 @@ jobs: environment: __ENV_NAME__ timeout-minutes: 30 steps: - - name: Checkout + - name: Checkout trusted base uses: actions/checkout@v6 + with: + ref: ${{ github.event.pull_request.base.sha || github.sha }} + persist-credentials: false - name: Download candidate deployment record uses: actions/download-artifact@v7 @@ -114,13 +128,6 @@ jobs: name: foundry-agent-__ENV_KEY__-candidate path: .agentops/deployments - - name: Azure login (OIDC) - uses: azure/login@v3 - with: - client-id: ${{ vars.AZURE_CLIENT_ID }} - tenant-id: ${{ vars.AZURE_TENANT_ID }} - subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} - - name: Set up Python uses: actions/setup-python@v6 with: @@ -135,10 +142,19 @@ jobs: run: | uv pip install --system "agentops-accelerator[cockpit]__AGENTOPS_INSTALL_SPEC__" +__PR_INPUT_STEPS__ + + - name: Azure login (OIDC) + uses: azure/login@v3 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + __EVAL_STEPS__ - name: Generate release evidence - if: always() + if: always() && steps.pr_input.outcome == 'success' env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} @@ -182,7 +198,7 @@ __EVAL_STEPS__ } >> "$GITHUB_STEP_SUMMARY" - name: Comment AgentOps report on PR - if: always() && github.event_name == 'pull_request' + if: always() && github.event_name == 'pull_request_target' uses: actions/github-script@v8 with: script: | @@ -208,3 +224,5 @@ __EVAL_STEPS__ } else { await github.rest.issues.createComment({ owner, repo, issue_number, body }); } + +__PR_STATUS_STEP__ diff --git a/src/agentops/templates/workflows/agentops-pr.yml b/src/agentops/templates/workflows/agentops-pr.yml index 98902b63..8a0be410 100644 --- a/src/agentops/templates/workflows/agentops-pr.yml +++ b/src/agentops/templates/workflows/agentops-pr.yml @@ -14,7 +14,7 @@ name: AgentOps PR on: - pull_request: + pull_request_target: branches: - develop - "release/**" @@ -29,12 +29,16 @@ on: permissions: contents: read pull-requests: write + statuses: write id-token: write concurrency: - group: agentops-pr-${{ github.workflow }}-${{ github.ref }} + group: agentops-pr-${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +env: + AGENTOPS_PR_CONFIG: ${{ inputs.config || 'agentops.yaml' }} + jobs: eval: name: AgentOps eval (PR gate) @@ -44,15 +48,13 @@ jobs: environment: dev timeout-minutes: 30 steps: - - name: Checkout + # Never checkout or execute the PR. The trusted workflow reads only + # explicitly selected data blobs from the immutable candidate commit. + - name: Checkout trusted base uses: actions/checkout@v6 - - - name: Azure login (OIDC) - uses: azure/login@v3 with: - client-id: ${{ vars.AZURE_CLIENT_ID }} - tenant-id: ${{ vars.AZURE_TENANT_ID }} - subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + ref: ${{ github.event.pull_request.base.sha || github.sha }} + persist-credentials: false - name: Set up Python uses: actions/setup-python@v6 @@ -71,10 +73,19 @@ jobs: run: | uv pip install --system "agentops-accelerator[cockpit]__AGENTOPS_INSTALL_SPEC__" +__PR_INPUT_STEPS__ + + - name: Azure login (OIDC) + uses: azure/login@v3 + with: + client-id: ${{ vars.AZURE_CLIENT_ID }} + tenant-id: ${{ vars.AZURE_TENANT_ID }} + subscription-id: ${{ vars.AZURE_SUBSCRIPTION_ID }} + __EVAL_STEPS__ - name: Generate release evidence - if: always() + if: always() && steps.pr_input.outcome == 'success' env: AZURE_AI_FOUNDRY_PROJECT_ENDPOINT: ${{ vars.AZURE_AI_FOUNDRY_PROJECT_ENDPOINT }} AZURE_OPENAI_ENDPOINT: ${{ vars.AZURE_OPENAI_ENDPOINT }} @@ -118,7 +129,7 @@ __EVAL_STEPS__ } >> "$GITHUB_STEP_SUMMARY" - name: Comment AgentOps report on PR - if: always() && github.event_name == 'pull_request' + if: always() && github.event_name == 'pull_request_target' uses: actions/github-script@v8 with: script: | @@ -144,3 +155,5 @@ __EVAL_STEPS__ } else { await github.rest.issues.createComment({ owner, repo, issue_number, body }); } + +__PR_STATUS_STEP__ diff --git a/tests/unit/test_agentops_config.py b/tests/unit/test_agentops_config.py index bd29f607..61e7b85e 100644 --- a/tests/unit/test_agentops_config.py +++ b/tests/unit/test_agentops_config.py @@ -106,10 +106,9 @@ def test_foundry_hosted_invocations(self) -> None: assert result.kind == "foundry_hosted" assert result.protocol == "invocations" - def test_foundry_hosted_rejects_http_json_protocol(self) -> None: + def test_foundry_hosted_allows_explicit_http_json_protocol(self) -> None: url = "https://my-project.services.ai.azure.com/agents/foo" - with pytest.raises(ValueError, match="responses"): - classify_agent(url, protocol="http-json") + assert classify_agent(url, protocol="http-json").kind == "http_json" def test_http_json_default_protocol(self) -> None: url = "https://my-app.azurecontainerapps.io/chat" @@ -117,10 +116,25 @@ def test_http_json_default_protocol(self) -> None: assert result.kind == "http_json" assert result.protocol == "http-json" - def test_http_json_rejects_responses_protocol(self) -> None: + def test_custom_host_allows_explicit_responses_protocol(self) -> None: url = "https://my-app.azurecontainerapps.io/chat" - with pytest.raises(ValueError, match="http-json"): - classify_agent(url, protocol="responses") + assert classify_agent(url, protocol="responses").kind == "foundry_hosted" + + @pytest.mark.parametrize("url", [ + "https://receiver.invalid/.services.ai.azure.com/agents/bot", + "https://receiver.invalid/?target=https://a.services.ai.azure.com", + "https://a.services.ai.azure.com@receiver.invalid/agents/bot", + "https://a.services.ai.azure.com.receiver.invalid/agents/bot", + ]) + def test_foundry_domains_outside_hostname_do_not_enable_auth(self, url) -> None: + assert classify_agent(url).kind == "http_json" + + @pytest.mark.parametrize("host", [ + "a.services.ai.azure.us", "a.services.ai.azure.cn", + "a.services.ai.azure.com", "A.SERVICES.AI.AZURE.COM.", + ]) + def test_sovereign_and_normalized_foundry_hosts(self, host) -> None: + assert classify_agent(f"https://{host}/agents/bot").kind == "foundry_hosted" def test_unrecognized_value(self) -> None: with pytest.raises(ValueError, match="unrecognized"): diff --git a/tests/unit/test_cicd.py b/tests/unit/test_cicd.py index cf72edde..78e607c8 100644 --- a/tests/unit/test_cicd.py +++ b/tests/unit/test_cicd.py @@ -133,7 +133,7 @@ def test_github_templates_include_optional_governance_gates(tmp_path: Path) -> N ] if rel == _PR_PATH: assert redteam_lines == [ - "agentops redteam run --config \"${{ inputs.config || 'agentops.yaml' }}\"" + 'agentops redteam run --config "$AGENTOPS_PR_CONFIG"' ] else: assert redteam_lines == ['agentops redteam run --config "agentops.yaml"'] @@ -460,7 +460,7 @@ def test_azure_devops_pr_template_autodetects_committed_baseline( directory=tmp_path, kinds=["pr"], platform="azure-devops" ) content = ( - tmp_path / ".azuredevops/pipelines/agentops-pr.yml" + tmp_path / _ADO_PR ).read_text(encoding="utf-8") assert 'BASELINE_ARG=""' in content @@ -923,7 +923,8 @@ def test_azure_devops_azd_eval_stage_configures_azd_auth(tmp_path: Path) -> None assert "curl -fsSL https://aka.ms/install-azd.sh | bash" in section assert 'azd extension install azure.ai.agents --version "' in section assert 'azd config set auth.useAzCliAuth "true"' in section - assert "azureSubscription: $(AZURE_SERVICE_CONNECTION)" in section + connection = "agentops-pr-azure" if kind == "pr" else "$(AZURE_SERVICE_CONNECTION)" + assert f"azureSubscription: {connection}" in section # GitHub Actions expression syntax must never leak into ADO output. assert "${{" not in section @@ -967,6 +968,48 @@ def test_azure_devops_pipelines_stay_valid_yaml_for_every_eval_runner( assert line.split(":", 1)[0].isidentifier(), f"{rel}:{number}: {line}" +@pytest.mark.parametrize("deploy_mode", ["placeholder", "prompt-agent"]) +def test_ado_pr_cloud_config_output_is_allowed_only_for_its_producer( + tmp_path: Path, deploy_mode: str, +) -> None: + _write_cloud_eval_project(tmp_path) + result = generate_cicd_workflows( + directory=tmp_path, platform="azure-devops", kinds=["pr"], + deploy_mode=deploy_mode, + ) + assert result.eval_runner == AGENTOPS_CLOUD_RUNNER + stages = _read_yaml(tmp_path / _ADO_PR)["stages"] + eval_stage = next(stage for stage in stages if stage.get("stage") == "eval") + steps = eval_stage["jobs"][0]["steps"] + producer = next( + step for step in steps + if step.get("displayName") == "Prepare AgentOps cloud eval config" + ) + consumer = next( + step for step in steps + if step.get("displayName") == "Run AgentOps Foundry cloud eval" + ) + assert producer["target"] == { + "commands": "restricted", "settableVariables": ["AGENTOPS_CI_CONFIG"], + } + assert "task.setvariable variable=AGENTOPS_CI_CONFIG" in producer["bash"] + assert "AGENTOPS_EVAL_EXIT_CODE" not in producer["bash"] + assert '--config "$(AGENTOPS_CI_CONFIG)"' in consumer["inputs"]["inlineScript"] + assert consumer["target"] == { + "commands": "restricted", "settableVariables": ["AGENTOPS_EVAL_EXIT_CODE"], + } + for step in steps: + if step is not producer: + assert "AGENTOPS_CI_CONFIG" not in step.get("target", {}).get("settableVariables", []) + if deploy_mode == "prompt-agent": + candidate_stage = next(stage for stage in stages if stage.get("stage") == "stage_candidate") + candidate = next( + step for step in candidate_stage["jobs"][0]["steps"] + if step.get("displayName") == "Create candidate agent version from prompt_file" + ) + assert candidate["target"] == {"commands": "restricted", "settableVariables": []} + + def test_official_eval_prepare_step_forwards_the_agent_override(tmp_path: Path) -> None: """The official runner must honor `--agent` / `AGENTOPS_AGENT` like the others. @@ -1274,7 +1317,8 @@ def test_cli_next_steps_for_pr_only_do_not_request_deploy_envs( _ADO_DIR = ".azuredevops/pipelines" -_ADO_PR = f"{_ADO_DIR}/agentops-pr.yml" +_ADO_PR_INPUT = f"{_ADO_DIR}/agentops-pr.yml" +_ADO_PR = ".azuredevops/templates/agentops-pr-trusted.yml" _ADO_DEV = f"{_ADO_DIR}/agentops-deploy-dev.yml" _ADO_QA = f"{_ADO_DIR}/agentops-deploy-qa.yml" _ADO_PROD = f"{_ADO_DIR}/agentops-deploy-prod.yml" @@ -1300,15 +1344,17 @@ def test_azure_devops_pr_template_uses_ado_idioms(tmp_path: Path) -> None: content = (tmp_path / _ADO_PR).read_text(encoding="utf-8") # ADO-specific idioms. - assert "trigger: none" in content - assert "pr:" in content + entry = (tmp_path / _ADO_PR_INPUT).read_text(encoding="utf-8") + assert "trigger: none" in entry + assert "pr:" in entry + assert "extends:" in entry assert "pool:" in content assert "vmImage: ubuntu-latest" in content assert "AzureCLI@2" in content assert "UsePythonVersion@0" in content # Variable group + service connection wiring. - assert "group: agentops" in content - assert "AZURE_SERVICE_CONNECTION" in content + assert "group: agentops-pr" in content + assert "azureSubscription: agentops-pr-azure" in content # PR comment marker preserved across platforms. assert "" in content # Default --doctor-gate is critical. @@ -1603,8 +1649,9 @@ def test_azure_devops_prompt_agent_pr_template_stages_candidate(tmp_path: Path) assert "prompt_deploy record" not in content assert "--severity-fail critical" in content # Pipeline uses pr: trigger, not push - assert "trigger: none" in content - assert "pr:" in content + entry = (tmp_path / _ADO_PR_INPUT).read_text(encoding="utf-8") + assert "trigger: none" in entry + assert "pr:" in entry def test_azure_devops_prompt_agent_pr_template_is_valid_yaml(tmp_path: Path) -> None: @@ -1642,6 +1689,42 @@ def test_azure_devops_prompt_agent_pr_template_propagates_doctor_gate(tmp_path: # Version-pinning of the AgentOps install spec # --------------------------------------------------------------------------- +def test_ado_pr_gate_requires_protected_template_and_exclusive_status_identity(tmp_path: Path) -> None: + generate_cicd_workflows( + directory=tmp_path, platform="azure-devops", kinds=["pr"], + ) + entry = (tmp_path / _ADO_PR_INPUT).read_text(encoding="utf-8") + content = (tmp_path / _ADO_PR).read_text(encoding="utf-8") + assert "agentops-pr-trusted.yml@agentops_trusted" in entry + assert "ref: refs/heads/develop" in entry + assert "parameters:" not in entry + assert "parameters:" not in content + assert "checkout: git://YOUR_PROJECT/YOUR_REPOSITORY@refs/heads/develop" in content + assert "checkout: self" not in content + assert "checkout: agentops_trusted" not in content + assert "commands: restricted" in content + assert "settableVariables: []" in content + assert "each variable in variables" in content + assert "AGENTOPS_PR_GIT_URL: $(AGENTOPS_ADO_REPOSITORY_URL)" in content + assert "AGENTOPS_ADO_COLLECTION_URI: $(AGENTOPS_ADO_COLLECTION_URI)" in content + status = content.split("displayName: Publish required AgentOps PR status", 1)[1] + assert "azureSubscription: agentops-pr-azure" in status + assert "SYSTEM_ACCESSTOKEN" not in status + + +def test_github_pr_gate_has_optional_exclusive_status_app(tmp_path: Path) -> None: + generate_cicd_workflows(directory=tmp_path, kinds=["pr"]) + content = (tmp_path / _PR_PATH).read_text(encoding="utf-8") + assert "pull_request_target:" in content + assert "github.event.pull_request.base.sha" in content + assert "persist-credentials: false" in content + assert "AGENTOPS_STATUS_APP_CLIENT_ID" in content + assert "AGENTOPS_STATUS_APP_PRIVATE_KEY" in content + assert "permission-statuses: write" in content + assert "steps.status_token.outputs.token || github.token" in content + assert "sha: context.payload.pull_request.head.sha" in content + + def test_agentops_install_spec_pins_clean_release() -> None: from agentops.services.cicd import _agentops_install_spec @@ -1694,7 +1777,7 @@ def test_workflow_install_lines_pin_to_release_version(tmp_path: Path) -> None: tmp_path / _DEV_PATH, tmp_path / _QA_PATH, tmp_path / _PROD_PATH, - tmp_path / ".azuredevops/pipelines/agentops-pr.yml", + tmp_path / _ADO_PR, tmp_path / ".azuredevops/pipelines/agentops-deploy-dev.yml", tmp_path / ".azuredevops/pipelines/agentops-deploy-qa.yml", tmp_path / ".azuredevops/pipelines/agentops-deploy-prod.yml", diff --git a/tests/unit/test_http_streaming.py b/tests/unit/test_http_streaming.py index 4f102772..b945f54c 100644 --- a/tests/unit/test_http_streaming.py +++ b/tests/unit/test_http_streaming.py @@ -70,9 +70,7 @@ def _fake(request, timeout=None): # noqa: ANN001 ) yield response - # urlopen is used as ``with urllib.request.urlopen(...) as r``; a context - # manager factory mirrors the stdlib contract. - monkeypatch.setattr(invocations.urllib.request, "urlopen", _fake) + monkeypatch.setattr(invocations, "_open_http", _fake) def _invoke(config: AgentOpsConfig, row: Dict[str, Any]) -> InvocationResult: diff --git a/tests/unit/test_invocations.py b/tests/unit/test_invocations.py index 0eaec5e2..41ef6afb 100644 --- a/tests/unit/test_invocations.py +++ b/tests/unit/test_invocations.py @@ -3,6 +3,8 @@ from __future__ import annotations from pathlib import Path +from unittest.mock import Mock +from urllib.request import Request import pytest @@ -110,3 +112,46 @@ def test_hosted_responses_target_leaves_custom_endpoints_without_api_version(): assert url == "https://my-agent.azurewebsites.net/chat/responses" assert agent_reference is None + + +@pytest.mark.parametrize("header", ["Authorization", "X-API-KEY"]) +@pytest.mark.parametrize("destination", [ + "https://receiver.invalid/next", + "http://trusted.example/next", + "https://trusted.example:444/next", + "https://trusted.example:0/next", +]) +def test_authenticated_redirects_cannot_change_origin(header, destination): + request = Request("https://trusted.example/start", headers={header: "dummy"}) + with pytest.raises(RuntimeError, match="another origin"): + invocations._SameOriginRedirectHandler().redirect_request( + request, None, 302, "Found", {}, destination, + ) + + +def test_authenticated_same_origin_redirect_is_preserved(): + request = Request("https://trusted.example/start", headers={"Authorization": "Bearer dummy"}) + redirected = invocations._SameOriginRedirectHandler().redirect_request( + request, None, 302, "Found", {}, "https://trusted.example:443/next", + ) + assert redirected.get_header("Authorization") == "Bearer dummy" + + +@pytest.mark.parametrize("streaming", [False, True]) +@pytest.mark.parametrize("header", ["X-API-KEY", "Accept", "Content-Type", "User-Agent"]) +def test_json_and_streaming_use_credential_safe_opener(monkeypatch, streaming, header): + response = Mock() + response.read.return_value = b'{"text":"ok"}' + response.__iter__ = Mock(return_value=iter([b"ok"])) + context = Mock() + context.__enter__ = Mock(return_value=response) + context.__exit__ = Mock(return_value=False) + opener = Mock() + opener.open.return_value = context + build = Mock(return_value=opener) + monkeypatch.setattr(invocations.urllib.request, "build_opener", build) + monkeypatch.setattr(invocations.urllib.request, "urlopen", lambda *a, **k: pytest.fail("unsafe opener")) + invoke = invocations._http_request_stream if streaming else invocations._http_request_json + invoke(method="GET", url="https://trusted.example", headers={header: "dummy"}, timeout=1) + assert isinstance(build.call_args.args[0], invocations._SameOriginRedirectHandler) + opener.open.assert_called_once() diff --git a/tests/unit/test_pr_gate.py b/tests/unit/test_pr_gate.py new file mode 100644 index 00000000..5878636a --- /dev/null +++ b/tests/unit/test_pr_gate.py @@ -0,0 +1,284 @@ +"""Offline trust-boundary tests for generated PR evaluation inputs.""" + +import json +import subprocess +from pathlib import Path + +import pytest + +from agentops.services import pr_gate +from agentops.utils.yaml import load_yaml, save_yaml + + +def _git(root: Path, *args: str) -> str: + return subprocess.run( + ["git", *args], cwd=root, check=True, capture_output=True, text=True, + ).stdout.strip() + + +@pytest.fixture +def repository(tmp_path, monkeypatch): + for key in ("AGENTOPS_AGENT_PROMPT_FILE", "AGENTOPS_PR_PROMPT_CANDIDATE"): + monkeypatch.delenv(key, raising=False) + monkeypatch.setenv("AZURE_AI_FOUNDRY_PROJECT_ENDPOINT", "https://project.internal.example") + monkeypatch.setenv("AZURE_OPENAI_ENDPOINT", "https://judge.internal.example") + _git(tmp_path, "init") + _git(tmp_path, "config", "user.email", "test@example.invalid") + _git(tmp_path, "config", "user.name", "Test") + save_yaml(tmp_path / "agentops.yaml", { + "version": 1, + "agent": "https://agent.internal.example/chat", + "protocol": "http-json", + "dataset": "data.jsonl", + "prompt_file": "instructions.md", + "auth_header_env": "DUMMY_TOKEN", + "project_endpoint": "https://project.internal.example", + }) + (tmp_path / "data.jsonl").write_text('{"input":"base"}\n') + (tmp_path / "instructions.md").write_text("base instructions") + (tmp_path / ".env").write_text("DUMMY=base\n") + _git(tmp_path, "add", ".") + _git(tmp_path, "commit", "-m", "Trusted base") + return tmp_path, _git(tmp_path, "rev-parse", "HEAD") + + +def _candidate(repository, **changes): + root, base = repository + config = load_yaml(root / "agentops.yaml") + config.update(changes) + save_yaml(root / "agentops.yaml", config) + (root / "data.jsonl").write_text('{"input":"candidate","expected":"dummy"}\n') + (root / "instructions.md").write_text("candidate instructions") + (root / ".env").write_text("DUMMY=candidate\n") + (root / "sitecustomize.py").write_text("raise RuntimeError('must never execute')") + _git(root, "add", ".") + _git(root, "commit", "-m", "Untrusted candidate") + sha = _git(root, "rev-parse", "HEAD") + _git(root, "checkout", "--detach", base) + return sha + + +def test_only_data_is_imported_and_config_destinations_stay_trusted(repository): + root, base = repository + sha = _candidate(repository) + pr_gate.prepare_inputs(root, candidate_sha=sha) + config = load_yaml(root / "agentops.yaml") + assert _git(root, "rev-parse", "HEAD") == base + assert config["agent"] == "https://agent.internal.example/chat" + assert config["auth_header_env"] == "DUMMY_TOKEN" + assert config["project_endpoint"] == "https://project.internal.example" + assert json.loads((root / "data.jsonl").read_text())["input"] == "candidate" + assert (root / config["prompt_file"]).read_text() == "candidate instructions" + assert (root / "instructions.md").read_text() == "base instructions" + assert (root / ".env").read_text() == "DUMMY=base\n" + assert not (root / "sitecustomize.py").exists() + + +@pytest.mark.parametrize("changes", [ + {"agent": "https://receiver.invalid/.services.ai.azure.com/agents/bot"}, + {"project_endpoint": "https://receiver.invalid"}, + {"auth_header_env": "OTHER_DUMMY_TOKEN"}, + {"auth_header_name": "X-API-KEY"}, + {"auth_value_template": "{token}"}, + {"headers": {"X-Secret": "dummy"}}, + {"execution": "azd"}, + {"assert": {"config": "evil.yaml", "env": {"AZURE_API_BASE": "https://receiver.invalid"}}}, + {"dataset": "different.jsonl"}, + {"prompt_agent_bootstrap": {"model": "other", "tools": []}}, +]) +def test_candidate_cannot_change_policy(repository, changes): + root, _ = repository + sha = _candidate(repository, **changes) + with pytest.raises(ValueError, match="trusted-base configuration"): + pr_gate.prepare_inputs(root, candidate_sha=sha) + assert json.loads((root / "data.jsonl").read_text())["input"] == "base" + + +@pytest.mark.parametrize("path", ["../outside.jsonl", "/tmp/input", "C:\\input", ".env\nINJECT=1"]) +def test_repository_paths_are_not_escapes(path): + with pytest.raises(ValueError, match="relative repository"): + pr_gate._repo_path(path) + + +def test_git_symlink_is_rejected_without_following_it(repository): + root, _base = repository + oid = _git(root, "hash-object", "-w", "instructions.md") + _git(root, "update-index", "--add", "--cacheinfo", f"120000,{oid},data.jsonl") + _git(root, "commit", "-m", "Candidate Git symlink") + sha = _git(root, "rev-parse", "HEAD") + with pytest.raises(ValueError, match="regular Git file"): + pr_gate.prepare_inputs(root, candidate_sha=sha) + + +def test_size_is_checked_before_blob_contents(repository, monkeypatch): + root, _ = repository + sha = _candidate(repository) + monkeypatch.setattr(pr_gate, "_MAX_DATASET", 1) + with pytest.raises(ValueError, match="size limit"): + pr_gate.prepare_inputs(root, candidate_sha=sha) + + +def test_malformed_dataset_does_not_partially_overlay_inputs(repository): + root, base = repository + (root / "data.jsonl").write_text('not json\n') + _git(root, "commit", "-am", "Malformed candidate data") + sha = _git(root, "rev-parse", "HEAD") + _git(root, "checkout", "--detach", base) + with pytest.raises(ValueError): + pr_gate.prepare_inputs(root, candidate_sha=sha) + assert (root / "data.jsonl").read_text() == '{"input":"base"}\n' + + +def test_trusted_remote_dataset_reference_is_not_a_candidate_download(repository): + root, _ = repository + config = load_yaml(root / "agentops.yaml") + config["dataset"] = "https://data.blob.core.windows.net/evals/data.jsonl" + save_yaml(root / "agentops.yaml", config) + _git(root, "commit", "-am", "Approve remote dataset") + repository = root, _git(root, "rev-parse", "HEAD") + sha = _candidate(repository) + pr_gate.prepare_inputs(root, candidate_sha=sha) + assert load_yaml(root / "agentops.yaml")["dataset"] == config["dataset"] + assert (root / "data.jsonl").read_text() == '{"input":"base"}\n' + + +def test_credential_destinations_require_https_in_pr_gate(repository): + root, _ = repository + config = load_yaml(root / "agentops.yaml") + config["project_endpoint"] = "http://project.internal.example" + save_yaml(root / "agentops.yaml", config) + _git(root, "commit", "-am", "Invalid CI transport policy") + repository = root, _git(root, "rev-parse", "HEAD") + sha = _candidate(repository) + with pytest.raises(ValueError, match="require HTTPS"): + pr_gate.prepare_inputs(root, candidate_sha=sha) + + +def test_ado_refuses_non_pr_ref_before_fetch(tmp_path, monkeypatch): + monkeypatch.setenv("AGENTOPS_PR_SHA", "a" * 40) + monkeypatch.setenv("AGENTOPS_PR_SOURCE_REF", "refs/heads/untrusted") + monkeypatch.setattr(pr_gate, "_git", lambda *a, **k: pytest.fail("must not fetch")) + with pytest.raises(ValueError, match="PR merge ref"): + pr_gate.prepare_from_environment(tmp_path) + + +def test_moving_candidate_is_rejected_before_data_or_auth(tmp_path, monkeypatch): + monkeypatch.setenv("AGENTOPS_PR_SHA", "a" * 40) + monkeypatch.delenv("AGENTOPS_PR_SOURCE_REF", raising=False) + monkeypatch.setattr(pr_gate, "_git", lambda root, command, *args, **kw: ( + b"" if command == "fetch" else ("b" * 40).encode() + )) + monkeypatch.setattr(pr_gate, "prepare_inputs", lambda *a, **k: pytest.fail("must not prepare")) + with pytest.raises(ValueError, match="revision changed"): + pr_gate.prepare_from_environment(tmp_path) + + +def test_ado_pins_git_destination_and_keeps_protected_checkout(tmp_path, monkeypatch, capsys): + merge, target, source = "a" * 40, "b" * 40, "c" * 40 + approved = "https://dev.azure.com/example/project/_git/repository" + for key, value in { + "AGENTOPS_PR_SHA": merge, + "AGENTOPS_PR_SOURCE_REF": "refs/pull/42/merge", + "AGENTOPS_PR_GIT_URL": approved, + "AGENTOPS_PR_GIT_TOKEN": "dummy-read-token", + "BUILD_REPOSITORY_URI": "https://untrusted.invalid/repository", + }.items(): + monkeypatch.setenv(key, value) + calls = [] + + def git(root, command, *args, **kwargs): + calls.append((command, args)) + if command == "rev-parse": + return merge.encode() + if command == "rev-list": + return f"{merge} {target} {source}".encode() + assert command == "fetch" + return b"" + + prepared = [] + monkeypatch.setattr(pr_gate, "_git", git) + monkeypatch.setattr(pr_gate, "prepare_inputs", lambda *args, **kwargs: prepared.append(kwargs)) + pr_gate.prepare_from_environment(tmp_path) + assert approved in calls[0][1] + assert not any(command == "checkout" for command, _ in calls) + assert prepared[0]["candidate_sha"] == merge + assert f"AGENTOPS_PR_SHA;isReadOnly=true]{source}" in capsys.readouterr().out + + +def test_error_diagnostics_cannot_inject_ado_logging_commands(monkeypatch, capsys): + monkeypatch.setattr(pr_gate.sys, "argv", ["pr_gate"]) + + def fail(root): + raise ValueError("invalid YAML\n##vso[task.setvariable variable=AGENTOPS_PR_PREPARED]true") + + monkeypatch.setattr(pr_gate, "prepare_from_environment", fail) + with pytest.raises(SystemExit) as error: + pr_gate.main() + assert error.value.code == 1 + output = capsys.readouterr().err + assert "##vso[" not in output + assert len(output.splitlines()) == 1 + + +def test_trusted_dispatch_stages_prompt_without_pr_fetch(repository, monkeypatch): + root, base = repository + monkeypatch.setenv("GITHUB_EVENT_NAME", "workflow_dispatch") + monkeypatch.delenv("AGENTOPS_PR_SHA", raising=False) + pr_gate.prepare_from_environment(root) + config = load_yaml(root / "agentops.yaml") + assert (root / config["prompt_file"]).read_text() == "base instructions" + assert _git(root, "rev-parse", "HEAD") == base + + +def test_ado_status_is_bound_to_iteration_and_never_approves_newer_head(tmp_path, monkeypatch): + directory = tmp_path / ".agentops/pr-input" + directory.mkdir(parents=True) + (directory / "revision.json").write_text(json.dumps({"source_sha": "a" * 40, "pr": 42})) + for key, value in { + "AGENTOPS_ADO_COLLECTION_URI": "https://dev.azure.com/example/", + "AGENTOPS_ADO_PROJECT_ID": "project-id", + "AGENTOPS_ADO_REPOSITORY_ID": "source-id", + "SYSTEM_COLLECTIONURI": "https://untrusted.invalid/", + "BUILD_REPOSITORY_ID": "untrusted-repository", + "BUILD_BUILDID": "123", + "AGENT_JOBSTATUS": "Succeeded", + }.items(): + monkeypatch.setenv(key, value) + monkeypatch.setattr(pr_gate, "_ado_status_token", lambda: "dummy-exclusive-principal-token") + captured = [] + current_sha = "b" * 40 + + class Response: + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + def read(self): + return json.dumps({"value": [{ + "id": 3, "sourceRefCommit": {"commitId": current_sha}, + }]}).encode() + + class Opener: + def open(self, request, timeout): + captured.append(request) + return Response() + + monkeypatch.setattr(pr_gate.urllib.request, "build_opener", lambda *args: Opener()) + with pytest.raises(ValueError, match="refusing to approve a newer revision"): + pr_gate.publish_ado_status(tmp_path) + assert len(captured) == 1 + assert captured[0].full_url.startswith("https://dev.azure.com/example/project-id/") + assert captured[0].get_header("Authorization") == "Bearer dummy-exclusive-principal-token" + current_sha = "a" * 40 + captured.clear() + pr_gate.publish_ado_status(tmp_path) + status = json.loads(captured[-1].data) + assert status["state"] == "succeeded" + assert status["iterationId"] == 3 + assert status["context"] == {"name": "pr-eval", "genre": "agentops"} + monkeypatch.setenv("AGENT_JOBSTATUS", "Failed") + captured.clear() + pr_gate.publish_ado_status(tmp_path) + assert json.loads(captured[-1].data)["state"] == "failed" diff --git a/tests/unit/test_prompt_deploy.py b/tests/unit/test_prompt_deploy.py index 5edbc9f8..3a13d63f 100644 --- a/tests/unit/test_prompt_deploy.py +++ b/tests/unit/test_prompt_deploy.py @@ -71,7 +71,11 @@ def fake_create(endpoint, name, definition, *, metadata, description): encoding="utf-8" ) assert "agent: support-agent:4" in candidate_config - assert str(dataset) in candidate_config + data = prompt_deploy.load_yaml( + tmp_path / ".agentops/deployments/agentops.candidate.yaml" + ) + assert not Path(data["dataset"]).is_absolute() + assert (tmp_path / ".agentops/deployments" / data["dataset"]).resolve() == dataset def test_candidate_eval_config_preserves_remote_dataset_uri(tmp_path: Path) -> None: