From 55dc81581aa2afb15832d2789f620a5b29d0ab95 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Tue, 29 Sep 2026 13:35:55 -0700 Subject: [PATCH 1/2] uptake scorer_invoke grant from orbit --- engine/src/agent_control_engine/core.py | 7 +- engine/tests/test_core.py | 19 +- .../src/agent_control_evaluators/_base.py | 25 + evaluators/builtin/tests/test_base.py | 36 +- evaluators/contrib/galileo/README.md | 15 +- .../luna/client.py | 172 +++++- .../luna/evaluator.py | 51 +- .../galileo/tests/test_luna_coverage_gaps.py | 110 ++-- .../galileo/tests/test_luna_evaluator.py | 493 ++++++++++++++++-- examples/galileo_luna/README.md | 9 +- examples/galileo_luna/demo_agent.py | 34 +- examples/galileo_luna/setup_controls.py | 2 +- 12 files changed, 843 insertions(+), 130 deletions(-) diff --git a/engine/src/agent_control_engine/core.py b/engine/src/agent_control_engine/core.py index 7dbba3c72..23a6b9599 100644 --- a/engine/src/agent_control_engine/core.py +++ b/engine/src/agent_control_engine/core.py @@ -320,7 +320,12 @@ async def _evaluate_leaf( timeout = DEFAULT_EVALUATOR_TIMEOUT result = await asyncio.wait_for( - evaluator.evaluate_with_context(data, request.step), + evaluator.evaluate_with_request_context( + data, + request.step, + target_type=request.target_type, + target_id=request.target_id, + ), timeout=timeout, ) except TimeoutError: diff --git a/engine/tests/test_core.py b/engine/tests/test_core.py index e16d437f6..45330120c 100644 --- a/engine/tests/test_core.py +++ b/engine/tests/test_core.py @@ -39,7 +39,7 @@ class SimpleConfig(BaseModel): # Shared state for coordination between test evaluators _execution_log: list[str] = [] _blocker_event: asyncio.Event | None = None -_context_calls: list[tuple[Any, Step]] = [] +_context_calls: list[tuple[Any, Step, str | None, str | None]] = [] def reset_test_state() -> None: @@ -178,8 +178,15 @@ class ContextEvaluator(Evaluator[SimpleConfig]): async def evaluate(self, data: Any) -> EvaluatorResult: raise AssertionError("engine should call evaluate_with_context") - async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: - _context_calls.append((data, step)) + async def evaluate_with_request_context( + self, + data: Any, + step: Step, + *, + target_type: str | None = None, + target_id: str | None = None, + ) -> EvaluatorResult: + _context_calls.append((data, step, target_type, target_id)) return EvaluatorResult(matched=False, confidence=1.0, message="context received") @@ -296,11 +303,13 @@ async def test_context_evaluator_receives_selected_data_and_complete_step() -> N agent_name="00000000-0000-0000-0000-000000000001", step=step, stage="pre", + target_type="log_stream", + target_id="run-1", ) ) # Then: selector behavior is unchanged and the complete Step is separate - assert _context_calls == [("answer", step)] + assert _context_calls == [("answer", step, "log_stream", "run-1")] @pytest.mark.asyncio @@ -331,7 +340,7 @@ async def test_cached_context_evaluator_handles_concurrent_steps_without_retaini ) # Then: each selected value remains paired with its own full Step - assert {(data, step.ground_truth) for data, step in _context_calls} == { + assert {(data, step.ground_truth) for data, step, _, _ in _context_calls} == { ("first", "one"), ("second", "two"), } diff --git a/evaluators/builtin/src/agent_control_evaluators/_base.py b/evaluators/builtin/src/agent_control_evaluators/_base.py index d83c0c3ab..a072ef7be 100644 --- a/evaluators/builtin/src/agent_control_evaluators/_base.py +++ b/evaluators/builtin/src/agent_control_evaluators/_base.py @@ -178,6 +178,31 @@ async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: """ return await self.evaluate(data) + async def evaluate_with_request_context( + self, + data: Any, + step: Step, + *, + target_type: str | None = None, + target_id: str | None = None, + ) -> EvaluatorResult: + """Evaluate selected data with step and opaque request target metadata. + + The default implementation delegates to :meth:`evaluate_with_context` + so existing evaluators that override that hook keep working unchanged. + + Args: + data: Data extracted by the configured selector. + step: Complete runtime step for the current request. + target_type: Optional target kind attached to the evaluation request. + target_id: Optional opaque target ID attached to the evaluation request. + + Returns: + EvaluatorResult produced by this evaluator. + """ + del target_type, target_id + return await self.evaluate_with_context(data, step) + def get_timeout_seconds(self) -> float: """Get timeout in seconds from config or metadata default.""" timeout_ms: int = getattr(self.config, "timeout_ms", self.metadata.timeout_ms) diff --git a/evaluators/builtin/tests/test_base.py b/evaluators/builtin/tests/test_base.py index d81cc3f5b..21d6d26c7 100644 --- a/evaluators/builtin/tests/test_base.py +++ b/evaluators/builtin/tests/test_base.py @@ -6,7 +6,6 @@ from typing import Any import pytest - from agent_control_evaluators import Evaluator, EvaluatorConfig, EvaluatorMetadata from agent_control_models import EvaluatorResult, Step @@ -40,6 +39,15 @@ async def evaluate(self, data: Any) -> EvaluatorResult: ) +class LegacyContextOverrideEvaluator(MockEvaluator): + """Models an installed evaluator overriding the pre-existing context hook.""" + + async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: + result = await self.evaluate(data) + result.metadata["step_name"] = step.name + return result + + class TestEvaluatorMetadata: """Tests for EvaluatorMetadata dataclass.""" @@ -114,13 +122,35 @@ async def test_contextual_evaluation_delegates_to_existing_evaluate(self): evaluator = MockEvaluator.from_dict({"should_match": True}) step = Step(type="llm", name="answer", input="full input") - # When: the engine-facing contextual hook is called - result = await evaluator.evaluate_with_context("selected data", step) + # When: the engine-facing hook includes opaque request target metadata + result = await evaluator.evaluate_with_request_context( + "selected data", + step, + target_type="log_stream", + target_id="run-1", + ) # Then: the legacy evaluate implementation handles the selected data assert result.matched is True assert result.metadata == {"data": "selected data"} + @pytest.mark.asyncio + async def test_request_context_preserves_existing_context_overrides(self): + """The new request hook delegates to older evaluate_with_context overrides.""" + evaluator = LegacyContextOverrideEvaluator.from_dict({"should_match": True}) + step = Step(type="llm", name="answer", input="full input") + + result = await evaluator.evaluate_with_request_context( + "selected data", + step, + target_type="log_stream", + target_id="run-1", + ) + + assert result.matched is True + assert result.metadata["data"] == "selected data" + assert result.metadata["step_name"] == "answer" + def test_evaluator_config_stored(self): """Test that evaluator stores config.""" evaluator = MockEvaluator.from_dict({"should_match": True}) diff --git a/evaluators/contrib/galileo/README.md b/evaluators/contrib/galileo/README.md index 8191bb73d..6702b438b 100644 --- a/evaluators/contrib/galileo/README.md +++ b/evaluators/contrib/galileo/README.md @@ -9,10 +9,17 @@ The `galileo.luna2` evaluator ID has been removed. Existing controls that use configuration to use the direct Luna scorer fields. `scorer_id` is required; `scorer_label` and `scorer_version_id` are optional. `scorer_version_id` is a deprecated optional compatibility identifier; Orbit currently invokes the -scorer's current default version. The evaluator calls the -URL configured by `GALILEO_LUNA_INVOKE_URL`; the target must support the Luna -scorer invoke request/response contract and internal Galileo secret auth. Also -set `threshold` and `operator` as needed. If you still need the legacy Luna2 +scorer's current default version. When `GALILEO_API_KEY` and `GALILEO_API_URL` +are both configured, an invoked evaluator exchanges the application key for +one short-lived Orbit scorer grant scoped to the `target_id` on the Agent +Control evaluation request. Configure `agent_control.init()` with +`target_type="log_stream"` and that run's `target_id`; Orbit receives the ID as +`run_id`. The grant is reused across scorers for that run and sent to the Luna +invoke URL configured by `GALILEO_LUNA_INVOKE_URL`. The application key is +sent only to Orbit. If only +`GALILEO_API_SECRET_KEY` or `GALILEO_API_SECRET` is configured, the legacy +internal JWT flow remains active and does not request a scorer grant. Also set +`threshold` and `operator` as needed. If you still need the legacy Luna2 evaluator, pin `agent-control-evaluator-galileo <8`. diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py index d00d14571..ae1fc6f5b 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/client.py @@ -7,6 +7,7 @@ import ssl from asyncio import Lock from base64 import urlsafe_b64encode +from datetime import UTC, datetime from hashlib import sha256 from hmac import new as hmac_new from json import dumps @@ -26,7 +27,12 @@ SERVER_TIMEOUT_RATIO = 0.8 DEFAULT_INTERNAL_TOKEN_TTL_SECS = 3600 DEFAULT_LUNA_SCORER_INVOKE_PATH = "/api/v1/scorers/invoke" +SCORER_GRANT_PATH = "/internal/auth/scorer_grant" LUNA_INVOKE_URL_ENV = "GALILEO_LUNA_INVOKE_URL" +ORBIT_API_URL_ENV = "GALILEO_API_URL" +GALILEO_API_KEY_ENV = "GALILEO_API_KEY" +GALILEO_API_SECRET_KEY_ENV = "GALILEO_API_SECRET_KEY" +GALILEO_API_SECRET_ENV = "GALILEO_API_SECRET" LUNA_INVOKE_CA_FILE_ENV = "GALILEO_LUNA_INVOKE_CA_FILE" AUTH_UPSTREAM_CA_FILE_ENV = "AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE" @@ -59,6 +65,20 @@ SUPPORTED_SCORER_INVOKE_RECORD_TYPES = frozenset(get_args(ScorerInvokeRecordType)) +def _normalize_luna_invoke_url(raw_url: str) -> str: + """Use full invoke URLs as-is and append the default path to bare service roots.""" + url = raw_url.strip().rstrip("/") + parsed = urlsplit(url) + if parsed.path not in ("", "/") or parsed.query or parsed.fragment: + return url + return f"{url}{DEFAULT_LUNA_SCORER_INVOKE_PATH}" + + +def _normalize_orbit_url(raw_url: str) -> str: + """Normalize the configured Galileo API root used for grant exchange.""" + return raw_url.strip().rstrip("/") + + def _b64url(data: bytes) -> str: return urlsafe_b64encode(data).rstrip(b"=").decode("ascii") @@ -67,7 +87,7 @@ def _internal_auth_token( api_secret: str, ttl_seconds: int = DEFAULT_INTERNAL_TOKEN_TTL_SECS, ) -> str: - """Create the internal JWT expected by Luna scorer invoke routes.""" + """Create the legacy internal JWT expected by Luna scorer invoke routes.""" now = int(time()) header = {"alg": "HS256", "typ": "JWT"} payload = { @@ -86,13 +106,27 @@ def _internal_auth_token( return f"{signing_input}.{_b64url(signature)}" -def _normalize_luna_invoke_url(raw_url: str) -> str: - """Use full invoke URLs as-is and append the default path to bare service roots.""" - url = raw_url.strip().rstrip("/") - parsed = urlsplit(url) - if parsed.path not in ("", "/") or parsed.query or parsed.fragment: - return url - return f"{url}{DEFAULT_LUNA_SCORER_INVOKE_PATH}" +def _run_id_from_target(target_type: str | None, target_id: str | None) -> str: + """Resolve the Orbit run ID from Agent Control's opaque evaluation target.""" + if target_type is None or target_id is None: + raise ValueError("target_type and target_id must be supplied together for scorer grants.") + if target_type != "log_stream": + raise ValueError("Galileo scorer grants require target_type='log_stream'.") + resolved_target_id = target_id.strip() + if not resolved_target_id: + raise ValueError("target_id must be a non-empty log-stream ID for scorer grants.") + return resolved_target_id + + +class _ScorerGrant(BaseModel): + """Orbit-issued grant and its expiration time.""" + + token: str + expires_at: datetime + + +_scorer_grant_cache: dict[tuple[str, str, str, str], _ScorerGrant] = {} +_scorer_grant_cache_lock = Lock() def _load_float_env(env_name: str, default: float) -> float: @@ -337,7 +371,9 @@ class GalileoLunaClient: """Thin HTTP client for Galileo Luna scorer invocation. Environment Variables: - GALILEO_API_SECRET_KEY or GALILEO_API_SECRET: JWT signing secret for internal auth. + GALILEO_API_KEY and GALILEO_API_URL: Application API key and Galileo API + root for Orbit grant exchange. Both must be configured together. + GALILEO_API_SECRET_KEY or GALILEO_API_SECRET: Legacy internal JWT secret. GALILEO_LUNA_INVOKE_URL: Luna scorer invoke URL or service root (required). GALILEO_LUNA_INVOKE_CA_FILE: CA bundle used to verify Luna invoke TLS. AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE: Shared internal CA fallback. @@ -352,12 +388,19 @@ def __init__( api_secret: str | None = None, luna_invoke_url: str | None = None, luna_invoke_ca_file: str | None = None, + *, + api_key: str | None = None, + orbit_url: str | None = None, ) -> None: """Initialize the Galileo Luna client. Args: - api_secret: Internal JWT signing secret. If not provided, reads from - GALILEO_API_SECRET_KEY or GALILEO_API_SECRET. + api_key: Galileo application API key. If not provided, reads from + GALILEO_API_KEY. The key is sent only to Orbit's grant endpoint. + orbit_url: Galileo API root. If not provided, reads from GALILEO_API_URL. + api_secret: Legacy internal JWT secret. If not provided, reads from + GALILEO_API_SECRET_KEY or GALILEO_API_SECRET. New grant credentials + take precedence when both modes are configured. luna_invoke_url: Luna scorer invoke URL or service root. If not provided, reads from GALILEO_LUNA_INVOKE_URL. luna_invoke_ca_file: Optional CA bundle used to verify Luna invoke TLS. If not @@ -365,17 +408,29 @@ def __init__( AGENT_CONTROL_AUTH_UPSTREAM_CA_FILE. Raises: - ValueError: If the API secret, Luna invoke URL, CA bundle, or connection - tuning configuration is invalid. + ValueError: If the grant credentials are incomplete, neither auth mode is + configured, or the Luna URL, CA bundle, or connection tuning is invalid. """ + configured_api_key = api_key if api_key is not None else os.getenv(GALILEO_API_KEY_ENV) + configured_orbit_url = orbit_url if orbit_url is not None else os.getenv(ORBIT_API_URL_ENV) + has_api_key = bool(configured_api_key and configured_api_key.strip()) + has_orbit_url = bool(configured_orbit_url and configured_orbit_url.strip()) + if has_api_key != has_orbit_url: + raise ValueError( + "GALILEO_API_KEY and GALILEO_API_URL must be configured together " + "for scorer-grant authentication." + ) + resolved_api_secret = ( - api_secret or os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") + api_secret + or os.getenv(GALILEO_API_SECRET_KEY_ENV) + or os.getenv(GALILEO_API_SECRET_ENV) ) - if not resolved_api_secret: + if not has_api_key and not resolved_api_secret: raise ValueError( - "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET is required for Luna " - "scorer invocation. Set one as an environment variable or pass it " - "to the constructor." + "Configure GALILEO_API_KEY with GALILEO_API_URL for scorer-grant " + "authentication, or set GALILEO_API_SECRET_KEY or GALILEO_API_SECRET " + "for legacy Luna authentication." ) resolved_luna_invoke_url = luna_invoke_url or os.getenv(LUNA_INVOKE_URL_ENV) @@ -385,7 +440,17 @@ def __init__( "Set it as an environment variable or pass it to the constructor." ) + self.api_key = configured_api_key.strip() if configured_api_key else None self.api_secret = resolved_api_secret + self.auth_mode: Literal["scorer_grant", "legacy"] = ( + "scorer_grant" if has_api_key else "legacy" + ) + self.orbit_url = ( + _normalize_orbit_url(configured_orbit_url) if configured_orbit_url else None + ) + self.scorer_grant_url = ( + f"{self.orbit_url}{SCORER_GRANT_PATH}" if self.orbit_url is not None else None + ) self.luna_invoke_url = _normalize_luna_invoke_url(resolved_luna_invoke_url) self.luna_invoke_ca_file = ( luna_invoke_ca_file @@ -461,7 +526,58 @@ async def _get_client(self) -> httpx.AsyncClient: return self._select_pooled_client() + async def _get_scorer_grant(self, *, target_type: str, run_id: str) -> str: + """Exchange the application key for a cached Orbit-issued run grant.""" + if self.auth_mode != "scorer_grant" or self.api_key is None or self.orbit_url is None: + raise RuntimeError("Scorer-grant exchange is unavailable in legacy Luna auth mode.") + caller_context = sha256(self.api_key.encode("utf-8")).hexdigest() + cache_key = (self.orbit_url, target_type, run_id, caller_context) + + async with _scorer_grant_cache_lock: + cached = _scorer_grant_cache.get(cache_key) + now = datetime.now(tz=UTC) + if cached is not None: + expires_at = cached.expires_at + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=UTC) + if expires_at > now: + return cached.token + _scorer_grant_cache.pop(cache_key, None) + + client = await self._get_client() + response = await client.post( + f"{self.orbit_url}{SCORER_GRANT_PATH}", + json={"target_type": target_type, "run_id": run_id}, + headers={"Galileo-API-Key": self.api_key}, + timeout=DEFAULT_TIMEOUT_SECS, + ) + response.raise_for_status() + try: + payload = response.json() + if not isinstance(payload, dict): + raise ValueError("response must be a JSON object") + grant_value = payload.get("grant") + expires_value = payload.get("expires_at") + if not isinstance(grant_value, str) or not grant_value.strip(): + raise ValueError("response field 'grant' must be a non-empty string") + if not isinstance(expires_value, str): + raise ValueError("response field 'expires_at' must be an ISO timestamp") + expires_at = datetime.fromisoformat(expires_value.replace("Z", "+00:00")) + if expires_at.tzinfo is None: + expires_at = expires_at.replace(tzinfo=UTC) + if expires_at <= datetime.now(tz=UTC): + raise ValueError("response grant is already expired") + except (TypeError, ValueError) as exc: + raise RuntimeError(f"Invalid Orbit scorer grant response: {exc}") from exc + + grant = _ScorerGrant(token=grant_value, expires_at=expires_at) + _scorer_grant_cache[cache_key] = grant + return grant.token + def _endpoint_and_auth_header(self) -> tuple[str, str]: + """Return the invoke endpoint and legacy bearer token.""" + if self.auth_mode != "legacy" or self.api_secret is None: + raise RuntimeError("Legacy Luna auth is unavailable in scorer-grant mode.") token = _internal_auth_token(self.api_secret) return self.luna_invoke_url, f"Bearer {token}" @@ -477,6 +593,8 @@ async def invoke( config: ScorerInvokeConfig | JSONObject | None = None, timeout: float = DEFAULT_TIMEOUT_SECS, headers: dict[str, str] | None = None, + target_type: str | None = None, + target_id: str | None = None, ) -> ScorerInvokeResponse: """Invoke a Galileo Luna scorer. @@ -491,6 +609,9 @@ async def invoke( config: Optional Orbit-supported scorer invocation configuration. timeout: Request timeout in seconds. headers: Additional request headers. + target_type: Opaque evaluation target kind. Grant mode supports + ``log_stream`` and uses ``target_id`` as Orbit's run ID. + target_id: Opaque evaluation target ID supplied by Agent Control. Returns: Parsed scorer invocation response. @@ -534,19 +655,28 @@ async def invoke( config=invoke_config, ).to_dict() - endpoint, auth_header = self._endpoint_and_auth_header() + if self.auth_mode == "scorer_grant": + resolved_run_id = _run_id_from_target(target_type, target_id) + assert target_type is not None + scorer_grant = await self._get_scorer_grant( + target_type=target_type, + run_id=resolved_run_id, + ) + auth_header = f"Bearer {scorer_grant}" + else: + _, auth_header = self._endpoint_and_auth_header() request_headers = { k: v for k, v in (headers or {}).items() if k.lower() not in _BLOCKED_REQUEST_HEADERS } request_headers["Authorization"] = auth_header - logger.debug("[GalileoLunaClient] POST %s", endpoint) + logger.debug("[GalileoLunaClient] POST %s", self.luna_invoke_url) logger.debug("[GalileoLunaClient] Request body: %s", request_body) try: client = await self._get_client() response = await client.post( - endpoint, + self.luna_invoke_url, json=request_body, headers=request_headers, timeout=timeout, diff --git a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py index e4147a9ad..204267b6c 100644 --- a/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py +++ b/evaluators/contrib/galileo/src/agent_control_evaluator_galileo/luna/evaluator.py @@ -127,14 +127,24 @@ def __init__(self, config: LunaEvaluatorConfig) -> None: config: Validated LunaEvaluatorConfig instance. Raises: - ValueError: If neither GALILEO_API_SECRET_KEY nor GALILEO_API_SECRET is set. + ValueError: If neither complete scorer-grant credentials nor a legacy + Galileo API secret is configured. """ - has_secret = os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") - if not has_secret: + has_api_key = bool(os.getenv("GALILEO_API_KEY", "").strip()) + has_api_url = bool(os.getenv("GALILEO_API_URL", "").strip()) + has_legacy_secret = bool( + os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") + ) + if has_api_key != has_api_url: + raise ValueError( + "GALILEO_API_KEY and GALILEO_API_URL must be configured together " + "for scorer-grant authentication." + ) + if not has_api_key and not has_legacy_secret: raise ValueError( - "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET is required for Luna " - "scorer invocation. Set one as an environment variable before using " - "galileo.luna." + "Configure GALILEO_API_KEY with GALILEO_API_URL for scorer-grant " + "authentication, or set GALILEO_API_SECRET_KEY or GALILEO_API_SECRET " + "for legacy Luna authentication." ) super().__init__(config) @@ -212,7 +222,30 @@ async def evaluate_with_context(self, data: Any, step: Step) -> EvaluatorResult: """ return await self._evaluate(data, step=step) - async def _evaluate(self, data: Any, *, step: Step | None) -> EvaluatorResult: + async def evaluate_with_request_context( + self, + data: Any, + step: Step, + *, + target_type: str | None = None, + target_id: str | None = None, + ) -> EvaluatorResult: + """Evaluate using the request's Agent Control target metadata.""" + return await self._evaluate( + data, + step=step, + target_type=target_type, + target_id=target_id, + ) + + async def _evaluate( + self, + data: Any, + *, + step: Step | None, + target_type: str | None = None, + target_id: str | None = None, + ) -> EvaluatorResult: """Run a Luna evaluation with optional structured runtime context.""" input_text, output_text = self._prepare_payload(data) if not (_has_text(input_text) or _has_text(output_text)): @@ -227,6 +260,10 @@ async def _evaluate(self, data: Any, *, step: Step | None) -> EvaluatorResult: scorer_kwargs = self._scorer_kwargs() if step is not None: scorer_kwargs["step"] = step + if target_type is not None: + scorer_kwargs["target_type"] = target_type + if target_id is not None: + scorer_kwargs["target_id"] = target_id response = await self._get_client().invoke( **scorer_kwargs, input=input_text if _has_text(input_text) else None, diff --git a/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py b/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py index c5b65e7b4..e7f605ef5 100644 --- a/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py +++ b/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py @@ -7,22 +7,33 @@ from __future__ import annotations import json -from base64 import urlsafe_b64decode from unittest.mock import AsyncMock, MagicMock, patch import httpx import pytest LUNA_ENV = { - "GALILEO_API_SECRET_KEY": "test-secret", + "GALILEO_API_KEY": "test-api-key", + "GALILEO_API_URL": "http://orbit:8088", "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", } +TEST_RUN_ID = "coverage-run" -def _decode_jwt_payload(token: str) -> dict[str, object]: - payload_segment = token.split(".")[1] - padded = payload_segment + ("=" * (-len(payload_segment) % 4)) - return json.loads(urlsafe_b64decode(padded.encode()).decode()) +@pytest.fixture(autouse=True) +def _clear_scorer_grant_cache() -> None: + from agent_control_evaluator_galileo.luna import client as luna_client + + luna_client._scorer_grant_cache.clear() + + +def _mock_grant_response(request: httpx.Request) -> httpx.Response | None: + if request.url.path == "/internal/auth/scorer_grant": + return httpx.Response( + 200, + json={"grant": "coverage-test-grant", "expires_at": "2099-01-01T00:00:00Z"}, + ) + return None # ============================================================================= @@ -419,20 +430,21 @@ def test_missing_scorer_id_raises(self): ScorerInvokeRequest(inputs=ScorerInvokeInputs(query="hello")) -def test_client_raises_when_no_api_secret(monkeypatch): - """The client requires GALILEO_API_SECRET_KEY or GALILEO_API_SECRET.""" - for name in ("GALILEO_API_SECRET_KEY", "GALILEO_API_SECRET"): - monkeypatch.delenv(name, raising=False) +def test_client_raises_when_no_api_key(monkeypatch): + """The client requires the application API key for grant exchange.""" + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.setenv("GALILEO_API_URL", "http://orbit:8088") monkeypatch.setenv("GALILEO_LUNA_INVOKE_URL", "http://luna-invoke:8090") from agent_control_evaluator_galileo.luna.client import GalileoLunaClient - with pytest.raises(ValueError, match="GALILEO_API_SECRET_KEY or GALILEO_API_SECRET"): + with pytest.raises(ValueError, match="GALILEO_API_KEY"): GalileoLunaClient() def test_client_raises_when_no_luna_invoke_url(monkeypatch): """The client requires GALILEO_LUNA_INVOKE_URL.""" - monkeypatch.setenv("GALILEO_API_SECRET_KEY", "test-secret") + monkeypatch.setenv("GALILEO_API_KEY", "test-api-key") + monkeypatch.setenv("GALILEO_API_URL", "http://orbit:8088") monkeypatch.delenv("GALILEO_LUNA_INVOKE_URL", raising=False) from agent_control_evaluator_galileo.luna.client import GalileoLunaClient @@ -440,31 +452,24 @@ def test_client_raises_when_no_luna_invoke_url(monkeypatch): GalileoLunaClient() -def test_client_jwt_has_internal_scope(monkeypatch): - """JWT produced by the client must carry internal=True and scope=scorers.invoke.""" +def test_client_targets_the_orbit_grant_endpoint(monkeypatch): + """Grant exchange uses Orbit's credential-passthrough endpoint.""" for key, value in LUNA_ENV.items(): monkeypatch.setenv(key, value) from agent_control_evaluator_galileo.luna.client import GalileoLunaClient client = GalileoLunaClient() - _, auth_header = client._endpoint_and_auth_header() - - assert auth_header.startswith("Bearer ") - payload = _decode_jwt_payload(auth_header.removeprefix("Bearer ")) - assert payload["internal"] is True - assert payload["scope"] == "scorers.invoke" + assert client.scorer_grant_url == "http://orbit:8088/internal/auth/scorer_grant" def test_client_posts_to_correct_luna_invoke_endpoint(monkeypatch): - """_endpoint_and_auth_header must return the Luna invoke endpoint path.""" + """The runner endpoint remains the configured Luna invoke path.""" for key, value in LUNA_ENV.items(): monkeypatch.setenv(key, value) from agent_control_evaluator_galileo.luna.client import GalileoLunaClient client = GalileoLunaClient() - endpoint, _ = client._endpoint_and_auth_header() - - assert endpoint == "http://luna-invoke:8090/api/v1/scorers/invoke" + assert client.luna_invoke_url == "http://luna-invoke:8090/api/v1/scorers/invoke" def test_client_does_not_use_old_api_paths(monkeypatch): @@ -474,11 +479,9 @@ def test_client_does_not_use_old_api_paths(monkeypatch): from agent_control_evaluator_galileo.luna.client import GalileoLunaClient client = GalileoLunaClient() - endpoint, _ = client._endpoint_and_auth_header() - - assert "/scorers/invoke" in endpoint - assert endpoint.startswith("http://luna-invoke:8090/api/v1/") - assert "/internal/scorers/invoke" not in endpoint + assert "/scorers/invoke" in client.luna_invoke_url + assert client.luna_invoke_url.startswith("http://luna-invoke:8090/api/v1/") + assert "/internal/scorers/invoke" not in client.luna_invoke_url @pytest.mark.asyncio @@ -554,13 +557,23 @@ async def test_invoke_raises_when_response_is_not_a_json_object(monkeypatch): fake_response.json = MagicMock(return_value=["not", "an", "object"]) fake_http = AsyncMock() - fake_http.post = AsyncMock(return_value=fake_response) + grant_response = MagicMock() + grant_response.raise_for_status = MagicMock() + grant_response.json = MagicMock( + return_value={"grant": "test-grant", "expires_at": "2099-01-01T00:00:00Z"} + ) + fake_http.post = AsyncMock(side_effect=[grant_response, fake_response]) fake_http.is_closed = False client._client = fake_http try: with pytest.raises(RuntimeError, match="not a JSON object"): - await client.invoke(scorer_id="scorer-123", input="hello") + await client.invoke( + scorer_id="scorer-123", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) finally: await client.close() @@ -584,13 +597,23 @@ async def test_invoke_propagates_http_status_error(monkeypatch): ) fake_http = AsyncMock() - fake_http.post = AsyncMock(return_value=fake_response) + grant_response = MagicMock() + grant_response.raise_for_status = MagicMock() + grant_response.json = MagicMock( + return_value={"grant": "test-grant", "expires_at": "2099-01-01T00:00:00Z"} + ) + fake_http.post = AsyncMock(side_effect=[grant_response, fake_response]) fake_http.is_closed = False client._client = fake_http try: with pytest.raises(httpx.HTTPStatusError): - await client.invoke(scorer_id="scorer-123", input="hello") + await client.invoke( + scorer_id="scorer-123", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) finally: await client.close() @@ -611,7 +634,12 @@ async def test_invoke_propagates_request_error(monkeypatch): try: with pytest.raises(httpx.RequestError): - await client.invoke(scorer_id="scorer-123", input="hello") + await client.invoke( + scorer_id="scorer-123", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) finally: await client.close() @@ -640,6 +668,9 @@ async def test_invoke_strips_caller_supplied_galileo_api_key_header(monkeypatch) captured: dict[str, object] = {} def handler(request: httpx.Request) -> httpx.Response: + grant_response = _mock_grant_response(request) + if grant_response is not None: + return grant_response captured["headers"] = dict(request.headers) return httpx.Response(200, json={"score": 0.9, "status": "success"}) @@ -651,6 +682,7 @@ def handler(request: httpx.Request) -> httpx.Response: scorer_id="scorer-123", input="hello", headers={"Galileo-API-Key": "should-be-stripped", "X-Custom": "keep-me"}, + target_type="log_stream", target_id=TEST_RUN_ID, ) finally: await client.close() @@ -671,6 +703,9 @@ async def test_invoke_always_emits_config_field(monkeypatch): captured: dict[str, object] = {} def handler(request: httpx.Request) -> httpx.Response: + grant_response = _mock_grant_response(request) + if grant_response is not None: + return grant_response captured["body"] = json.loads(request.content.decode()) return httpx.Response(200, json={"score": 0.5, "status": "success"}) @@ -678,7 +713,12 @@ def handler(request: httpx.Request) -> httpx.Response: client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) try: - await client.invoke(scorer_id="scorer-123", input="hello") + await client.invoke( + scorer_id="scorer-123", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) finally: await client.close() diff --git a/evaluators/contrib/galileo/tests/test_luna_evaluator.py b/evaluators/contrib/galileo/tests/test_luna_evaluator.py index 4334ac5e8..6323dbf2b 100644 --- a/evaluators/contrib/galileo/tests/test_luna_evaluator.py +++ b/evaluators/contrib/galileo/tests/test_luna_evaluator.py @@ -6,6 +6,8 @@ import json import os from base64 import urlsafe_b64decode +from datetime import UTC, datetime +from hashlib import sha256 from typing import Literal from unittest.mock import AsyncMock, patch @@ -15,11 +17,35 @@ from pydantic import UUID4, BaseModel, ConfigDict, ValidationError LUNA_ENV = { - "GALILEO_API_SECRET_KEY": "test-secret", + "GALILEO_API_KEY": "test-api-key", + "GALILEO_API_URL": "http://orbit:8088", "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", } SCORER_ID = "3d45ef0d-5f14-4f1a-a8f1-8ab758da18b4" SCORER_VERSION_ID = "07fb9c96-9752-4cf5-a253-1a396100e9d2" +TEST_RUN_ID = "run-for-grant-tests" + + +@pytest.fixture(autouse=True) +def _clear_scorer_grant_cache() -> None: + from agent_control_evaluator_galileo.luna import client as luna_client + + luna_client._scorer_grant_cache.clear() + + +def _mock_grant_response(request: httpx.Request) -> httpx.Response | None: + if request.url.path == "/internal/auth/scorer_grant": + return httpx.Response( + 200, + json={"grant": "orbit-signed-test-grant", "expires_at": "2099-01-01T00:00:00Z"}, + ) + return None + + +def _decode_jwt_payload(token: str) -> dict[str, object]: + payload_segment = token.split(".")[1] + padded = payload_segment + ("=" * (-len(payload_segment) % 4)) + return json.loads(urlsafe_b64decode(padded.encode()).decode()) class _LegacyOrbitInputs(BaseModel): @@ -83,17 +109,6 @@ class _Orbit1720Request(BaseModel): config: _Orbit1720Config | None = None -def _decode_jwt_segment(segment: str) -> dict[str, object]: - """Decode one base64url JSON segment from an internal JWT.""" - padded = segment + ("=" * (-len(segment) % 4)) - return json.loads(urlsafe_b64decode(padded.encode()).decode()) - - -def _decode_jwt_payload(token: str) -> dict[str, object]: - """Decode the claims segment from an internal JWT.""" - return _decode_jwt_segment(token.split(".")[1]) - - class TestLunaEvaluatorConfig: """Tests for direct Luna evaluator configuration.""" @@ -415,7 +430,8 @@ def recording_client(**kwargs: object) -> httpx.AsyncClient: with patch.dict( os.environ, { - "GALILEO_API_SECRET_KEY": "test-secret", + "GALILEO_API_KEY": "test-api-key", + "GALILEO_API_URL": "http://orbit:8088", "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", "GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS": "0.25", "GALILEO_LUNA_MAX_CONNECTIONS": "17", @@ -454,7 +470,8 @@ def test_client_ignores_empty_connection_tuning_env(self) -> None: with patch.dict( os.environ, { - "GALILEO_API_SECRET_KEY": "test-secret", + "GALILEO_API_KEY": "test-api-key", + "GALILEO_API_URL": "http://orbit:8088", "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", "GALILEO_LUNA_KEEPALIVE_EXPIRY_SECONDS": "", "GALILEO_LUNA_MAX_CONNECTIONS": " ", @@ -493,7 +510,8 @@ def recording_client(**kwargs: object) -> FakeAsyncClient: with patch.dict( os.environ, { - "GALILEO_API_SECRET_KEY": "test-secret", + "GALILEO_API_KEY": "test-api-key", + "GALILEO_API_URL": "http://orbit:8088", "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", "GALILEO_LUNA_CLIENT_POOL_SIZE": "3", }, @@ -525,7 +543,8 @@ async def aclose(self) -> None: with patch.dict( os.environ, { - "GALILEO_API_SECRET_KEY": "test-secret", + "GALILEO_API_KEY": "test-api-key", + "GALILEO_API_URL": "http://orbit:8088", "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", "GALILEO_LUNA_CLIENT_POOL_SIZE": "2", }, @@ -631,6 +650,9 @@ async def test_client_posts_to_luna_invoke_scorer_invoke(self) -> None: captured: dict[str, object] = {} def handler(request: httpx.Request) -> httpx.Response: + grant_response = _mock_grant_response(request) + if grant_response is not None: + return grant_response captured["url"] = str(request.url) captured["headers"] = dict(request.headers) captured["body"] = json.loads(request.content.decode()) @@ -656,11 +678,12 @@ def handler(request: httpx.Request) -> httpx.Response: output="model answer", config={"request_timeout_seconds": 7}, headers={"Galileo-API-Key": "blocked", "X-Request-ID": "safe-id"}, + target_type="log_stream", target_id=TEST_RUN_ID, ) finally: await client.close() - # Then: posts to luna invoke endpoint /api/v1/scorers/invoke with JWT, no Galileo-API-Key + # Then: posts to the runner with Orbit's grant and no application API key assert response.score == 0.82 assert captured["url"] == "http://luna-invoke:8090/api/v1/scorers/invoke" expected_body = { @@ -673,18 +696,7 @@ def handler(request: httpx.Request) -> httpx.Response: assert isinstance(headers, dict) assert "galileo-api-key" not in headers assert headers["x-request-id"] == "safe-id" - auth_header = headers["authorization"] - assert isinstance(auth_header, str) - assert auth_header.startswith("Bearer ") - token = auth_header.removeprefix("Bearer ") - jwt_header = _decode_jwt_segment(token.split(".")[0]) - payload = _decode_jwt_payload(token) - assert jwt_header["alg"] == "HS256" - assert payload["internal"] is True - assert payload["scope"] == "scorers.invoke" - assert isinstance(payload["iat"], int) - assert isinstance(payload["exp"], int) - assert payload["exp"] > payload["iat"] + assert headers["authorization"] == "Bearer orbit-signed-test-grant" @pytest.mark.asyncio async def test_client_derives_server_timeout_from_custom_http_deadline(self) -> None: @@ -693,6 +705,9 @@ async def test_client_derives_server_timeout_from_custom_http_deadline(self) -> captured: dict[str, object] = {} def handler(request: httpx.Request) -> httpx.Response: + grant_response = _mock_grant_response(request) + if grant_response is not None: + return grant_response captured["body"] = json.loads(request.content.decode()) return httpx.Response(200, json={"score": 0.5, "status": "success"}) @@ -703,7 +718,13 @@ def handler(request: httpx.Request) -> httpx.Response: # When: invoking the scorer with a five-second HTTP deadline try: - await client.invoke(scorer_id=SCORER_ID, input="hello", timeout=5) + await client.invoke( + scorer_id=SCORER_ID, + input="hello", + timeout=5, + target_type="log_stream", + target_id=TEST_RUN_ID, + ) finally: await client.close() @@ -739,6 +760,7 @@ def handler(_request: httpx.Request) -> httpx.Response: input="hello", timeout=10, config={"request_timeout_seconds": server_timeout}, + target_type="log_stream", target_id=TEST_RUN_ID, ) finally: await client.close() @@ -751,6 +773,9 @@ async def test_client_dual_writes_legacy_inputs_and_structured_record(self) -> N captured: dict[str, object] = {} def handler(request: httpx.Request) -> httpx.Response: + grant_response = _mock_grant_response(request) + if grant_response is not None: + return grant_response captured["body"] = json.loads(request.content.decode()) return httpx.Response( 200, @@ -782,11 +807,12 @@ def handler(request: httpx.Request) -> httpx.Response: try: response = await client.invoke( scorer_id=SCORER_ID, - scorer_version_id=SCORER_VERSION_ID, - scorer_label="toxicity", - input="selected question", - output="selected answer", - step=step, + scorer_version_id=SCORER_VERSION_ID, + scorer_label="toxicity", + input="selected question", + output="selected answer", + step=step, + target_type="log_stream", target_id=TEST_RUN_ID, ) finally: await client.close() @@ -851,6 +877,9 @@ async def test_client_omits_record_for_step_type_orbit_does_not_support(self) -> captured: dict[str, object] = {} def handler(request: httpx.Request) -> httpx.Response: + grant_response = _mock_grant_response(request) + if grant_response is not None: + return grant_response captured["body"] = json.loads(request.content.decode()) return httpx.Response(200, json={"score": 0.4, "status": "success"}) @@ -862,7 +891,13 @@ def handler(request: httpx.Request) -> httpx.Response: # When: the scorer is invoked with selector-selected input try: - await client.invoke(scorer_id="scorer-123", input="selected input", step=step) + await client.invoke( + scorer_id="scorer-123", + input="selected input", + step=step, + target_type="log_stream", + target_id=TEST_RUN_ID, + ) finally: await client.close() @@ -880,6 +915,9 @@ async def test_client_forwards_scorer_version_id_when_configured(self) -> None: captured: dict[str, object] = {} def handler(request: httpx.Request) -> httpx.Response: + grant_response = _mock_grant_response(request) + if grant_response is not None: + return grant_response captured["body"] = json.loads(request.content.decode()) return httpx.Response(200, json={"score": 0.5, "status": "success"}) @@ -892,6 +930,7 @@ def handler(request: httpx.Request) -> httpx.Response: scorer_id="scorer-123", scorer_version_id="version-456", input="hello", + target_type="log_stream", target_id=TEST_RUN_ID, ) finally: await client.close() @@ -905,6 +944,9 @@ async def test_client_omits_galileo_api_key_even_when_env_is_set(self) -> None: captured: dict[str, object] = {} def handler(request: httpx.Request) -> httpx.Response: + grant_response = _mock_grant_response(request) + if grant_response is not None: + return grant_response captured["headers"] = dict(request.headers) return httpx.Response(200, json={"score": 0.5, "status": "success"}) @@ -914,7 +956,12 @@ def handler(request: httpx.Request) -> httpx.Response: client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) try: - await client.invoke(scorer_id="scorer-123", input="hello") + await client.invoke( + scorer_id="scorer-123", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) finally: await client.close() @@ -936,6 +983,365 @@ async def test_client_rejects_missing_input_and_output_values( await client.invoke(scorer_id="scorer-123", input=empty_value, output=empty_value) +class TestScorerGrantExchange: + """Orbit grants are acquired lazily and reused by run and caller context.""" + + @pytest.mark.asyncio + async def test_configured_luna_requests_grant_only_when_invoked(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + grant_response = _mock_grant_response(request) + if grant_response is not None: + return grant_response + return httpx.Response(200, json={"score": 0.9, "status": "success"}) + + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + + assert requests == [] + try: + await client.invoke( + scorer_id="scorer-one", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) + finally: + await client.close() + + grant_requests = [ + request for request in requests if request.url.path.endswith("scorer_grant") + ] + runner_requests = [ + request for request in requests if request.url.path.endswith("scorers/invoke") + ] + assert len(grant_requests) == 1 + assert grant_requests[0].url == "http://orbit:8088/internal/auth/scorer_grant" + assert grant_requests[0].headers["Galileo-API-Key"] == "test-api-key" + assert json.loads(grant_requests[0].content) == { + "target_type": "log_stream", + "run_id": TEST_RUN_ID, + } + assert len(runner_requests) == 1 + assert "galileo-api-key" not in runner_requests[0].headers + assert runner_requests[0].headers["authorization"] == "Bearer orbit-signed-test-grant" + + @pytest.mark.asyncio + async def test_multiple_scorers_reuse_one_grant_for_a_run(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + grant_requests = 0 + invoke_requests = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal grant_requests, invoke_requests + grant_response = _mock_grant_response(request) + if grant_response is not None: + grant_requests += 1 + return grant_response + invoke_requests += 1 + return httpx.Response(200, json={"score": 0.9, "status": "success"}) + + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + for scorer_id in ("scorer-one", "scorer-two"): + await client.invoke( + scorer_id=scorer_id, + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) + finally: + await client.close() + + assert grant_requests == 1 + assert invoke_requests == 2 + + @pytest.mark.asyncio + async def test_different_runs_get_different_grants(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + grant_runs: list[str] = [] + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("scorer_grant"): + body = json.loads(request.content) + grant_runs.append(body["run_id"]) + return httpx.Response( + 200, + json={ + "grant": f"grant-for-{body['run_id']}", + "expires_at": "2099-01-01T00:00:00Z", + }, + ) + return httpx.Response(200, json={"score": 0.9, "status": "success"}) + + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + for run_id in ("run-one", "run-two"): + await client.invoke( + scorer_id="scorer-one", + input="hello", + target_type="log_stream", + target_id=run_id, + ) + finally: + await client.close() + + assert grant_runs == ["run-one", "run-two"] + + @pytest.mark.asyncio + async def test_different_application_callers_do_not_share_grants(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + grant_requests = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal grant_requests + grant_response = _mock_grant_response(request) + if grant_response is not None: + grant_requests += 1 + return grant_response + return httpx.Response(200, json={"score": 0.9, "status": "success"}) + + with patch.dict(os.environ, LUNA_ENV, clear=True): + clients = [ + GalileoLunaClient(api_key="caller-one-key"), + GalileoLunaClient(api_key="caller-two-key"), + ] + for client in clients: + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + for client in clients: + await client.invoke( + scorer_id="scorer-one", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) + finally: + for client in clients: + await client.close() + + assert grant_requests == 2 + + @pytest.mark.asyncio + async def test_expired_grant_is_refreshed(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + from agent_control_evaluator_galileo.luna import client as luna_client + + grant_requests = 0 + + def handler(request: httpx.Request) -> httpx.Response: + nonlocal grant_requests + if request.url.path.endswith("scorer_grant"): + grant_requests += 1 + return httpx.Response( + 200, + json={"grant": f"grant-{grant_requests}", "expires_at": "2099-01-01T00:00:00Z"}, + ) + return httpx.Response(200, json={"score": 0.9, "status": "success"}) + + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + caller_context = sha256(client.api_key.encode("utf-8")).hexdigest() + cache_key = (client.orbit_url, "log_stream", TEST_RUN_ID, caller_context) + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + await client.invoke( + scorer_id="scorer-one", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) + luna_client._scorer_grant_cache[cache_key].expires_at = datetime( + 2000, 1, 1, tzinfo=UTC + ) + await client.invoke( + scorer_id="scorer-two", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) + finally: + await client.close() + + assert grant_requests == 2 + + @pytest.mark.asyncio + async def test_invalid_grant_response_fails_clearly(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + def handler(request: httpx.Request) -> httpx.Response: + if request.url.path.endswith("scorer_grant"): + return httpx.Response(200, json={"expires_at": "2099-01-01T00:00:00Z"}) + return httpx.Response(200, json={"score": 0.9, "status": "success"}) + + with patch.dict(os.environ, LUNA_ENV, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + with pytest.raises(RuntimeError, match="Invalid Orbit scorer grant response.*grant"): + await client.invoke( + scorer_id="scorer-one", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) + finally: + await client.close() + + @pytest.mark.asyncio + async def test_generic_evaluator_does_not_request_a_scorer_grant(self) -> None: + from agent_control_evaluator_galileo.luna import client as luna_client + from agent_control_evaluators.regex import RegexEvaluator + + evaluator = RegexEvaluator.from_dict({"pattern": "secret"}) + result = await evaluator.evaluate("not a secret") + + assert result.matched is True + assert luna_client._scorer_grant_cache == {} + + +class TestLegacyLunaAuthentication: + """Secret-only deployments retain the original local JWT invocation flow.""" + + @pytest.mark.asyncio + async def test_legacy_secret_only_invocation_sends_internal_jwt_without_grant(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(200, json={"score": 0.9, "status": "success"}) + + with patch.dict( + os.environ, + { + "GALILEO_API_SECRET_KEY": "legacy-secret", + "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", + }, + clear=True, + ): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + await client.invoke(scorer_id="scorer-one", input="hello") + finally: + await client.close() + + assert len(requests) == 1 + assert requests[0].url.path == "/api/v1/scorers/invoke" + authorization = requests[0].headers["authorization"] + assert authorization.startswith("Bearer ") + payload = _decode_jwt_payload(authorization.removeprefix("Bearer ")) + assert payload["internal"] is True + assert payload["scope"] == "scorers.invoke" + assert "galileo-api-key" not in requests[0].headers + + @pytest.mark.asyncio + async def test_new_grant_mode_takes_precedence_when_both_modes_are_configured(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + grant_response = _mock_grant_response(request) + if grant_response is not None: + return grant_response + return httpx.Response(200, json={"score": 0.9, "status": "success"}) + + env = {**LUNA_ENV, "GALILEO_API_SECRET_KEY": "legacy-secret"} + with patch.dict(os.environ, env, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + await client.invoke( + scorer_id="scorer-one", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) + finally: + await client.close() + + assert client.auth_mode == "scorer_grant" + assert [request.url.path for request in requests] == [ + "/internal/auth/scorer_grant", + "/api/v1/scorers/invoke", + ] + assert requests[1].headers["authorization"] == "Bearer orbit-signed-test-grant" + + def test_partial_grant_configuration_fails_even_with_legacy_secret(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + with patch.dict( + os.environ, + { + "GALILEO_API_KEY": "application-key", + "GALILEO_API_SECRET_KEY": "legacy-secret", + "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", + }, + clear=True, + ): + with pytest.raises(ValueError, match="GALILEO_API_KEY and GALILEO_API_URL"): + GalileoLunaClient() + + @pytest.mark.asyncio + async def test_grant_failure_never_falls_back_to_legacy_jwt(self) -> None: + from agent_control_evaluator_galileo.luna import GalileoLunaClient + + requests: list[httpx.Request] = [] + + def handler(request: httpx.Request) -> httpx.Response: + requests.append(request) + return httpx.Response(401, json={"detail": "invalid API key"}) + + env = {**LUNA_ENV, "GALILEO_API_SECRET_KEY": "legacy-secret"} + with patch.dict(os.environ, env, clear=True): + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + with pytest.raises(httpx.HTTPStatusError): + await client.invoke( + scorer_id="scorer-one", + input="hello", + target_type="log_stream", + target_id=TEST_RUN_ID, + ) + finally: + await client.close() + + assert len(requests) == 1 + assert requests[0].url.path == "/internal/auth/scorer_grant" + + def test_luna_evaluator_accepts_legacy_secret_only_configuration(self) -> None: + from agent_control_evaluator_galileo.luna import LunaEvaluator + + with patch.dict( + os.environ, + { + "GALILEO_API_SECRET": "legacy-secret", + "GALILEO_LUNA_INVOKE_URL": "http://luna-invoke:8090", + }, + clear=True, + ): + evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-one"}) + + assert evaluator._client.auth_mode == "legacy" + + class TestLunaEvaluator: """Tests for direct Luna evaluator behavior.""" @@ -950,11 +1356,11 @@ def test_evaluator_metadata(self) -> None: def test_evaluator_init_without_auth_raises(self) -> None: from agent_control_evaluator_galileo.luna import LunaEvaluator - with pytest.raises(ValueError, match="GALILEO_API_SECRET_KEY or GALILEO_API_SECRET"): + with pytest.raises(ValueError, match="GALILEO_API_KEY"): LunaEvaluator.from_dict({"scorer_id": "scorer-123", "threshold": 0.5}) @patch.dict(os.environ, LUNA_ENV, clear=True) - def test_evaluator_init_accepts_api_secret(self) -> None: + def test_evaluator_init_accepts_application_api_key(self) -> None: from agent_control_evaluator_galileo.luna import LunaEvaluator evaluator = LunaEvaluator.from_dict({"scorer_id": "scorer-123", "threshold": 0.5}) @@ -1036,13 +1442,20 @@ async def test_evaluator_contextual_hook_forwards_complete_step(self) -> None: # When: evaluating through the contextual hook with patch.object(GalileoLunaClient, "invoke", new_callable=AsyncMock) as mock_invoke: mock_invoke.return_value = ScorerInvokeResponse(score=0.8, status="success") - result = await evaluator.evaluate_with_context("selected input", step) + result = await evaluator.evaluate_with_request_context( + "selected input", + step, + target_type="log_stream", + target_id=TEST_RUN_ID, + ) # Then: selector-selected data and the complete Step are both forwarded assert result.matched is True mock_invoke.assert_awaited_once_with( scorer_id="scorer-123", step=step, + target_type="log_stream", + target_id=TEST_RUN_ID, input="selected input", output=None, config=None, diff --git a/examples/galileo_luna/README.md b/examples/galileo_luna/README.md index 5d9ada076..c80a466cf 100644 --- a/examples/galileo_luna/README.md +++ b/examples/galileo_luna/README.md @@ -20,11 +20,12 @@ make server-run Configure Luna invoke credentials: ```bash -export GALILEO_API_SECRET_KEY="your-api-secret" -export GALILEO_LUNA_INVOKE_URL="http://luna-invoke.internal/api/v1/scorers/invoke" +export GALILEO_API_KEY="your-application-api-key" +export GALILEO_API_URL="https://your-galileo-api.example" +export GALILEO_LUNA_INVOKE_URL="https://your-luna-invoke.example/api/v1/scorers/invoke" ``` -`GALILEO_API_SECRET` can be used instead of `GALILEO_API_SECRET_KEY` if that is how your deployment exposes the internal Galileo JWT signing secret. `GALILEO_LUNA_INVOKE_URL` can be either the full scorer invoke URL or a service root that serves `/api/v1/scorers/invoke`. +The application API key is used only to obtain a short-lived, run-scoped scorer grant from Orbit. Configure `agent_control.init()` with `target_type="log_stream"` and the target log-stream ID; Agent Control forwards that target on each evaluation request, and Orbit uses `target_id` as `run_id`. `GALILEO_API_URL` is the Orbit API root. `GALILEO_LUNA_INVOKE_URL` can be either the full scorer invoke URL or a service root that serves `/api/v1/scorers/invoke`. Existing deployments can instead set `GALILEO_API_SECRET_KEY` or `GALILEO_API_SECRET` to keep using the legacy internal JWT flow. Required scorer setting: @@ -58,5 +59,5 @@ Run: ```bash cd examples/galileo_luna uv run python setup_controls.py -uv run python demo_agent.py +uv run python demo_agent.py --target-id "your-existing-log-stream-id" ``` diff --git a/examples/galileo_luna/demo_agent.py b/examples/galileo_luna/demo_agent.py index c5fab4c17..e2aa7916d 100644 --- a/examples/galileo_luna/demo_agent.py +++ b/examples/galileo_luna/demo_agent.py @@ -5,7 +5,7 @@ 1. Start server: make server-run 2. Create controls: uv run python setup_controls.py 3. Set Galileo credentials where this script runs: - GALILEO_API_SECRET_KEY or GALILEO_API_SECRET + GALILEO_API_KEY and GALILEO_API_URL, or a legacy API secret GALILEO_LUNA_INVOKE_URL Usage: @@ -14,6 +14,7 @@ from __future__ import annotations +import argparse import asyncio import logging import os @@ -72,12 +73,14 @@ async def run_case(label: str, message: str) -> None: print(f"Metadata: {exc.metadata}") -def init_agent() -> None: +def init_agent(target_id: str) -> None: """Initialize Agent Control and fetch controls created by setup_controls.py.""" agent_control.init( agent_name=AGENT_NAME, agent_description="Demo agent protected by direct Galileo Luna scorer controls", server_url=SERVER_URL, + target_type="log_stream", + target_id=target_id, steps=[ { "type": "llm", @@ -90,15 +93,20 @@ def init_agent() -> None: ) -async def run_demo() -> None: +async def run_demo(target_id: str) -> None: """Run scripted scenarios.""" - api_secret = os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") + api_key = os.getenv("GALILEO_API_KEY") + api_url = os.getenv("GALILEO_API_URL") + legacy_secret = os.getenv("GALILEO_API_SECRET_KEY") or os.getenv("GALILEO_API_SECRET") luna_invoke_url = os.getenv("GALILEO_LUNA_INVOKE_URL") - if not api_secret: + if bool(api_key) != bool(api_url): + print("GALILEO_API_KEY and GALILEO_API_URL must be set together.") + return + if not ((api_key and api_url) or legacy_secret): print( - "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET is required for the " - "galileo.luna evaluator." + "Set GALILEO_API_KEY and GALILEO_API_URL, or configure " + "GALILEO_API_SECRET_KEY or GALILEO_API_SECRET." ) return if not luna_invoke_url: @@ -111,9 +119,10 @@ async def run_demo() -> None: print(f"Server: {SERVER_URL}") print(f"Agent: {AGENT_NAME}") print(f"Luna invoke: {luna_invoke_url}") + print(f"Log stream: {target_id}") print() - init_agent() + init_agent(target_id) try: await run_case( "Safe request: no composite prefilter match, Luna is not called", @@ -133,7 +142,14 @@ async def run_demo() -> None: def main() -> None: """Run the demo.""" - asyncio.run(run_demo()) + parser = argparse.ArgumentParser(description="Run the Galileo Luna evaluator demo.") + parser.add_argument( + "--target-id", + required=True, + help="ID of the existing Galileo log stream used for this evaluation run.", + ) + args = parser.parse_args() + asyncio.run(run_demo(args.target_id)) if __name__ == "__main__": diff --git a/examples/galileo_luna/setup_controls.py b/examples/galileo_luna/setup_controls.py index 753805dfc..28953ec96 100644 --- a/examples/galileo_luna/setup_controls.py +++ b/examples/galileo_luna/setup_controls.py @@ -4,7 +4,7 @@ Prerequisites: - Agent Control server running at AGENT_CONTROL_URL, default http://localhost:8000 - Galileo credentials set where demo_agent.py will run: - GALILEO_API_SECRET_KEY or GALILEO_API_SECRET + GALILEO_API_KEY and GALILEO_API_URL GALILEO_LUNA_INVOKE_URL GALILEO_LUNA_SCORER_ID (required) From 89f969d12e46944e3563dd58e73a05b3a3785328 Mon Sep 17 00:00:00 2001 From: Namrata Ghadi Date: Tue, 29 Sep 2026 15:42:36 -0700 Subject: [PATCH 2/2] test(galileo): cover scorer grant edge cases --- .../galileo/tests/test_luna_coverage_gaps.py | 145 ++++++++++++++++++ 1 file changed, 145 insertions(+) diff --git a/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py b/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py index e7f605ef5..7ea52f111 100644 --- a/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py +++ b/evaluators/contrib/galileo/tests/test_luna_coverage_gaps.py @@ -441,6 +441,151 @@ def test_client_raises_when_no_api_key(monkeypatch): GalileoLunaClient() +def test_client_raises_when_no_auth_mode_is_configured(monkeypatch): + """The client still rejects configurations with neither grant nor legacy auth.""" + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.delenv("GALILEO_API_SECRET_KEY", raising=False) + monkeypatch.delenv("GALILEO_API_SECRET", raising=False) + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.delenv("GALILEO_API_URL", raising=False) + monkeypatch.setenv("GALILEO_LUNA_INVOKE_URL", "http://luna-invoke:8090") + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + with pytest.raises(ValueError, match="Configure GALILEO_API_KEY"): + GalileoLunaClient() + + +def test_evaluator_rejects_partial_grant_configuration(monkeypatch): + monkeypatch.setenv("GALILEO_API_KEY", "test-api-key") + monkeypatch.delenv("GALILEO_API_URL", raising=False) + monkeypatch.delenv("GALILEO_API_SECRET_KEY", raising=False) + monkeypatch.setenv("GALILEO_LUNA_INVOKE_URL", "http://luna-invoke:8090") + from agent_control_evaluator_galileo.luna import LunaEvaluator + + with pytest.raises(ValueError, match="GALILEO_API_KEY and GALILEO_API_URL"): + LunaEvaluator.from_dict({"scorer_id": "scorer-123"}) + + +@pytest.mark.parametrize( + ("target_type", "target_id", "message"), + [ + (None, None, "supplied together"), + ("log_stream", None, "supplied together"), + ("trace", "run-123", "target_type='log_stream'"), + ("log_stream", " ", "non-empty log-stream ID"), + ], +) +def test_run_id_from_target_rejects_invalid_target(target_type, target_id, message): + from agent_control_evaluator_galileo.luna.client import _run_id_from_target + + with pytest.raises(ValueError, match=message): + _run_id_from_target(target_type, target_id) + + +@pytest.mark.asyncio +async def test_scorer_grant_exchange_is_rejected_in_legacy_mode(monkeypatch): + monkeypatch.delenv("GALILEO_API_KEY", raising=False) + monkeypatch.delenv("GALILEO_API_URL", raising=False) + monkeypatch.setenv("GALILEO_API_SECRET_KEY", "legacy-secret") + monkeypatch.setenv("GALILEO_LUNA_INVOKE_URL", "http://luna-invoke:8090") + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + client = GalileoLunaClient() + with pytest.raises(RuntimeError, match="unavailable in legacy Luna auth mode"): + await client._get_scorer_grant(target_type="log_stream", run_id=TEST_RUN_ID) + + +def test_legacy_auth_header_is_unavailable_in_scorer_grant_mode(monkeypatch): + for key, value in LUNA_ENV.items(): + monkeypatch.setenv(key, value) + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + client = GalileoLunaClient() + with pytest.raises(RuntimeError, match="Legacy Luna auth is unavailable"): + client._endpoint_and_auth_header() + + +@pytest.mark.asyncio +async def test_scorer_grant_exchange_refreshes_naive_expiration(monkeypatch): + for key, value in LUNA_ENV.items(): + monkeypatch.setenv(key, value) + from datetime import datetime, timedelta + + from agent_control_evaluator_galileo.luna import client as luna_client + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient, _ScorerGrant + + client = GalileoLunaClient() + cache_key = ( + client.orbit_url, + "log_stream", + TEST_RUN_ID, + luna_client.sha256(client.api_key.encode("utf-8")).hexdigest(), + ) + luna_client._scorer_grant_cache[cache_key] = _ScorerGrant( + token="expired-grant", + expires_at=datetime.now() - timedelta(seconds=1), + ) + client._client = httpx.AsyncClient( + transport=httpx.MockTransport(_mock_grant_response) + ) + + try: + assert await client._get_scorer_grant( + target_type="log_stream", run_id=TEST_RUN_ID + ) == "coverage-test-grant" + finally: + await client.close() + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + ("payload", "message"), + [ + (["not", "an", "object"], "JSON object"), + ({"expires_at": "2099-01-01T00:00:00Z"}, "'grant'"), + ({"grant": "test-grant"}, "'expires_at'"), + ({"grant": "test-grant", "expires_at": "2000-01-01T00:00:00Z"}, "already expired"), + ], +) +async def test_scorer_grant_exchange_rejects_invalid_responses(monkeypatch, payload, message): + for key, value in LUNA_ENV.items(): + monkeypatch.setenv(key, value) + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response(200, json=payload) + + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + with pytest.raises(RuntimeError, match=f"Invalid Orbit scorer grant response.*{message}"): + await client._get_scorer_grant(target_type="log_stream", run_id=TEST_RUN_ID) + finally: + await client.close() + + +@pytest.mark.asyncio +async def test_scorer_grant_exchange_accepts_naive_expiration(monkeypatch): + for key, value in LUNA_ENV.items(): + monkeypatch.setenv(key, value) + from agent_control_evaluator_galileo.luna.client import GalileoLunaClient + + def handler(request: httpx.Request) -> httpx.Response: + return httpx.Response( + 200, + json={"grant": "naive-expiry-grant", "expires_at": "2099-01-01T00:00:00"}, + ) + + client = GalileoLunaClient() + client._client = httpx.AsyncClient(transport=httpx.MockTransport(handler)) + try: + assert await client._get_scorer_grant( + target_type="log_stream", run_id=TEST_RUN_ID + ) == "naive-expiry-grant" + finally: + await client.close() + + def test_client_raises_when_no_luna_invoke_url(monkeypatch): """The client requires GALILEO_LUNA_INVOKE_URL.""" monkeypatch.setenv("GALILEO_API_KEY", "test-api-key")