diff --git a/.github/scripts/check_strict_types.sh b/.github/scripts/check_strict_types.sh index 4e41c9e8f..f27e39519 100755 --- a/.github/scripts/check_strict_types.sh +++ b/.github/scripts/check_strict_types.sh @@ -20,9 +20,11 @@ from posthog.ai.evaluations import ( AsyncOfflineEvaluations, BooleanScorerConfig, EvaluationItem, + EvaluationResult, NumericPassingRule, NumericScorerConfig, OfflineEvaluations, + ScorerComposition, ) client = Posthog("phc_test") @@ -73,6 +75,21 @@ async def offline_evaluation_types( async_experiment = await async_client.create_experiment(name="async run") await async_experiment.upload_result(item=item, scorer_version_id=boolean.current_version_id, value=False) _ = config + composition: ScorerComposition = { + "expression": "relevance", + "inputs": [{"alias": "relevance", "scorer_version_id": numeric.current_version_id}], + } + overall = sync_client.scorers.create( + name="Overall", kind="numeric", config=NumericScorerConfig(min=0, max=1), + calculation="composite", composition=composition, + ) + valid: bool = await async_client.scorers.validate_composition(kind="numeric", composition=composition) + sync_client.scorers.list(calculation="composite", input_definition_id=numeric.id) + experiment.upload_results([ + EvaluationResult(item=item, scorer_version_id=numeric.current_version_id, value=0.9), + EvaluationResult(item=item, scorer_version_id=overall.current_version_id, calculate=True), + ]) + _ = valid PY cat > "$tmp/invalid_types.py" <<'PY' diff --git a/.sampo/changesets/composite-offline-scorers.md b/.sampo/changesets/composite-offline-scorers.md new file mode 100644 index 000000000..07c1cb9ac --- /dev/null +++ b/.sampo/changesets/composite-offline-scorers.md @@ -0,0 +1,5 @@ +--- +pypi/posthog: minor +--- + +Support versioned composite scorers for offline evaluations, including formula validation, dependency metadata, explicit server calculations, stored outcome receipts, and composite-safe bulk uploads. diff --git a/examples/example-offline-evaluations/README.md b/examples/example-offline-evaluations/README.md index 00dfec78e..188b2e22e 100644 --- a/examples/example-offline-evaluations/README.md +++ b/examples/example-offline-evaluations/README.md @@ -10,6 +10,8 @@ Offline evaluations must be enabled for your PostHog project. Use the project's | --- | --- | | Create experiments, upload results, complete/fail | Personal API key or project secret key with `offline_evaluation_ingestion:write` | | Fetch scorers and versions | Personal API key with `llm_analytics:read` | +| Validate a composite formula without saving | Personal API key with `llm_analytics:read` | +| Calculate composites using previously stored inputs | Personal API key with ingestion access, `evaluation:read`, and `llm_analytics:read` | | Create/update scorers and create versions | Personal API key with `llm_analytics:write` | | Fetch datasets, items, and history | Personal API key with `dataset:read` | | Create/update/archive/restore datasets and items | Personal API key with `dataset:write` | @@ -92,6 +94,79 @@ EvaluationItem(input=None) # Payload contains a null input. The client context manager releases connections. Call `complete()` explicitly after uploads succeed, or `fail()` when you deliberately want to close the experiment as failed. Neither exceptions nor async cancellation automatically close an experiment. Expected counts include every outcome status; the server checks them on completion. +## Composite item scores + +Composite scorers calculate one score per item on the PostHog server from exact reported scorer versions. They can return numeric, boolean, or categorical values. This requires a backend with composite scorer support; ordinary uploads remain compatible with older backends. + +Provision a composite once, using a personal key with scorer write access. This example assumes `correctness_version_id` identifies a reported boolean scorer and `relevance_version_id` a reported numeric scorer: + +```python +from posthog.ai.evaluations import ScorerComposition + +composition: ScorerComposition = { + "expression": "(number(correct) + relevance) / 2", + "inputs": [ + {"alias": "correct", "scorer_version_id": correctness_version_id}, + {"alias": "relevance", "scorer_version_id": relevance_version_id}, + ], +} +# Optional; read scope is sufficient and no scorer/version is created. +evaluations.scorers.validate_composition( + kind="numeric", composition=composition, config={"min": 0, "max": 1}, +) +overall = evaluations.scorers.create( + name="Overall", + kind="numeric", + calculation="composite", + config={"min": 0, "max": 1, "passing_rule": {"operator": "gte", "threshold": 0.8}}, + composition=composition, +) +overall_version_id = overall.current_version_id +``` + +Save that version UUID for your runs. Creating scorers/versions is not idempotent, so a lost response requires inspecting server state before repeating creation. Update a formula, output configuration, or pinned input through `create_version(scorer_id, config=..., composition=..., base_version=...)`. Supply the complete composition each time. New source versions never update a composite automatically. + +Upload the reported inputs and explicitly request the calculation: + +```python +experiment = evaluations.create_experiment( + name="Composite example", expected_item_count=1, expected_result_count=3, +) +item = EvaluationItem(input="Capital of France?", output="Paris") +receipt = experiment.upload_results([ + EvaluationResult(item=item, scorer_version_id=correctness_version_id, value=True), + EvaluationResult(item=item, scorer_version_id=relevance_version_id, value=0.8), + EvaluationResult(item=item, scorer_version_id=overall_version_id, calculate=True), +]) +for chunk in receipt.chunks: + for outcome in chunk.results: + print(outcome.scorer_version_id, outcome.status, outcome.error_code) +experiment.complete() +``` + +Calculation declarations contain only the item reference, scorer version, and `calculate=True`. Do not supply a value, reasoning, metadata, evaluator trace, or evaluation time. An item may also be referenced by its already accepted UUID. + +Project secret keys and ingestion-only personal keys must supply **every pinned input result in the same HTTP request**, including on retries. Previously uploaded inputs may be resubmitted with their original values and metadata. Reusing stored inputs without resubmitting them requires both `evaluation:read` and `llm_analytics:read`, in addition to ingestion access. Composite requests and retries also require current access to the composite and its inputs. + +The SDK keeps all supplied results for a composite-bearing item together without fetching scorer definitions. It preserves order, so interleaved composite items can form a larger indivisible group. Group each item's results together for large batches. If a group exceeds 1,000 items/results or 5 MiB, the entire call fails locally before any upload. The SDK cannot check that your supplied inputs match the pinned formula without read access; the server validates this. Missing inputs reject the whole HTTP request. Inputs can appear before or after their composite in that request. + +HTTP success acknowledges persistence, not successful calculation. Input errors become `composite_input_error`; formula errors or invalid output values become `composite_formula_error`. Input statuses propagate in the order `error`, `skipped`, `not_applicable`. Check each receipt's `status` and `error_code`; no result-read permission is needed. For ordinary results on an older backend, `status=None` means the backend omitted the outcome. Composite acknowledgments require an outcome. + +All accepted composite outcomes count toward `expected_result_count`, including errors and skips. Derived results add no uploaded-result billing usage. Omitting a composite leaves it unrequested. Exact retries return the stored outcome, including after completion; formulas are not recalculated on retry or read. Persist declarations with `to_dict` and restore them with `from_dict` for exact recovery. + +Formulas support 1–16 reported inputs and at most 1,024 characters; aliases start with a lowercase letter and contain lowercase letters, digits, or underscores, up to 32 characters. Nested composites are unsupported. Use `number(flag)` for explicit boolean conversion and `"good" in categories` for categorical membership. Boolean inputs retain their raw meaning regardless of passing rules. Numeric composites support bounds and passing thresholds, but no non-null `step`. Formula syntax, input access, and output types are validated by the server; runtime values and bounds are checked during ingestion. + +Read `scorer.calculation`, `scorer.composition`, and `scorer.input_scorers` to inspect saved inputs and their pinned/latest versions. `dependent_composite_count` counts accessible active dependents; `scorers.list(calculation="composite", input_definition_id=source_id)` retrieves them. Historical versions retain their saved composition. All operations also exist on `AsyncOfflineEvaluations`. + +The runnable capital-city example can include a composite consuming only its configured boolean scorer version. Provision one with expression `number(correct)`, pin that boolean version as `correct`, and pass its composite version UUID when creating a new state file: + +```bash +uv run python examples/example-offline-evaluations/main.py \ + --state composite-run.json --composite-version-id 'your-composite-version-uuid' +``` + +The state file includes both reported and calculation declarations and the combined expected count. Reusing the file replays that saved run regardless of newly supplied scorer flags. + ## Datasets and reproducible evaluations Use `evaluations.datasets` to manage PostHog-hosted datasets with the same client configuration as scorers and experiments. Dataset names are unique within a project; keep the returned UUID for subsequent operations. diff --git a/examples/example-offline-evaluations/main.py b/examples/example-offline-evaluations/main.py index 6c2e7826f..de0832cba 100644 --- a/examples/example-offline-evaluations/main.py +++ b/examples/example-offline-evaluations/main.py @@ -29,8 +29,8 @@ def resolve_version(evaluations: OfflineEvaluations) -> str: "Set POSTHOG_SCORER_ID or POSTHOG_SCORER_VERSION_ID, or run --create-scorer once." ) scorer = evaluations.scorers.get(scorer_id) - if scorer.kind != "boolean": - raise ValueError("This example requires a boolean scorer.") + if scorer.kind != "boolean" or scorer.calculation != "reported": + raise ValueError("This example requires a reported boolean scorer.") return scorer.current_version_id @@ -67,6 +67,7 @@ def prepare_state( *, dataset_id: str | None = None, dataset_revision: int | None = None, + composite_version_id: str | None = None, ) -> dict[str, Any]: version_id = resolve_version(evaluations) cases = [ @@ -100,7 +101,17 @@ def prepare_state( ) for case in cases ] - results = [exact_match_result(item, version_id) for item in items] + results = [] + for item in items: + results.append(exact_match_result(item, version_id)) + if composite_version_id is not None: + results.append( + EvaluationResult( + item=item, + scorer_version_id=composite_version_id, + calculate=True, + ) + ) return { "project_id": project_id, "host": host, @@ -153,6 +164,10 @@ def main() -> None: type=int, help="Pin this numeric revision; defaults to latest once.", ) + parser.add_argument( + "--composite-version-id", + help="Also calculate this composite version, which must pin the configured boolean scorer version.", + ) args = parser.parse_args() if args.dataset_revision is not None and args.dataset_id is None: parser.error("--dataset-revision requires --dataset-id") @@ -186,6 +201,7 @@ def main() -> None: host, dataset_id=args.dataset_id, dataset_revision=args.dataset_revision, + composite_version_id=args.composite_version_id, ) save_state(args.state, state) @@ -215,6 +231,16 @@ def main() -> None: print( f"Acknowledged {sum(len(chunk.results) for chunk in receipt.chunks)} results across {len(receipt.chunks)} chunks." ) + errors = [ + result + for chunk in receipt.chunks + for result in chunk.results + if result.status == "error" + ] + if errors: + print( + f"Accepted {len(errors)} error outcomes: {[result.error_code for result in errors]}" + ) print( f"Keep {args.state} for exact replay; choose a different --state path for a new experiment." ) diff --git a/posthog/ai/evaluations/__init__.py b/posthog/ai/evaluations/__init__.py index 2dbd0cc35..aa8c3e227 100644 --- a/posthog/ai/evaluations/__init__.py +++ b/posthog/ai/evaluations/__init__.py @@ -36,7 +36,11 @@ NumericPassingRule, NumericScorerConfig, Scorer, + ScorerCalculation, + ScorerComposition, + ScorerCompositionInput, ScorerConfig, + ScorerInput, ScorerKind, ScorerOrder, ScorerPage, @@ -99,7 +103,11 @@ "RunSource", "ScoreValue", "Scorer", + "ScorerCalculation", + "ScorerComposition", + "ScorerCompositionInput", "ScorerConfig", + "ScorerInput", "ScorerKind", "ScorerOrder", "ScorerPage", diff --git a/posthog/ai/evaluations/_client.py b/posthog/ai/evaluations/_client.py index db0a5d695..475a856a3 100644 --- a/posthog/ai/evaluations/_client.py +++ b/posthog/ai/evaluations/_client.py @@ -95,6 +95,30 @@ def _upload_body(items: list[bytes], results: list[bytes]) -> bytes: ) +def _group_ends(results: tuple[EvaluationResult, ...]) -> list[int]: + # A calculation only names a version, so ingestion-only clients cannot + # discover its dependencies. Keep every supplied result for that item in + # one request. Merge overlapping spans to retain the original order. + first: dict[str, int] = {} + last: dict[str, int] = {} + composite_items: set[str] = set() + for index, result in enumerate(results): + first.setdefault(result.item_id, index) + last[result.item_id] = index + 1 + if result.calculate: + composite_items.add(result.item_id) + spans = {first[item_id]: last[item_id] for item_id in composite_items} + ends: list[int] = [] + index = 0 + while index < len(results): + end = index + 1 + while index < end: + end = max(end, spans.get(index, index + 1)) + index += 1 + ends.append(end) + return ends + + def _chunks(results: tuple[EvaluationResult, ...]) -> list[_Chunk]: # Validate the finite submission before sending any chunk. Receipts and # prepared bodies use memory proportional to this caller-sized sequence. @@ -120,38 +144,50 @@ def _chunks(results: tuple[EvaluationResult, ...]) -> list[_Chunk]: encoded_results: list[bytes] = [] start = 0 size = len(_upload_body([], [])) - for index, result in enumerate(results): - # Include declarations wherever their results occur. This makes every - # chunk independently replayable even after a lost acknowledgment. - item = ( - declarations.get(result.item_id) if result.item_id not in item_ids else None - ) - extra = len(result._json) + bool(encoded_results) - if item is not None: - extra += len(item) + bool(items) - if encoded_results and ( - len(encoded_results) >= _MAX_RESULTS - or len(items) + (item is not None) > _MAX_ITEMS - or size + extra > _MAX_REQUEST_BYTES - ): + group_start = 0 + for end in _group_ends(results): + group = results[group_start:end] + group_results = [result._json for result in group] + group_items = { + result.item_id: declarations[result.item_id] + for result in group + if result.item_id in declarations + } + while True: + new_items = [body for id, body in group_items.items() if id not in item_ids] + extra = ( + sum(map(len, group_results)) + + len(group_results) + - (not encoded_results) + ) + if new_items: + extra += sum(map(len, new_items)) + len(new_items) - (not items) + fits = ( + len(encoded_results) + len(group) <= _MAX_RESULTS + and len(items) + len(new_items) <= _MAX_ITEMS + and size + extra <= _MAX_REQUEST_BYTES + ) + if fits: + break + if not encoded_results: + raise ValueError( + "One result or composite item group exceeds the upload request limit." + ) chunks.append( _Chunk( - start, results[start:index], _upload_body(items, encoded_results) + start, + results[start:group_start], + _upload_body(items, encoded_results), ) ) - start, size = index, len(_upload_body([], [])) + start, size = group_start, len(_upload_body([], [])) item_ids, items, encoded_results = set(), [], [] - item = declarations.get(result.item_id) - extra = len(result._json) + (len(item) if item is not None else 0) - if size + extra > _MAX_REQUEST_BYTES: - raise ValueError( - "One result and its item exceed the upload request byte limit." - ) - if item is not None: - item_ids.add(result.item_id) - items.append(item) - encoded_results.append(result._json) + # Repeat available declarations in every request for exact recovery. + item_ids.update(group_items) + items.extend(new_items) + encoded_results.extend(group_results) size += extra + group_start = end if encoded_results: chunks.append( _Chunk(start, results[start:], _upload_body(items, encoded_results)) @@ -245,6 +281,11 @@ def upload_results(self, results: Sequence[EvaluationResult]) -> BulkUploadRecei Returns per-chunk receipts; an empty sequence makes no requests. Memory scales with the caller's sequence. For large datasets, call in batches. BulkUploadError exposes committed chunks and pending_results to retry. + Items with calculations keep all their supplied results in one request. + Interleaved items may form a larger indivisible group; an oversized + group fails before any I/O. Group each item's results together when + preparing large batches. Stored inputs require evaluation:read and + llm_analytics:read; ingestion-only keys must supply every pinned input. """ prepared = tuple(results) chunks = _chunks(prepared) diff --git a/posthog/ai/evaluations/_scorer_types.py b/posthog/ai/evaluations/_scorer_types.py index e7aedd95b..7651047a6 100644 --- a/posthog/ai/evaluations/_scorer_types.py +++ b/posthog/ai/evaluations/_scorer_types.py @@ -10,6 +10,7 @@ ScorerKind = Literal["boolean", "numeric", "categorical"] +ScorerCalculation = Literal["reported", "composite"] ScorerOrder = Literal[ "name", "-name", @@ -89,6 +90,43 @@ class CategoricalScorerConfig(TypedDict, total=False): _ConfigT = TypeVar("_ConfigT", bound=ScorerConfig, covariant=True) +class ScorerCompositionInput(TypedDict): + """A formula alias and an exact reported scorer-version UUID.""" + + alias: str + scorer_version_id: str + + +class ScorerComposition(TypedDict): + """Server-evaluated formula and its pinned inputs for the same item. + + Formulas accept 1–16 reported inputs; nested composites are unsupported. + Boolean inputs retain their raw polarity and categorical inputs are key + lists. Use ``number(flag)`` and ``"key" in categories`` explicitly. + """ + + expression: str + inputs: list[ScorerCompositionInput] + + +@dataclass(frozen=True) +class ScorerInput: + """Pinned input metadata alongside the source's latest version. + + A new source version never changes the composite's pinned configuration. + """ + + alias: str + scorer_version_id: str + definition_id: str + name: str + kind: ScorerKind + version: int + config: ScorerConfig + latest_version_id: str | None + latest_version: int | None + + @dataclass(frozen=True) class Scorer(Generic[_ConfigT]): """Scorer metadata and the configuration of its current immutable version. @@ -112,6 +150,10 @@ class Scorer(Generic[_ConfigT]): created_by: dict[str, Any] | None team: int tags: tuple[str, ...] = () + calculation: ScorerCalculation = "reported" + composition: ScorerComposition | None = None + input_scorers: tuple[ScorerInput, ...] = () + dependent_composite_count: int = 0 @dataclass(frozen=True) @@ -125,6 +167,8 @@ class ScorerVersion(Generic[_ConfigT]): config: _ConfigT created_at: datetime created_by: dict[str, Any] | None + calculation: ScorerCalculation = "reported" + composition: ScorerComposition | None = None @dataclass(frozen=True) diff --git a/posthog/ai/evaluations/_scorers.py b/posthog/ai/evaluations/_scorers.py index a99bd7ddf..72ae60b9e 100644 --- a/posthog/ai/evaluations/_scorers.py +++ b/posthog/ai/evaluations/_scorers.py @@ -4,6 +4,7 @@ import copy import json +import re from datetime import datetime from typing import Any, Callable, Literal, Sequence, TypeVar, cast, overload from uuid import UUID @@ -13,7 +14,10 @@ CategoricalScorerConfig, NumericScorerConfig, Scorer, + ScorerCalculation, + ScorerComposition, ScorerConfig, + ScorerInput, ScorerKind, ScorerOrder, ScorerPage, @@ -56,6 +60,89 @@ def _kind(value: Any) -> ScorerKind: return cast(ScorerKind, value) +def _calculation(value: Any) -> ScorerCalculation: + if value not in ("reported", "composite"): + raise ValueError("Unsupported scorer calculation.") + return cast(ScorerCalculation, value) + + +def _alias(value: Any) -> str: + value = _text(value) + if not re.fullmatch(r"[a-z][a-z0-9_]{0,31}", value) or value in { + "number", + "min", + "max", + "abs", + "and", + "or", + "not", + "if", + "else", + "in", + "is", + "lambda", + "for", + "await", + }: + raise ValueError("Invalid composite input alias.") + return value + + +def _composition(value: Any) -> ScorerComposition: + data = _mapping(value) + expression = _text(data.get("expression")) + if not expression.strip() or len(expression) > 1024: + raise ValueError("A formula must contain 1–1024 characters.") + inputs = data.get("inputs") + if not isinstance(inputs, list) or not 1 <= len(inputs) <= 16: + raise ValueError("A composite requires 1–16 reported inputs.") + prepared = [ + { + "alias": _alias(_mapping(row).get("alias")), + "scorer_version_id": _uuid(row.get("scorer_version_id")), + } + for row in inputs + ] + if len({row["alias"] for row in prepared}) != len(prepared): + raise ValueError("Composite input aliases must be unique.") + return cast(ScorerComposition, {"expression": expression, "inputs": prepared}) + + +def _scoring( + data: dict[str, Any], +) -> tuple[ScorerCalculation, ScorerComposition | None]: + calculation = _calculation(data.get("calculation", "reported")) + composition = data.get("composition") + if calculation == "composite": + return calculation, _composition(composition) + if composition is not None: + raise ValueError("Reported scorers cannot contain a composition.") + return calculation, None + + +def _input_scorers(value: Any) -> tuple[ScorerInput, ...]: + if not isinstance(value, list): + raise ValueError("Expected input scorer metadata.") + return tuple( + ScorerInput( + alias=_alias(_mapping(row)["alias"]), + scorer_version_id=_uuid(row["scorer_version_id"]), + definition_id=_uuid(row["definition_id"]), + name=_text(row["name"]), + kind=_kind(row["kind"]), + version=_integer(row["version"], minimum=1), + config=cast(ScorerConfig, _mapping(row["config"])), + latest_version_id=None + if row["latest_version_id"] is None + else _uuid(row["latest_version_id"]), + latest_version=None + if row["latest_version"] is None + else _integer(row["latest_version"], minimum=1), + ) + for row in value + ) + + def _timestamp(value: Any) -> datetime: parsed = datetime.fromisoformat(_text(value).replace("Z", "+00:00")) if parsed.tzinfo is None or parsed.utcoffset() is None: @@ -70,6 +157,7 @@ def _mapping(value: Any) -> dict[str, Any]: def _scorer(data: dict[str, Any]) -> Scorer[ScorerConfig]: + calculation, composition = _scoring(data) archived = data["archived"] if not isinstance(archived, bool): raise ValueError("Expected boolean archived state.") @@ -89,10 +177,17 @@ def _scorer(data: dict[str, Any]) -> Scorer[ScorerConfig]: created_by=None if created_by is None else _mapping(created_by), team=_integer(data["team"], minimum=1), tags=_response_tags(data.get("tags", [])), + calculation=calculation, + composition=composition, + input_scorers=_input_scorers(data.get("input_scorers", [])), + dependent_composite_count=_integer( + data.get("dependent_composite_count", 0), minimum=0 + ), ) def _version(data: dict[str, Any]) -> ScorerVersion[ScorerConfig]: + calculation, composition = _scoring(data) created_by = data["created_by"] return ScorerVersion( id=_uuid(data["id"]), @@ -102,6 +197,8 @@ def _version(data: dict[str, Any]) -> ScorerVersion[ScorerConfig]: config=cast(ScorerConfig, _mapping(data["config"])), created_at=_timestamp(data["created_at"]), created_by=None if created_by is None else _mapping(created_by), + calculation=calculation, + composition=composition, ) @@ -154,12 +251,20 @@ def _create_body( config: ScorerConfig, description: str | None | _Omitted, tags: Sequence[str] | _Omitted, + calculation: ScorerCalculation, + composition: ScorerComposition | None, ) -> bytes: data: dict[str, Any] = { "name": name, "kind": _kind(kind), "config": _mapping(config), } + if _calculation(calculation) == "composite": + data["calculation"] = calculation + data["composition"] = _composition(composition) + _composite_config(kind, config) + elif composition is not None: + raise ValueError("Only composite scorers accept a composition.") if not isinstance(description, _Omitted): data["description"] = description if not isinstance(tags, _Omitted): @@ -192,9 +297,14 @@ def _version_body( base_version: int | None, name: str | _Omitted, description: str | None | _Omitted, + composition: ScorerComposition | None, ) -> bytes: data = _metadata(name, description) data["config"] = _mapping(config) + if composition is not None: + data["composition"] = _composition(composition) + if config.get("step") is not None: + raise ValueError("Numeric composite scorers do not accept a step.") if base_version is not None: data["base_version"] = _integer(base_version, minimum=1) return _encode(data) @@ -209,6 +319,8 @@ def _list_params( order_by: ScorerOrder | None, tags: Sequence[str] | None, tag_match: Literal["any", "all"], + calculation: ScorerCalculation | None, + input_definition_id: str | UUID | None, ) -> dict[str, str | int]: tag_params = _tag_params(tags, tag_match) params: dict[str, str | int] = { @@ -219,6 +331,10 @@ def _list_params( params["search"] = search if kind is not None: params["kind"] = _kind(kind) + if calculation is not None: + params["calculation"] = _calculation(calculation) + if input_definition_id is not None: + params["input_definition_id"] = _uuid(input_definition_id) if archived is not None: if not isinstance(archived, bool): raise ValueError("archived must be a boolean.") @@ -229,6 +345,30 @@ def _list_params( return params +def _composite_config(kind: ScorerKind, config: ScorerConfig) -> None: + if kind == "numeric" and config.get("step") is not None: + raise ValueError("Numeric composite scorers do not accept a step.") + + +def _validation_body( + kind: ScorerKind, composition: ScorerComposition, config: ScorerConfig | None +) -> bytes: + data: dict[str, Any] = { + "kind": _kind(kind), + "composition": _composition(composition), + } + if config is not None: + data["config"] = _mapping(config) + _composite_config(kind, config) + return _encode(data) + + +def _validation(data: dict[str, Any]) -> bool: + if data.get("valid") is not True: + raise ValueError("Expected successful formula validation.") + return True + + def _version_params(limit: int, cursor: str | None) -> dict[str, str | int]: if _integer(limit, minimum=1) > 100: raise ValueError("Version page limit must be between 1 and 100.") @@ -262,6 +402,8 @@ def create( config: BooleanScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED, + calculation: ScorerCalculation = "reported", + composition: ScorerComposition | None = None, ) -> Scorer[BooleanScorerConfig]: ... @overload @@ -273,6 +415,8 @@ def create( config: NumericScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED, + calculation: ScorerCalculation = "reported", + composition: ScorerComposition | None = None, ) -> Scorer[NumericScorerConfig]: ... @overload @@ -284,6 +428,8 @@ def create( config: CategoricalScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED, + calculation: ScorerCalculation = "reported", + composition: ScorerComposition | None = None, ) -> Scorer[CategoricalScorerConfig]: ... def create( @@ -294,17 +440,23 @@ def create( config: ScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED, + calculation: ScorerCalculation = "reported", + composition: ScorerComposition | None = None, ) -> Scorer[ScorerConfig]: """Create a scorer and its first version, optionally assigning tags. Tag names are trimmed, lowercased, deduplicated, and sorted by the server. + Set calculation="composite" with a composition to calculate item scores + on the server. Calculation and output kind cannot change after creation. """ return _parse( _scorer, self._transport.request( "POST", self._base, - body=_create_body(name, kind, config, description, tags), + body=_create_body( + name, kind, config, description, tags, calculation, composition + ), retry_safe=False, ), ) @@ -324,6 +476,8 @@ def list( order_by: ScorerOrder | None = None, tags: Sequence[str] | None = None, tag_match: Literal["any", "all"] = "any", + calculation: ScorerCalculation | None = None, + input_definition_id: str | UUID | None = None, ) -> ScorerPage: """Fetch one page. Omit ``archived`` to list active scorers. @@ -331,6 +485,8 @@ def list( Search matches names and descriptions; scorer names are not unique. ``tags`` accepts up to 20 names. ``tag_match`` matches any (default) or all selected tags; an empty sequence disables tag filtering. + Use calculation to select reported/composite scorers, and + input_definition_id to find composites depending on a source scorer. """ return _parse( _scorer_page, @@ -338,7 +494,16 @@ def list( "GET", self._base, params=_list_params( - limit, offset, search, kind, archived, order_by, tags, tag_match + limit, + offset, + search, + kind, + archived, + order_by, + tags, + tag_match, + calculation, + input_definition_id, ), ), ) @@ -376,12 +541,15 @@ def create_version( base_version: int | None = None, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, + composition: ScorerComposition | None = None, ) -> Scorer[_ConfigT]: """Create an immutable version and return the updated scorer. Pass the observed ``current_version`` as ``base_version`` to detect concurrent writes. Conflicts are surfaced without automatic refresh. Identical configuration is allowed, for example after a prompt change. + Composite versions require composition, including every pinned input, + even when only changing the output configuration. """ return cast( Scorer[_ConfigT], @@ -390,7 +558,9 @@ def create_version( self._transport.request( "POST", self._path(scorer_id) + "new_version/", - body=_version_body(config, base_version, name, description), + body=_version_body( + config, base_version, name, description, composition + ), retry_safe=False, ), ), @@ -407,6 +577,28 @@ def get_version( ), ) + def validate_composition( + self, + *, + kind: ScorerKind, + composition: ScorerComposition, + config: ScorerConfig | None = None, + ) -> bool: + """Validate a formula without saving, using llm_analytics:read access. + + Returns True on validation success; invalid formulas raise + EvaluationAPIError. Omit config to check syntax and types only. + Runtime values and output bounds are checked during calculation. + """ + return _parse( + _validation, + self._transport.request( + "POST", + self._base + "validate_composition/", + body=_validation_body(kind, composition, config), + ), + ) + def list_versions( self, scorer_id: str | UUID, *, limit: int = 50, cursor: str | None = None ) -> ScorerVersionPage: @@ -440,6 +632,8 @@ async def create( config: BooleanScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED, + calculation: ScorerCalculation = "reported", + composition: ScorerComposition | None = None, ) -> Scorer[BooleanScorerConfig]: ... @overload @@ -451,6 +645,8 @@ async def create( config: NumericScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED, + calculation: ScorerCalculation = "reported", + composition: ScorerComposition | None = None, ) -> Scorer[NumericScorerConfig]: ... @overload @@ -462,6 +658,8 @@ async def create( config: CategoricalScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED, + calculation: ScorerCalculation = "reported", + composition: ScorerComposition | None = None, ) -> Scorer[CategoricalScorerConfig]: ... async def create( @@ -472,6 +670,8 @@ async def create( config: ScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED, + calculation: ScorerCalculation = "reported", + composition: ScorerComposition | None = None, ) -> Scorer[ScorerConfig]: """Create a scorer with optional tags. See Scorers.create.""" return _parse( @@ -479,7 +679,9 @@ async def create( await self._transport.request( "POST", self._base, - body=_create_body(name, kind, config, description, tags), + body=_create_body( + name, kind, config, description, tags, calculation, composition + ), retry_safe=False, ), ) @@ -501,6 +703,8 @@ async def list( order_by: ScorerOrder | None = None, tags: Sequence[str] | None = None, tag_match: Literal["any", "all"] = "any", + calculation: ScorerCalculation | None = None, + input_definition_id: str | UUID | None = None, ) -> ScorerPage: """Fetch one page with optional tag filters. See Scorers.list.""" return _parse( @@ -509,7 +713,16 @@ async def list( "GET", self._base, params=_list_params( - limit, offset, search, kind, archived, order_by, tags, tag_match + limit, + offset, + search, + kind, + archived, + order_by, + tags, + tag_match, + calculation, + input_definition_id, ), ), ) @@ -542,6 +755,7 @@ async def create_version( base_version: int | None = None, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, + composition: ScorerComposition | None = None, ) -> Scorer[_ConfigT]: """Create a version; ``base_version`` detects concurrent configuration edits. @@ -554,7 +768,9 @@ async def create_version( await self._transport.request( "POST", self._path(scorer_id) + "new_version/", - body=_version_body(config, base_version, name, description), + body=_version_body( + config, base_version, name, description, composition + ), retry_safe=False, ), ), @@ -571,6 +787,23 @@ async def get_version( ), ) + async def validate_composition( + self, + *, + kind: ScorerKind, + composition: ScorerComposition, + config: ScorerConfig | None = None, + ) -> bool: + """Validate without saving. See Scorers.validate_composition.""" + return _parse( + _validation, + await self._transport.request( + "POST", + self._base + "validate_composition/", + body=_validation_body(kind, composition, config), + ), + ) + async def list_versions( self, scorer_id: str | UUID, *, limit: int = 50, cursor: str | None = None ) -> ScorerVersionPage: diff --git a/posthog/ai/evaluations/_types.py b/posthog/ai/evaluations/_types.py index 595f06920..7365ffb8a 100644 --- a/posthog/ai/evaluations/_types.py +++ b/posthog/ai/evaluations/_types.py @@ -3,7 +3,7 @@ import json from dataclasses import dataclass, field from datetime import datetime -from typing import Any, Literal, Mapping +from typing import Any, Literal, Mapping, cast from uuid import UUID, uuid4 from typing_extensions import TypedDict @@ -59,11 +59,13 @@ class ExperimentOptions(TypedDict, total=False): class ResultOptions(TypedDict, total=False): """Score outcome, trace references, and optional result payload fields. - ``value`` is required for ``ok`` (the default). ``error_code`` and + ``calculate=True`` requests a composite with no value or result metadata. + Otherwise ``value`` is required for ``ok`` (the default). ``error_code`` and ``error_message`` are permitted only for ``error``. Use either ``payload`` or individual payload fields; omitted values differ from explicit None. """ + calculate: bool value: ScoreValue | None status: ResultStatus error_code: str | None @@ -199,10 +201,14 @@ class EvaluationResult: determines whether the value passes. Other statuses contain no score. Pass an item object to declare it, or an item UUID already accepted by the experiment. Payloads and values are snapshotted during construction. + Set ``calculate=True`` to request a composite with no value or metadata. + Include its pinned reported inputs in the same upload unless the caller + has result-read access to reuse stored inputs. """ item: EvaluationItem | str scorer_version_id: str + calculate: bool _json: bytes = field(repr=False) def __init__( @@ -210,6 +216,7 @@ def __init__( *, item: EvaluationItem | str | UUID, scorer_version_id: str | UUID, + calculate: bool = False, value: ScoreValue | None = None, status: ResultStatus = "ok", error_code: str | None = None, @@ -222,6 +229,22 @@ def __init__( ) -> None: item_id = item.id if isinstance(item, EvaluationItem) else _uuid(item) version_id = _uuid(scorer_version_id) + if not isinstance(calculate, bool): + raise ValueError("calculate must be a boolean.") + if calculate and ( + value is not None + or status != "ok" + or error_code is not None + or evaluator_trace_id is not None + or evaluated_at is not None + or any( + not isinstance(field, _Unset) + for field in (reasoning, error_message, metadata, payload) + ) + ): + raise ValueError( + "A calculation accepts only item, scorer_version_id, and calculate=True." + ) if status not in ("ok", "error", "skipped", "not_applicable"): raise ValueError("Unsupported result status.") if status != "ok" and value is not None: @@ -231,9 +254,12 @@ def __init__( data: dict[str, Any] = { "item_id": item_id, "scorer_version_id": version_id, - "status": status, } - if status == "ok": + if calculate: + data["calculate"] = True + else: + data["status"] = status + if status == "ok" and not calculate: data["value"] = _score(value) if error_code is not None: data["error_code"] = _identifier(error_code, "error_code", 128) @@ -260,6 +286,7 @@ def __init__( self, "item", item if isinstance(item, EvaluationItem) else item_id ) object.__setattr__(self, "scorer_version_id", version_id) + object.__setattr__(self, "calculate", calculate) object.__setattr__(self, "_json", _encode(data)) @property @@ -310,10 +337,16 @@ class ItemReceipt: @dataclass(frozen=True) class ResultReceipt(ItemReceipt): - """Persisted result UUID and the item/scorer-version pair it acknowledges.""" + """Persisted result identity and stored outcome, including formula errors. + + ``status=None`` means an older backend omitted the outcome of a reported + result. Composite acknowledgments always require a server outcome. + """ item_id: str scorer_version_id: str + status: ResultStatus | None = None + error_code: str | None = None @dataclass(frozen=True) @@ -401,6 +434,30 @@ def _experiment_receipt(data: dict[str, Any], identity: str) -> ExperimentReceip ) from None +def _result_receipt(data: dict[str, Any]) -> ResultReceipt: + status = data.get("status") + error_code = data.get("error_code") + if "status" in data or "error_code" in data: + if ( + status not in ("ok", "error", "skipped", "not_applicable") + or "error_code" not in data + ): + raise ValueError("Invalid stored result outcome.") + if error_code is not None: + if status != "error": + raise ValueError("Only an error outcome may contain an error code.") + error_code = _identifier(error_code, "error_code", 128) + return ResultReceipt( + id=_uuid(data["id"]), + created=_boolean(data["created"]), + accepted_at=_date(data["accepted_at"]), + item_id=_uuid(data["item_id"]), + scorer_version_id=_uuid(data["scorer_version_id"]), + status=cast(ResultStatus | None, status), + error_code=error_code, + ) + + def _upload_receipt( data: dict[str, Any], results: tuple[EvaluationResult, ...] ) -> UploadReceipt: @@ -415,16 +472,7 @@ def _upload_receipt( ) for row in data["items"] ) - outcomes = tuple( - ResultReceipt( - id=_uuid(_object(row)["id"]), - created=_boolean(row["created"]), - accepted_at=_date(row["accepted_at"]), - item_id=_uuid(row["item_id"]), - scorer_version_id=_uuid(row["scorer_version_id"]), - ) - for row in data["results"] - ) + outcomes = tuple(_result_receipt(_object(row)) for row in data["results"]) if len(items) != len({item.id for item in items}) or { item.id for item in items } != {result.item_id for result in results}: @@ -435,6 +483,13 @@ def _upload_receipt( raise ValueError("Incomplete result acknowledgments.") if len({result.id for result in outcomes}) != len(outcomes): raise ValueError("Duplicate result identities in acknowledgment.") + if any( + result.calculate and outcome.status is None + for result, outcome in zip(results, outcomes) + ): + raise ValueError( + "A composite acknowledgment must include its stored outcome." + ) return UploadReceipt(items=items, results=outcomes) except (KeyError, TypeError, ValueError): raise EvaluationAPIError( diff --git a/posthog/test/ai/test_evaluations.py b/posthog/test/ai/test_evaluations.py index 99bcd962c..a132fa370 100644 --- a/posthog/test/ai/test_evaluations.py +++ b/posthog/test/ai/test_evaluations.py @@ -114,6 +114,283 @@ def result(item=None, **kwargs): ) +def composite(item): + return EvaluationResult( + item=item, scorer_version_id=OTHER_VERSION_ID, calculate=True + ) + + +def composite_response(method, path, *, body): + response = upload_response(body) + for row, declaration in zip(response["results"], json.loads(body)["results"]): + row.update( + status=declaration.get("status", "ok"), + error_code=declaration.get("error_code"), + ) + return response + + +def test_calculation_declarations_round_trip_without_metadata(): + item = EvaluationItem(input="question", model_version="test-model") + declaration = composite(item) + expected = { + "item_id": item.id, + "scorer_version_id": OTHER_VERSION_ID, + "calculate": True, + } + assert declaration.to_dict() == expected + restored = EvaluationResult.from_dict(expected, item=item) + expected["calculate"] = False + assert restored == declaration + assert restored.calculate is True + assert ( + EvaluationResult.from_dict(declaration.to_dict()).to_dict() + == declaration.to_dict() + ) + assert "calculate" not in result(item, calculate=False).to_dict() + + +@pytest.mark.parametrize( + "options", + [ + {"calculate": 1}, + {"calculate": "true"}, + {"calculate": None}, + {"value": False}, + {"value": 0}, + {"status": "error"}, + {"status": "skipped"}, + {"error_code": "bad"}, + {"reasoning": None}, + {"metadata": {}}, + {"error_message": None}, + {"payload": {}}, + {"evaluator_trace_id": "trace"}, + {"evaluated_at": ACCEPTED_AT}, + ], +) +def test_invalid_calculation_fields_fail_before_upload(client, options): + evaluations, transport = client + with pytest.raises(ValueError): + invoke( + evaluations.resume_experiment(EXPERIMENT_ID).upload_result, + **( + { + "item": EvaluationItem(), + "scorer_version_id": OTHER_VERSION_ID, + "calculate": True, + } + | options + ), + ) + transport.request.assert_not_called() + + +@pytest.mark.parametrize( + "status,error_code", + [ + ("ok", None), + ("error", "composite_input_error"), + ("error", "composite_formula_error"), + ("skipped", None), + ("not_applicable", None), + ], +) +def test_calculated_outcome_is_visible_on_initial_upload_and_replay( + client, status, error_code +): + evaluations, transport = client + experiment = evaluations.resume_experiment(EXPERIMENT_ID) + item = EvaluationItem() + + def acknowledge(method, path, *, body): + response = composite_response(method, path, body=body) + response["results"][0].update( + status=status, + error_code=error_code, + created=transport.request.call_count == 1, + ) + return response + + transport.request.side_effect = acknowledge + for created in [True, False]: + receipt = invoke( + experiment.upload_result, + item=item, + scorer_version_id=OTHER_VERSION_ID, + calculate=True, + ) + assert (receipt.results[0].status, receipt.results[0].error_code) == ( + status, + error_code, + ) + assert receipt.results[0].created is created + assert json.loads(transport.request.call_args.kwargs["body"])["results"] == [ + composite(item).to_dict() + ] + assert transport.request.call_args_list[0] == transport.request.call_args_list[1] + + +@pytest.mark.parametrize( + "outcome", + [ + {}, + {"status": "ok"}, + {"status": "unknown", "error_code": None}, + {"status": None, "error_code": None}, + {"status": "ok", "error_code": "unexpected"}, + {"status": "error", "error_code": 42}, + ], +) +def test_invalid_composite_acknowledgments_preserve_uncertain_pending_work( + client, outcome +): + evaluations, transport = client + declarations = [composite(EvaluationItem())] + + def acknowledge(method, path, *, body): + response = upload_response(body) + response["results"][0].update(outcome) + return response + + transport.request.side_effect = acknowledge + with pytest.raises(BulkUploadError) as caught: + invoke( + evaluations.resume_experiment(EXPERIMENT_ID).upload_results, declarations + ) + assert caught.value.code == "invalid_response" + assert caught.value.persistence == "unknown" + assert caught.value.pending_results == tuple(declarations) + + +def test_reported_receipts_accept_legacy_and_new_outcomes(client): + evaluations, transport = client + experiment = evaluations.resume_experiment(EXPERIMENT_ID) + kwargs = dict( + item=EvaluationItem(), + scorer_version_id=VERSION_ID, + status="error", + error_code="judge_failed", + ) + assert invoke(experiment.upload_result, **kwargs).results[0].status is None + transport.request.side_effect = composite_response + receipt = invoke(experiment.upload_result, **kwargs).results[0] + assert (receipt.status, receipt.error_code) == ("error", "judge_failed") + + +@pytest.mark.parametrize("reverse", [False, True]) +def test_real_count_boundary_keeps_inputs_with_calculation_in_original_order( + client, reverse +): + evaluations, transport = client + transport.request.side_effect = composite_response + item = EvaluationItem() + group = [result(item), composite(item)] + if reverse: + group.reverse() + declarations = [result() for _ in range(999)] + group + receipt = invoke( + evaluations.resume_experiment(EXPERIMENT_ID).upload_results, declarations + ) + assert [len(chunk.results) for chunk in receipt.chunks] == [999, 2] + sent = [ + json.loads(call.kwargs["body"]) for call in transport.request.call_args_list + ] + assert sent[1] == { + "items": [item.to_dict()], + "results": [row.to_dict() for row in group], + } + assert [ + (row.item_id, row.scorer_version_id) + for chunk in receipt.chunks + for row in chunk.results + ] == [(row.item_id, row.scorer_version_id) for row in declarations] + + +def test_overlapping_composite_spans_preserve_order_and_item_declarations( + client, monkeypatch +): + evaluations, transport = client + transport.request.side_effect = composite_response + monkeypatch.setattr(_client, "_MAX_RESULTS", 4) + a, b = EvaluationItem(), EvaluationItem() + declarations = [result(), result(a), composite(b.id), composite(a.id), result(b)] + receipt = invoke( + evaluations.resume_experiment(EXPERIMENT_ID).upload_results, declarations + ) + assert [len(chunk.results) for chunk in receipt.chunks] == [1, 4] + body = json.loads(transport.request.call_args.kwargs["body"]) + assert body["results"] == [row.to_dict() for row in declarations[1:]] + assert body["items"] == [a.to_dict(), b.to_dict()] + + +def test_utf8_limit_accounts_for_whole_composite_group(client, monkeypatch): + evaluations, transport = client + transport.request.side_effect = composite_response + experiment = evaluations.resume_experiment(EXPERIMENT_ID) + item = EvaluationItem(input="🙂" * 100) + group = [result(item), composite(item)] + invoke(experiment.upload_results, group) + exact_limit = len(transport.request.call_args.kwargs["body"]) + monkeypatch.setattr(_client, "_MAX_REQUEST_BYTES", exact_limit) + transport.request.reset_mock() + receipt = invoke(experiment.upload_results, [result()] + group) + assert [len(chunk.results) for chunk in receipt.chunks] == [1, 2] + assert len(transport.request.call_args.kwargs["body"]) == exact_limit + monkeypatch.setattr(_client, "_MAX_REQUEST_BYTES", exact_limit - 1) + transport.request.reset_mock() + with pytest.raises(ValueError, match="limit"): + invoke(experiment.upload_results, [result()] + group) + transport.request.assert_not_called() + + +@pytest.mark.parametrize("limit,value", [("_MAX_RESULTS", 3), ("_MAX_ITEMS", 1)]) +def test_unsendable_interleaved_group_rejects_entire_bulk_before_io( + client, monkeypatch, limit, value +): + evaluations, transport = client + a, b = EvaluationItem(), EvaluationItem() + declarations = [result(), result(a), result(b), composite(a), composite(b)] + monkeypatch.setattr(_client, limit, value) + with pytest.raises(ValueError, match="limit"): + invoke( + evaluations.resume_experiment(EXPERIMENT_ID).upload_results, declarations + ) + transport.request.assert_not_called() + + +@pytest.mark.parametrize("persistence", ["unknown", "rejected"]) +def test_composite_partial_failure_replays_exact_group_and_original_index( + client, monkeypatch, persistence +): + evaluations, transport = client + monkeypatch.setattr(_client, "_MAX_RESULTS", 2) + item = EvaluationItem(input="question", model_version="model-b") + declarations = [result(), result(), composite(item), result(item), result()] + + def fail_second(method, path, *, body): + if transport.request.call_count == 2: + raise EvaluationAPIError(status=503, persistence=persistence) + return composite_response(method, path, body=body) + + transport.request.side_effect = fail_second + experiment = evaluations.resume_experiment(EXPERIMENT_ID) + with pytest.raises(BulkUploadError) as caught: + invoke(experiment.upload_results, declarations) + error = caught.value + assert error.failed_index == 2 + assert error.pending_results == tuple(declarations[2:]) + assert len(error.completed.chunks) == 1 + failed_body = transport.request.call_args.kwargs["body"] + assert error.failed_chunk["results"] == [row.to_dict() for row in declarations[2:4]] + transport.request.side_effect = composite_response + invoke(experiment.upload_results, error.pending_results) + assert transport.request.call_args_list[2].kwargs["body"] == failed_body + assert all( + call.args[1].endswith("/upload/") for call in transport.request.call_args_list + ) + + def test_experiment_tags_are_separate_from_creation_and_lifecycle_receipts(client): evaluations, transport = client experiment = invoke(evaluations.create_experiment, name="Tagged run") @@ -854,7 +1131,10 @@ async def run(): asyncio.run(run()) -def test_example_persists_before_writes_and_replays_the_same_run(monkeypatch, tmp_path): +@pytest.mark.parametrize("with_composite", [False, True]) +def test_example_persists_before_writes_and_replays_the_same_run( + monkeypatch, tmp_path, with_composite +): example = Path(__file__).parents[3] / "examples/example-offline-evaluations/main.py" spec = importlib.util.spec_from_file_location("offline_evaluation_example", example) module = importlib.util.module_from_spec(spec) @@ -864,18 +1144,34 @@ def test_example_persists_before_writes_and_replays_the_same_run(monkeypatch, tm monkeypatch.setenv("POSTHOG_SECRET_KEY", "never-persist-this-secret") monkeypatch.setenv("POSTHOG_HOST", "https://us.posthog.com") monkeypatch.setenv("POSTHOG_SCORER_VERSION_ID", VERSION_ID) - monkeypatch.setattr("sys.argv", [str(example), "--state", str(state_path)]) + monkeypatch.setattr( + "sys.argv", + [str(example), "--state", str(state_path)] + + (["--composite-version-id", OTHER_VERSION_ID] if with_composite else []), + ) transport = Mock(spec=SyncTransport) def check_state_before_write(method, path, **kwargs): assert state_path.exists() assert "never-persist-this-secret" not in state_path.read_text() + if with_composite and path.endswith("/upload/"): + body = json.loads(kwargs["body"]) + assert [row.get("calculate", False) for row in body["results"]] == [ + False, + True, + False, + True, + ] + return composite_response(method, path, **kwargs) return respond(method, path, **kwargs) transport.request.side_effect = check_state_before_write monkeypatch.setattr(_client, "SyncTransport", lambda *args: transport) module.main() prepared_state = state_path.read_bytes() + assert json.loads(prepared_state)["experiment"]["expected_result_count"] == ( + 4 if with_composite else 2 + ) module.main() assert state_path.read_bytes() == prepared_state requests = transport.request.call_args_list diff --git a/posthog/test/ai/test_evaluations_scorers.py b/posthog/test/ai/test_evaluations_scorers.py index 54cf97bac..21a37d4bb 100644 --- a/posthog/test/ai/test_evaluations_scorers.py +++ b/posthog/test/ai/test_evaluations_scorers.py @@ -26,6 +26,298 @@ BASE = "/api/projects/123/llm_analytics/score_definitions/" +def composition(): + return { + "expression": "number(correct)", + "inputs": [{"alias": "correct", "scorer_version_id": VERSION_ID}], + } + + +def input_scorer(): + return { + "alias": "correct", + "scorer_version_id": VERSION_ID, + "definition_id": SCORER_ID, + "name": "Correct", + "kind": "boolean", + "version": 1, + "config": {"true_is_failure": True}, + "latest_version_id": "f72481ec-1310-41de-b5bb-d20e1c59f122", + "latest_version": 2, + } + + +@pytest.mark.parametrize( + "kind,config,expression", + [ + ("numeric", {"min": 0, "max": 1}, "number(correct)"), + ("boolean", {"true_is_failure": False}, "correct"), + ( + "categorical", + {"options": [{"key": "good", "label": "Good"}]}, + '"good" if correct else "good"', + ), + ], +) +def test_composite_creation_and_detached_pinned_metadata( + scorers, kind, config, expression +): + namespace, transport = scorers + formula = composition() + formula["expression"] = expression + response = scorer_response( + kind=kind, + config=config, + calculation="composite", + composition=formula, + input_scorers=[input_scorer()], + dependent_composite_count=2, + ) + transport.request.return_value = response + scorer = invoke( + namespace.create, + name="Overall", + kind=kind, + config=config, + calculation="composite", + composition=formula, + ) + sent = json.loads(transport.request.call_args.kwargs["body"]) + assert sent == { + "name": "Overall", + "kind": kind, + "config": config, + "calculation": "composite", + "composition": formula, + } + assert transport.request.call_args.kwargs["retry_safe"] is False + assert scorer.calculation == "composite" + assert scorer.composition == formula + assert scorer.dependent_composite_count == 2 + pinned = scorer.input_scorers[0] + assert (pinned.version, pinned.latest_version) == (1, 2) + assert pinned.scorer_version_id == VERSION_ID + assert pinned.config == {"true_is_failure": True} + formula["inputs"][0]["alias"] = "changed" + response["input_scorers"][0]["config"]["true_is_failure"] = False + assert scorer.composition["inputs"][0]["alias"] == "correct" + assert pinned.config["true_is_failure"] is True + assert json.loads(transport.request.call_args.kwargs["body"]) == sent + + +def test_composite_versioning_history_and_dependency_filters(scorers): + namespace, transport = scorers + formula = composition() + transport.request.return_value = scorer_response( + calculation="composite", composition=formula + ) + invoke( + namespace.create_version, + SCORER_ID, + config={}, + composition=formula, + base_version=1, + ) + assert json.loads(transport.request.call_args.kwargs["body"]) == { + "config": {}, + "composition": formula, + "base_version": 1, + } + assert transport.request.call_args.kwargs["retry_safe"] is False + transport.request.return_value = version_response( + calculation="composite", composition=formula + ) + version = invoke(namespace.get_version, SCORER_ID, VERSION_ID) + assert version.composition == formula + assert version.calculation == "composite" + transport.request.return_value = { + "count": 1, + "results": [version_response(calculation="composite", composition=formula)], + "next_cursor": None, + } + assert invoke(namespace.list_versions, SCORER_ID).results == (version,) + transport.request.return_value = { + "count": 0, + "results": [], + "next": None, + "previous": None, + } + invoke( + namespace.list, + calculation="composite", + input_definition_id=UUID(SCORER_ID), + kind="numeric", + tags=["quality"], + offset=100, + ) + params = transport.request.call_args.kwargs["params"] + assert params["calculation"] == "composite" + assert params["input_definition_id"] == SCORER_ID + assert params["kind"] == "numeric" and params["offset"] == 100 + + +def test_legacy_scorers_and_versions_default_to_reported(scorers): + namespace, transport = scorers + scorer = invoke( + namespace.create, + name="Reported", + kind="numeric", + config={}, + calculation="reported", + ) + assert json.loads(transport.request.call_args.kwargs["body"]) == { + "name": "Reported", + "kind": "numeric", + "config": {}, + } + assert ( + scorer.calculation, + scorer.composition, + scorer.input_scorers, + scorer.dependent_composite_count, + ) == ( + "reported", + None, + (), + 0, + ) + transport.request.return_value = version_response() + version = invoke(namespace.get_version, SCORER_ID, VERSION_ID) + assert (version.calculation, version.composition) == ("reported", None) + + +@pytest.mark.parametrize("config", [None, {}, {"min": 0, "max": 1}]) +def test_validate_composition_is_read_only_and_preserves_optional_config( + scorers, config +): + namespace, transport = scorers + transport.request.return_value = {"valid": True} + assert ( + invoke( + namespace.validate_composition, + kind="numeric", + composition=composition(), + config=config, + ) + is True + ) + assert transport.request.call_args.args == ("POST", BASE + "validate_composition/") + body = json.loads(transport.request.call_args.kwargs["body"]) + assert body == { + "kind": "numeric", + "composition": composition(), + **({"config": config} if config is not None else {}), + } + transport.request.assert_called_once() + + +@pytest.mark.parametrize( + "response", [{}, {"valid": False}, {"valid": 1}, {"valid": "true"}] +) +def test_malformed_formula_validation_is_not_success(scorers, response): + namespace, transport = scorers + transport.request.return_value = response + with pytest.raises(EvaluationAPIError) as caught: + invoke( + namespace.validate_composition, kind="numeric", composition=composition() + ) + assert caught.value.code == "invalid_response" + + +def test_formula_validation_surfaces_server_access_and_formula_errors(scorers): + namespace, transport = scorers + for status, code in [(403, "permission_denied"), (400, "invalid_input")]: + error = EvaluationAPIError(status=status, code=code, persistence="rejected") + transport.request.side_effect = error + with pytest.raises(EvaluationAPIError) as caught: + invoke( + namespace.validate_composition, + kind="numeric", + composition=composition(), + ) + assert caught.value is error + + +@pytest.mark.parametrize( + "formula", + [ + {}, + {"expression": "", "inputs": []}, + {"expression": "x" * 1025, "inputs": composition()["inputs"]}, + {"expression": "correct", "inputs": []}, + {"expression": "correct", "inputs": composition()["inputs"] * 17}, + {"expression": "correct", "inputs": composition()["inputs"] * 2}, + { + "expression": "correct", + "inputs": [{"alias": "number", "scorer_version_id": VERSION_ID}], + }, + { + "expression": "correct", + "inputs": [{"alias": "bad-name", "scorer_version_id": VERSION_ID}], + }, + { + "expression": "correct", + "inputs": [{"alias": "correct", "scorer_version_id": "not-a-uuid"}], + }, + ], +) +def test_invalid_compositions_fail_before_io(scorers, formula): + namespace, transport = scorers + with pytest.raises(ValueError): + invoke( + namespace.create, + name="Overall", + kind="numeric", + config={}, + calculation="composite", + composition=formula, + ) + transport.request.assert_not_called() + + +@pytest.mark.parametrize( + "options", + [ + {"calculation": "unknown"}, + {"calculation": "reported", "composition": composition()}, + {"calculation": "composite"}, + { + "calculation": "composite", + "composition": composition(), + "config": {"step": 0.1}, + }, + ], +) +def test_invalid_calculation_configuration_fails_before_io(scorers, options): + namespace, transport = scorers + with pytest.raises(ValueError): + invoke( + namespace.create, + **({"name": "Overall", "kind": "numeric", "config": {}} | options), + ) + transport.request.assert_not_called() + + +@pytest.mark.parametrize( + "overrides", + [ + {"calculation": "unknown"}, + {"calculation": "composite"}, + {"composition": composition()}, + {"dependent_composite_count": -1}, + {"input_scorers": None}, + {"input_scorers": [input_scorer() | {"scorer_version_id": "invalid"}]}, + {"input_scorers": [input_scorer() | {"version": True}]}, + ], +) +def test_malformed_composite_metadata_is_not_acknowledged(scorers, overrides): + namespace, transport = scorers + transport.request.return_value = scorer_response(**overrides) + with pytest.raises(EvaluationAPIError) as caught: + invoke(namespace.get, SCORER_ID) + assert caught.value.code == "invalid_response" + + def scorer_response(**overrides): return { "id": SCORER_ID, diff --git a/references/public_api_snapshot.txt b/references/public_api_snapshot.txt index bfffd5d64..7b5f841fc 100644 --- a/references/public_api_snapshot.txt +++ b/references/public_api_snapshot.txt @@ -488,6 +488,7 @@ attribute posthog.ai.evaluations.EvaluationAPIError.retry_after = retry_after attribute posthog.ai.evaluations.EvaluationAPIError.status = status attribute posthog.ai.evaluations.EvaluationAPIError.submission = deepcopy(submission) attribute posthog.ai.evaluations.EvaluationItem.id: str +attribute posthog.ai.evaluations.EvaluationResult.calculate: bool attribute posthog.ai.evaluations.EvaluationResult.item: EvaluationItem | str attribute posthog.ai.evaluations.EvaluationResult.item_id: str attribute posthog.ai.evaluations.EvaluationResult.scorer_version_id: str @@ -529,6 +530,7 @@ attribute posthog.ai.evaluations.NumericScorerConfig.passing_rule: NumericPassin attribute posthog.ai.evaluations.NumericScorerConfig.step: float | None attribute posthog.ai.evaluations.OfflineEvaluations.datasets = Datasets(self._transport, project_id) attribute posthog.ai.evaluations.OfflineEvaluations.scorers = Scorers(self._transport, project_id) +attribute posthog.ai.evaluations.ResultOptions.calculate: bool attribute posthog.ai.evaluations.ResultOptions.error_code: str | None attribute posthog.ai.evaluations.ResultOptions.error_message: str | None attribute posthog.ai.evaluations.ResultOptions.evaluated_at: datetime | str | None @@ -538,25 +540,45 @@ attribute posthog.ai.evaluations.ResultOptions.payload: dict[str, JSONValue] attribute posthog.ai.evaluations.ResultOptions.reasoning: str | None attribute posthog.ai.evaluations.ResultOptions.status: ResultStatus attribute posthog.ai.evaluations.ResultOptions.value: ScoreValue | None +attribute posthog.ai.evaluations.ResultReceipt.error_code: str | None = None attribute posthog.ai.evaluations.ResultReceipt.item_id: str attribute posthog.ai.evaluations.ResultReceipt.scorer_version_id: str +attribute posthog.ai.evaluations.ResultReceipt.status: ResultStatus | None = None attribute posthog.ai.evaluations.ResultStatus = Literal['ok', 'error', 'skipped', 'not_applicable'] attribute posthog.ai.evaluations.RunSource = Literal['ci', 'local', 'scheduled'] attribute posthog.ai.evaluations.ScoreValue = bool | int | float | list[str] attribute posthog.ai.evaluations.Scorer.archived: bool +attribute posthog.ai.evaluations.Scorer.calculation: ScorerCalculation = 'reported' +attribute posthog.ai.evaluations.Scorer.composition: ScorerComposition | None = None attribute posthog.ai.evaluations.Scorer.config: _ConfigT attribute posthog.ai.evaluations.Scorer.created_at: datetime attribute posthog.ai.evaluations.Scorer.created_by: dict[str, Any] | None attribute posthog.ai.evaluations.Scorer.current_version: int attribute posthog.ai.evaluations.Scorer.current_version_id: str +attribute posthog.ai.evaluations.Scorer.dependent_composite_count: int = 0 attribute posthog.ai.evaluations.Scorer.description: str attribute posthog.ai.evaluations.Scorer.id: str +attribute posthog.ai.evaluations.Scorer.input_scorers: tuple[ScorerInput, ...] = () attribute posthog.ai.evaluations.Scorer.kind: ScorerKind attribute posthog.ai.evaluations.Scorer.name: str attribute posthog.ai.evaluations.Scorer.tags: tuple[str, ...] = () attribute posthog.ai.evaluations.Scorer.team: int attribute posthog.ai.evaluations.Scorer.updated_at: datetime +attribute posthog.ai.evaluations.ScorerCalculation = Literal['reported', 'composite'] +attribute posthog.ai.evaluations.ScorerComposition.expression: str +attribute posthog.ai.evaluations.ScorerComposition.inputs: list[ScorerCompositionInput] +attribute posthog.ai.evaluations.ScorerCompositionInput.alias: str +attribute posthog.ai.evaluations.ScorerCompositionInput.scorer_version_id: str attribute posthog.ai.evaluations.ScorerConfig = BooleanScorerConfig | NumericScorerConfig | CategoricalScorerConfig +attribute posthog.ai.evaluations.ScorerInput.alias: str +attribute posthog.ai.evaluations.ScorerInput.config: ScorerConfig +attribute posthog.ai.evaluations.ScorerInput.definition_id: str +attribute posthog.ai.evaluations.ScorerInput.kind: ScorerKind +attribute posthog.ai.evaluations.ScorerInput.latest_version: int | None +attribute posthog.ai.evaluations.ScorerInput.latest_version_id: str | None +attribute posthog.ai.evaluations.ScorerInput.name: str +attribute posthog.ai.evaluations.ScorerInput.scorer_version_id: str +attribute posthog.ai.evaluations.ScorerInput.version: int attribute posthog.ai.evaluations.ScorerKind = Literal['boolean', 'numeric', 'categorical'] attribute posthog.ai.evaluations.ScorerOrder = Literal['name', '-name', 'kind', '-kind', 'created_at', '-created_at', 'updated_at', '-updated_at', 'current_version', '-current_version'] attribute posthog.ai.evaluations.ScorerPage.count: int @@ -564,6 +586,8 @@ attribute posthog.ai.evaluations.ScorerPage.next: str | None attribute posthog.ai.evaluations.ScorerPage.next_offset: int | None attribute posthog.ai.evaluations.ScorerPage.previous: str | None attribute posthog.ai.evaluations.ScorerPage.results: tuple[Scorer[ScorerConfig], ...] +attribute posthog.ai.evaluations.ScorerVersion.calculation: ScorerCalculation = 'reported' +attribute posthog.ai.evaluations.ScorerVersion.composition: ScorerComposition | None = None attribute posthog.ai.evaluations.ScorerVersion.config: _ConfigT attribute posthog.ai.evaluations.ScorerVersion.created_at: datetime attribute posthog.ai.evaluations.ScorerVersion.created_by: dict[str, Any] | None @@ -1269,7 +1293,7 @@ class posthog.ai.evaluations.DatasetSnapshot(dataset: Dataset, revision: int, re class posthog.ai.evaluations.Datasets(transport: SyncTransport, project_id: int) class posthog.ai.evaluations.EvaluationAPIError(*, status: int | None = None, code: str | None = None, detail: Any = None, attr: str | None = None, errors: Any = None, response: dict[str, Any] | None = None, retry_after: float | None = None, persistence: Literal['rejected', 'unknown', 'not_sent'] = 'unknown', submission: dict[str, Any] | None = None) class posthog.ai.evaluations.EvaluationItem(*, id: str | UUID | None = None, input: JSONValue | _Unset = _UNSET, output: JSONValue | _Unset = _UNSET, expected_output: JSONValue | _Unset = _UNSET, metadata: dict[str, JSONValue] | None | _Unset = _UNSET, case_key: str | None = None, trial: str | None = None, dataset_item_identifier: str | None = None, dataset_item_version_identifier: str | None = None, dataset_item_version_id: str | UUID | None = None, application_trace_id: str | None = None, model_version: str | None = None, payload: dict[str, JSONValue] | _Unset = _UNSET) -class posthog.ai.evaluations.EvaluationResult(*, item: EvaluationItem | str | UUID, scorer_version_id: str | UUID, value: ScoreValue | None = None, status: ResultStatus = 'ok', error_code: str | None = None, reasoning: str | None | _Unset = _UNSET, error_message: str | None | _Unset = _UNSET, metadata: dict[str, JSONValue] | None | _Unset = _UNSET, evaluator_trace_id: str | None = None, evaluated_at: datetime | str | None = None, payload: dict[str, JSONValue] | _Unset = _UNSET) +class posthog.ai.evaluations.EvaluationResult(*, item: EvaluationItem | str | UUID, scorer_version_id: str | UUID, calculate: bool = False, value: ScoreValue | None = None, status: ResultStatus = 'ok', error_code: str | None = None, reasoning: str | None | _Unset = _UNSET, error_message: str | None | _Unset = _UNSET, metadata: dict[str, JSONValue] | None | _Unset = _UNSET, evaluator_trace_id: str | None = None, evaluated_at: datetime | str | None = None, payload: dict[str, JSONValue] | _Unset = _UNSET) class posthog.ai.evaluations.Experiment(transport: SyncTransport, project_id: int, id: str, *, receipt: ExperimentReceipt | None = None, submission: bytes | None = None) class posthog.ai.evaluations.ExperimentOptions class posthog.ai.evaluations.ExperimentReceipt(id: str, status: Literal['uploading', 'completed', 'failed'], created: bool, started_at: datetime, created_at: datetime, finished_at: datetime | None, expected_item_count: int | None, expected_result_count: int | None, accepted_item_count: int, accepted_result_count: int) @@ -1278,10 +1302,13 @@ class posthog.ai.evaluations.NumericPassingRule class posthog.ai.evaluations.NumericScorerConfig class posthog.ai.evaluations.OfflineEvaluations(*, project_id: int, secret_key: str, host: str | None = None, timeout: float = 15, max_retries: int = 3) class posthog.ai.evaluations.ResultOptions -class posthog.ai.evaluations.ResultReceipt(id: str, created: bool, accepted_at: datetime, item_id: str, scorer_version_id: str) -class posthog.ai.evaluations.Scorer(id: str, name: str, description: str, kind: ScorerKind, archived: bool, current_version: int, current_version_id: str, config: _ConfigT, created_at: datetime, updated_at: datetime, created_by: dict[str, Any] | None, team: int, tags: tuple[str, ...] = ()) +class posthog.ai.evaluations.ResultReceipt(id: str, created: bool, accepted_at: datetime, item_id: str, scorer_version_id: str, status: ResultStatus | None = None, error_code: str | None = None) +class posthog.ai.evaluations.Scorer(id: str, name: str, description: str, kind: ScorerKind, archived: bool, current_version: int, current_version_id: str, config: _ConfigT, created_at: datetime, updated_at: datetime, created_by: dict[str, Any] | None, team: int, tags: tuple[str, ...] = (), calculation: ScorerCalculation = 'reported', composition: ScorerComposition | None = None, input_scorers: tuple[ScorerInput, ...] = (), dependent_composite_count: int = 0) +class posthog.ai.evaluations.ScorerComposition +class posthog.ai.evaluations.ScorerCompositionInput +class posthog.ai.evaluations.ScorerInput(alias: str, scorer_version_id: str, definition_id: str, name: str, kind: ScorerKind, version: int, config: ScorerConfig, latest_version_id: str | None, latest_version: int | None) class posthog.ai.evaluations.ScorerPage(count: int, results: tuple[Scorer[ScorerConfig], ...], next: str | None, previous: str | None, next_offset: int | None) -class posthog.ai.evaluations.ScorerVersion(id: str, definition_id: str, version: int, kind: ScorerKind, config: _ConfigT, created_at: datetime, created_by: dict[str, Any] | None) +class posthog.ai.evaluations.ScorerVersion(id: str, definition_id: str, version: int, kind: ScorerKind, config: _ConfigT, created_at: datetime, created_by: dict[str, Any] | None, calculation: ScorerCalculation = 'reported', composition: ScorerComposition | None = None) class posthog.ai.evaluations.ScorerVersionPage(count: int, results: tuple[ScorerVersion[ScorerConfig], ...], next_cursor: str | None) class posthog.ai.evaluations.Scorers(transport: SyncTransport, project_id: int) class posthog.ai.evaluations.UploadReceipt(items: tuple[ItemReceipt, ...], results: tuple[ResultReceipt, ...]) @@ -1669,13 +1696,14 @@ method posthog.ai.evaluations.AsyncExperiment.upload_results(results: Sequence[E method posthog.ai.evaluations.AsyncOfflineEvaluations.aclose() method posthog.ai.evaluations.AsyncOfflineEvaluations.create_experiment(*, name: str, **options: Unpack[ExperimentOptions]) method posthog.ai.evaluations.AsyncOfflineEvaluations.resume_experiment(id: str | UUID) -method posthog.ai.evaluations.AsyncScorers.create(*, name: str, kind: ScorerKind, config: ScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED) -method posthog.ai.evaluations.AsyncScorers.create_version(scorer_id: str | UUID, *, config: _ConfigT, base_version: int | None = None, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED) +method posthog.ai.evaluations.AsyncScorers.create(*, name: str, kind: ScorerKind, config: ScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED, calculation: ScorerCalculation = 'reported', composition: ScorerComposition | None = None) +method posthog.ai.evaluations.AsyncScorers.create_version(scorer_id: str | UUID, *, config: _ConfigT, base_version: int | None = None, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, composition: ScorerComposition | None = None) method posthog.ai.evaluations.AsyncScorers.get(scorer_id: str | UUID) method posthog.ai.evaluations.AsyncScorers.get_version(scorer_id: str | UUID, version_id: str | UUID) -method posthog.ai.evaluations.AsyncScorers.list(*, limit: int = 100, offset: int = 0, search: str | None = None, kind: ScorerKind | None = None, archived: bool | None = None, order_by: ScorerOrder | None = None, tags: Sequence[str] | None = None, tag_match: Literal['any', 'all'] = 'any') +method posthog.ai.evaluations.AsyncScorers.list(*, limit: int = 100, offset: int = 0, search: str | None = None, kind: ScorerKind | None = None, archived: bool | None = None, order_by: ScorerOrder | None = None, tags: Sequence[str] | None = None, tag_match: Literal['any', 'all'] = 'any', calculation: ScorerCalculation | None = None, input_definition_id: str | UUID | None = None) method posthog.ai.evaluations.AsyncScorers.list_versions(scorer_id: str | UUID, *, limit: int = 50, cursor: str | None = None) method posthog.ai.evaluations.AsyncScorers.update(scorer_id: str | UUID, *, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, archived: bool | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED) +method posthog.ai.evaluations.AsyncScorers.validate_composition(*, kind: ScorerKind, composition: ScorerComposition, config: ScorerConfig | None = None) method posthog.ai.evaluations.DatasetSnapshot.iter_items(*, page_size: int = 25) method posthog.ai.evaluations.Datasets.archive(dataset_id: str | UUID) method posthog.ai.evaluations.Datasets.archive_item(item_id: str | UUID, *, base_version: int) @@ -1706,13 +1734,14 @@ method posthog.ai.evaluations.Experiment.upload_results(results: Sequence[Evalua method posthog.ai.evaluations.OfflineEvaluations.close() method posthog.ai.evaluations.OfflineEvaluations.create_experiment(*, name: str, **options: Unpack[ExperimentOptions]) method posthog.ai.evaluations.OfflineEvaluations.resume_experiment(id: str | UUID) -method posthog.ai.evaluations.Scorers.create(*, name: str, kind: ScorerKind, config: ScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED) -method posthog.ai.evaluations.Scorers.create_version(scorer_id: str | UUID, *, config: _ConfigT, base_version: int | None = None, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED) +method posthog.ai.evaluations.Scorers.create(*, name: str, kind: ScorerKind, config: ScorerConfig, description: str | None | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED, calculation: ScorerCalculation = 'reported', composition: ScorerComposition | None = None) +method posthog.ai.evaluations.Scorers.create_version(scorer_id: str | UUID, *, config: _ConfigT, base_version: int | None = None, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, composition: ScorerComposition | None = None) method posthog.ai.evaluations.Scorers.get(scorer_id: str | UUID) method posthog.ai.evaluations.Scorers.get_version(scorer_id: str | UUID, version_id: str | UUID) -method posthog.ai.evaluations.Scorers.list(*, limit: int = 100, offset: int = 0, search: str | None = None, kind: ScorerKind | None = None, archived: bool | None = None, order_by: ScorerOrder | None = None, tags: Sequence[str] | None = None, tag_match: Literal['any', 'all'] = 'any') +method posthog.ai.evaluations.Scorers.list(*, limit: int = 100, offset: int = 0, search: str | None = None, kind: ScorerKind | None = None, archived: bool | None = None, order_by: ScorerOrder | None = None, tags: Sequence[str] | None = None, tag_match: Literal['any', 'all'] = 'any', calculation: ScorerCalculation | None = None, input_definition_id: str | UUID | None = None) method posthog.ai.evaluations.Scorers.list_versions(scorer_id: str | UUID, *, limit: int = 50, cursor: str | None = None) method posthog.ai.evaluations.Scorers.update(scorer_id: str | UUID, *, name: str | _Omitted = _OMITTED, description: str | None | _Omitted = _OMITTED, archived: bool | _Omitted = _OMITTED, tags: Sequence[str] | _Omitted = _OMITTED) +method posthog.ai.evaluations.Scorers.validate_composition(*, kind: ScorerKind, composition: ScorerComposition, config: ScorerConfig | None = None) method posthog.ai.gemini.gemini.Models.embed_content(model: str, contents, posthog_distinct_id: Optional[str] = None, posthog_trace_id: Optional[str] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_privacy_mode: Optional[bool] = None, posthog_groups: Optional[Dict[str, Any]] = None, **kwargs: Any) method posthog.ai.gemini.gemini.Models.generate_content(model: str, contents, posthog_distinct_id: Optional[str] = None, posthog_trace_id: Optional[str] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_privacy_mode: Optional[bool] = None, posthog_groups: Optional[Dict[str, Any]] = None, **kwargs: Any) method posthog.ai.gemini.gemini.Models.generate_content_stream(model: str, contents, posthog_distinct_id: Optional[str] = None, posthog_trace_id: Optional[str] = None, posthog_properties: Optional[Dict[str, Any]] = None, posthog_privacy_mode: Optional[bool] = None, posthog_groups: Optional[Dict[str, Any]] = None, **kwargs: Any)