From 8d027f8147b1fa4c9951cb32eebbaff2c70b5f31 Mon Sep 17 00:00:00 2001 From: pateti Chandu Date: Wed, 23 Sep 2026 12:59:42 +0530 Subject: [PATCH 1/7] feat: add production safety foundations --- opendecision/__init__.py | 20 ++-- opendecision/audit.py | 52 +++++++++ opendecision/errors.py | 16 +++ opendecision/evaluation.py | 31 +++++ opendecision/guard.py | 176 ++++++++++++----------------- opendecision/models.py | 94 +++++++++++---- opendecision/policies.py | 52 ++++++--- opendecision/providers/__init__.py | 4 +- opendecision/providers/base.py | 13 +-- opendecision/providers/chain.py | 28 +++++ opendecision/providers/laya.py | 74 ++++-------- opendecision/providers/rules.py | 48 ++++++++ opendecision/review.py | 39 +++++++ 13 files changed, 435 insertions(+), 212 deletions(-) create mode 100644 opendecision/audit.py create mode 100644 opendecision/errors.py create mode 100644 opendecision/evaluation.py create mode 100644 opendecision/providers/chain.py create mode 100644 opendecision/providers/rules.py create mode 100644 opendecision/review.py diff --git a/opendecision/__init__.py b/opendecision/__init__.py index 1a3224e..5936208 100644 --- a/opendecision/__init__.py +++ b/opendecision/__init__.py @@ -1,15 +1,19 @@ -"""OpenDecision: a typed decision layer for AI agents.""" +"""OpenDecision: an auditable typed decision layer for AI agents.""" +from .audit import AuditEvent, JsonlAuditSink, MemoryAuditSink +from .evaluation import EvaluationCase, EvaluationReport, evaluate from .guard import DecisionGuard -from .models import DecisionQuestion, DecisionResult +from .models import DecisionContext, DecisionQuestion, DecisionResult from .policies import DecisionPolicy, load_policy +from .providers import LayaProvider, ProviderChain, Rule, RuleProvider +from .review import InMemoryReviewStore, ReviewRequest, ReviewStatus __all__ = [ - "DecisionGuard", - "DecisionQuestion", - "DecisionResult", - "DecisionPolicy", - "load_policy", + "AuditEvent", "DecisionContext", "DecisionGuard", "DecisionPolicy", + "DecisionQuestion", "DecisionResult", "EvaluationCase", "EvaluationReport", + "InMemoryReviewStore", "JsonlAuditSink", "LayaProvider", "MemoryAuditSink", + "ProviderChain", "ReviewRequest", "ReviewStatus", "Rule", "RuleProvider", + "evaluate", "load_policy", ] -__version__ = "0.1.0" +__version__ = "0.2.0" diff --git a/opendecision/audit.py b/opendecision/audit.py new file mode 100644 index 0000000..c371379 --- /dev/null +++ b/opendecision/audit.py @@ -0,0 +1,52 @@ +"""Privacy-preserving decision audit sinks.""" + +from __future__ import annotations + +import json +from pathlib import Path +from threading import Lock +from typing import Any, Protocol + +from pydantic import BaseModel, ConfigDict, Field + +from .models import DecisionResult + +class AuditEvent(BaseModel): + model_config = ConfigDict(extra="forbid") + decision_id: str + created_at: str + context_hash: str + provider: str + decision: str | float | bool + confidence: float | None + policy_id: str | None + policy_version: str | None + requires_review: bool + actor: str | None = None + tool: str | None = None + action_id: str | None = None + metadata: dict[str, Any] = Field(default_factory=dict) + + @classmethod + def from_result(cls, result: DecisionResult, *, actor: str | None = None, tool: str | None = None, action_id: str | None = None) -> AuditEvent: + return cls(decision_id=result.decision_id, created_at=result.created_at.isoformat(), context_hash=result.context_hash, provider=result.provider, decision=result.decision, confidence=result.confidence, policy_id=result.policy_id, policy_version=result.policy_version, requires_review=result.requires_review, actor=actor, tool=tool, action_id=action_id) + +class AuditSink(Protocol): + def write(self, event: AuditEvent) -> None: ... + +class MemoryAuditSink: + def __init__(self) -> None: + self.events: list[AuditEvent] = [] + def write(self, event: AuditEvent) -> None: + self.events.append(event) + +class JsonlAuditSink: + """Append redacted events; raw state and raw provider output are never written.""" + def __init__(self, path: str | Path) -> None: + self.path = Path(path) + self.path.parent.mkdir(parents=True, exist_ok=True) + self._lock = Lock() + def write(self, event: AuditEvent) -> None: + line = json.dumps(event.model_dump(mode="json"), sort_keys=True) + with self._lock, self.path.open("a", encoding="utf-8") as handle: + handle.write(line + "\n") diff --git a/opendecision/errors.py b/opendecision/errors.py new file mode 100644 index 0000000..3a8bcb7 --- /dev/null +++ b/opendecision/errors.py @@ -0,0 +1,16 @@ +"""OpenDecision exception hierarchy.""" + +class OpenDecisionError(Exception): + """Base error for OpenDecision.""" + +class ContractValidationError(OpenDecisionError): + """Raised when decision input or provider output violates its contract.""" + +class ProviderError(OpenDecisionError): + """Raised when a decision provider cannot return a valid answer.""" + +class ProvidersExhaustedError(ProviderError): + """Raised when every provider in a fallback chain fails.""" + +class ReviewLoopError(OpenDecisionError): + """Raised when an action exceeds the configured review-attempt limit.""" diff --git a/opendecision/evaluation.py b/opendecision/evaluation.py new file mode 100644 index 0000000..85d027d --- /dev/null +++ b/opendecision/evaluation.py @@ -0,0 +1,31 @@ +"""Small reproducible evaluation primitives for decision packs.""" + +from __future__ import annotations +from collections import Counter +from pydantic import BaseModel, ConfigDict, Field + +class EvaluationCase(BaseModel): + model_config = ConfigDict(extra="forbid") + case_id: str + expected: str + predicted: str + probabilities: dict[str, float] = Field(default_factory=dict) + +class EvaluationReport(BaseModel): + total: int + correct: int + accuracy: float + brier_score: float | None + confusion: dict[str, int] + +def evaluate(cases: list[EvaluationCase]) -> EvaluationReport: + if not cases: + raise ValueError("at least one evaluation case is required") + correct = sum(case.expected == case.predicted for case in cases) + confusion = Counter(f"{case.expected}->{case.predicted}" for case in cases) + brier_values: list[float] = [] + for case in cases: + if case.probabilities: + labels = set(case.probabilities) | {case.expected} + brier_values.append(sum((case.probabilities.get(label, 0.0) - (1.0 if label == case.expected else 0.0)) ** 2 for label in labels) / len(labels)) + return EvaluationReport(total=len(cases), correct=correct, accuracy=correct / len(cases), brier_score=(sum(brier_values) / len(brier_values) if brier_values else None), confusion=dict(confusion)) diff --git a/opendecision/guard.py b/opendecision/guard.py index f03e64e..d68bbb3 100644 --- a/opendecision/guard.py +++ b/opendecision/guard.py @@ -1,114 +1,78 @@ -"""DecisionGuard and provider-output normalization.""" - +"""Decision guard with strict validation, audit traces, and fail-safe behavior.""" from __future__ import annotations - +import hashlib, json from collections.abc import Mapping -from typing import Any - -from .models import DecisionQuestion, DecisionResult +from datetime import datetime, timezone +from typing import Any, Literal +from uuid import uuid4 +from .audit import AuditEvent, AuditSink +from .errors import ContractValidationError, ProviderError +from .models import DecisionContext, DecisionQuestion, DecisionResult, ProviderAttempt from .providers.base import DecisionProvider from .providers.laya import LayaProvider - +FailureMode = Literal["raise", "review", "block"] class DecisionGuard: - """Evaluate typed decision contracts before an agent takes an action.""" - - def __init__(self, provider: DecisionProvider | None = None, *, model: str = "router") -> None: - self.provider = provider or LayaProvider(model=model) - - def decide( - self, - state: Mapping[str, Any] | str, - question: DecisionQuestion | Mapping[str, Any] | str, - ) -> DecisionResult: - """Evaluate a choice, score, or noul question and normalize its result.""" + def __init__(self, provider: DecisionProvider | None = None, *, model: str = "router", audit_sink: AuditSink | None = None, failure_mode: FailureMode = "raise", max_state_bytes: int = 100_000) -> None: + self.provider, self.audit_sink, self.failure_mode, self.max_state_bytes = provider or LayaProvider(model=model), audit_sink, failure_mode, max_state_bytes + def decide(self, state: Mapping[str, Any] | str, question: DecisionQuestion | Mapping[str, Any] | str, *, context: DecisionContext | Mapping[str, Any] | None = None) -> DecisionResult: contract = DecisionQuestion.from_input(question) - output = self.provider.predict(state, contract) - answer = output.get("answer", output) - raw_output = output.get("raw", output) - return self._normalize(answer, raw_output, contract) - - def check(self, state: Mapping[str, Any] | str, *, decision: str) -> DecisionResult: - """Convenience method for a boolean allow/block gate.""" - return self.decide(state, DecisionQuestion(type="noul", instructions=decision)) - - def _normalize( - self, - answer: Mapping[str, Any], - raw_output: Any, - question: DecisionQuestion, - ) -> DecisionResult: - if not isinstance(answer, Mapping): - raise TypeError("decision provider must return a mapping for its answer") - - if question.type == "choice": - probabilities = _float_mapping(answer.get("probabilities") or answer.get("probs")) - decision = _first(answer, "choice", "label", "decision") - if decision is None: - raise ValueError("provider response did not contain a choice decision") - confidence = _as_probability(_first(answer, "confidence")) - if confidence is None and probabilities: - confidence = max(probabilities.values()) - return DecisionResult( - decision=str(decision), - probabilities=probabilities, - confidence=confidence, - question_type="choice", - provider=self.provider.name, - metadata={"raw_answer": dict(answer)}, - raw_output=raw_output, - ) - - if question.type == "score": + ctx = context if isinstance(context, DecisionContext) else DecisionContext.model_validate(context or {}) + context_hash = self._hash(state) + try: + output = self.provider.predict(state, contract) + result = self._normalize(output.get("answer", output), output.get("raw", output), contract, ctx, context_hash, str(output.get("provider", self.provider.name)), [ProviderAttempt.model_validate(x) for x in output.get("attempts", [])]) + except Exception as exc: + if self.failure_mode == "raise": + if isinstance(exc, (ContractValidationError, ProviderError)): raise + raise ProviderError(str(exc)) from exc + result = self._failure(contract, ctx, context_hash, exc) + if self.audit_sink: self.audit_sink.write(AuditEvent.from_result(result, actor=ctx.actor, tool=ctx.tool, action_id=ctx.action_id)) + return result + def check(self, state: Mapping[str, Any] | str, *, decision: str, context: DecisionContext | Mapping[str, Any] | None = None) -> DecisionResult: + return self.decide(state, DecisionQuestion(type="noul", instructions=decision), context=context) + def _hash(self, state: Mapping[str, Any] | str) -> str: + if isinstance(state, str): + if not state.strip(): raise ContractValidationError("state string cannot be empty") + data = state.encode() + elif isinstance(state, Mapping): + try: data = json.dumps(state, sort_keys=True, separators=(",", ":")).encode() + except (TypeError, ValueError) as exc: raise ContractValidationError("state must be JSON serializable") from exc + else: raise ContractValidationError("state must be a string or mapping") + if len(data) > self.max_state_bytes: raise ContractValidationError(f"state exceeds configured limit of {self.max_state_bytes} bytes") + return hashlib.sha256(data).hexdigest() + def _normalize(self, answer: Mapping[str, Any], raw: Any, q: DecisionQuestion, ctx: DecisionContext, context_hash: str, provider: str, attempts: list[ProviderAttempt]) -> DecisionResult: + if not isinstance(answer, Mapping): raise ContractValidationError("provider answer must be a mapping") + common = dict(provider=provider, decision_id=str(uuid4()), created_at=datetime.now(timezone.utc), context_hash=context_hash, policy_id=ctx.policy_id, policy_version=ctx.policy_version, provider_attempts=attempts, raw_output=raw) + if q.type == "choice": + probs = _probs(answer.get("probabilities") or answer.get("probs")); decision = _first(answer, "choice", "label", "decision") + choices = q.options or (q.criteria if isinstance(q.criteria, dict) else {}); labels = set(choices) + if decision not in labels: raise ContractValidationError(f"provider choice {decision!r} is not one of {sorted(labels)}") + if probs and set(probs) != labels: raise ContractValidationError("provider probabilities must match declared options") + confidence = _prob(_first(answer, "confidence")) or (probs.get(str(decision)) if probs else None) + return DecisionResult(decision=str(decision), probabilities=probs, confidence=confidence, question_type="choice", requires_review=str(decision) == "review", metadata={"raw_answer": dict(answer)}, **common) + if q.type == "score": score = _first(answer, "score", "value", "decision") - if score is None: - raise ValueError("provider response did not contain a score decision") - distribution = _float_mapping(answer.get("distribution") or answer.get("probabilities")) - confidence = _as_probability(_first(answer, "confidence")) - return DecisionResult( - decision=float(score), - probabilities=distribution, - confidence=confidence, - question_type="score", - provider=self.provider.name, - metadata={"raw_answer": dict(answer)}, - raw_output=raw_output, - ) - - probability = _as_probability( - _first(answer, "noul", "probability", "confidence", "decision") - ) - if probability is None: - raise ValueError("provider response did not contain a noul probability") - decision = probability >= question.threshold - return DecisionResult( - decision=decision, - probabilities={"true": probability, "false": 1.0 - probability}, - confidence=max(probability, 1.0 - probability), - question_type="noul", - provider=self.provider.name, - metadata={"threshold": question.threshold, "raw_answer": dict(answer)}, - raw_output=raw_output, - ) - - -def _first(mapping: Mapping[str, Any], *keys: str) -> Any: - for key in keys: - if mapping.get(key) is not None: - return mapping[key] - return None - - -def _as_probability(value: Any) -> float | None: - if value is None: - return None - value = float(value) - if not 0.0 <= value <= 1.0: - raise ValueError(f"probability must be between 0 and 1, got {value}") - return value - - -def _float_mapping(value: Any) -> dict[str, float]: - if not isinstance(value, Mapping): - return {} - return {str(key): float(probability) for key, probability in value.items()} + if score is None: raise ContractValidationError("provider response did not contain a score") + return DecisionResult(decision=float(score), probabilities=_probs(answer.get("distribution") or answer.get("probabilities")), confidence=_prob(_first(answer, "confidence")), question_type="score", metadata={"raw_answer": dict(answer)}, **common) + p = _prob(_first(answer, "noul", "probability", "confidence", "decision")) + if p is None: raise ContractValidationError("provider response did not contain a noul probability") + return DecisionResult(decision=p >= q.threshold, probabilities={"true": p, "false": 1-p}, confidence=max(p,1-p), question_type="noul", metadata={"threshold": q.threshold, "raw_answer": dict(answer)}, **common) + def _failure(self, q: DecisionQuestion, ctx: DecisionContext, context_hash: str, error: Exception) -> DecisionResult: + review = self.failure_mode == "review" + if q.type == "choice": + choices = q.options or (q.criteria if isinstance(q.criteria, dict) else {}); labels = list(choices); preferred = "review" if review and "review" in labels else "block"; decision: str | bool = preferred if preferred in labels else labels[-1] + else: decision = False + return DecisionResult(decision=decision, probabilities={}, confidence=None, question_type=q.type, provider="failure_policy", decision_id=str(uuid4()), created_at=datetime.now(timezone.utc), context_hash=context_hash, policy_id=ctx.policy_id, policy_version=ctx.policy_version, requires_review=review, metadata={"failure_mode": self.failure_mode, "error": f"{type(error).__name__}: {error}"}) + +def _first(m: Mapping[str, Any], *keys: str) -> Any: + return next((m[k] for k in keys if m.get(k) is not None), None) +def _prob(value: Any) -> float | None: + if value is None: return None + p=float(value) + if not 0<=p<=1: raise ContractValidationError(f"probability must be between 0 and 1, got {value}") + return p +def _probs(value: Any) -> dict[str,float]: + if value is None: return {} + if not isinstance(value, Mapping): raise ContractValidationError("probabilities must be a mapping") + return {str(k): float(_prob(v)) for k,v in value.items()} diff --git a/opendecision/models.py b/opendecision/models.py index 04f1c60..57b00f6 100644 --- a/opendecision/models.py +++ b/opendecision/models.py @@ -1,32 +1,51 @@ -"""Public data models and input normalization for OpenDecision.""" +"""Strict public data models for OpenDecision.""" from __future__ import annotations - +import re from collections.abc import Mapping +from datetime import datetime from typing import Any, Literal - -from pydantic import BaseModel, ConfigDict, Field, model_validator +from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator DecisionType = Literal["choice", "score", "noul"] - +RiskLevel = Literal["low", "medium", "high", "critical"] +_OPTION_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") class DecisionQuestion(BaseModel): - """A typed question sent to a decision provider.""" - - model_config = ConfigDict(extra="allow") - + model_config = ConfigDict(extra="forbid") type: DecisionType - instructions: str + instructions: str = Field(min_length=3, max_length=2000) options: dict[str, str] | None = None criteria: list[str] | dict[str, str] | None = None threshold: float = Field(default=0.5, ge=0.0, le=1.0) + @field_validator("options") + @classmethod + def validate_option_names(cls, value: dict[str, str] | None) -> dict[str, str] | None: + if value is None: + return value + if len(value) < 2: + raise ValueError("choice questions require at least two options") + for name, description in value.items(): + if not _OPTION_RE.fullmatch(name): + raise ValueError("option names must be lowercase snake_case and at most 64 characters") + if not description.strip(): + raise ValueError(f"option {name!r} requires a description") + return value + @model_validator(mode="after") def validate_shape(self) -> DecisionQuestion: - if self.type == "choice" and not self.options and not self.criteria: - raise ValueError("choice questions require options or criteria") - if self.type == "score" and not self.criteria: - raise ValueError("score questions require criteria") + if self.type == "choice": + values = self.options or self.criteria + if not isinstance(values, dict) or len(values) < 2: + raise ValueError("choice questions require at least two named options") + elif self.type == "score": + if not isinstance(self.criteria, list) or len(self.criteria) < 2: + raise ValueError("score questions require an ordered list of at least two criteria") + if len({item.strip().casefold() for item in self.criteria}) != len(self.criteria): + raise ValueError("score criteria must be unique") + elif self.options is not None or self.criteria is not None: + raise ValueError("noul questions do not accept options or criteria") return self @classmethod @@ -38,23 +57,54 @@ def from_input(cls, question: DecisionQuestion | Mapping[str, Any] | str) -> Dec return cls.model_validate(question) def to_laya(self) -> dict[str, Any]: - """Return the question shape expected by Laya.""" - return self.model_dump(exclude_none=True) - + payload: dict[str, Any] = {"type": self.type, "instructions": self.instructions} + if self.type == "choice": + payload["criteria"] = self.options or self.criteria + elif self.type == "score": + payload["criteria"] = self.criteria + return payload + +class DecisionContext(BaseModel): + model_config = ConfigDict(extra="forbid") + actor: str | None = Field(default=None, max_length=200) + tool: str | None = Field(default=None, max_length=200) + action_id: str | None = Field(default=None, max_length=200) + policy_id: str | None = Field(default=None, max_length=200) + policy_version: str | None = Field(default=None, max_length=50) + risk: RiskLevel = "medium" + +class ProviderAttempt(BaseModel): + provider: str + success: bool + latency_ms: float = Field(ge=0) + error: str | None = None class DecisionResult(BaseModel): - """A normalized, provider-independent decision result.""" - - model_config = ConfigDict(arbitrary_types_allowed=True) - + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") decision: str | float | bool probabilities: dict[str, float] = Field(default_factory=dict) confidence: float | None = Field(default=None, ge=0.0, le=1.0) question_type: DecisionType provider: str + decision_id: str + created_at: datetime + context_hash: str + policy_id: str | None = None + policy_version: str | None = None + requires_review: bool = False + provider_attempts: list[ProviderAttempt] = Field(default_factory=list) metadata: dict[str, Any] = Field(default_factory=dict) raw_output: Any = None + @field_validator("probabilities") + @classmethod + def validate_probabilities(cls, value: dict[str, float]) -> dict[str, float]: + for name, probability in value.items(): + if not 0.0 <= probability <= 1.0: + raise ValueError(f"probability {name!r} must be between 0 and 1") + if value and abs(sum(value.values()) - 1.0) > 0.05: + raise ValueError("probabilities must sum to approximately 1") + return value + def matches(self, value: str | float | bool) -> bool: - """Return whether the normalized decision equals a value.""" return self.decision == value diff --git a/opendecision/policies.py b/opendecision/policies.py index 21e16d9..d1cc6d3 100644 --- a/opendecision/policies.py +++ b/opendecision/policies.py @@ -1,28 +1,52 @@ -"""Small YAML-backed decision-pack loader.""" +"""Versioned YAML policy loading and composition.""" from __future__ import annotations - from pathlib import Path -from typing import Any - +from typing import Any, Literal import yaml -from pydantic import BaseModel, Field - -from .models import DecisionQuestion - +from pydantic import BaseModel, ConfigDict, Field, model_validator +from .models import DecisionQuestion, RiskLevel class DecisionPolicy(BaseModel): - """A named, reviewable decision contract stored in a decision pack.""" - + model_config = ConfigDict(extra="forbid") + id: str = "" + version: str = "1.0.0" name: str description: str = "" + risk: RiskLevel = "medium" + default_action: Literal["allow", "review", "block"] = "review" question: DecisionQuestion tags: list[str] = Field(default_factory=list) + metadata: dict[str, Any] = Field(default_factory=dict) + @model_validator(mode="before") + @classmethod + def derive_legacy_id(cls, value: Any) -> Any: + if isinstance(value, dict) and not value.get("id") and value.get("name"): + value = dict(value) + value["id"] = str(value["name"]).strip().lower().replace(" ", "-") + return value def load_policy(path: str | Path) -> DecisionPolicy: - """Load a policy from a YAML file.""" - policy_path = Path(path) - with policy_path.open("r", encoding="utf-8") as handle: + return DecisionPolicy.model_validate(_load_payload(Path(path).resolve(), set())) + +def _load_payload(path: Path, seen: set[Path]) -> dict[str, Any]: + if path in seen: + raise ValueError(f"policy inheritance cycle detected at {path}") + seen.add(path) + with path.open("r", encoding="utf-8") as handle: payload: dict[str, Any] = yaml.safe_load(handle) or {} - return DecisionPolicy.model_validate(payload) + extends = payload.pop("extends", None) + if not extends: + return payload + base = _load_payload((path.parent / str(extends)).resolve(), seen) + return _merge(base, payload) + +def _merge(base: dict[str, Any], overlay: dict[str, Any]) -> dict[str, Any]: + merged = dict(base) + for key, value in overlay.items(): + if isinstance(value, dict) and isinstance(merged.get(key), dict): + merged[key] = _merge(merged[key], value) + else: + merged[key] = value + return merged diff --git a/opendecision/providers/__init__.py b/opendecision/providers/__init__.py index 707e9db..8f45ada 100644 --- a/opendecision/providers/__init__.py +++ b/opendecision/providers/__init__.py @@ -1,4 +1,6 @@ from .base import DecisionProvider +from .chain import ProviderChain from .laya import LayaProvider +from .rules import Rule, RuleProvider -__all__ = ["DecisionProvider", "LayaProvider"] +__all__ = ["DecisionProvider", "LayaProvider", "ProviderChain", "Rule", "RuleProvider"] diff --git a/opendecision/providers/base.py b/opendecision/providers/base.py index 818119a..3ac41f4 100644 --- a/opendecision/providers/base.py +++ b/opendecision/providers/base.py @@ -1,19 +1,8 @@ -"""Provider protocol used by the decision guard.""" - from __future__ import annotations - from collections.abc import Mapping from typing import Any, Protocol - from ..models import DecisionQuestion - class DecisionProvider(Protocol): name: str - - def predict( - self, - state: Mapping[str, Any] | str, - question: DecisionQuestion, - ) -> Mapping[str, Any]: - """Return a provider-native answer and the raw response.""" + def predict(self, state: Mapping[str, Any] | str, question: DecisionQuestion) -> Mapping[str, Any]: ... diff --git a/opendecision/providers/chain.py b/opendecision/providers/chain.py new file mode 100644 index 0000000..af60ad4 --- /dev/null +++ b/opendecision/providers/chain.py @@ -0,0 +1,28 @@ +from __future__ import annotations +import time +from collections.abc import Mapping +from typing import Any +from ..errors import ProvidersExhaustedError +from ..models import DecisionQuestion +from .base import DecisionProvider + +class ProviderChain: + name = "provider_chain" + def __init__(self, providers: list[DecisionProvider], *, terminal_answer: Mapping[str, Any] | None = None) -> None: + if not providers: + raise ValueError("provider chain requires at least one provider") + self.providers = providers + self.terminal_answer = dict(terminal_answer) if terminal_answer else None + def predict(self, state: Mapping[str, Any] | str, question: DecisionQuestion) -> Mapping[str, Any]: + attempts: list[dict[str, Any]] = [] + for provider in self.providers: + started = time.perf_counter() + try: + response = provider.predict(state, question) + attempts.append({"provider": provider.name, "success": True, "latency_ms": (time.perf_counter() - started) * 1000}) + return {"answer": response.get("answer", response), "raw": response.get("raw", response), "provider": provider.name, "attempts": attempts} + except Exception as exc: + attempts.append({"provider": provider.name, "success": False, "latency_ms": (time.perf_counter() - started) * 1000, "error": f"{type(exc).__name__}: {exc}"}) + if self.terminal_answer is not None: + return {"answer": self.terminal_answer, "raw": {"terminal_fallback": True}, "provider": "terminal_fallback", "attempts": attempts} + raise ProvidersExhaustedError(f"all providers failed: {attempts}") diff --git a/opendecision/providers/laya.py b/opendecision/providers/laya.py index d254b26..b46cf41 100644 --- a/opendecision/providers/laya.py +++ b/opendecision/providers/laya.py @@ -1,63 +1,39 @@ -"""Lazy Laya provider. - -Laya is optional so the core package can be installed and tested without model -weights. The import and checkpoint loading happen only on the first prediction. -""" - from __future__ import annotations - from collections.abc import Mapping from typing import Any - +from ..errors import ProviderError from ..models import DecisionQuestion - class LayaProvider: - """Use Laya's Router or a named checkpoint as an OpenDecision provider.""" - name = "laya" - - def __init__( - self, - *, - model: str = "router", - preload: bool = True, - device: str | None = None, - ) -> None: - self.model = model - self.preload = preload - self.device = device - self._agent: Any = None - + def __init__(self, *, model: str = "router", preload: bool = True, device: str | None = None) -> None: + self.model, self.preload, self.device, self._agent = model, preload, device, None def _load(self) -> Any: if self._agent is not None: return self._agent try: - import laya # type: ignore + import laya except ImportError as exc: - raise RuntimeError( - "Laya is not installed. Install it with `pip install opendecision[laya]`." - ) from exc - - if self.model == "router": - kwargs: dict[str, Any] = {"preload": self.preload} - if self.device is not None: - kwargs["device"] = self.device - self._agent = laya.Router(**kwargs) - else: - kwargs = {} - if self.device is not None: - kwargs["device"] = self.device - self._agent = laya.load("convaiinnovations/laya", subfolder=self.model, **kwargs) + raise ProviderError("Laya is not installed. Install it with `pip install opendecision[laya]`.") from exc + try: + if self.model == "router": + kwargs: dict[str, Any] = {"preload": self.preload} + if self.device is not None: kwargs["device"] = self.device + self._agent = laya.Router(**kwargs) + else: + kwargs = {} + if self.device is not None: kwargs["device"] = self.device + self._agent = laya.load("convaiinnovations/laya", subfolder=self.model, **kwargs) + except Exception as exc: + raise ProviderError(f"failed to load Laya model {self.model!r}: {exc}") from exc return self._agent - - def predict( - self, - state: Mapping[str, Any] | str, - question: DecisionQuestion, - ) -> Mapping[str, Any]: + def predict(self, state: Mapping[str, Any] | str, question: DecisionQuestion) -> Mapping[str, Any]: agent = self._load() - state_payload = state if isinstance(state, Mapping) else {"input": state} - response = agent.predict(state_payload, {"decision": question.to_laya()}) - answer = response.get("answers", {}).get("decision", {}) - return {"answer": answer, "raw": response} + try: + response = agent.predict(state if isinstance(state, Mapping) else {"input": state}, {"decision": question.to_laya()}) + except Exception as exc: + raise ProviderError(f"Laya prediction failed: {exc}") from exc + answer = response.get("answers", {}).get("decision") + if not isinstance(answer, Mapping): + raise ProviderError("Laya response did not include answers.decision") + return {"answer": answer, "raw": response, "provider": f"laya:{self.model}"} diff --git a/opendecision/providers/rules.py b/opendecision/providers/rules.py new file mode 100644 index 0000000..26ac2ab --- /dev/null +++ b/opendecision/providers/rules.py @@ -0,0 +1,48 @@ +from __future__ import annotations +import re +from collections.abc import Mapping +from typing import Any, Literal +from pydantic import BaseModel, ConfigDict, Field +from ..models import DecisionQuestion +RuleOperator = Literal["equals", "contains", "in", "regex", "exists"] + +class Rule(BaseModel): + model_config = ConfigDict(extra="forbid") + field: str + operator: RuleOperator + value: Any = None + decision: str | bool | float + confidence: float = Field(default=1.0, ge=0.0, le=1.0) + def matches(self, state: Mapping[str, Any]) -> bool: + actual: Any = state + for part in self.field.split("."): + if not isinstance(actual, Mapping) or part not in actual: return False + actual = actual[part] + if self.operator == "exists": return actual is not None + if self.operator == "equals": return bool(actual == self.value) + if self.operator == "contains": return str(self.value).casefold() in str(actual).casefold() + if self.operator == "in": return bool(actual in self.value) + if self.operator == "regex": return re.search(str(self.value), str(actual), re.IGNORECASE) is not None + return False + +class RuleProvider: + name = "rules" + def __init__(self, rules: list[Rule], *, default: str | bool | float | None = None) -> None: + self.rules, self.default = rules, default + def predict(self, state: Mapping[str, Any] | str, question: DecisionQuestion) -> Mapping[str, Any]: + payload = state if isinstance(state, Mapping) else {"input": state} + for rule in self.rules: + if rule.matches(payload): return {"answer": _answer(question, rule.decision, rule.confidence), "raw": {"rule": rule.model_dump()}} + if self.default is None: raise LookupError("no deterministic rule matched") + return {"answer": _answer(question, self.default, 1.0), "raw": {"default": True}} + +def _answer(question: DecisionQuestion, decision: str | bool | float, confidence: float) -> dict[str, Any]: + if question.type == "choice": + choices = question.options or (question.criteria if isinstance(question.criteria, dict) else {}) + labels = list(choices.keys()) + if str(decision) not in labels: raise ValueError(f"rule decision {decision!r} is not a declared option") + remaining = (1.0 - confidence) / max(len(labels) - 1, 1) + probabilities = {label: (confidence if label == decision else remaining) for label in labels} + return {"choice": str(decision), "probabilities": probabilities, "confidence": confidence} + if question.type == "score": return {"score": float(decision), "confidence": confidence} + return {"noul": confidence if bool(decision) else 1.0 - confidence} diff --git a/opendecision/review.py b/opendecision/review.py new file mode 100644 index 0000000..3e18d31 --- /dev/null +++ b/opendecision/review.py @@ -0,0 +1,39 @@ +from __future__ import annotations +from datetime import datetime, timezone +from enum import Enum +from uuid import uuid4 +from pydantic import BaseModel, ConfigDict, Field +from .errors import ReviewLoopError +from .models import DecisionResult + +class ReviewStatus(str, Enum): + pending, approved, rejected = "pending", "approved", "rejected" + +class ReviewRequest(BaseModel): + model_config = ConfigDict(extra="forbid") + review_id: str = Field(default_factory=lambda: str(uuid4())) + action_id: str + decision_id: str + status: ReviewStatus = ReviewStatus.pending + created_at: datetime = Field(default_factory=lambda: datetime.now(timezone.utc)) + resolved_at: datetime | None = None + reviewer: str | None = None + reason: str | None = None + final_decision: str | bool | float | None = None + +class InMemoryReviewStore: + def __init__(self, *, max_attempts_per_action: int = 3) -> None: + self.max_attempts_per_action = max_attempts_per_action + self.requests: dict[str, ReviewRequest] = {} + def create(self, action_id: str, result: DecisionResult) -> ReviewRequest: + if sum(item.action_id == action_id for item in self.requests.values()) >= self.max_attempts_per_action: + raise ReviewLoopError(f"action {action_id!r} exceeded review attempt limit") + request = ReviewRequest(action_id=action_id, decision_id=result.decision_id) + self.requests[request.review_id] = request + return request + def resolve(self, review_id: str, *, approved: bool, reviewer: str, reason: str, final_decision: str | bool | float | None = None) -> ReviewRequest: + request = self.requests[review_id] + if request.status is not ReviewStatus.pending: raise ValueError("review request is already resolved") + request.status = ReviewStatus.approved if approved else ReviewStatus.rejected + request.resolved_at, request.reviewer, request.reason, request.final_decision = datetime.now(timezone.utc), reviewer, reason, final_decision + return request From bca164609c31ee05678541d4dfc419783c5b1661 Mon Sep 17 00:00:00 2001 From: pateti Chandu Date: Wed, 23 Sep 2026 13:00:39 +0530 Subject: [PATCH 2/7] docs: add production guidance and policy coverage --- .github/workflows/tests.yml | 10 +- CHANGELOG.md | 19 +++ README.md | 155 ++++++------------ decision_packs/data/data_access.yaml | 14 ++ decision_packs/finance/payment_action.yaml | 14 ++ decision_packs/hr/permission_change.yaml | 14 ++ decision_packs/integrations/external_api.yaml | 14 ++ .../security/destructive_command.yaml | 14 ++ decision_packs/security/privacy_exposure.yaml | 14 ++ docs/production-readiness.md | 48 ++++++ pyproject.toml | 31 +++- 11 files changed, 235 insertions(+), 112 deletions(-) create mode 100644 CHANGELOG.md create mode 100644 decision_packs/data/data_access.yaml create mode 100644 decision_packs/finance/payment_action.yaml create mode 100644 decision_packs/hr/permission_change.yaml create mode 100644 decision_packs/integrations/external_api.yaml create mode 100644 decision_packs/security/destructive_command.yaml create mode 100644 decision_packs/security/privacy_exposure.yaml create mode 100644 docs/production-readiness.md diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 1cc0fd8..f55a93e 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -5,10 +5,14 @@ on: branches: [main, "feature/**"] pull_request: +permissions: + contents: read + jobs: test: runs-on: ubuntu-latest strategy: + fail-fast: false matrix: python-version: ["3.10", "3.11", "3.12", "3.13"] steps: @@ -16,7 +20,11 @@ jobs: - uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} + cache: pip - run: python -m pip install --upgrade pip - run: python -m pip install -e ".[dev]" - - run: pytest + - run: coverage run -m pytest + - run: coverage report + - run: ruff check . + - run: mypy opendecision - run: python -m compileall opendecision diff --git a/CHANGELOG.md b/CHANGELOG.md new file mode 100644 index 0000000..5404453 --- /dev/null +++ b/CHANGELOG.md @@ -0,0 +1,19 @@ +# Changelog + +## 0.2.0 - Unreleased + +### Added + +- Strict decision contracts and provider-output validation. +- Deterministic rules provider and ordered provider fallback chains. +- Fail-safe review and block modes. +- Versioned, inheritable policies with risk and default-action metadata. +- Privacy-preserving audit sinks and provider-attempt telemetry. +- Human-review lifecycle with repeated-loop protection. +- Accuracy, confusion, and Brier-score evaluation primitives. +- Coverage, linting, strict typing, and expanded CI gates. + +### Changed + +- Laya choice contracts now translate `options` to Laya's `criteria` schema. +- Package maturity remains pre-1.0; this release does not claim production safety guarantees. diff --git a/README.md b/README.md index 5d16a23..5fd5183 100644 --- a/README.md +++ b/README.md @@ -1,132 +1,77 @@ # OpenDecision -**A framework-agnostic decision layer for AI agents.** +An auditable, typed decision layer for AI agents. -OpenDecision places a typed, testable contract between an AI agent and the action it wants to take. It uses [Laya](https://github.com/NandhaKishorM/laya) for fast `choice`, `score`, and `noul` decisions without adding another text-generation step. +OpenDecision places explicit contracts, deterministic rules, provider fallback, human review, and privacy-preserving audit traces between an agent and consequential actions. It is a control-layer library, not a safety guarantee. -> Status: early MVP. Do not treat uncalibrated model confidence as a production safety guarantee. +## Production-foundation preview -## Why OpenDecision? +Version 0.2 adds: -Agents repeatedly need to decide whether to continue, stop, call a tool, ask a person, retrieve more context, or escalate. OpenDecision makes those branches explicit and reusable: +- strict decision-contract and provider-output validation; +- deterministic rules plus ordered provider fallback chains; +- fail-safe `raise`, `review`, and `block` behavior; +- policy IDs, semantic versions, risk levels, defaults, and inheritance; +- context hashes, decision IDs, provider attempts, and JSONL audit sinks; +- an in-memory human-review workflow with loop protection; +- reproducible accuracy, confusion, and Brier-score evaluation primitives; +- coverage, lint, strict typing, and Python 3.10–3.13 CI gates. -- typed decision contracts instead of free-form generated text; -- normalized results across decision providers; -- community-maintained YAML decision packs; -- adapters that do not couple the core SDK to one agent framework; -- optional Laya loading, so importing OpenDecision never downloads a model. +## Example -## Install - -```bash -pip install -e ".[laya]" -``` - -Laya and OpenDecision require Python 3.10 or newer. For the examples: - -```bash -pip install -e ".[laya,langgraph,fastapi]" -``` +```python +from opendecision import ( + DecisionContext, + DecisionGuard, + JsonlAuditSink, + ProviderChain, + Rule, + RuleProvider, +) -## Quick start +rules = RuleProvider( + [Rule(field="command", operator="contains", value="rm -rf", decision="block")], +) +providers = ProviderChain([rules], terminal_answer={"choice": "review"}) -```python -from opendecision import DecisionGuard +guard = DecisionGuard( + providers, + audit_sink=JsonlAuditSink("audit/decisions.jsonl"), + failure_mode="review", +) -guard = DecisionGuard() result = guard.decide( - state={"tool": "send_email", "recipient": "customer@example.com"}, - question={ + {"command": "rm -rf /tmp/cache"}, + { "type": "choice", - "instructions": "Should the agent execute this tool action?", + "instructions": "Should this command execute?", "options": { - "allow": "Proceed automatically", - "review": "Require human approval", - "block": "Stop the action", + "allow": "Authorized and low risk", + "review": "Needs human approval", + "block": "Unauthorized or destructive", }, }, + context=DecisionContext( + actor="agent:ops", + tool="shell", + action_id="run-123", + policy_id="shell-command-risk", + policy_version="1.0.0", + risk="critical", + ), ) - -print(result.decision) -print(result.probabilities) -print(result.confidence) -``` - -OpenDecision returns model decisions and evidence; it does **not** ask Laya to generate a reason. Applications may add an explanation separately without confusing generated prose with decision-model output. - -## Decision packs - -```python -from opendecision import DecisionGuard, load_policy - -policy = load_policy("decision_packs/security/tool_risk.yaml") -result = DecisionGuard().decide(agent_state, policy.question) -``` - -The first pack includes a tool-risk contract with `allow`, `review`, and `block` outcomes. - -## LangGraph - -```python -from opendecision import DecisionGuard -from opendecision.integrations.langgraph import decision_node, route_by_decision - -graph.add_node("tool_guard", decision_node(DecisionGuard(), question)) -graph.add_conditional_edges( - "tool_guard", - route_by_decision(), - {"allow": "execute_tool", "review": "human_review", "block": "stop"}, -) -``` - -## FastAPI - -```bash -uvicorn app.main:app --reload ``` -Then send `POST /v1/decide` with `state` and a typed `question`. - -## Architecture - -```text -Agent / application - | -DecisionGuard + contract - | -Provider adapter (Laya first) - | -Normalized DecisionResult - | -ALLOW | REVIEW | BLOCK -``` - -## Scope of v0.1 - -- Python SDK and Pydantic contracts -- lazy Laya provider -- choice, score, and noul normalization -- YAML decision packs -- LangGraph node and routing helpers -- FastAPI example -- unit tests that run without downloading model weights - -A dashboard, TypeScript SDK, additional frameworks, model-generated explanations, and evaluation tooling are intentionally deferred. - -## Calibration and safety - -Laya's documentation notes meaningful limitations: base checkpoints can be weak zero-shot on typed workflows, confidence may require domain calibration, and high-cardinality choices need special handling. Benchmark decision packs on representative data before automating consequential actions. Prefer human review when evidence or authorization is insufficient. +Raw state is hashed for correlation and is not written by the built-in audit sinks. Applications remain responsible for authentication, authorization, durable review storage, encryption, retention, monitoring, calibration, and incident response. ## Development ```bash pip install -e ".[dev]" -pytest +coverage run -m pytest +coverage report ruff check . +mypy opendecision ``` -See [CONTRIBUTING.md](CONTRIBUTING.md) for contribution guidelines. - -## License - -Apache License 2.0. Laya is a separate Apache-2.0 project and remains subject to its own license and notices. +See `docs/production-readiness.md` for operating guidance. diff --git a/decision_packs/data/data_access.yaml b/decision_packs/data/data_access.yaml new file mode 100644 index 0000000..29651a3 --- /dev/null +++ b/decision_packs/data/data_access.yaml @@ -0,0 +1,14 @@ +id: data-access +version: 1.0.0 +name: Data access gate +description: Classify agent reads and writes by authorization and sensitivity. +risk: high +default_action: review +tags: [data, authorization, privacy] +question: + type: choice + instructions: Should the agent perform the requested data operation? + options: + allow: The actor is authorized, scope is minimal, and the operation is low risk. + review: Authorization, necessity, or data sensitivity needs human verification. + block: The operation is unauthorized, excessive, or exposes restricted data. diff --git a/decision_packs/finance/payment_action.yaml b/decision_packs/finance/payment_action.yaml new file mode 100644 index 0000000..b8bae07 --- /dev/null +++ b/decision_packs/finance/payment_action.yaml @@ -0,0 +1,14 @@ +id: payment-action +version: 1.0.0 +name: Payment action gate +description: Route refunds, transfers, and payment changes through appropriate controls. +risk: critical +default_action: block +tags: [finance, payments, approval] +question: + type: choice + instructions: Should this payment-related action proceed? + options: + allow: The action is within an approved low-risk policy and fully authorized. + review: A verified finance reviewer must approve the amount, recipient, or exception. + block: The action is unauthorized, anomalous, irreversible, or outside policy. diff --git a/decision_packs/hr/permission_change.yaml b/decision_packs/hr/permission_change.yaml new file mode 100644 index 0000000..c95d921 --- /dev/null +++ b/decision_packs/hr/permission_change.yaml @@ -0,0 +1,14 @@ +id: permission-change +version: 1.0.0 +name: HR and permission change gate +description: Guard role, access, employment, and entitlement changes. +risk: critical +default_action: block +tags: [hr, iam, permissions] +question: + type: choice + instructions: Should this identity or employment-related change execute? + options: + allow: The change is pre-approved, least-privilege, and supported by authoritative records. + review: An authorized HR or security reviewer must validate the change. + block: The change is unsupported, over-privileged, discriminatory, or unauthorized. diff --git a/decision_packs/integrations/external_api.yaml b/decision_packs/integrations/external_api.yaml new file mode 100644 index 0000000..6f362e7 --- /dev/null +++ b/decision_packs/integrations/external_api.yaml @@ -0,0 +1,14 @@ +id: external-api-call +version: 1.0.0 +name: External API call gate +description: Guard outbound requests, side effects, and data sharing with third parties. +risk: high +default_action: review +tags: [api, integrations, egress] +question: + type: choice + instructions: Should the agent make this external API call? + options: + allow: The destination, scope, payload, and side effect are approved and expected. + review: The destination, data classification, cost, or side effect needs approval. + block: The call targets an untrusted service, leaks restricted data, or exceeds authorization. diff --git a/decision_packs/security/destructive_command.yaml b/decision_packs/security/destructive_command.yaml new file mode 100644 index 0000000..db1ec42 --- /dev/null +++ b/decision_packs/security/destructive_command.yaml @@ -0,0 +1,14 @@ +id: destructive-command +version: 1.0.0 +name: Destructive command gate +description: Guard shell, infrastructure, database, and filesystem actions. +risk: critical +default_action: block +tags: [security, shell, infrastructure] +question: + type: choice + instructions: Should this potentially destructive command execute? + options: + allow: The command is scoped, reversible, authorized, and validated in the target environment. + review: A qualified operator must confirm scope, backup, target, and rollback readiness. + block: The command is destructive, ambiguous, unbounded, unauthorized, or lacks recovery controls. diff --git a/decision_packs/security/privacy_exposure.yaml b/decision_packs/security/privacy_exposure.yaml new file mode 100644 index 0000000..1d6df2c --- /dev/null +++ b/decision_packs/security/privacy_exposure.yaml @@ -0,0 +1,14 @@ +id: privacy-exposure +version: 1.0.0 +name: Privacy exposure gate +description: Guard output that may reveal personal, secret, or regulated information. +risk: critical +default_action: block +tags: [privacy, pii, secrets] +question: + type: choice + instructions: Should the agent disclose or transmit this information? + options: + allow: The disclosure is authorized, necessary, minimized, and sent to an approved destination. + review: Data classification, consent, recipient, or retention requires verification. + block: The content contains restricted personal data, credentials, secrets, or unauthorized records. diff --git a/docs/production-readiness.md b/docs/production-readiness.md new file mode 100644 index 0000000..6389089 --- /dev/null +++ b/docs/production-readiness.md @@ -0,0 +1,48 @@ +# Production readiness + +OpenDecision is a decision-control primitive, not a complete security boundary. Deploy it behind application authentication and authorization, and preserve independent enforcement at the protected tool or service. + +## Recommended decision flow + +1. Validate state and a versioned decision contract. +2. Apply deterministic deny or allow rules where policy is unambiguous. +3. Use a calibrated model provider only for the residual decision space. +4. Route uncertainty and consequential actions to human review. +5. Enforce the final decision at the tool boundary. +6. Record a redacted audit event and monitor overrides and failure rates. + +## Failure policy + +- `raise`: fail the request and let the caller handle the outage. +- `review`: produce a human-review outcome when the contract contains one. +- `block`: deny automatically when provider output is unavailable or invalid. + +Critical actions should generally use `block` or a durable review queue. The built-in in-memory review store is for prototypes and tests; production needs durable storage, authentication, RBAC, idempotency, notifications, and retention controls. + +## Calibration + +Tune thresholds on held-out, representative data for each policy and checkpoint. Report sample size, class balance, false positives, false negatives, Brier score, ECE, model version, hardware, and preprocessing. Never transfer thresholds between domains without evaluation. + +## Audit and privacy + +Built-in audit sinks store a SHA-256 context hash rather than raw state. Operators must define data classification, encryption, retention, access controls, deletion, and incident response. Do not place secrets or personal data into metadata fields. + +## Risk matrix + +| Risk | Default | Human review | Example | +| --- | --- | --- | --- | +| Low | allow after deterministic checks | optional | read public documentation | +| Medium | model or rules | on uncertainty | send internal notification | +| High | review | required by default | send external email or refund | +| Critical | block | explicit authorized approval | destructive command or production deletion | + +## Deployment checklist + +- Pin and scan dependencies. +- Protect tool credentials independently of the agent. +- Use versioned policies and immutable release artifacts. +- Run adversarial and regression fixtures. +- Configure fallback and outage behavior. +- Use a durable review service with RBAC. +- Monitor provider errors, overrides, calibration drift, and latency. +- Test rollback and emergency disable paths. diff --git a/pyproject.toml b/pyproject.toml index e1e757e..aa1fc00 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,30 +4,49 @@ build-backend = "hatchling.build" [project] name = "opendecision" -version = "0.1.0" -description = "A typed decision layer for AI agents" +version = "0.2.0" +description = "An auditable typed decision layer for AI agents" readme = "README.md" requires-python = ">=3.10" license = { text = "Apache-2.0" } authors = [{ name = "OpenDecision contributors" }] -dependencies = ["pydantic>=2.7,<3", "PyYAML>=6.0"] +dependencies = ["pydantic>=2.7,<3", "PyYAML>=6.0,<7"] [project.optional-dependencies] laya = ["laya"] langgraph = ["langgraph>=0.2"] fastapi = ["fastapi>=0.115", "uvicorn>=0.30"] -dev = ["pytest>=8.0", "ruff>=0.6", "mypy>=1.10"] +dev = [ + "coverage[toml]>=7.6", + "mypy>=1.10", + "pytest>=8.0", + "ruff>=0.6", + "types-PyYAML>=6.0", +] [tool.hatch.build.targets.wheel] packages = ["opendecision"] [tool.pytest.ini_options] testpaths = ["tests"] -addopts = "-q" +addopts = "-q --strict-markers" + +[tool.coverage.run] +branch = true +source = ["opendecision"] + +[tool.coverage.report] +fail_under = 85 +show_missing = true + +[tool.mypy] +python_version = "3.10" +strict = true +packages = ["opendecision"] [tool.ruff] line-length = 100 target-version = "py310" [tool.ruff.lint] -select = ["E", "F", "I", "UP"] +select = ["E", "F", "I", "UP", "B", "SIM"] From 642a54e4a8a78de863a3dca46fa50dc79077ff63 Mon Sep 17 00:00:00 2001 From: pateti Chandu Date: Wed, 23 Sep 2026 13:01:30 +0530 Subject: [PATCH 3/7] test: add production foundation coverage --- tests/test_bundled_policies.py | 14 +++++ tests/test_guard.py | 89 +++++++++++++------------------- tests/test_laya_provider.py | 42 +++++++++++++++ tests/test_models.py | 21 ++++---- tests/test_policies_legacy.py | 7 +++ tests/test_policy_review_eval.py | 50 ++++++++++++++++++ tests/test_providers.py | 21 ++++++++ 7 files changed, 182 insertions(+), 62 deletions(-) create mode 100644 tests/test_bundled_policies.py create mode 100644 tests/test_laya_provider.py create mode 100644 tests/test_policies_legacy.py create mode 100644 tests/test_policy_review_eval.py create mode 100644 tests/test_providers.py diff --git a/tests/test_bundled_policies.py b/tests/test_bundled_policies.py new file mode 100644 index 0000000..fa73dcc --- /dev/null +++ b/tests/test_bundled_policies.py @@ -0,0 +1,14 @@ +from pathlib import Path + +from opendecision import load_policy + + +def test_all_bundled_policies_are_valid_and_fail_safe(): + paths = sorted(Path("decision_packs").rglob("*.yaml")) + assert len(paths) >= 7 + policies = [load_policy(path) for path in paths] + assert len({policy.id for policy in policies}) == len(policies) + for policy in policies: + assert policy.question.type in {"choice", "score", "noul"} + if policy.risk in {"high", "critical"}: + assert policy.default_action in {"review", "block"} diff --git a/tests/test_guard.py b/tests/test_guard.py index efff26e..1790560 100644 --- a/tests/test_guard.py +++ b/tests/test_guard.py @@ -1,57 +1,42 @@ -from opendecision import DecisionGuard +import pytest +from opendecision import DecisionContext, DecisionGuard, MemoryAuditSink +from opendecision.errors import ContractValidationError class FakeProvider: name = "fake" - - def __init__(self, answer): - self.answer = answer - + def __init__(self, answer=None, error=None): self.answer, self.error = answer, error def predict(self, state, question): - return {"answer": self.answer, "raw": {"state": state}} - - -def test_choice_result_is_normalized(): - guard = DecisionGuard( - FakeProvider( - { - "choice": "review", - "probabilities": {"allow": 0.1, "review": 0.8, "block": 0.1}, - } - ) - ) - result = guard.decide( - {"tool": "send_email"}, - { - "type": "choice", - "instructions": "Should this action execute?", - "options": {"allow": "Proceed", "review": "Review", "block": "Block"}, - }, - ) - assert result.decision == "review" - assert result.confidence == 0.8 - assert result.provider == "fake" - - -def test_noul_threshold_controls_boolean_decision(): - guard = DecisionGuard(FakeProvider({"noul": 0.72})) - result = guard.decide( - {"command": "rm -rf /"}, - {"type": "noul", "instructions": "Is this destructive?", "threshold": 0.7}, - ) - assert result.decision is True - assert result.probabilities == {"true": 0.72, "false": 0.28} - - -def test_score_is_normalized(): - guard = DecisionGuard(FakeProvider({"score": 1.5, "confidence": 0.9})) - result = guard.decide( - {"ticket": "Production is unavailable"}, - { - "type": "score", - "instructions": "How urgent is this?", - "criteria": ["low", "medium", "critical"], - }, - ) - assert result.decision == 1.5 - assert result.confidence == 0.9 + if self.error: raise self.error + return {"answer": self.answer, "raw": {"model": "fake-v1"}} + +def choice_question(): + return {"type":"choice","instructions":"Should this action execute?","options":{"allow":"Proceed","review":"Review","block":"Stop"}} + +def test_choice_normalizes_and_audits_without_raw_state(): + sink = MemoryAuditSink() + guard = DecisionGuard(FakeProvider({"choice":"review","probabilities":{"allow":0.1,"review":0.8,"block":0.1}}), audit_sink=sink) + result = guard.decide({"secret":"do-not-log","tool":"send_email"}, choice_question(), context=DecisionContext(actor="user-1",tool="send_email",policy_id="tool-risk",policy_version="1.0.0")) + assert result.decision == "review" and result.requires_review + assert len(result.context_hash) == 64 and sink.events[0].actor == "user-1" + assert "secret" not in sink.events[0].model_dump_json() + +def test_invalid_choice_output_is_rejected(): + with pytest.raises(ContractValidationError, match="not one of"): + DecisionGuard(FakeProvider({"choice":"invented"})).decide({"tool":"x"}, choice_question()) + +def test_invalid_state_is_rejected_before_provider(): + with pytest.raises(ContractValidationError, match="JSON serializable"): + DecisionGuard(FakeProvider({"noul":0.5})).check({"bad":object()}, decision="Is this risky?") + +def test_failure_modes_review_and_block(): + provider = FakeProvider(error=RuntimeError("down")) + review = DecisionGuard(provider,failure_mode="review").decide({},choice_question()) + block = DecisionGuard(provider,failure_mode="block").decide({},choice_question()) + assert review.decision == "review" and review.requires_review + assert block.decision == "block" and not block.requires_review + +def test_noul_and_score_normalization(): + noul=DecisionGuard(FakeProvider({"noul":0.75})).check({},decision="Risky?") + score=DecisionGuard(FakeProvider({"score":1.5,"confidence":0.9})).decide({}, {"type":"score","instructions":"How urgent is this?","criteria":["low","high"]}) + assert noul.decision is True and score.decision == 1.5 diff --git a/tests/test_laya_provider.py b/tests/test_laya_provider.py new file mode 100644 index 0000000..3437169 --- /dev/null +++ b/tests/test_laya_provider.py @@ -0,0 +1,42 @@ +import sys +from types import SimpleNamespace +import pytest +from opendecision import DecisionQuestion +from opendecision.errors import ProviderError +from opendecision.providers.laya import LayaProvider + +class FakeAgent: + def __init__(self, *, fail=False, invalid=False): self.fail,self.invalid=fail,invalid + def predict(self,state,questions): + if self.fail: raise RuntimeError("prediction exploded") + if self.invalid: return {"answers":{}} + return {"answers":{"decision":{"choice":"allow","probabilities":{"allow":0.9,"block":0.1}}},"routing":{"model":"english"}} + +def question(): + return DecisionQuestion(type="choice",instructions="Should this execute?",options={"allow":"Proceed","block":"Stop"}) + +def test_router_load_is_lazy_and_cached(monkeypatch): + calls=[] + def router(**kwargs): calls.append(kwargs); return FakeAgent() + monkeypatch.setitem(sys.modules,"laya",SimpleNamespace(Router=router)) + provider=LayaProvider(preload=False,device="cpu") + assert provider.predict("hello",question())["provider"] == "laya:router" + assert provider.predict({},question())["answer"]["choice"] == "allow" + assert calls == [{"preload":False,"device":"cpu"}] + +def test_named_checkpoint_and_provider_errors(monkeypatch): + loaded=[] + def load(repo,**kwargs): loaded.append((repo,kwargs)); return FakeAgent() + monkeypatch.setitem(sys.modules,"laya",SimpleNamespace(load=load)) + provider=LayaProvider(model="typed-decisions") + assert provider.predict({},question())["answer"]["choice"] == "allow" + assert loaded[0][1]["subfolder"] == "typed-decisions" + provider._agent=FakeAgent(fail=True) + with pytest.raises(ProviderError,match="prediction failed"): provider.predict({},question()) + provider._agent=FakeAgent(invalid=True) + with pytest.raises(ProviderError,match="answers.decision"): provider.predict({},question()) + +def test_model_load_failure(monkeypatch): + def router(**kwargs): raise RuntimeError("cannot load") + monkeypatch.setitem(sys.modules,"laya",SimpleNamespace(Router=router)) + with pytest.raises(ProviderError,match="failed to load"): LayaProvider().predict({},question()) diff --git a/tests/test_models.py b/tests/test_models.py index 9e15da0..4616f47 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,15 +1,16 @@ import pytest from pydantic import ValidationError - from opendecision import DecisionQuestion +def test_laya_choice_uses_criteria_not_options(): + question=DecisionQuestion(type="choice",instructions="Choose an outcome",options={"allow":"Proceed","block":"Stop"}) + assert question.to_laya()=={"type":"choice","instructions":"Choose an outcome","criteria":{"allow":"Proceed","block":"Stop"}} -def test_string_question_becomes_noul_contract(): - question = DecisionQuestion.from_input("Is this safe?") - assert question.type == "noul" - assert question.instructions == "Is this safe?" - - -def test_choice_requires_options_or_criteria(): - with pytest.raises(ValidationError): - DecisionQuestion(type="choice", instructions="Choose") +@pytest.mark.parametrize("payload",[ + {"type":"choice","instructions":"Choose","options":{"Bad-Name":"x","ok":"y"}}, + {"type":"score","instructions":"Rate","criteria":["same","same"]}, + {"type":"noul","instructions":"Risk?","options":{"yes":"x","no":"y"}}, + {"type":"noul","instructions":"x","unknown":True}, +]) +def test_invalid_contracts_fail_early(payload): + with pytest.raises(ValidationError): DecisionQuestion.model_validate(payload) diff --git a/tests/test_policies_legacy.py b/tests/test_policies_legacy.py new file mode 100644 index 0000000..714b3e1 --- /dev/null +++ b/tests/test_policies_legacy.py @@ -0,0 +1,7 @@ +from opendecision import load_policy + +def test_v01_policy_remains_compatible(): + policy=load_policy("decision_packs/security/tool_risk.yaml") + assert policy.id == "tool-risk" + assert policy.version == "1.0.0" + assert policy.question.type == "choice" diff --git a/tests/test_policy_review_eval.py b/tests/test_policy_review_eval.py new file mode 100644 index 0000000..89ecf11 --- /dev/null +++ b/tests/test_policy_review_eval.py @@ -0,0 +1,50 @@ +from pathlib import Path +import pytest +from opendecision import DecisionGuard,EvaluationCase,InMemoryReviewStore,JsonlAuditSink,RuleProvider,evaluate,load_policy +from opendecision.errors import ReviewLoopError + +class FakeProvider: + name="fake" + def predict(self,state,question): return {"answer":{"choice":"review","probabilities":{"allow":0.1,"review":0.8,"block":0.1}}} + +def test_policy_inheritance(tmp_path:Path): + (tmp_path/"base.yaml").write_text("""id: base +version: 1.0.0 +name: Base +risk: high +default_action: block +question: + type: choice + instructions: Should this action run? + options: + allow: Proceed + review: Human review + block: Stop +""") + (tmp_path/"child.yaml").write_text("""extends: base.yaml +id: child +version: 1.1.0 +name: Child +description: Specialized policy +""") + policy=load_policy(tmp_path/"child.yaml") + assert policy.id=="child" and policy.risk=="high" and policy.question.options["block"]=="Stop" + +def test_policy_cycle_is_rejected(tmp_path:Path): + (tmp_path/"a.yaml").write_text("extends: b.yaml\n"); (tmp_path/"b.yaml").write_text("extends: a.yaml\n") + with pytest.raises(ValueError,match="cycle"): load_policy(tmp_path/"a.yaml") + +def test_review_lifecycle_and_loop_guard(): + result=DecisionGuard(FakeProvider()).decide({}, {"type":"choice","instructions":"Should this execute?","options":{"allow":"Proceed","review":"Review","block":"Stop"}}) + store=InMemoryReviewStore(max_attempts_per_action=1); request=store.create("action-1",result) + resolved=store.resolve(request.review_id,approved=False,reviewer="security@example.com",reason="Missing authorization",final_decision="block") + assert resolved.status.value=="rejected" + with pytest.raises(ReviewLoopError): store.create("action-1",result) + +def test_jsonl_audit_and_evaluation(tmp_path:Path): + sink=JsonlAuditSink(tmp_path/"audit.jsonl") + DecisionGuard(RuleProvider([],default="review"),audit_sink=sink).decide({}, {"type":"choice","instructions":"Should this execute?","options":{"allow":"Proceed","review":"Review","block":"Stop"}}) + assert "context_hash" in (tmp_path/"audit.jsonl").read_text() + report=evaluate([EvaluationCase(case_id="1",expected="block",predicted="block",probabilities={"allow":0.1,"block":0.9}),EvaluationCase(case_id="2",expected="allow",predicted="block")]) + assert report.accuracy==0.5 and report.brier_score is not None + with pytest.raises(ValueError): evaluate([]) diff --git a/tests/test_providers.py b/tests/test_providers.py new file mode 100644 index 0000000..3468b83 --- /dev/null +++ b/tests/test_providers.py @@ -0,0 +1,21 @@ +import pytest +from opendecision import DecisionQuestion,ProviderChain,Rule,RuleProvider +from opendecision.errors import ProvidersExhaustedError + +class BrokenProvider: + name="broken" + def predict(self,state,question): raise RuntimeError("unavailable") + +def question(): return DecisionQuestion(type="choice",instructions="Should this execute?",options={"allow":"Proceed","review":"Review","block":"Stop"}) + +def test_rules_provider_supports_nested_fields(): + provider=RuleProvider([Rule(field="action.command",operator="contains",value="rm -rf",decision="block")],default="review") + assert provider.predict({"action":{"command":"sudo rm -rf /tmp/x"}},question())["answer"]["choice"]=="block" + +def test_chain_falls_back_and_records_attempts(): + result=ProviderChain([BrokenProvider(),RuleProvider([],default="review")]).predict({},question()) + assert result["provider"]=="rules" and [x["success"] for x in result["attempts"]]==[False,True] + +def test_terminal_fallback_and_exhaustion(): + assert ProviderChain([BrokenProvider()],terminal_answer={"choice":"block"}).predict({},question())["provider"]=="terminal_fallback" + with pytest.raises(ProvidersExhaustedError): ProviderChain([BrokenProvider()]).predict({},question()) From d393e44e7bd32d20d6ec764a1939bf47097ec5c8 Mon Sep 17 00:00:00 2001 From: pateti Chandu Date: Wed, 23 Sep 2026 13:08:08 +0530 Subject: [PATCH 4/7] style: restore formatted core contracts --- opendecision/guard.py | 289 +++++++++++++++++++++++++++++++++-------- opendecision/models.py | 20 ++- 2 files changed, 254 insertions(+), 55 deletions(-) diff --git a/opendecision/guard.py b/opendecision/guard.py index d68bbb3..64977ed 100644 --- a/opendecision/guard.py +++ b/opendecision/guard.py @@ -1,78 +1,259 @@ -"""Decision guard with strict validation, audit traces, and fail-safe behavior.""" +"""DecisionGuard with strict validation, audit traces, and fail-safe behavior.""" + from __future__ import annotations -import hashlib, json + +import hashlib +import json from collections.abc import Mapping from datetime import datetime, timezone from typing import Any, Literal from uuid import uuid4 + from .audit import AuditEvent, AuditSink from .errors import ContractValidationError, ProviderError from .models import DecisionContext, DecisionQuestion, DecisionResult, ProviderAttempt from .providers.base import DecisionProvider from .providers.laya import LayaProvider + FailureMode = Literal["raise", "review", "block"] + class DecisionGuard: - def __init__(self, provider: DecisionProvider | None = None, *, model: str = "router", audit_sink: AuditSink | None = None, failure_mode: FailureMode = "raise", max_state_bytes: int = 100_000) -> None: - self.provider, self.audit_sink, self.failure_mode, self.max_state_bytes = provider or LayaProvider(model=model), audit_sink, failure_mode, max_state_bytes - def decide(self, state: Mapping[str, Any] | str, question: DecisionQuestion | Mapping[str, Any] | str, *, context: DecisionContext | Mapping[str, Any] | None = None) -> DecisionResult: + """Evaluate typed contracts before an agent acts.""" + + def __init__( + self, + provider: DecisionProvider | None = None, + *, + model: str = "router", + audit_sink: AuditSink | None = None, + failure_mode: FailureMode = "raise", + max_state_bytes: int = 100_000, + ) -> None: + self.provider = provider or LayaProvider(model=model) + self.audit_sink = audit_sink + self.failure_mode = failure_mode + self.max_state_bytes = max_state_bytes + + def decide( + self, + state: Mapping[str, Any] | str, + question: DecisionQuestion | Mapping[str, Any] | str, + *, + context: DecisionContext | Mapping[str, Any] | None = None, + ) -> DecisionResult: contract = DecisionQuestion.from_input(question) - ctx = context if isinstance(context, DecisionContext) else DecisionContext.model_validate(context or {}) - context_hash = self._hash(state) + decision_context = ( + context + if isinstance(context, DecisionContext) + else DecisionContext.model_validate(context or {}) + ) + context_hash = self._validate_and_hash_state(state) try: output = self.provider.predict(state, contract) - result = self._normalize(output.get("answer", output), output.get("raw", output), contract, ctx, context_hash, str(output.get("provider", self.provider.name)), [ProviderAttempt.model_validate(x) for x in output.get("attempts", [])]) + answer = output.get("answer", output) + raw_output = output.get("raw", output) + provider_name = str(output.get("provider", self.provider.name)) + attempts = [ + ProviderAttempt.model_validate(item) for item in output.get("attempts", []) + ] + result = self._normalize( + answer, + raw_output, + contract, + decision_context, + context_hash, + provider_name, + attempts, + ) except Exception as exc: if self.failure_mode == "raise": - if isinstance(exc, (ContractValidationError, ProviderError)): raise + if isinstance(exc, (ContractValidationError, ProviderError)): + raise raise ProviderError(str(exc)) from exc - result = self._failure(contract, ctx, context_hash, exc) - if self.audit_sink: self.audit_sink.write(AuditEvent.from_result(result, actor=ctx.actor, tool=ctx.tool, action_id=ctx.action_id)) + result = self._failure_result(contract, decision_context, context_hash, exc) + self._audit(result, decision_context) return result - def check(self, state: Mapping[str, Any] | str, *, decision: str, context: DecisionContext | Mapping[str, Any] | None = None) -> DecisionResult: - return self.decide(state, DecisionQuestion(type="noul", instructions=decision), context=context) - def _hash(self, state: Mapping[str, Any] | str) -> str: + + def check( + self, + state: Mapping[str, Any] | str, + *, + decision: str, + context: DecisionContext | Mapping[str, Any] | None = None, + ) -> DecisionResult: + return self.decide( + state, + DecisionQuestion(type="noul", instructions=decision), + context=context, + ) + + def _validate_and_hash_state(self, state: Mapping[str, Any] | str) -> str: if isinstance(state, str): - if not state.strip(): raise ContractValidationError("state string cannot be empty") - data = state.encode() + if not state.strip(): + raise ContractValidationError("state string cannot be empty") + encoded = state.encode() elif isinstance(state, Mapping): - try: data = json.dumps(state, sort_keys=True, separators=(",", ":")).encode() - except (TypeError, ValueError) as exc: raise ContractValidationError("state must be JSON serializable") from exc - else: raise ContractValidationError("state must be a string or mapping") - if len(data) > self.max_state_bytes: raise ContractValidationError(f"state exceeds configured limit of {self.max_state_bytes} bytes") - return hashlib.sha256(data).hexdigest() - def _normalize(self, answer: Mapping[str, Any], raw: Any, q: DecisionQuestion, ctx: DecisionContext, context_hash: str, provider: str, attempts: list[ProviderAttempt]) -> DecisionResult: - if not isinstance(answer, Mapping): raise ContractValidationError("provider answer must be a mapping") - common = dict(provider=provider, decision_id=str(uuid4()), created_at=datetime.now(timezone.utc), context_hash=context_hash, policy_id=ctx.policy_id, policy_version=ctx.policy_version, provider_attempts=attempts, raw_output=raw) - if q.type == "choice": - probs = _probs(answer.get("probabilities") or answer.get("probs")); decision = _first(answer, "choice", "label", "decision") - choices = q.options or (q.criteria if isinstance(q.criteria, dict) else {}); labels = set(choices) - if decision not in labels: raise ContractValidationError(f"provider choice {decision!r} is not one of {sorted(labels)}") - if probs and set(probs) != labels: raise ContractValidationError("provider probabilities must match declared options") - confidence = _prob(_first(answer, "confidence")) or (probs.get(str(decision)) if probs else None) - return DecisionResult(decision=str(decision), probabilities=probs, confidence=confidence, question_type="choice", requires_review=str(decision) == "review", metadata={"raw_answer": dict(answer)}, **common) - if q.type == "score": + try: + encoded = json.dumps(state, sort_keys=True, separators=(",", ":")).encode() + except (TypeError, ValueError) as exc: + raise ContractValidationError("state must be JSON serializable") from exc + else: + raise ContractValidationError("state must be a string or mapping") + if len(encoded) > self.max_state_bytes: + raise ContractValidationError( + f"state exceeds configured limit of {self.max_state_bytes} bytes" + ) + return hashlib.sha256(encoded).hexdigest() + + def _normalize( + self, + answer: Mapping[str, Any], + raw_output: Any, + question: DecisionQuestion, + context: DecisionContext, + context_hash: str, + provider_name: str, + attempts: list[ProviderAttempt], + ) -> DecisionResult: + if not isinstance(answer, Mapping): + raise ContractValidationError("provider answer must be a mapping") + common = { + "provider": provider_name, + "decision_id": str(uuid4()), + "created_at": datetime.now(timezone.utc), + "context_hash": context_hash, + "policy_id": context.policy_id, + "policy_version": context.policy_version, + "provider_attempts": attempts, + "raw_output": raw_output, + } + if question.type == "choice": + probabilities = _probabilities(answer.get("probabilities") or answer.get("probs")) + decision = _first(answer, "choice", "label", "decision") + choice_options = question.options or ( + question.criteria if isinstance(question.criteria, dict) else {} + ) + labels = set(choice_options.keys()) + if decision not in labels: + raise ContractValidationError( + f"provider choice {decision!r} is not one of {sorted(labels)}" + ) + if probabilities and set(probabilities) != labels: + raise ContractValidationError("provider probabilities must match declared options") + confidence = _probability(_first(answer, "confidence")) + if confidence is None and probabilities: + confidence = probabilities[str(decision)] + requires_review = str(decision) == "review" + return DecisionResult( + decision=str(decision), + probabilities=probabilities, + confidence=confidence, + question_type="choice", + requires_review=requires_review, + metadata={"raw_answer": dict(answer)}, + **common, + ) + if question.type == "score": score = _first(answer, "score", "value", "decision") - if score is None: raise ContractValidationError("provider response did not contain a score") - return DecisionResult(decision=float(score), probabilities=_probs(answer.get("distribution") or answer.get("probabilities")), confidence=_prob(_first(answer, "confidence")), question_type="score", metadata={"raw_answer": dict(answer)}, **common) - p = _prob(_first(answer, "noul", "probability", "confidence", "decision")) - if p is None: raise ContractValidationError("provider response did not contain a noul probability") - return DecisionResult(decision=p >= q.threshold, probabilities={"true": p, "false": 1-p}, confidence=max(p,1-p), question_type="noul", metadata={"threshold": q.threshold, "raw_answer": dict(answer)}, **common) - def _failure(self, q: DecisionQuestion, ctx: DecisionContext, context_hash: str, error: Exception) -> DecisionResult: + if score is None: + raise ContractValidationError("provider response did not contain a score") + confidence = _probability(_first(answer, "confidence")) + distribution = _probabilities( + answer.get("distribution") or answer.get("probabilities") + ) + return DecisionResult( + decision=float(score), + probabilities=distribution, + confidence=confidence, + question_type="score", + metadata={"raw_answer": dict(answer)}, + **common, + ) + probability = _probability( + _first(answer, "noul", "probability", "confidence", "decision") + ) + if probability is None: + raise ContractValidationError("provider response did not contain a noul probability") + decision = probability >= question.threshold + return DecisionResult( + decision=decision, + probabilities={"true": probability, "false": 1.0 - probability}, + confidence=max(probability, 1.0 - probability), + question_type="noul", + metadata={"threshold": question.threshold, "raw_answer": dict(answer)}, + **common, + ) + + def _failure_result( + self, + question: DecisionQuestion, + context: DecisionContext, + context_hash: str, + error: Exception, + ) -> DecisionResult: review = self.failure_mode == "review" - if q.type == "choice": - choices = q.options or (q.criteria if isinstance(q.criteria, dict) else {}); labels = list(choices); preferred = "review" if review and "review" in labels else "block"; decision: str | bool = preferred if preferred in labels else labels[-1] - else: decision = False - return DecisionResult(decision=decision, probabilities={}, confidence=None, question_type=q.type, provider="failure_policy", decision_id=str(uuid4()), created_at=datetime.now(timezone.utc), context_hash=context_hash, policy_id=ctx.policy_id, policy_version=ctx.policy_version, requires_review=review, metadata={"failure_mode": self.failure_mode, "error": f"{type(error).__name__}: {error}"}) - -def _first(m: Mapping[str, Any], *keys: str) -> Any: - return next((m[k] for k in keys if m.get(k) is not None), None) -def _prob(value: Any) -> float | None: - if value is None: return None - p=float(value) - if not 0<=p<=1: raise ContractValidationError(f"probability must be between 0 and 1, got {value}") - return p -def _probs(value: Any) -> dict[str,float]: - if value is None: return {} - if not isinstance(value, Mapping): raise ContractValidationError("probabilities must be a mapping") - return {str(k): float(_prob(v)) for k,v in value.items()} + if question.type == "choice": + choice_options = question.options or ( + question.criteria if isinstance(question.criteria, dict) else {} + ) + labels = list(choice_options.keys()) + preferred = "review" if review and "review" in labels else "block" + decision: str | bool = preferred if preferred in labels else labels[-1] + else: + decision = False + return DecisionResult( + decision=decision, + probabilities={}, + confidence=None, + question_type=question.type, + provider="failure_policy", + decision_id=str(uuid4()), + created_at=datetime.now(timezone.utc), + context_hash=context_hash, + policy_id=context.policy_id, + policy_version=context.policy_version, + requires_review=review, + metadata={ + "failure_mode": self.failure_mode, + "error": f"{type(error).__name__}: {error}", + }, + ) + + def _audit(self, result: DecisionResult, context: DecisionContext) -> None: + if self.audit_sink is None: + return + self.audit_sink.write( + AuditEvent.from_result( + result, + actor=context.actor, + tool=context.tool, + action_id=context.action_id, + ) + ) + + +def _first(mapping: Mapping[str, Any], *keys: str) -> Any: + for key in keys: + if mapping.get(key) is not None: + return mapping[key] + return None + + +def _probability(value: Any) -> float | None: + if value is None: + return None + probability = float(value) + if not 0.0 <= probability <= 1.0: + raise ContractValidationError(f"probability must be between 0 and 1, got {value}") + return probability + + +def _probabilities(value: Any) -> dict[str, float]: + if value is None: + return {} + if not isinstance(value, Mapping): + raise ContractValidationError("probabilities must be a mapping") + return { + str(key): _probability(probability) for key, probability in value.items() + } # type: ignore[misc] diff --git a/opendecision/models.py b/opendecision/models.py index 57b00f6..c540722 100644 --- a/opendecision/models.py +++ b/opendecision/models.py @@ -1,18 +1,24 @@ """Strict public data models for OpenDecision.""" from __future__ import annotations + import re from collections.abc import Mapping from datetime import datetime from typing import Any, Literal + from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator DecisionType = Literal["choice", "score", "noul"] RiskLevel = Literal["low", "medium", "high", "critical"] _OPTION_RE = re.compile(r"^[a-z][a-z0-9_]{0,63}$") + class DecisionQuestion(BaseModel): + """A strict typed question sent to a decision provider.""" + model_config = ConfigDict(extra="forbid") + type: DecisionType instructions: str = Field(min_length=3, max_length=2000) options: dict[str, str] | None = None @@ -28,7 +34,9 @@ def validate_option_names(cls, value: dict[str, str] | None) -> dict[str, str] | raise ValueError("choice questions require at least two options") for name, description in value.items(): if not _OPTION_RE.fullmatch(name): - raise ValueError("option names must be lowercase snake_case and at most 64 characters") + raise ValueError( + "option names must be lowercase snake_case and at most 64 characters" + ) if not description.strip(): raise ValueError(f"option {name!r} requires a description") return value @@ -57,6 +65,7 @@ def from_input(cls, question: DecisionQuestion | Mapping[str, Any] | str) -> Dec return cls.model_validate(question) def to_laya(self) -> dict[str, Any]: + """Return the exact question shape expected by Laya.""" payload: dict[str, Any] = {"type": self.type, "instructions": self.instructions} if self.type == "choice": payload["criteria"] = self.options or self.criteria @@ -64,8 +73,12 @@ def to_laya(self) -> dict[str, Any]: payload["criteria"] = self.criteria return payload + class DecisionContext(BaseModel): + """Non-sensitive operational context attached to a decision trace.""" + model_config = ConfigDict(extra="forbid") + actor: str | None = Field(default=None, max_length=200) tool: str | None = Field(default=None, max_length=200) action_id: str | None = Field(default=None, max_length=200) @@ -73,14 +86,19 @@ class DecisionContext(BaseModel): policy_version: str | None = Field(default=None, max_length=50) risk: RiskLevel = "medium" + class ProviderAttempt(BaseModel): provider: str success: bool latency_ms: float = Field(ge=0) error: str | None = None + class DecisionResult(BaseModel): + """A normalized, auditable, provider-independent result.""" + model_config = ConfigDict(arbitrary_types_allowed=True, extra="forbid") + decision: str | float | bool probabilities: dict[str, float] = Field(default_factory=dict) confidence: float | None = Field(default=None, ge=0.0, le=1.0) From 94610c6fe914f9bb6660cb2af6ad33d0c74812ec Mon Sep 17 00:00:00 2001 From: pateti Chandu Date: Wed, 23 Sep 2026 13:09:02 +0530 Subject: [PATCH 5/7] style: format providers and production modules --- opendecision/__init__.py | 24 +++++++--- opendecision/audit.py | 34 ++++++++++++-- opendecision/errors.py | 5 +++ opendecision/evaluation.py | 26 ++++++++++- opendecision/policies.py | 10 +++++ opendecision/providers/base.py | 13 +++++- opendecision/providers/chain.py | 53 +++++++++++++++++++--- opendecision/providers/laya.py | 52 ++++++++++++++++----- opendecision/providers/rules.py | 80 +++++++++++++++++++++++++-------- opendecision/review.py | 36 ++++++++++++--- 10 files changed, 282 insertions(+), 51 deletions(-) diff --git a/opendecision/__init__.py b/opendecision/__init__.py index 5936208..170a924 100644 --- a/opendecision/__init__.py +++ b/opendecision/__init__.py @@ -9,11 +9,25 @@ from .review import InMemoryReviewStore, ReviewRequest, ReviewStatus __all__ = [ - "AuditEvent", "DecisionContext", "DecisionGuard", "DecisionPolicy", - "DecisionQuestion", "DecisionResult", "EvaluationCase", "EvaluationReport", - "InMemoryReviewStore", "JsonlAuditSink", "LayaProvider", "MemoryAuditSink", - "ProviderChain", "ReviewRequest", "ReviewStatus", "Rule", "RuleProvider", - "evaluate", "load_policy", + "AuditEvent", + "DecisionContext", + "DecisionGuard", + "DecisionPolicy", + "DecisionQuestion", + "DecisionResult", + "EvaluationCase", + "EvaluationReport", + "InMemoryReviewStore", + "JsonlAuditSink", + "LayaProvider", + "MemoryAuditSink", + "ProviderChain", + "ReviewRequest", + "ReviewStatus", + "Rule", + "RuleProvider", + "evaluate", + "load_policy", ] __version__ = "0.2.0" diff --git a/opendecision/audit.py b/opendecision/audit.py index c371379..117fbbc 100644 --- a/opendecision/audit.py +++ b/opendecision/audit.py @@ -11,8 +11,10 @@ from .models import DecisionResult + class AuditEvent(BaseModel): model_config = ConfigDict(extra="forbid") + decision_id: str created_at: str context_hash: str @@ -28,24 +30,50 @@ class AuditEvent(BaseModel): metadata: dict[str, Any] = Field(default_factory=dict) @classmethod - def from_result(cls, result: DecisionResult, *, actor: str | None = None, tool: str | None = None, action_id: str | None = None) -> AuditEvent: - return cls(decision_id=result.decision_id, created_at=result.created_at.isoformat(), context_hash=result.context_hash, provider=result.provider, decision=result.decision, confidence=result.confidence, policy_id=result.policy_id, policy_version=result.policy_version, requires_review=result.requires_review, actor=actor, tool=tool, action_id=action_id) + def from_result( + cls, + result: DecisionResult, + *, + actor: str | None = None, + tool: str | None = None, + action_id: str | None = None, + ) -> AuditEvent: + return cls( + decision_id=result.decision_id, + created_at=result.created_at.isoformat(), + context_hash=result.context_hash, + provider=result.provider, + decision=result.decision, + confidence=result.confidence, + policy_id=result.policy_id, + policy_version=result.policy_version, + requires_review=result.requires_review, + actor=actor, + tool=tool, + action_id=action_id, + ) + class AuditSink(Protocol): def write(self, event: AuditEvent) -> None: ... + class MemoryAuditSink: def __init__(self) -> None: self.events: list[AuditEvent] = [] + def write(self, event: AuditEvent) -> None: self.events.append(event) + class JsonlAuditSink: - """Append redacted events; raw state and raw provider output are never written.""" + """Append redacted audit events; raw state and raw provider output are never written.""" + def __init__(self, path: str | Path) -> None: self.path = Path(path) self.path.parent.mkdir(parents=True, exist_ok=True) self._lock = Lock() + def write(self, event: AuditEvent) -> None: line = json.dumps(event.model_dump(mode="json"), sort_keys=True) with self._lock, self.path.open("a", encoding="utf-8") as handle: diff --git a/opendecision/errors.py b/opendecision/errors.py index 3a8bcb7..b616cbc 100644 --- a/opendecision/errors.py +++ b/opendecision/errors.py @@ -1,16 +1,21 @@ """OpenDecision exception hierarchy.""" + class OpenDecisionError(Exception): """Base error for OpenDecision.""" + class ContractValidationError(OpenDecisionError): """Raised when decision input or provider output violates its contract.""" + class ProviderError(OpenDecisionError): """Raised when a decision provider cannot return a valid answer.""" + class ProvidersExhaustedError(ProviderError): """Raised when every provider in a fallback chain fails.""" + class ReviewLoopError(OpenDecisionError): """Raised when an action exceeds the configured review-attempt limit.""" diff --git a/opendecision/evaluation.py b/opendecision/evaluation.py index 85d027d..bb7eeba 100644 --- a/opendecision/evaluation.py +++ b/opendecision/evaluation.py @@ -1,16 +1,21 @@ """Small reproducible evaluation primitives for decision packs.""" from __future__ import annotations + from collections import Counter + from pydantic import BaseModel, ConfigDict, Field + class EvaluationCase(BaseModel): model_config = ConfigDict(extra="forbid") + case_id: str expected: str predicted: str probabilities: dict[str, float] = Field(default_factory=dict) + class EvaluationReport(BaseModel): total: int correct: int @@ -18,6 +23,7 @@ class EvaluationReport(BaseModel): brier_score: float | None confusion: dict[str, int] + def evaluate(cases: list[EvaluationCase]) -> EvaluationReport: if not cases: raise ValueError("at least one evaluation case is required") @@ -27,5 +33,21 @@ def evaluate(cases: list[EvaluationCase]) -> EvaluationReport: for case in cases: if case.probabilities: labels = set(case.probabilities) | {case.expected} - brier_values.append(sum((case.probabilities.get(label, 0.0) - (1.0 if label == case.expected else 0.0)) ** 2 for label in labels) / len(labels)) - return EvaluationReport(total=len(cases), correct=correct, accuracy=correct / len(cases), brier_score=(sum(brier_values) / len(brier_values) if brier_values else None), confusion=dict(confusion)) + brier_values.append( + sum( + ( + case.probabilities.get(label, 0.0) + - (1.0 if label == case.expected else 0.0) + ) + ** 2 + for label in labels + ) + / len(labels) + ) + return EvaluationReport( + total=len(cases), + correct=correct, + accuracy=correct / len(cases), + brier_score=(sum(brier_values) / len(brier_values) if brier_values else None), + confusion=dict(confusion), + ) diff --git a/opendecision/policies.py b/opendecision/policies.py index d1cc6d3..626691d 100644 --- a/opendecision/policies.py +++ b/opendecision/policies.py @@ -1,14 +1,19 @@ """Versioned YAML policy loading and composition.""" from __future__ import annotations + from pathlib import Path from typing import Any, Literal + import yaml from pydantic import BaseModel, ConfigDict, Field, model_validator + from .models import DecisionQuestion, RiskLevel + class DecisionPolicy(BaseModel): model_config = ConfigDict(extra="forbid") + id: str = "" version: str = "1.0.0" name: str @@ -22,14 +27,18 @@ class DecisionPolicy(BaseModel): @model_validator(mode="before") @classmethod def derive_legacy_id(cls, value: Any) -> Any: + """Keep v0.1 packs valid by deriving an ID from their name.""" if isinstance(value, dict) and not value.get("id") and value.get("name"): value = dict(value) value["id"] = str(value["name"]).strip().lower().replace(" ", "-") return value + def load_policy(path: str | Path) -> DecisionPolicy: + """Load a policy and recursively compose an optional relative `extends` chain.""" return DecisionPolicy.model_validate(_load_payload(Path(path).resolve(), set())) + def _load_payload(path: Path, seen: set[Path]) -> dict[str, Any]: if path in seen: raise ValueError(f"policy inheritance cycle detected at {path}") @@ -42,6 +51,7 @@ def _load_payload(path: Path, seen: set[Path]) -> dict[str, Any]: base = _load_payload((path.parent / str(extends)).resolve(), seen) return _merge(base, payload) + def _merge(base: dict[str, Any], overlay: dict[str, Any]) -> dict[str, Any]: merged = dict(base) for key, value in overlay.items(): diff --git a/opendecision/providers/base.py b/opendecision/providers/base.py index 3ac41f4..f21130a 100644 --- a/opendecision/providers/base.py +++ b/opendecision/providers/base.py @@ -1,8 +1,19 @@ +"""Provider protocol used by the decision guard.""" + from __future__ import annotations + from collections.abc import Mapping from typing import Any, Protocol + from ..models import DecisionQuestion + class DecisionProvider(Protocol): name: str - def predict(self, state: Mapping[str, Any] | str, question: DecisionQuestion) -> Mapping[str, Any]: ... + + def predict( + self, + state: Mapping[str, Any] | str, + question: DecisionQuestion, + ) -> Mapping[str, Any]: + """Return a provider-native answer and optional raw response.""" diff --git a/opendecision/providers/chain.py b/opendecision/providers/chain.py index af60ad4..6e96bed 100644 --- a/opendecision/providers/chain.py +++ b/opendecision/providers/chain.py @@ -1,28 +1,67 @@ +"""Ordered provider fallback chains with attempt telemetry.""" + from __future__ import annotations + import time from collections.abc import Mapping from typing import Any + from ..errors import ProvidersExhaustedError from ..models import DecisionQuestion from .base import DecisionProvider + class ProviderChain: name = "provider_chain" - def __init__(self, providers: list[DecisionProvider], *, terminal_answer: Mapping[str, Any] | None = None) -> None: + + def __init__( + self, + providers: list[DecisionProvider], + *, + terminal_answer: Mapping[str, Any] | None = None, + ) -> None: if not providers: raise ValueError("provider chain requires at least one provider") self.providers = providers self.terminal_answer = dict(terminal_answer) if terminal_answer else None - def predict(self, state: Mapping[str, Any] | str, question: DecisionQuestion) -> Mapping[str, Any]: + + def predict( + self, + state: Mapping[str, Any] | str, + question: DecisionQuestion, + ) -> Mapping[str, Any]: attempts: list[dict[str, Any]] = [] for provider in self.providers: started = time.perf_counter() try: response = provider.predict(state, question) - attempts.append({"provider": provider.name, "success": True, "latency_ms": (time.perf_counter() - started) * 1000}) - return {"answer": response.get("answer", response), "raw": response.get("raw", response), "provider": provider.name, "attempts": attempts} - except Exception as exc: - attempts.append({"provider": provider.name, "success": False, "latency_ms": (time.perf_counter() - started) * 1000, "error": f"{type(exc).__name__}: {exc}"}) + attempts.append( + { + "provider": provider.name, + "success": True, + "latency_ms": (time.perf_counter() - started) * 1000, + } + ) + return { + "answer": response.get("answer", response), + "raw": response.get("raw", response), + "provider": provider.name, + "attempts": attempts, + } + except Exception as exc: # provider boundaries intentionally isolate failures + attempts.append( + { + "provider": provider.name, + "success": False, + "latency_ms": (time.perf_counter() - started) * 1000, + "error": f"{type(exc).__name__}: {exc}", + } + ) if self.terminal_answer is not None: - return {"answer": self.terminal_answer, "raw": {"terminal_fallback": True}, "provider": "terminal_fallback", "attempts": attempts} + return { + "answer": self.terminal_answer, + "raw": {"terminal_fallback": True}, + "provider": "terminal_fallback", + "attempts": attempts, + } raise ProvidersExhaustedError(f"all providers failed: {attempts}") diff --git a/opendecision/providers/laya.py b/opendecision/providers/laya.py index b46cf41..70512e0 100644 --- a/opendecision/providers/laya.py +++ b/opendecision/providers/laya.py @@ -1,39 +1,71 @@ +"""Lazy Laya provider.""" + from __future__ import annotations + from collections.abc import Mapping from typing import Any + from ..errors import ProviderError from ..models import DecisionQuestion + class LayaProvider: name = "laya" - def __init__(self, *, model: str = "router", preload: bool = True, device: str | None = None) -> None: - self.model, self.preload, self.device, self._agent = model, preload, device, None + + def __init__( + self, + *, + model: str = "router", + preload: bool = True, + device: str | None = None, + ) -> None: + self.model = model + self.preload = preload + self.device = device + self._agent: Any = None + def _load(self) -> Any: if self._agent is not None: return self._agent try: - import laya + import laya # type: ignore except ImportError as exc: - raise ProviderError("Laya is not installed. Install it with `pip install opendecision[laya]`.") from exc + raise ProviderError( + "Laya is not installed. Install it with `pip install opendecision[laya]`." + ) from exc try: if self.model == "router": kwargs: dict[str, Any] = {"preload": self.preload} - if self.device is not None: kwargs["device"] = self.device + if self.device is not None: + kwargs["device"] = self.device self._agent = laya.Router(**kwargs) else: kwargs = {} - if self.device is not None: kwargs["device"] = self.device - self._agent = laya.load("convaiinnovations/laya", subfolder=self.model, **kwargs) + if self.device is not None: + kwargs["device"] = self.device + self._agent = laya.load( + "convaiinnovations/laya", subfolder=self.model, **kwargs + ) except Exception as exc: raise ProviderError(f"failed to load Laya model {self.model!r}: {exc}") from exc return self._agent - def predict(self, state: Mapping[str, Any] | str, question: DecisionQuestion) -> Mapping[str, Any]: + + def predict( + self, + state: Mapping[str, Any] | str, + question: DecisionQuestion, + ) -> Mapping[str, Any]: agent = self._load() + state_payload = state if isinstance(state, Mapping) else {"input": state} try: - response = agent.predict(state if isinstance(state, Mapping) else {"input": state}, {"decision": question.to_laya()}) + response = agent.predict(state_payload, {"decision": question.to_laya()}) except Exception as exc: raise ProviderError(f"Laya prediction failed: {exc}") from exc answer = response.get("answers", {}).get("decision") if not isinstance(answer, Mapping): raise ProviderError("Laya response did not include answers.decision") - return {"answer": answer, "raw": response, "provider": f"laya:{self.model}"} + return { + "answer": answer, + "raw": response, + "provider": f"laya:{self.model}", + } diff --git a/opendecision/providers/rules.py b/opendecision/providers/rules.py index 26ac2ab..de81f7e 100644 --- a/opendecision/providers/rules.py +++ b/opendecision/providers/rules.py @@ -1,48 +1,92 @@ +"""Deterministic rules provider for high-confidence and fallback decisions.""" + from __future__ import annotations + import re from collections.abc import Mapping from typing import Any, Literal + from pydantic import BaseModel, ConfigDict, Field + from ..models import DecisionQuestion + RuleOperator = Literal["equals", "contains", "in", "regex", "exists"] + class Rule(BaseModel): model_config = ConfigDict(extra="forbid") + field: str operator: RuleOperator value: Any = None decision: str | bool | float confidence: float = Field(default=1.0, ge=0.0, le=1.0) + def matches(self, state: Mapping[str, Any]) -> bool: actual: Any = state for part in self.field.split("."): - if not isinstance(actual, Mapping) or part not in actual: return False + if not isinstance(actual, Mapping) or part not in actual: + return False actual = actual[part] - if self.operator == "exists": return actual is not None - if self.operator == "equals": return bool(actual == self.value) - if self.operator == "contains": return str(self.value).casefold() in str(actual).casefold() - if self.operator == "in": return bool(actual in self.value) - if self.operator == "regex": return re.search(str(self.value), str(actual), re.IGNORECASE) is not None + if self.operator == "exists": + return actual is not None + if self.operator == "equals": + return bool(actual == self.value) + if self.operator == "contains": + return str(self.value).casefold() in str(actual).casefold() + if self.operator == "in": + return bool(actual in self.value) + if self.operator == "regex": + return re.search(str(self.value), str(actual), re.IGNORECASE) is not None return False + class RuleProvider: name = "rules" + def __init__(self, rules: list[Rule], *, default: str | bool | float | None = None) -> None: - self.rules, self.default = rules, default - def predict(self, state: Mapping[str, Any] | str, question: DecisionQuestion) -> Mapping[str, Any]: + self.rules = rules + self.default = default + + def predict( + self, + state: Mapping[str, Any] | str, + question: DecisionQuestion, + ) -> Mapping[str, Any]: payload = state if isinstance(state, Mapping) else {"input": state} for rule in self.rules: - if rule.matches(payload): return {"answer": _answer(question, rule.decision, rule.confidence), "raw": {"rule": rule.model_dump()}} - if self.default is None: raise LookupError("no deterministic rule matched") + if rule.matches(payload): + return { + "answer": _answer(question, rule.decision, rule.confidence), + "raw": {"rule": rule.model_dump()}, + } + if self.default is None: + raise LookupError("no deterministic rule matched") return {"answer": _answer(question, self.default, 1.0), "raw": {"default": True}} -def _answer(question: DecisionQuestion, decision: str | bool | float, confidence: float) -> dict[str, Any]: + +def _answer( + question: DecisionQuestion, + decision: str | bool | float, + confidence: float, +) -> dict[str, Any]: if question.type == "choice": - choices = question.options or (question.criteria if isinstance(question.criteria, dict) else {}) - labels = list(choices.keys()) - if str(decision) not in labels: raise ValueError(f"rule decision {decision!r} is not a declared option") + choice_options = question.options or ( + question.criteria if isinstance(question.criteria, dict) else {} + ) + labels = list(choice_options.keys()) + if str(decision) not in labels: + raise ValueError(f"rule decision {decision!r} is not a declared option") remaining = (1.0 - confidence) / max(len(labels) - 1, 1) - probabilities = {label: (confidence if label == decision else remaining) for label in labels} - return {"choice": str(decision), "probabilities": probabilities, "confidence": confidence} - if question.type == "score": return {"score": float(decision), "confidence": confidence} - return {"noul": confidence if bool(decision) else 1.0 - confidence} + probabilities = { + label: (confidence if label == decision else remaining) for label in labels + } + return { + "choice": str(decision), + "probabilities": probabilities, + "confidence": confidence, + } + if question.type == "score": + return {"score": float(decision), "confidence": confidence} + probability = confidence if bool(decision) else 1.0 - confidence + return {"noul": probability} diff --git a/opendecision/review.py b/opendecision/review.py index 3e18d31..0cbebac 100644 --- a/opendecision/review.py +++ b/opendecision/review.py @@ -1,16 +1,26 @@ +"""Minimal human-review workflow and loop protection.""" + from __future__ import annotations + from datetime import datetime, timezone from enum import Enum from uuid import uuid4 + from pydantic import BaseModel, ConfigDict, Field + from .errors import ReviewLoopError from .models import DecisionResult + class ReviewStatus(str, Enum): - pending, approved, rejected = "pending", "approved", "rejected" + pending = "pending" + approved = "approved" + rejected = "rejected" + class ReviewRequest(BaseModel): model_config = ConfigDict(extra="forbid") + review_id: str = Field(default_factory=lambda: str(uuid4())) action_id: str decision_id: str @@ -21,19 +31,35 @@ class ReviewRequest(BaseModel): reason: str | None = None final_decision: str | bool | float | None = None + class InMemoryReviewStore: def __init__(self, *, max_attempts_per_action: int = 3) -> None: self.max_attempts_per_action = max_attempts_per_action self.requests: dict[str, ReviewRequest] = {} + def create(self, action_id: str, result: DecisionResult) -> ReviewRequest: - if sum(item.action_id == action_id for item in self.requests.values()) >= self.max_attempts_per_action: + attempts = sum(item.action_id == action_id for item in self.requests.values()) + if attempts >= self.max_attempts_per_action: raise ReviewLoopError(f"action {action_id!r} exceeded review attempt limit") request = ReviewRequest(action_id=action_id, decision_id=result.decision_id) self.requests[request.review_id] = request return request - def resolve(self, review_id: str, *, approved: bool, reviewer: str, reason: str, final_decision: str | bool | float | None = None) -> ReviewRequest: + + def resolve( + self, + review_id: str, + *, + approved: bool, + reviewer: str, + reason: str, + final_decision: str | bool | float | None = None, + ) -> ReviewRequest: request = self.requests[review_id] - if request.status is not ReviewStatus.pending: raise ValueError("review request is already resolved") + if request.status is not ReviewStatus.pending: + raise ValueError("review request is already resolved") request.status = ReviewStatus.approved if approved else ReviewStatus.rejected - request.resolved_at, request.reviewer, request.reason, request.final_decision = datetime.now(timezone.utc), reviewer, reason, final_decision + request.resolved_at = datetime.now(timezone.utc) + request.reviewer = reviewer + request.reason = reason + request.final_decision = final_decision return request From dec730a05b060c09ce9ea0df128bf9f919a23c0a Mon Sep 17 00:00:00 2001 From: pateti Chandu Date: Wed, 23 Sep 2026 13:09:44 +0530 Subject: [PATCH 6/7] style: format production test suite --- tests/test_guard.py | 73 +++++++++++++++---- tests/test_laya_provider.py | 91 +++++++++++++++++------- tests/test_models.py | 41 ++++++++--- tests/test_policies_legacy.py | 3 +- tests/test_policy_review_eval.py | 116 ++++++++++++++++++++++++------- tests/test_providers.py | 50 ++++++++++--- 6 files changed, 289 insertions(+), 85 deletions(-) diff --git a/tests/test_guard.py b/tests/test_guard.py index 1790560..30c113a 100644 --- a/tests/test_guard.py +++ b/tests/test_guard.py @@ -3,40 +3,85 @@ from opendecision import DecisionContext, DecisionGuard, MemoryAuditSink from opendecision.errors import ContractValidationError + class FakeProvider: name = "fake" - def __init__(self, answer=None, error=None): self.answer, self.error = answer, error + + def __init__(self, answer=None, error=None): + self.answer = answer + self.error = error + def predict(self, state, question): - if self.error: raise self.error + if self.error: + raise self.error return {"answer": self.answer, "raw": {"model": "fake-v1"}} + def choice_question(): - return {"type":"choice","instructions":"Should this action execute?","options":{"allow":"Proceed","review":"Review","block":"Stop"}} + return { + "type": "choice", + "instructions": "Should this action execute?", + "options": {"allow": "Proceed", "review": "Review", "block": "Stop"}, + } + def test_choice_normalizes_and_audits_without_raw_state(): sink = MemoryAuditSink() - guard = DecisionGuard(FakeProvider({"choice":"review","probabilities":{"allow":0.1,"review":0.8,"block":0.1}}), audit_sink=sink) - result = guard.decide({"secret":"do-not-log","tool":"send_email"}, choice_question(), context=DecisionContext(actor="user-1",tool="send_email",policy_id="tool-risk",policy_version="1.0.0")) - assert result.decision == "review" and result.requires_review - assert len(result.context_hash) == 64 and sink.events[0].actor == "user-1" + guard = DecisionGuard( + FakeProvider( + { + "choice": "review", + "probabilities": {"allow": 0.1, "review": 0.8, "block": 0.1}, + } + ), + audit_sink=sink, + ) + result = guard.decide( + {"secret": "do-not-log", "tool": "send_email"}, + choice_question(), + context=DecisionContext( + actor="user-1", + tool="send_email", + policy_id="tool-risk", + policy_version="1.0.0", + ), + ) + assert result.decision == "review" + assert result.requires_review is True + assert len(result.context_hash) == 64 + assert sink.events[0].actor == "user-1" assert "secret" not in sink.events[0].model_dump_json() + def test_invalid_choice_output_is_rejected(): + guard = DecisionGuard(FakeProvider({"choice": "invented"})) with pytest.raises(ContractValidationError, match="not one of"): - DecisionGuard(FakeProvider({"choice":"invented"})).decide({"tool":"x"}, choice_question()) + guard.decide({"tool": "x"}, choice_question()) + def test_invalid_state_is_rejected_before_provider(): + guard = DecisionGuard(FakeProvider({"noul": 0.5})) with pytest.raises(ContractValidationError, match="JSON serializable"): - DecisionGuard(FakeProvider({"noul":0.5})).check({"bad":object()}, decision="Is this risky?") + guard.check({"bad": object()}, decision="Is this risky?") + def test_failure_modes_review_and_block(): provider = FakeProvider(error=RuntimeError("down")) - review = DecisionGuard(provider,failure_mode="review").decide({},choice_question()) - block = DecisionGuard(provider,failure_mode="block").decide({},choice_question()) + review = DecisionGuard(provider, failure_mode="review").decide({}, choice_question()) + block = DecisionGuard(provider, failure_mode="block").decide({}, choice_question()) assert review.decision == "review" and review.requires_review assert block.decision == "block" and not block.requires_review + def test_noul_and_score_normalization(): - noul=DecisionGuard(FakeProvider({"noul":0.75})).check({},decision="Risky?") - score=DecisionGuard(FakeProvider({"score":1.5,"confidence":0.9})).decide({}, {"type":"score","instructions":"How urgent is this?","criteria":["low","high"]}) - assert noul.decision is True and score.decision == 1.5 + noul = DecisionGuard(FakeProvider({"noul": 0.75})).check({}, decision="Risky?") + score = DecisionGuard(FakeProvider({"score": 1.5, "confidence": 0.9})).decide( + {}, + { + "type": "score", + "instructions": "How urgent is this?", + "criteria": ["low", "high"], + }, + ) + assert noul.decision is True + assert score.decision == 1.5 diff --git a/tests/test_laya_provider.py b/tests/test_laya_provider.py index 3437169..12aa310 100644 --- a/tests/test_laya_provider.py +++ b/tests/test_laya_provider.py @@ -1,42 +1,83 @@ import sys from types import SimpleNamespace + import pytest + from opendecision import DecisionQuestion from opendecision.errors import ProviderError from opendecision.providers.laya import LayaProvider + class FakeAgent: - def __init__(self, *, fail=False, invalid=False): self.fail,self.invalid=fail,invalid - def predict(self,state,questions): - if self.fail: raise RuntimeError("prediction exploded") - if self.invalid: return {"answers":{}} - return {"answers":{"decision":{"choice":"allow","probabilities":{"allow":0.9,"block":0.1}}},"routing":{"model":"english"}} + def __init__(self, *, fail=False, invalid=False): + self.fail = fail + self.invalid = invalid + + def predict(self, state, questions): + if self.fail: + raise RuntimeError("prediction exploded") + if self.invalid: + return {"answers": {}} + return { + "answers": { + "decision": { + "choice": "allow", + "probabilities": {"allow": 0.9, "block": 0.1}, + } + }, + "routing": {"model": "english"}, + } + def question(): - return DecisionQuestion(type="choice",instructions="Should this execute?",options={"allow":"Proceed","block":"Stop"}) + return DecisionQuestion( + type="choice", + instructions="Should this execute?", + options={"allow": "Proceed", "block": "Stop"}, + ) + def test_router_load_is_lazy_and_cached(monkeypatch): - calls=[] - def router(**kwargs): calls.append(kwargs); return FakeAgent() - monkeypatch.setitem(sys.modules,"laya",SimpleNamespace(Router=router)) - provider=LayaProvider(preload=False,device="cpu") - assert provider.predict("hello",question())["provider"] == "laya:router" - assert provider.predict({},question())["answer"]["choice"] == "allow" - assert calls == [{"preload":False,"device":"cpu"}] + calls = [] + + def router(**kwargs): + calls.append(kwargs) + return FakeAgent() + + monkeypatch.setitem(sys.modules, "laya", SimpleNamespace(Router=router)) + provider = LayaProvider(preload=False, device="cpu") + first = provider.predict("hello", question()) + second = provider.predict({}, question()) + assert first["provider"] == "laya:router" + assert second["answer"]["choice"] == "allow" + assert calls == [{"preload": False, "device": "cpu"}] + def test_named_checkpoint_and_provider_errors(monkeypatch): - loaded=[] - def load(repo,**kwargs): loaded.append((repo,kwargs)); return FakeAgent() - monkeypatch.setitem(sys.modules,"laya",SimpleNamespace(load=load)) - provider=LayaProvider(model="typed-decisions") - assert provider.predict({},question())["answer"]["choice"] == "allow" + loaded = [] + + def load(repo, **kwargs): + loaded.append((repo, kwargs)) + return FakeAgent() + + monkeypatch.setitem(sys.modules, "laya", SimpleNamespace(load=load)) + provider = LayaProvider(model="typed-decisions") + assert provider.predict({}, question())["answer"]["choice"] == "allow" assert loaded[0][1]["subfolder"] == "typed-decisions" - provider._agent=FakeAgent(fail=True) - with pytest.raises(ProviderError,match="prediction failed"): provider.predict({},question()) - provider._agent=FakeAgent(invalid=True) - with pytest.raises(ProviderError,match="answers.decision"): provider.predict({},question()) + + provider._agent = FakeAgent(fail=True) + with pytest.raises(ProviderError, match="prediction failed"): + provider.predict({}, question()) + + provider._agent = FakeAgent(invalid=True) + with pytest.raises(ProviderError, match="answers.decision"): + provider.predict({}, question()) + def test_model_load_failure(monkeypatch): - def router(**kwargs): raise RuntimeError("cannot load") - monkeypatch.setitem(sys.modules,"laya",SimpleNamespace(Router=router)) - with pytest.raises(ProviderError,match="failed to load"): LayaProvider().predict({},question()) + def router(**kwargs): + raise RuntimeError("cannot load") + + monkeypatch.setitem(sys.modules, "laya", SimpleNamespace(Router=router)) + with pytest.raises(ProviderError, match="failed to load"): + LayaProvider().predict({}, question()) diff --git a/tests/test_models.py b/tests/test_models.py index 4616f47..5c146b0 100644 --- a/tests/test_models.py +++ b/tests/test_models.py @@ -1,16 +1,39 @@ import pytest from pydantic import ValidationError + from opendecision import DecisionQuestion + def test_laya_choice_uses_criteria_not_options(): - question=DecisionQuestion(type="choice",instructions="Choose an outcome",options={"allow":"Proceed","block":"Stop"}) - assert question.to_laya()=={"type":"choice","instructions":"Choose an outcome","criteria":{"allow":"Proceed","block":"Stop"}} + question = DecisionQuestion( + type="choice", + instructions="Choose an outcome", + options={"allow": "Proceed", "block": "Stop"}, + ) + assert question.to_laya() == { + "type": "choice", + "instructions": "Choose an outcome", + "criteria": {"allow": "Proceed", "block": "Stop"}, + } + -@pytest.mark.parametrize("payload",[ - {"type":"choice","instructions":"Choose","options":{"Bad-Name":"x","ok":"y"}}, - {"type":"score","instructions":"Rate","criteria":["same","same"]}, - {"type":"noul","instructions":"Risk?","options":{"yes":"x","no":"y"}}, - {"type":"noul","instructions":"x","unknown":True}, -]) +@pytest.mark.parametrize( + "payload", + [ + { + "type": "choice", + "instructions": "Choose", + "options": {"Bad-Name": "x", "ok": "y"}, + }, + {"type": "score", "instructions": "Rate", "criteria": ["same", "same"]}, + { + "type": "noul", + "instructions": "Risk?", + "options": {"yes": "x", "no": "y"}, + }, + {"type": "noul", "instructions": "x", "unknown": True}, + ], +) def test_invalid_contracts_fail_early(payload): - with pytest.raises(ValidationError): DecisionQuestion.model_validate(payload) + with pytest.raises(ValidationError): + DecisionQuestion.model_validate(payload) diff --git a/tests/test_policies_legacy.py b/tests/test_policies_legacy.py index 714b3e1..a90cc09 100644 --- a/tests/test_policies_legacy.py +++ b/tests/test_policies_legacy.py @@ -1,7 +1,8 @@ from opendecision import load_policy + def test_v01_policy_remains_compatible(): - policy=load_policy("decision_packs/security/tool_risk.yaml") + policy = load_policy("decision_packs/security/tool_risk.yaml") assert policy.id == "tool-risk" assert policy.version == "1.0.0" assert policy.question.type == "choice" diff --git a/tests/test_policy_review_eval.py b/tests/test_policy_review_eval.py index 89ecf11..4f6e041 100644 --- a/tests/test_policy_review_eval.py +++ b/tests/test_policy_review_eval.py @@ -1,14 +1,34 @@ from pathlib import Path + import pytest -from opendecision import DecisionGuard,EvaluationCase,InMemoryReviewStore,JsonlAuditSink,RuleProvider,evaluate,load_policy + +from opendecision import ( + DecisionGuard, + EvaluationCase, + InMemoryReviewStore, + JsonlAuditSink, + RuleProvider, + evaluate, + load_policy, +) from opendecision.errors import ReviewLoopError + class FakeProvider: - name="fake" - def predict(self,state,question): return {"answer":{"choice":"review","probabilities":{"allow":0.1,"review":0.8,"block":0.1}}} + name = "fake" + + def predict(self, state, question): + return { + "answer": { + "choice": "review", + "probabilities": {"allow": 0.1, "review": 0.8, "block": 0.1}, + } + } + -def test_policy_inheritance(tmp_path:Path): - (tmp_path/"base.yaml").write_text("""id: base +def test_policy_inheritance(tmp_path: Path): + (tmp_path / "base.yaml").write_text( + """id: base version: 1.0.0 name: Base risk: high @@ -20,31 +40,75 @@ def test_policy_inheritance(tmp_path:Path): allow: Proceed review: Human review block: Stop -""") - (tmp_path/"child.yaml").write_text("""extends: base.yaml +""" + ) + (tmp_path / "child.yaml").write_text( + """extends: base.yaml id: child version: 1.1.0 name: Child description: Specialized policy -""") - policy=load_policy(tmp_path/"child.yaml") - assert policy.id=="child" and policy.risk=="high" and policy.question.options["block"]=="Stop" +""" + ) + policy = load_policy(tmp_path / "child.yaml") + assert policy.id == "child" + assert policy.risk == "high" + assert policy.question.options["block"] == "Stop" + + +def test_policy_cycle_is_rejected(tmp_path: Path): + (tmp_path / "a.yaml").write_text("extends: b.yaml\n") + (tmp_path / "b.yaml").write_text("extends: a.yaml\n") + with pytest.raises(ValueError, match="cycle"): + load_policy(tmp_path / "a.yaml") -def test_policy_cycle_is_rejected(tmp_path:Path): - (tmp_path/"a.yaml").write_text("extends: b.yaml\n"); (tmp_path/"b.yaml").write_text("extends: a.yaml\n") - with pytest.raises(ValueError,match="cycle"): load_policy(tmp_path/"a.yaml") def test_review_lifecycle_and_loop_guard(): - result=DecisionGuard(FakeProvider()).decide({}, {"type":"choice","instructions":"Should this execute?","options":{"allow":"Proceed","review":"Review","block":"Stop"}}) - store=InMemoryReviewStore(max_attempts_per_action=1); request=store.create("action-1",result) - resolved=store.resolve(request.review_id,approved=False,reviewer="security@example.com",reason="Missing authorization",final_decision="block") - assert resolved.status.value=="rejected" - with pytest.raises(ReviewLoopError): store.create("action-1",result) - -def test_jsonl_audit_and_evaluation(tmp_path:Path): - sink=JsonlAuditSink(tmp_path/"audit.jsonl") - DecisionGuard(RuleProvider([],default="review"),audit_sink=sink).decide({}, {"type":"choice","instructions":"Should this execute?","options":{"allow":"Proceed","review":"Review","block":"Stop"}}) - assert "context_hash" in (tmp_path/"audit.jsonl").read_text() - report=evaluate([EvaluationCase(case_id="1",expected="block",predicted="block",probabilities={"allow":0.1,"block":0.9}),EvaluationCase(case_id="2",expected="allow",predicted="block")]) - assert report.accuracy==0.5 and report.brier_score is not None - with pytest.raises(ValueError): evaluate([]) + result = DecisionGuard(FakeProvider()).decide( + {}, + { + "type": "choice", + "instructions": "Should this execute?", + "options": {"allow": "Proceed", "review": "Review", "block": "Stop"}, + }, + ) + store = InMemoryReviewStore(max_attempts_per_action=1) + request = store.create("action-1", result) + resolved = store.resolve( + request.review_id, + approved=False, + reviewer="security@example.com", + reason="Missing authorization", + final_decision="block", + ) + assert resolved.status.value == "rejected" + with pytest.raises(ReviewLoopError): + store.create("action-1", result) + + +def test_jsonl_audit_and_evaluation(tmp_path: Path): + sink = JsonlAuditSink(tmp_path / "audit.jsonl") + DecisionGuard(RuleProvider([], default="review"), audit_sink=sink).decide( + {}, + { + "type": "choice", + "instructions": "Should this execute?", + "options": {"allow": "Proceed", "review": "Review", "block": "Stop"}, + }, + ) + assert "context_hash" in (tmp_path / "audit.jsonl").read_text() + report = evaluate( + [ + EvaluationCase( + case_id="1", + expected="block", + predicted="block", + probabilities={"allow": 0.1, "block": 0.9}, + ), + EvaluationCase(case_id="2", expected="allow", predicted="block"), + ] + ) + assert report.accuracy == 0.5 + assert report.brier_score is not None + with pytest.raises(ValueError): + evaluate([]) diff --git a/tests/test_providers.py b/tests/test_providers.py index 3468b83..966c68d 100644 --- a/tests/test_providers.py +++ b/tests/test_providers.py @@ -1,21 +1,51 @@ import pytest -from opendecision import DecisionQuestion,ProviderChain,Rule,RuleProvider + +from opendecision import DecisionQuestion, ProviderChain, Rule, RuleProvider from opendecision.errors import ProvidersExhaustedError + class BrokenProvider: - name="broken" - def predict(self,state,question): raise RuntimeError("unavailable") + name = "broken" + + def predict(self, state, question): + raise RuntimeError("unavailable") + + +def question(): + return DecisionQuestion( + type="choice", + instructions="Should this execute?", + options={"allow": "Proceed", "review": "Review", "block": "Stop"}, + ) -def question(): return DecisionQuestion(type="choice",instructions="Should this execute?",options={"allow":"Proceed","review":"Review","block":"Stop"}) def test_rules_provider_supports_nested_fields(): - provider=RuleProvider([Rule(field="action.command",operator="contains",value="rm -rf",decision="block")],default="review") - assert provider.predict({"action":{"command":"sudo rm -rf /tmp/x"}},question())["answer"]["choice"]=="block" + provider = RuleProvider( + [ + Rule( + field="action.command", + operator="contains", + value="rm -rf", + decision="block", + ) + ], + default="review", + ) + result = provider.predict({"action": {"command": "sudo rm -rf /tmp/x"}}, question()) + assert result["answer"]["choice"] == "block" + def test_chain_falls_back_and_records_attempts(): - result=ProviderChain([BrokenProvider(),RuleProvider([],default="review")]).predict({},question()) - assert result["provider"]=="rules" and [x["success"] for x in result["attempts"]]==[False,True] + chain = ProviderChain( + [BrokenProvider(), RuleProvider([], default="review")], + ) + result = chain.predict({}, question()) + assert result["provider"] == "rules" + assert [attempt["success"] for attempt in result["attempts"]] == [False, True] + def test_terminal_fallback_and_exhaustion(): - assert ProviderChain([BrokenProvider()],terminal_answer={"choice":"block"}).predict({},question())["provider"]=="terminal_fallback" - with pytest.raises(ProvidersExhaustedError): ProviderChain([BrokenProvider()]).predict({},question()) + terminal = ProviderChain([BrokenProvider()], terminal_answer={"choice": "block"}) + assert terminal.predict({}, question())["provider"] == "terminal_fallback" + with pytest.raises(ProvidersExhaustedError): + ProviderChain([BrokenProvider()]).predict({}, question()) From 58d6d5e0ec32d125b3665417d23c09bceeae8f9d Mon Sep 17 00:00:00 2001 From: pateti Chandu Date: Wed, 23 Sep 2026 13:14:07 +0530 Subject: [PATCH 7/7] fix: normalize probability types for strict mypy --- opendecision/guard.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/opendecision/guard.py b/opendecision/guard.py index 64977ed..0aadba7 100644 --- a/opendecision/guard.py +++ b/opendecision/guard.py @@ -254,6 +254,10 @@ def _probabilities(value: Any) -> dict[str, float]: return {} if not isinstance(value, Mapping): raise ContractValidationError("probabilities must be a mapping") - return { - str(key): _probability(probability) for key, probability in value.items() - } # type: ignore[misc] + normalized: dict[str, float] = {} + for key, probability in value.items(): + validated = _probability(probability) + if validated is None: + raise ContractValidationError("probability cannot be null") + normalized[str(key)] = validated + return normalized