From 608f9c21ad1d5119b0171f17450ba4408b85715c Mon Sep 17 00:00:00 2001 From: Nicole Gomes Date: Mon, 19 Oct 2026 10:33:40 +0200 Subject: [PATCH 1/5] fix: content filtering with stream --- src/sap_cloud_sdk/aicore/completion.py | 75 ++++++++-- src/sap_cloud_sdk/aicore/filtering/_patch.py | 4 +- src/sap_cloud_sdk/aicore/user-guide.md | 36 +++-- tests/aicore/filtering/unit/test_filtering.py | 58 ++++++++ tests/aicore/unit/test_completion.py | 136 ++++++++++++++++++ 5 files changed, 284 insertions(+), 25 deletions(-) diff --git a/src/sap_cloud_sdk/aicore/completion.py b/src/sap_cloud_sdk/aicore/completion.py index 70dc6266..080a8203 100644 --- a/src/sap_cloud_sdk/aicore/completion.py +++ b/src/sap_cloud_sdk/aicore/completion.py @@ -40,9 +40,15 @@ ... These are intentionally thin — every other keyword argument is forwarded -verbatim to ``litellm.completion`` / ``litellm.acompletion``, including -``stream=True`` (the streaming iterator is returned unchanged). No -telemetry is recorded here; the wrappers fire on every LLM call and a +verbatim to ``litellm.completion`` / ``litellm.acompletion``. When content +filtering is inactive, or when only input filtering is configured, +``stream=True`` is allowed — input filter rejections surface as +:class:`ContentFilteredError` through the stream iterator regardless. +When output filtering is active, the wrappers wrap the returned iterator +so that a ``finish_reason == "content_filter"`` chunk raises +:class:`ContentFilteredError` rather than being yielded silently. + +No telemetry is recorded here; the wrappers fire on every LLM call and a counter at this level would be both noisy and uninformative about adoption of the SDK. """ @@ -58,10 +64,46 @@ from sap_cloud_sdk.core.telemetry.module import Module from sap_cloud_sdk.core.telemetry.operation import Operation from .filtering.filters import _parse_input_filter_error +from .filtering import _patch as _filtering_patch logger = logging.getLogger(__name__) +def _output_filtering_active() -> bool: + """Return True only when output filtering is configured. + + Input-filter rejections already surface correctly through the stream + iterator (the server sends an in-stream error event). Output filtering + needs extra handling because the server signals it via a normal chunk + with ``finish_reason == "content_filter"``, which would otherwise be + silently yielded to the caller. + """ + cfg = _filtering_patch._active_cfg + return cfg is not None and cfg.output_filtering is not None + + +def _wrap_stream_for_output_filter(iterator: Any) -> Any: + """Wrap a sync stream iterator to raise ContentFilteredError on filtered chunks.""" + from .filtering.exceptions import ContentFilteredError + + for chunk in iterator: + choices = getattr(chunk, "choices", None) or [] + if choices and getattr(choices[0], "finish_reason", None) == "content_filter": + raise ContentFilteredError(direction="output", details={}, request_id=None) + yield chunk + + +async def _awrap_stream_for_output_filter(iterator: Any) -> Any: + """Wrap an async stream iterator to raise ContentFilteredError on filtered chunks.""" + from .filtering.exceptions import ContentFilteredError + + async for chunk in iterator: + choices = getattr(chunk, "choices", None) or [] + if choices and getattr(choices[0], "finish_reason", None) == "content_filter": + raise ContentFilteredError(direction="output", details={}, request_id=None) + yield chunk + + @record_metrics(Module.AICORE, Operation.AICORE_REACTIVE_RELOAD) def _reload_reactive() -> None: # Local import avoids circular dep: completion ← __init__ ← completion @@ -94,35 +136,50 @@ def completion(*args: Any, **kwargs: Any) -> Any: Model strings (e.g. ``sap/``) are passed verbatim to LiteLLM in all routing modes — proxy routing is handled by ``litellm.api_base`` configured in :func:`set_aicore_config`, not by rewriting the model name. + + When output filtering is active and ``stream=True``, the returned iterator + is wrapped so that a ``finish_reason == "content_filter"`` chunk raises + :class:`ContentFilteredError` instead of being yielded silently. """ try: - return litellm.completion(*args, **kwargs) + result = litellm.completion(*args, **kwargs) except litellm.AuthenticationError: _reload_reactive() - return litellm.completion(*args, **kwargs) + result = litellm.completion(*args, **kwargs) except Exception as exc: translated = _maybe_translate_filter_error(exc) if translated is exc: raise raise translated from exc + if kwargs.get("stream") and _output_filtering_active(): + return _wrap_stream_for_output_filter(result) + return result async def acompletion(*args: Any, **kwargs: Any) -> Any: """Async wrapper around :func:`litellm.acompletion`. - Same credential-rotation semantics as :func:`completion`. - Model strings are passed verbatim to LiteLLM in all routing modes. + Same credential-rotation and output-filter streaming semantics as + :func:`completion`. Model strings are passed verbatim to LiteLLM in all + routing modes. + + When output filtering is active and ``stream=True``, the returned async + iterator is wrapped so that a ``finish_reason == "content_filter"`` chunk + raises :class:`ContentFilteredError` instead of being yielded silently. """ try: - return await litellm.acompletion(*args, **kwargs) + result = await litellm.acompletion(*args, **kwargs) except litellm.AuthenticationError: _reload_reactive() - return await litellm.acompletion(*args, **kwargs) + result = await litellm.acompletion(*args, **kwargs) except Exception as exc: translated = _maybe_translate_filter_error(exc) if translated is exc: raise raise translated from exc + if kwargs.get("stream") and _output_filtering_active(): + return _awrap_stream_for_output_filter(result) + return result __all__ = ["completion", "acompletion"] diff --git a/src/sap_cloud_sdk/aicore/filtering/_patch.py b/src/sap_cloud_sdk/aicore/filtering/_patch.py index c3c1ab1d..cdf730be 100644 --- a/src/sap_cloud_sdk/aicore/filtering/_patch.py +++ b/src/sap_cloud_sdk/aicore/filtering/_patch.py @@ -75,8 +75,8 @@ def transform_request( modules = body["config"]["modules"] if isinstance(modules, list): - # Fallback mode (list of configs) — inject into primary config only. - modules[0]["filtering"] = filtering_dict + for m in modules: + m["filtering"] = filtering_dict else: modules["filtering"] = filtering_dict diff --git a/src/sap_cloud_sdk/aicore/user-guide.md b/src/sap_cloud_sdk/aicore/user-guide.md index 095a9204..09d01ab2 100644 --- a/src/sap_cloud_sdk/aicore/user-guide.md +++ b/src/sap_cloud_sdk/aicore/user-guide.md @@ -360,10 +360,12 @@ except ContentFilteredError as e: return "Your request was blocked by content safety policy." ``` -The wrapper forwards every argument verbatim to `litellm.completion` -(including `stream=True`), and only intercepts the wrapped -`APIConnectionError` shape that LiteLLM produces for input-filter -rejections. All other exceptions surface unchanged. +The wrapper forwards every argument verbatim to `litellm.completion`, +including `stream=True`. When output filtering is active and `stream=True`, +the returned iterator is wrapped so that a `finish_reason == "content_filter"` +chunk raises `ContentFilteredError` instead of being silently yielded. Input +filtering works server-side regardless of streaming — no extra handling needed. +All other exceptions surface unchanged. `ContentFilteredError` exposes three attributes — `direction`, `details`, `request_id`. The `details` field contains severity scalars from the server, @@ -444,21 +446,27 @@ print(response.choices[0].message.content) ### Streaming Responses +Use `sap_cloud_sdk.aicore.completion` (not `litellm.completion` directly) so +that output-filter rejections surface as `ContentFilteredError` rather than +being silently yielded: + ```python -from sap_cloud_sdk.aicore import set_aicore_config -from litellm import completion +from sap_cloud_sdk.aicore import ContentFilteredError, completion, set_aicore_config set_aicore_config() -response = completion( - model="sap/gpt-4", - messages=[{"role": "user", "content": "Tell me a story"}], - stream=True, -) +try: + response = completion( + model="sap/gpt-4", + messages=[{"role": "user", "content": "Tell me a story"}], + stream=True, + ) -for chunk in response: - if chunk.choices[0].delta.content: - print(chunk.choices[0].delta.content, end="") + for chunk in response: + if chunk.choices[0].delta.content: + print(chunk.choices[0].delta.content, end="") +except ContentFilteredError as e: + print(f"Blocked ({e.direction}): {e.details}") ``` ### Using Different Models diff --git a/tests/aicore/filtering/unit/test_filtering.py b/tests/aicore/filtering/unit/test_filtering.py index 0f0fe6c9..b8954a18 100644 --- a/tests/aicore/filtering/unit/test_filtering.py +++ b/tests/aicore/filtering/unit/test_filtering.py @@ -133,3 +133,61 @@ def test_disable_when_never_enabled(self, monkeypatch): disable_filtering() assert litellm.GenAIHubOrchestrationConfig is _ORIGINAL_CONFIG assert _filters_mod._active_cfg is None + + +class TestTransformRequestMultiModule: + """Regression tests: filtering must be injected into every fallback module, not just modules[0].""" + + def _make_cfg(self) -> ContentFiltering: + return ContentFiltering( + input_filtering=InputFiltering(filters=[AzureContentFilter()]) + ) + + def _call_transform_request(self, modules_value): + """Call the filtering injection logic directly, bypassing the LiteLLM base class.""" + from sap_cloud_sdk.aicore.filtering import _patch as _p + from unittest.mock import patch + + body = {"config": {"modules": modules_value}} + instance = FilteringOrchestrationConfig() + + with patch( + "sap_cloud_sdk.aicore.filtering._patch.GenAIHubOrchestrationConfig.transform_request", + return_value=body, + ): + return instance.transform_request( + model="sap/x", + messages=[], + optional_params={}, + litellm_params={}, + headers={}, + ) + + def test_single_module_dict_gets_filtering(self, monkeypatch): + _clear_aicore_env(monkeypatch) + set_filtering(self._make_cfg()) + modules = {"model_id": "primary-llm"} + result = self._call_transform_request(modules) + assert "filtering" in result["config"]["modules"] + + def test_multi_module_list_all_get_filtering(self, monkeypatch): + _clear_aicore_env(monkeypatch) + set_filtering(self._make_cfg()) + modules = [ + {"model_id": "primary-llm"}, + {"model_id": "fallback-llm-1"}, + {"model_id": "fallback-llm-2"}, + ] + result = self._call_transform_request(modules) + for i, m in enumerate(result["config"]["modules"]): + assert "filtering" in m, f"modules[{i}] missing filtering block" + + def test_multi_module_filtering_dict_is_identical_across_modules(self, monkeypatch): + _clear_aicore_env(monkeypatch) + set_filtering(self._make_cfg()) + modules = [{"model_id": "a"}, {"model_id": "b"}] + result = self._call_transform_request(modules) + assert ( + result["config"]["modules"][0]["filtering"] + == result["config"]["modules"][1]["filtering"] + ) diff --git a/tests/aicore/unit/test_completion.py b/tests/aicore/unit/test_completion.py index 8238e922..5b89cb1c 100644 --- a/tests/aicore/unit/test_completion.py +++ b/tests/aicore/unit/test_completion.py @@ -275,6 +275,142 @@ def test_non_auth_error_does_not_trigger_reload(self): mock_reload.assert_not_called() +# --------------------------------------------------------------------------- +# Streaming guard — stream=True must not bypass output filtering +# --------------------------------------------------------------------------- + + +class TestStreamingGuard: + """Output-filter streaming: ContentFilteredError raised on content_filter chunk. + Input-only filtering: stream=True passes through unmodified. + """ + + @pytest.fixture(autouse=True) + def restore_filtering(self): + from sap_cloud_sdk.aicore.filtering._patch import _install + _install(None) + yield + _install(None) + + def _activate_input_only(self): + from sap_cloud_sdk.aicore.filtering import ( + AzureContentFilter, ContentFiltering, InputFiltering, set_filtering, + ) + set_filtering(ContentFiltering( + input_filtering=InputFiltering(filters=[AzureContentFilter()]) + )) + + def _activate_output(self): + from sap_cloud_sdk.aicore.filtering import ( + AzureContentFilter, ContentFiltering, OutputFiltering, set_filtering, + ) + set_filtering(ContentFiltering( + output_filtering=OutputFiltering(filters=[AzureContentFilter()]) + )) + + def _activate_input_and_output(self): + from sap_cloud_sdk.aicore.filtering import ( + AzureContentFilter, ContentFiltering, InputFiltering, OutputFiltering, + set_filtering, + ) + set_filtering(ContentFiltering( + input_filtering=InputFiltering(filters=[AzureContentFilter()]), + output_filtering=OutputFiltering(filters=[AzureContentFilter()]), + )) + + def _make_chunk(self, finish_reason=None): + """Build a minimal object that looks like an OpenAI stream chunk.""" + choice = MagicMock() + choice.finish_reason = finish_reason + chunk = MagicMock() + chunk.choices = [choice] + return chunk + + # --- input-only: stream allowed, iterator returned as-is ---------------- + + def test_no_filtering_stream_allowed(self): + sentinel = object() + with patch("sap_cloud_sdk.aicore.completion.litellm.completion", return_value=sentinel): + result = completion(model="sap/x", messages=[], stream=True) + assert result is sentinel + + def test_input_only_filtering_stream_allowed(self): + self._activate_input_only() + sentinel = object() + with patch("sap_cloud_sdk.aicore.completion.litellm.completion", return_value=sentinel): + result = completion(model="sap/x", messages=[], stream=True) + assert result is sentinel + + # --- output filtering: iterator wrapped, normal chunks pass through ------ + + def test_output_filter_stream_normal_chunks_pass_through(self): + self._activate_output() + chunks = [self._make_chunk(finish_reason=None), self._make_chunk(finish_reason="stop")] + with patch("sap_cloud_sdk.aicore.completion.litellm.completion", return_value=iter(chunks)): + stream = completion(model="sap/x", messages=[], stream=True) + collected = list(stream) + assert len(collected) == 2 + + def test_output_filter_stream_raises_on_content_filter_chunk(self): + from sap_cloud_sdk.aicore.filtering.exceptions import ContentFilteredError + self._activate_output() + chunks = [ + self._make_chunk(finish_reason=None), + self._make_chunk(finish_reason="content_filter"), + ] + with patch("sap_cloud_sdk.aicore.completion.litellm.completion", return_value=iter(chunks)): + stream = completion(model="sap/x", messages=[], stream=True) + with pytest.raises(ContentFilteredError) as ei: + list(stream) + assert ei.value.direction == "output" + + def test_input_and_output_stream_raises_on_content_filter_chunk(self): + from sap_cloud_sdk.aicore.filtering.exceptions import ContentFilteredError + self._activate_input_and_output() + chunks = [self._make_chunk(finish_reason="content_filter")] + with patch("sap_cloud_sdk.aicore.completion.litellm.completion", return_value=iter(chunks)): + stream = completion(model="sap/x", messages=[], stream=True) + with pytest.raises(ContentFilteredError): + list(stream) + + # --- async variants ------------------------------------------------------ + + def test_acompletion_input_only_stream_allowed(self): + sentinel = object() + + async def _fake_acompletion(*_, **__): # noqa: S7503 + return sentinel + + with patch("sap_cloud_sdk.aicore.completion.litellm.acompletion", side_effect=_fake_acompletion): + result = asyncio.run(acompletion(model="sap/x", messages=[], stream=True)) + assert result is sentinel + + def test_acompletion_output_filter_stream_raises_on_content_filter_chunk(self): + from sap_cloud_sdk.aicore.filtering.exceptions import ContentFilteredError + self._activate_output() + + chunks = [self._make_chunk(finish_reason="content_filter")] + + async def fake_iter(): + for c in chunks: + yield c + + async def fake(*args, **kwargs): + return fake_iter() + + with patch("sap_cloud_sdk.aicore.completion.litellm.acompletion", side_effect=fake): + stream = asyncio.run(acompletion(model="sap/x", messages=[], stream=True)) + + async def collect(): + results = [] + async for chunk in stream: + results.append(chunk) + return results + + with pytest.raises(ContentFilteredError): + asyncio.run(collect()) + + # --------------------------------------------------------------------------- # Reactive reload on AuthenticationError — async # --------------------------------------------------------------------------- From 689db5a3a8e1ef6bbad8118d56fb34aa8b674b29 Mon Sep 17 00:00:00 2001 From: Nicole Gomes Date: Mon, 19 Oct 2026 15:13:57 +0200 Subject: [PATCH 2/5] fix(aicore): extract request_id from filtered stream chunk The streaming ContentFilteredError was always raised with request_id=None. The SAP orchestration v2 SSE format includes request_id in every chunk (surfaced as chunk.id by the LiteLLM SAP handler), so extract it from the filtered chunk for auditability. --- src/sap_cloud_sdk/aicore/completion.py | 8 ++++++-- tests/aicore/unit/test_completion.py | 6 ++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/sap_cloud_sdk/aicore/completion.py b/src/sap_cloud_sdk/aicore/completion.py index 080a8203..bada323c 100644 --- a/src/sap_cloud_sdk/aicore/completion.py +++ b/src/sap_cloud_sdk/aicore/completion.py @@ -89,7 +89,9 @@ def _wrap_stream_for_output_filter(iterator: Any) -> Any: for chunk in iterator: choices = getattr(chunk, "choices", None) or [] if choices and getattr(choices[0], "finish_reason", None) == "content_filter": - raise ContentFilteredError(direction="output", details={}, request_id=None) + raise ContentFilteredError( + direction="output", details={}, request_id=getattr(chunk, "id", None) + ) yield chunk @@ -100,7 +102,9 @@ async def _awrap_stream_for_output_filter(iterator: Any) -> Any: async for chunk in iterator: choices = getattr(chunk, "choices", None) or [] if choices and getattr(choices[0], "finish_reason", None) == "content_filter": - raise ContentFilteredError(direction="output", details={}, request_id=None) + raise ContentFilteredError( + direction="output", details={}, request_id=getattr(chunk, "id", None) + ) yield chunk diff --git a/tests/aicore/unit/test_completion.py b/tests/aicore/unit/test_completion.py index 5b89cb1c..e4bf1cb0 100644 --- a/tests/aicore/unit/test_completion.py +++ b/tests/aicore/unit/test_completion.py @@ -318,12 +318,13 @@ def _activate_input_and_output(self): output_filtering=OutputFiltering(filters=[AzureContentFilter()]), )) - def _make_chunk(self, finish_reason=None): + def _make_chunk(self, finish_reason=None, request_id=None): """Build a minimal object that looks like an OpenAI stream chunk.""" choice = MagicMock() choice.finish_reason = finish_reason chunk = MagicMock() chunk.choices = [choice] + chunk.id = request_id return chunk # --- input-only: stream allowed, iterator returned as-is ---------------- @@ -356,13 +357,14 @@ def test_output_filter_stream_raises_on_content_filter_chunk(self): self._activate_output() chunks = [ self._make_chunk(finish_reason=None), - self._make_chunk(finish_reason="content_filter"), + self._make_chunk(finish_reason="content_filter", request_id="req-stream-123"), ] with patch("sap_cloud_sdk.aicore.completion.litellm.completion", return_value=iter(chunks)): stream = completion(model="sap/x", messages=[], stream=True) with pytest.raises(ContentFilteredError) as ei: list(stream) assert ei.value.direction == "output" + assert ei.value.request_id == "req-stream-123" def test_input_and_output_stream_raises_on_content_filter_chunk(self): from sap_cloud_sdk.aicore.filtering.exceptions import ContentFilteredError From 50e274d9c191bac2b93cf7b11d7426c78100121a Mon Sep 17 00:00:00 2001 From: Nicole Gomes Date: Mon, 19 Oct 2026 16:11:07 +0200 Subject: [PATCH 3/5] fix(aicore): block stream=True when output filtering is active MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Wrapping the stream iterator only protected callers using the SDK's completion() wrapper. ChatLiteLLM and LangGraph agents call litellm.completion directly, bypassing the wrapper entirely, so content_filter chunks would be silently yielded. Moving the guard into FilteringOrchestrationConfig.transform_request makes it universal — it fires before the HTTP request is sent regardless of the caller, because transform_request runs at the LiteLLM transport layer. --- src/sap_cloud_sdk/aicore/completion.py | 81 ++--------- src/sap_cloud_sdk/aicore/filtering/_patch.py | 28 ++++ src/sap_cloud_sdk/aicore/user-guide.md | 17 ++- tests/aicore/unit/test_completion.py | 137 ++++++++++--------- 4 files changed, 127 insertions(+), 136 deletions(-) diff --git a/src/sap_cloud_sdk/aicore/completion.py b/src/sap_cloud_sdk/aicore/completion.py index bada323c..4fb91668 100644 --- a/src/sap_cloud_sdk/aicore/completion.py +++ b/src/sap_cloud_sdk/aicore/completion.py @@ -40,13 +40,13 @@ ... These are intentionally thin — every other keyword argument is forwarded -verbatim to ``litellm.completion`` / ``litellm.acompletion``. When content -filtering is inactive, or when only input filtering is configured, -``stream=True`` is allowed — input filter rejections surface as -:class:`ContentFilteredError` through the stream iterator regardless. -When output filtering is active, the wrappers wrap the returned iterator -so that a ``finish_reason == "content_filter"`` chunk raises -:class:`ContentFilteredError` rather than being yielded silently. +verbatim to ``litellm.completion`` / ``litellm.acompletion``. When output +filtering is active, ``stream=True`` is not supported — the filtering +transport patch raises :class:`ContentFilteredError` upfront before the +request is sent. This applies universally to all callers (including +``ChatLiteLLM`` and LangGraph agents). Input filtering works with streaming +— filter rejections surface as :class:`ContentFilteredError` through the +stream iterator regardless. No telemetry is recorded here; the wrappers fire on every LLM call and a counter at this level would be both noisy and uninformative about @@ -64,50 +64,10 @@ from sap_cloud_sdk.core.telemetry.module import Module from sap_cloud_sdk.core.telemetry.operation import Operation from .filtering.filters import _parse_input_filter_error -from .filtering import _patch as _filtering_patch logger = logging.getLogger(__name__) -def _output_filtering_active() -> bool: - """Return True only when output filtering is configured. - - Input-filter rejections already surface correctly through the stream - iterator (the server sends an in-stream error event). Output filtering - needs extra handling because the server signals it via a normal chunk - with ``finish_reason == "content_filter"``, which would otherwise be - silently yielded to the caller. - """ - cfg = _filtering_patch._active_cfg - return cfg is not None and cfg.output_filtering is not None - - -def _wrap_stream_for_output_filter(iterator: Any) -> Any: - """Wrap a sync stream iterator to raise ContentFilteredError on filtered chunks.""" - from .filtering.exceptions import ContentFilteredError - - for chunk in iterator: - choices = getattr(chunk, "choices", None) or [] - if choices and getattr(choices[0], "finish_reason", None) == "content_filter": - raise ContentFilteredError( - direction="output", details={}, request_id=getattr(chunk, "id", None) - ) - yield chunk - - -async def _awrap_stream_for_output_filter(iterator: Any) -> Any: - """Wrap an async stream iterator to raise ContentFilteredError on filtered chunks.""" - from .filtering.exceptions import ContentFilteredError - - async for chunk in iterator: - choices = getattr(chunk, "choices", None) or [] - if choices and getattr(choices[0], "finish_reason", None) == "content_filter": - raise ContentFilteredError( - direction="output", details={}, request_id=getattr(chunk, "id", None) - ) - yield chunk - - @record_metrics(Module.AICORE, Operation.AICORE_REACTIVE_RELOAD) def _reload_reactive() -> None: # Local import avoids circular dep: completion ← __init__ ← completion @@ -140,50 +100,35 @@ def completion(*args: Any, **kwargs: Any) -> Any: Model strings (e.g. ``sap/``) are passed verbatim to LiteLLM in all routing modes — proxy routing is handled by ``litellm.api_base`` configured in :func:`set_aicore_config`, not by rewriting the model name. - - When output filtering is active and ``stream=True``, the returned iterator - is wrapped so that a ``finish_reason == "content_filter"`` chunk raises - :class:`ContentFilteredError` instead of being yielded silently. """ try: - result = litellm.completion(*args, **kwargs) + return litellm.completion(*args, **kwargs) except litellm.AuthenticationError: _reload_reactive() - result = litellm.completion(*args, **kwargs) + return litellm.completion(*args, **kwargs) except Exception as exc: translated = _maybe_translate_filter_error(exc) if translated is exc: raise raise translated from exc - if kwargs.get("stream") and _output_filtering_active(): - return _wrap_stream_for_output_filter(result) - return result async def acompletion(*args: Any, **kwargs: Any) -> Any: """Async wrapper around :func:`litellm.acompletion`. - Same credential-rotation and output-filter streaming semantics as - :func:`completion`. Model strings are passed verbatim to LiteLLM in all - routing modes. - - When output filtering is active and ``stream=True``, the returned async - iterator is wrapped so that a ``finish_reason == "content_filter"`` chunk - raises :class:`ContentFilteredError` instead of being yielded silently. + Same credential-rotation semantics as :func:`completion`. + Model strings are passed verbatim to LiteLLM in all routing modes. """ try: - result = await litellm.acompletion(*args, **kwargs) + return await litellm.acompletion(*args, **kwargs) except litellm.AuthenticationError: _reload_reactive() - result = await litellm.acompletion(*args, **kwargs) + return await litellm.acompletion(*args, **kwargs) except Exception as exc: translated = _maybe_translate_filter_error(exc) if translated is exc: raise raise translated from exc - if kwargs.get("stream") and _output_filtering_active(): - return _awrap_stream_for_output_filter(result) - return result __all__ = ["completion", "acompletion"] diff --git a/src/sap_cloud_sdk/aicore/filtering/_patch.py b/src/sap_cloud_sdk/aicore/filtering/_patch.py index cdf730be..ceeece89 100644 --- a/src/sap_cloud_sdk/aicore/filtering/_patch.py +++ b/src/sap_cloud_sdk/aicore/filtering/_patch.py @@ -26,6 +26,21 @@ back into :class:`ContentFilteredError` before it reaches caller code; the :func:`_parse_input_filter_error` helper in :mod:`.filters` does the JSON parsing. + +Streaming + output filtering +---------------------------- +The ``transform_response`` hook is never called for streaming responses — +LiteLLM routes those through a separate iterator path. This means output +filter rejections (``finish_reason == "content_filter"`` chunks) would be +silently yielded to the caller rather than raising +:class:`ContentFilteredError`. + +To prevent this bypass, ``transform_request`` raises +:class:`ContentFilteredError` upfront when both streaming and output +filtering are requested simultaneously. This guarantee is universal — it +applies to all callers (``sap_cloud_sdk.aicore.completion``, +``ChatLiteLLM``, LangGraph, etc.) because ``transform_request`` runs at +the LiteLLM transport layer. """ from __future__ import annotations @@ -73,6 +88,19 @@ def transform_request( if not filtering_dict: return body + # Streaming + output filtering: transform_response is never called for + # streams, so content_filter chunks would be silently yielded. Block + # the combination upfront — applies to all callers universally. + if ( + _active_cfg.output_filtering is not None + and body.get("config", {}).get("stream", {}).get("enabled") + ): + raise ContentFilteredError( + direction="output", + details={}, + request_id=None, + ) + modules = body["config"]["modules"] if isinstance(modules, list): for m in modules: diff --git a/src/sap_cloud_sdk/aicore/user-guide.md b/src/sap_cloud_sdk/aicore/user-guide.md index 09d01ab2..4fea8150 100644 --- a/src/sap_cloud_sdk/aicore/user-guide.md +++ b/src/sap_cloud_sdk/aicore/user-guide.md @@ -361,10 +361,12 @@ except ContentFilteredError as e: ``` The wrapper forwards every argument verbatim to `litellm.completion`, -including `stream=True`. When output filtering is active and `stream=True`, -the returned iterator is wrapped so that a `finish_reason == "content_filter"` -chunk raises `ContentFilteredError` instead of being silently yielded. Input -filtering works server-side regardless of streaming — no extra handling needed. +including `stream=True`. When output filtering is active, `stream=True` is +not supported — a `ContentFilteredError(direction="output")` is raised before +the request is sent. This block applies universally to all callers (including +`ChatLiteLLM` and LangGraph agents) because it fires inside the LiteLLM +transport patch. Input filtering works with streaming — filter rejections +surface as `ContentFilteredError` regardless. All other exceptions surface unchanged. `ContentFilteredError` exposes three attributes — `direction`, `details`, @@ -446,9 +448,10 @@ print(response.choices[0].message.content) ### Streaming Responses -Use `sap_cloud_sdk.aicore.completion` (not `litellm.completion` directly) so -that output-filter rejections surface as `ContentFilteredError` rather than -being silently yielded: +`stream=True` is supported when no output filtering is configured. If output +filtering is active, streaming is blocked upfront with +`ContentFilteredError(direction="output")` — this applies to all callers +including `ChatLiteLLM` and LangGraph agents. ```python from sap_cloud_sdk.aicore import ContentFilteredError, completion, set_aicore_config diff --git a/tests/aicore/unit/test_completion.py b/tests/aicore/unit/test_completion.py index e4bf1cb0..239eff03 100644 --- a/tests/aicore/unit/test_completion.py +++ b/tests/aicore/unit/test_completion.py @@ -281,7 +281,7 @@ def test_non_auth_error_does_not_trigger_reload(self): class TestStreamingGuard: - """Output-filter streaming: ContentFilteredError raised on content_filter chunk. + """stream=True is blocked upfront when output filtering is active. Input-only filtering: stream=True passes through unmodified. """ @@ -318,16 +318,7 @@ def _activate_input_and_output(self): output_filtering=OutputFiltering(filters=[AzureContentFilter()]), )) - def _make_chunk(self, finish_reason=None, request_id=None): - """Build a minimal object that looks like an OpenAI stream chunk.""" - choice = MagicMock() - choice.finish_reason = finish_reason - chunk = MagicMock() - chunk.choices = [choice] - chunk.id = request_id - return chunk - - # --- input-only: stream allowed, iterator returned as-is ---------------- + # --- input-only or no filtering: stream=True allowed -------------------- def test_no_filtering_stream_allowed(self): sentinel = object() @@ -342,75 +333,99 @@ def test_input_only_filtering_stream_allowed(self): result = completion(model="sap/x", messages=[], stream=True) assert result is sentinel - # --- output filtering: iterator wrapped, normal chunks pass through ------ + def test_acompletion_input_only_stream_allowed(self): + sentinel = object() - def test_output_filter_stream_normal_chunks_pass_through(self): - self._activate_output() - chunks = [self._make_chunk(finish_reason=None), self._make_chunk(finish_reason="stop")] - with patch("sap_cloud_sdk.aicore.completion.litellm.completion", return_value=iter(chunks)): - stream = completion(model="sap/x", messages=[], stream=True) - collected = list(stream) - assert len(collected) == 2 + async def _fake_acompletion(*_, **__): # noqa: S7503 + return sentinel + + with patch("sap_cloud_sdk.aicore.completion.litellm.acompletion", side_effect=_fake_acompletion): + result = asyncio.run(acompletion(model="sap/x", messages=[], stream=True)) + assert result is sentinel - def test_output_filter_stream_raises_on_content_filter_chunk(self): + # --- output filtering + stream=True: blocked upfront -------------------- + + def test_output_filter_stream_blocked(self): from sap_cloud_sdk.aicore.filtering.exceptions import ContentFilteredError + from sap_cloud_sdk.aicore.filtering._patch import FilteringOrchestrationConfig + from unittest.mock import patch as _patch + self._activate_output() - chunks = [ - self._make_chunk(finish_reason=None), - self._make_chunk(finish_reason="content_filter", request_id="req-stream-123"), - ] - with patch("sap_cloud_sdk.aicore.completion.litellm.completion", return_value=iter(chunks)): - stream = completion(model="sap/x", messages=[], stream=True) - with pytest.raises(ContentFilteredError) as ei: - list(stream) + + body = {"config": {"stream": {"enabled": True}, "modules": {"model_id": "x"}}} + instance = FilteringOrchestrationConfig() + + with _patch( + "sap_cloud_sdk.aicore.filtering._patch.GenAIHubOrchestrationConfig.transform_request", + return_value=body, + ): + with pytest.raises(ContentFilteredError) as ei: + instance.transform_request( + model="sap/x", messages=[], optional_params={}, + litellm_params={}, headers={}, + ) assert ei.value.direction == "output" - assert ei.value.request_id == "req-stream-123" - def test_input_and_output_stream_raises_on_content_filter_chunk(self): + def test_input_and_output_filter_stream_blocked(self): from sap_cloud_sdk.aicore.filtering.exceptions import ContentFilteredError + from sap_cloud_sdk.aicore.filtering._patch import FilteringOrchestrationConfig + from unittest.mock import patch as _patch + self._activate_input_and_output() - chunks = [self._make_chunk(finish_reason="content_filter")] - with patch("sap_cloud_sdk.aicore.completion.litellm.completion", return_value=iter(chunks)): - stream = completion(model="sap/x", messages=[], stream=True) - with pytest.raises(ContentFilteredError): - list(stream) - # --- async variants ------------------------------------------------------ + body = {"config": {"stream": {"enabled": True}, "modules": {"model_id": "x"}}} + instance = FilteringOrchestrationConfig() - def test_acompletion_input_only_stream_allowed(self): - sentinel = object() - - async def _fake_acompletion(*_, **__): # noqa: S7503 - return sentinel + with _patch( + "sap_cloud_sdk.aicore.filtering._patch.GenAIHubOrchestrationConfig.transform_request", + return_value=body, + ): + with pytest.raises(ContentFilteredError) as ei: + instance.transform_request( + model="sap/x", messages=[], optional_params={}, + litellm_params={}, headers={}, + ) + assert ei.value.direction == "output" - with patch("sap_cloud_sdk.aicore.completion.litellm.acompletion", side_effect=_fake_acompletion): - result = asyncio.run(acompletion(model="sap/x", messages=[], stream=True)) - assert result is sentinel + def test_output_filter_no_stream_not_blocked(self): + from sap_cloud_sdk.aicore.filtering._patch import FilteringOrchestrationConfig + from unittest.mock import patch as _patch - def test_acompletion_output_filter_stream_raises_on_content_filter_chunk(self): - from sap_cloud_sdk.aicore.filtering.exceptions import ContentFilteredError self._activate_output() - chunks = [self._make_chunk(finish_reason="content_filter")] + body = {"config": {"modules": {"model_id": "x"}}} + instance = FilteringOrchestrationConfig() - async def fake_iter(): - for c in chunks: - yield c + with _patch( + "sap_cloud_sdk.aicore.filtering._patch.GenAIHubOrchestrationConfig.transform_request", + return_value=body, + ): + result = instance.transform_request( + model="sap/x", messages=[], optional_params={}, + litellm_params={}, headers={}, + ) + assert "filtering" in result["config"]["modules"] - async def fake(*args, **kwargs): - return fake_iter() + def test_acompletion_output_filter_stream_blocked(self): + from sap_cloud_sdk.aicore.filtering.exceptions import ContentFilteredError + from sap_cloud_sdk.aicore.filtering._patch import FilteringOrchestrationConfig + from unittest.mock import patch as _patch - with patch("sap_cloud_sdk.aicore.completion.litellm.acompletion", side_effect=fake): - stream = asyncio.run(acompletion(model="sap/x", messages=[], stream=True)) + self._activate_output() - async def collect(): - results = [] - async for chunk in stream: - results.append(chunk) - return results + body = {"config": {"stream": {"enabled": True}, "modules": {"model_id": "x"}}} + instance = FilteringOrchestrationConfig() - with pytest.raises(ContentFilteredError): - asyncio.run(collect()) + with _patch( + "sap_cloud_sdk.aicore.filtering._patch.GenAIHubOrchestrationConfig.transform_request", + return_value=body, + ): + with pytest.raises(ContentFilteredError) as ei: + instance.transform_request( + model="sap/x", messages=[], optional_params={}, + litellm_params={}, headers={}, + ) + assert ei.value.direction == "output" # --------------------------------------------------------------------------- From 321b0ac16f54e3de0c431977bf4bb12061863b90 Mon Sep 17 00:00:00 2001 From: Nicole Gomes Date: Mon, 19 Oct 2026 16:39:16 +0200 Subject: [PATCH 4/5] chore(aicore): fix ruff lint/format issues in filtering patch and tests Remove unused imports flagged by ruff (F401) and reformat the multi-line if condition in _patch.py to satisfy ruff format. --- src/sap_cloud_sdk/aicore/filtering/_patch.py | 7 +++---- tests/aicore/filtering/unit/test_filtering.py | 1 - tests/aicore/unit/test_completion.py | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/sap_cloud_sdk/aicore/filtering/_patch.py b/src/sap_cloud_sdk/aicore/filtering/_patch.py index ceeece89..3f3d7575 100644 --- a/src/sap_cloud_sdk/aicore/filtering/_patch.py +++ b/src/sap_cloud_sdk/aicore/filtering/_patch.py @@ -91,10 +91,9 @@ def transform_request( # Streaming + output filtering: transform_response is never called for # streams, so content_filter chunks would be silently yielded. Block # the combination upfront — applies to all callers universally. - if ( - _active_cfg.output_filtering is not None - and body.get("config", {}).get("stream", {}).get("enabled") - ): + if _active_cfg.output_filtering is not None and body.get("config", {}).get( + "stream", {} + ).get("enabled"): raise ContentFilteredError( direction="output", details={}, diff --git a/tests/aicore/filtering/unit/test_filtering.py b/tests/aicore/filtering/unit/test_filtering.py index b8954a18..6cb16024 100644 --- a/tests/aicore/filtering/unit/test_filtering.py +++ b/tests/aicore/filtering/unit/test_filtering.py @@ -145,7 +145,6 @@ def _make_cfg(self) -> ContentFiltering: def _call_transform_request(self, modules_value): """Call the filtering injection logic directly, bypassing the LiteLLM base class.""" - from sap_cloud_sdk.aicore.filtering import _patch as _p from unittest.mock import patch body = {"config": {"modules": modules_value}} diff --git a/tests/aicore/unit/test_completion.py b/tests/aicore/unit/test_completion.py index 239eff03..88458c42 100644 --- a/tests/aicore/unit/test_completion.py +++ b/tests/aicore/unit/test_completion.py @@ -23,7 +23,7 @@ import asyncio import json -from unittest.mock import MagicMock, call, patch +from unittest.mock import MagicMock, patch import litellm import pytest From db6f33adb1d8c0e0a0c4bb876c79b024be27643a Mon Sep 17 00:00:00 2001 From: Nicole Gomes Date: Mon, 19 Oct 2026 16:41:27 +0200 Subject: [PATCH 5/5] docs(aicore): fix missing blank line in filtering paragraph in user-guide --- src/sap_cloud_sdk/aicore/user-guide.md | 1 + 1 file changed, 1 insertion(+) diff --git a/src/sap_cloud_sdk/aicore/user-guide.md b/src/sap_cloud_sdk/aicore/user-guide.md index 4fea8150..6304ec28 100644 --- a/src/sap_cloud_sdk/aicore/user-guide.md +++ b/src/sap_cloud_sdk/aicore/user-guide.md @@ -367,6 +367,7 @@ the request is sent. This block applies universally to all callers (including `ChatLiteLLM` and LangGraph agents) because it fires inside the LiteLLM transport patch. Input filtering works with streaming — filter rejections surface as `ContentFilteredError` regardless. + All other exceptions surface unchanged. `ContentFilteredError` exposes three attributes — `direction`, `details`,