From 885af304aafba40a706ed0f1cb8234c207908581 Mon Sep 17 00:00:00 2001 From: Jean Scherf Date: Thu, 15 Oct 2026 22:43:13 -0300 Subject: [PATCH] fix(telemetry): avoid duplicate span export under auto-instrumentation wrapper When an OTel auto-instrumentation agent (for example the OpenTelemetry Operator's inject-python) is active, it installs a TracerProvider that already exports spans via OTLP. auto_instrument() then adds the SDK's own Traceloop exporter to the same provider, so every span is exported twice. The log and metric providers already refuse a second exporter in this situation; the trace path did not. Drop the wrapper's span exporter when the SDK exporter is present, consolidating on the SDK exporter (which also applies GenAI attribute normalization). No-op when the SDK exporter is absent or no wrapper is present, so once the auto-instrumentation layer is retired this leaves the SDK exporter as the only one. Adds unit tests for both branches. --- .../core/telemetry/auto_instrument.py | 62 ++++++++++++++ .../unit/telemetry/test_auto_instrument.py | 84 ++++++++++++++++++- 2 files changed, 145 insertions(+), 1 deletion(-) diff --git a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py index 14f1ec3e..65023e58 100644 --- a/src/sap_cloud_sdk/core/telemetry/auto_instrument.py +++ b/src/sap_cloud_sdk/core/telemetry/auto_instrument.py @@ -93,6 +93,7 @@ def auto_instrument( ) _merge_resource_attrs_into_active_provider_if_wrapper_installed(resource) + _drop_wrapper_span_exporter_if_sdk_exports() _set_baggage_processor() _set_propagated_attributes_processor() @@ -226,3 +227,64 @@ def _merge_resource_attrs_into_active_provider_if_wrapper_installed( "Merged sap-cloud-sdk resource attrs onto wrapper-installed " "TracerProvider (marker: telemetry.auto.version)" ) + + +def _drop_wrapper_span_exporter_if_sdk_exports() -> None: + """Remove the auto-instrumentation wrapper's span exporter when the SDK already exports. + + The OTel operator ``inject-python`` auto-instrumentation installs a TracerProvider that + already carries its own OTLP span exporter (marked by ``telemetry.auto.version``). + ``Traceloop.init`` then adds the SDK's own exporting span processor to that same provider, + so every span leaves the process twice. We consolidate on the SDK exporter, which also + applies the GenAI attribute normalization, and remove the wrapper's duplicate exporter. + This anticipates retiring the operator auto-instrumentation layer: once it is gone this + becomes a no-op and the SDK exporter is simply the only one. + + Spans produced by the wrapper's own instrumentations still land on the shared provider and + are shipped by the SDK exporter, so nothing is lost. No-op when the SDK exporter is not + present, so the wrapper's exporter is never removed while it is the only one. In the + managed runtime the only non-SDK span exporter on the provider is the wrapper's; an + app-added custom exporter would also be treated as the wrapper's here. Touches + ``_active_span_processor`` because the OTel SDK exposes no public API to remove a span + processor, mirroring how the resource merge above mutates provider internals. + """ + provider = trace.get_tracer_provider() + if not isinstance(provider, TracerProvider): + return + + attributes = getattr(provider.resource, "attributes", None) + if ( + not isinstance(attributes, Mapping) + or "telemetry.auto.version" not in attributes + ): + return + + active = getattr(provider, "_active_span_processor", None) + if active is None: + return + processors = getattr(active, "_span_processors", None) + if not processors: + return + + exporting = [p for p in processors if getattr(p, "span_exporter", None) is not None] + sdk_added = [p for p in exporting if getattr(p, "_traceloop_processor", False)] + wrapper_added = [ + p for p in exporting if not getattr(p, "_traceloop_processor", False) + ] + + # Only drop the wrapper's exporter when the SDK is also exporting; otherwise the wrapper's + # exporter is the only one and must stay. + if not sdk_added or not wrapper_added: + return + + active._span_processors = tuple(p for p in processors if p not in wrapper_added) + for processor in wrapper_added: + try: + processor.shutdown() + except Exception: # noqa: BLE001 telemetry must never break startup + pass + + logger.info( + "SDK already exports spans; removed the auto-instrumentation wrapper's duplicate " + "span exporter to prevent double export" + ) diff --git a/tests/core/unit/telemetry/test_auto_instrument.py b/tests/core/unit/telemetry/test_auto_instrument.py index e95674b3..0fef9bfb 100644 --- a/tests/core/unit/telemetry/test_auto_instrument.py +++ b/tests/core/unit/telemetry/test_auto_instrument.py @@ -6,8 +6,13 @@ from opentelemetry.sdk.resources import Resource from opentelemetry.sdk.trace import TracerProvider as SDKTracerProvider +from opentelemetry.sdk.trace.export import SimpleSpanProcessor +from opentelemetry.sdk.trace.export.in_memory_span_exporter import InMemorySpanExporter -from sap_cloud_sdk.core.telemetry.auto_instrument import auto_instrument +from sap_cloud_sdk.core.telemetry.auto_instrument import ( + auto_instrument, + _drop_wrapper_span_exporter_if_sdk_exports, +) @pytest.fixture @@ -432,3 +437,80 @@ def test_setup_log_provider_called_with_console_exporter(self, mock_traceloop_co auto_instrument() mock_traceloop_components['setup_log_provider'].assert_called_once() + + +class TestDropWrapperSpanExporterIfSdkExports: + """Tests for the trace double-export guard _drop_wrapper_span_exporter_if_sdk_exports. + + The guard removes the auto-instrumentation wrapper's span exporter when the SDK's own + exporter is also present on the active provider, so each span is exported once. + """ + + PATCH_TARGET = "sap_cloud_sdk.core.telemetry.auto_instrument.trace.get_tracer_provider" + + @staticmethod + def _provider(with_marker: bool = True) -> SDKTracerProvider: + attrs = {"service.name": "agent"} + if with_marker: + attrs["telemetry.auto.version"] = "0.65b0" + return SDKTracerProvider(resource=Resource.create(attrs)) + + @staticmethod + def _sdk_processor() -> SimpleSpanProcessor: + # The SDK (Traceloop) processor is tagged with _traceloop_processor. + proc = SimpleSpanProcessor(InMemorySpanExporter()) + setattr(proc, "_traceloop_processor", True) + return proc + + @staticmethod + def _wrapper_processor() -> SimpleSpanProcessor: + # The inject-python wrapper processor is a plain exporting processor (untagged). + return SimpleSpanProcessor(InMemorySpanExporter()) + + @staticmethod + def _exporting(provider: SDKTracerProvider) -> list: + active = provider._active_span_processor + return [ + p + for p in active._span_processors + if getattr(p, "span_exporter", None) is not None + ] + + def test_drops_wrapper_exporter_and_keeps_sdk_when_both_present(self): + provider = self._provider(with_marker=True) + wrapper = self._wrapper_processor() + sdk = self._sdk_processor() + provider.add_span_processor(wrapper) + provider.add_span_processor(sdk) + + with patch(self.PATCH_TARGET, return_value=provider): + _drop_wrapper_span_exporter_if_sdk_exports() + + remaining = self._exporting(provider) + assert remaining == [sdk] + assert wrapper not in remaining + + def test_noop_when_only_sdk_exporter_present(self): + # Mirrors the state after the wrapper layer is retired: SDK exporter must stay. + provider = self._provider(with_marker=True) + sdk = self._sdk_processor() + provider.add_span_processor(sdk) + + with patch(self.PATCH_TARGET, return_value=provider): + _drop_wrapper_span_exporter_if_sdk_exports() + + assert self._exporting(provider) == [sdk] + + def test_noop_when_wrapper_marker_absent(self): + provider = self._provider(with_marker=False) + wrapper = self._wrapper_processor() + sdk = self._sdk_processor() + provider.add_span_processor(wrapper) + provider.add_span_processor(sdk) + + with patch(self.PATCH_TARGET, return_value=provider): + _drop_wrapper_span_exporter_if_sdk_exports() + + remaining = self._exporting(provider) + assert wrapper in remaining + assert sdk in remaining