feat: Mark override-affected evaluations in analytics events

Carries the override-affected marking on the evaluation event input, so the
event processor keys on the marking alone and never reads the evaluation
reason. An evaluation marked as override-affected produces no individual
feature event and no debug event, even when the flag requests them, and it is
counted in summary events like any other evaluation. The marking is part of
the summary counter key, so override-affected and ordinary evaluations of the
same flag, variation, and version accumulate into separate counters, and a
counter that aggregates marked evaluations carries the overrideAffected
marker, present only when true, like the unknown marker.

The evaluator passes each prerequisite record's own marking to its event, so
a marked prerequisite record produces no individual event while an unaffected
prerequisite inside a marked evaluation is recorded as usual. The client
passes the top-level marking to the evaluation event, keeps the marking on the
reason when a migration evaluation replaces it with a wrong-type error, and
presents an override-affected flag in the all-flags state with trackEvents
and trackReason false and no debugEventsUntilDate, so a consumer bootstrapped
from the state sends no individual events for it. The async client mirrors
these changes.

The OVERRIDE specification vector runner now also checks the marking each
evaluation contributes to its summary counter.

Author: kinyoklion

ldclient/async_client.py

@@ -486,7 +486,12 @@ async def evaluate():
                     tracker = OpTracker(key, flag, context, detail, default_stage)
                     return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker})
 
-            detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE'))
+            # The type mismatch replaces the reason. The evaluation read the same definitions,
+            # so the new reason keeps the override-affected marking.
+            reason = error_reason('WRONG_TYPE')
+            if detail.reason.get('overrideAffected') is True:
+                reason['overrideAffected'] = True
+            detail = EvaluationDetail(default_stage.value, None, reason)
             tracker = OpTracker(key, flag, context, detail, default_stage)
             return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker})
 
@@ -546,7 +551,7 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]:
                 detail = result.detail
                 if detail.is_default_value():
                     detail = EvaluationDetail(default, None, detail.reason)
-                self._send_event(event_factory.new_eval_event(flag, context, detail, default))
+                self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected))
                 return detail, flag
             except Exception as e:
                 log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e)))
@@ -630,23 +635,35 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState
                 result = await self._evaluator.evaluate(flag, context, self._event_factory_default)
                 detail = result.detail
                 prerequisites = result.prerequisites
+                override_affected = result.override_affected
             except Exception as e:
                 log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e)))
                 log.debug(traceback.format_exc())
                 reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}
                 detail = EvaluationDetail(None, None, reason)
                 prerequisites = []
+                override_affected = False
             requires_experiment_data = EventFactory.is_experiment(flag, detail.reason)
+            track_events = flag.get('trackEvents', False) or requires_experiment_data
+            track_reason = requires_experiment_data
+            debug_events_until_date = flag.get('debugEventsUntilDate', None)
+            if override_affected:
+                # A consumer of this state sends individual events according to these fields.
+                # An override-affected evaluation produces no individual events, so the state
+                # turns them off for this flag. The flag, its value, and its reason stay.
+                track_events = False
+                track_reason = False
+                debug_events_until_date = None
             flag_state = {
                 'key': flag['key'],
                 'value': detail.value,
                 'variation': detail.variation_index,
                 'reason': detail.reason,
                 'version': flag['version'],
                 'prerequisites': prerequisites,
-                'trackEvents': flag.get('trackEvents', False) or requires_experiment_data,
-                'trackReason': requires_experiment_data,
-                'debugEventsUntilDate': flag.get('debugEventsUntilDate', None),
+                'trackEvents': track_events,
+                'trackReason': track_reason,
+                'debugEventsUntilDate': debug_events_until_date,
             }
 
             state.add_flag(flag_state, with_reasons, details_only_if_tracked)

ldclient/client.py

@@ -455,7 +455,12 @@ def evaluate():
                     tracker = OpTracker(key, flag, context, detail, default_stage)
                     return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker})
 
-            detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE'))
+            # The type mismatch replaces the reason. The evaluation read the same definitions,
+            # so the new reason keeps the override-affected marking.
+            reason = error_reason('WRONG_TYPE')
+            if detail.reason.get('overrideAffected') is True:
+                reason['overrideAffected'] = True
+            detail = EvaluationDetail(default_stage.value, None, reason)
             tracker = OpTracker(key, flag, context, detail, default_stage)
             return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker})
 
@@ -517,7 +522,7 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]:
                 detail = result.detail
                 if detail.is_default_value():
                     detail = EvaluationDetail(default, None, detail.reason)
-                self._send_event(event_factory.new_eval_event(flag, context, detail, default))
+                self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected))
                 return detail, flag
             except Exception as e:
                 log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e)))
@@ -607,24 +612,36 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState:
                 result = self._evaluator.evaluate(flag, context, self._event_factory_default)
                 detail = result.detail
                 prerequisites = result.prerequisites
+                override_affected = result.override_affected
             except Exception as e:
                 log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e)))
                 log.debug(traceback.format_exc())
                 reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}
                 detail = EvaluationDetail(None, None, reason)
                 prerequisites = []
+                override_affected = False
 
             requires_experiment_data = EventFactory.is_experiment(flag, detail.reason)
+            track_events = flag.get('trackEvents', False) or requires_experiment_data
+            track_reason = requires_experiment_data
+            debug_events_until_date = flag.get('debugEventsUntilDate', None)
+            if override_affected:
+                # A consumer of this state sends individual events according to these fields.
+                # An override-affected evaluation produces no individual events, so the state
+                # turns them off for this flag. The flag, its value, and its reason stay.
+                track_events = False
+                track_reason = False
+                debug_events_until_date = None
             flag_state = {
                 'key': flag['key'],
                 'value': detail.value,
                 'variation': detail.variation_index,
                 'reason': detail.reason,
                 'version': flag['version'],
                 'prerequisites': prerequisites,
-                'trackEvents': flag.get('trackEvents', False) or requires_experiment_data,
-                'trackReason': requires_experiment_data,
-                'debugEventsUntilDate': flag.get('debugEventsUntilDate', None),
+                'trackEvents': track_events,
+                'trackReason': track_reason,
+                'debugEventsUntilDate': debug_events_until_date,
             }
 
             state.add_flag(flag_state, with_reasons, details_only_if_tracked)

ldclient/impl/async_evaluator.py

@@ -148,7 +148,7 @@ async def _check_prerequisites(self, flag: FeatureFlag, context: Context, state:
                     # off variation was. But we still need to evaluate it in order to generate an event.
                     if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation:
                         failed_prereq = prereq
-                    event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag)
+                    event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected)
                     state.add_event(event)
                 if failed_prereq:
                     return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key}

ldclient/impl/evaluator.py

@@ -146,7 +146,7 @@ def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: EvalR
                     # off variation was. But we still need to evaluate it in order to generate an event.
                     if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation:
                         failed_prereq = prereq
-                    event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag)
+                    event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected)
                     state.add_event(event)
                 if failed_prereq:
                     return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key}

ldclient/impl/events/event_processor_common.py

@@ -181,14 +181,17 @@ def make_summary_event(self, summary: EventSummary):
             }
             counters = []
             for ckey, cval in flag_data.counters.items():
-                variation, version = ckey
+                variation, version, override_affected = ckey
                 counter = {'count': cval.count, 'value': cval.value}
                 if variation is not None:
                     counter['variation'] = variation
                 if version is None:
                     counter['unknown'] = True
                 else:
                     counter['version'] = version
+                # The marker is present only when true, like the unknown marker.
+                if override_affected:
+                    counter['overrideAffected'] = True
                 counters.append(counter)
             flag_data_out['counters'] = counters
             flags_out[key] = flag_data_out
@@ -266,9 +269,11 @@ def _process_event(self, event: EventInput):
             context = event.context
             if not event.exclude_from_summaries:
                 self._outbox.add_to_summary(event)
-            if event.track_events:
+            # An override-affected evaluation appears only in the summary counters. It produces
+            # no individual feature event and no debug event, even when the flag requests them.
+            if event.track_events and not event.override_affected:
                 full_event = event
-            if self._should_debug_event(event):
+            if not event.override_affected and self._should_debug_event(event):
                 debug_event = DebugEvent(event)
         elif isinstance(event, EventInputIdentify):
             if self._omit_anonymous_contexts:

ldclient/impl/events/event_summarizer.py

@@ -22,10 +22,16 @@ def __repr__(self) -> str:  # used only in test debugging
         return "EventSummaryCounter(%d, %s)" % (self.count, self.value)
 
 
+# A counter is keyed by variation index, flag version, and whether the evaluations it counts
+# were override-affected. The marker is part of the key, so override-affected and ordinary
+# evaluations of the same flag, variation, and version accumulate into separate counters.
+CounterKey = Tuple[Optional[int], Optional[int], bool]
+
+
 class EventSummaryFlag:
     __slots__ = ['context_kinds', 'default', 'counters']
 
-    def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[Tuple[Optional[int], Optional[int]], EventSummaryCounter]):
+    def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[CounterKey, EventSummaryCounter]):
         self.context_kinds = context_kinds
         self.counters = counters
         self.default = default
@@ -71,7 +77,7 @@ def summarize_event(self, event: EventInputEvaluation):
             if c is not None:
                 flag_data.context_kinds.add(c.kind)
 
-        counter_key = (event.variation, None if event.flag is None else event.flag.version)
+        counter_key = (event.variation, None if event.flag is None else event.flag.version, event.override_affected)
         counter = flag_data.counters.get(counter_key)
         if counter is None:
             counter = EventSummaryCounter(1, event.value)

ldclient/impl/events/types.py

@@ -52,6 +52,7 @@ class EventInputEvaluation(EventInput):
         "track_events",
         "sampling_ratio",
         "exclude_from_summaries",
+        "override_affected",
     ]
 
     def __init__(
@@ -66,6 +67,7 @@ def __init__(
         default_value: Any,
         prereq_of: Optional[FeatureFlag] = None,
         track_events: bool = False,
+        override_affected: bool = False,
     ):
         super().__init__(timestamp, context, 1 if flag is None else flag.sampling_ratio)
         self.key = key
@@ -79,6 +81,11 @@ def __init__(
         self.exclude_from_summaries = (
             False if flag is None else flag.exclude_from_summaries
         )
+        # True when a flag override affected this evaluation, directly or through a
+        # prerequisite or segment. The event processor keys on this scalar alone, never on the
+        # reason: such an evaluation produces no individual feature or debug event and is
+        # counted in a summary counter that carries the override-affected marker.
+        self.override_affected = override_affected
 
     def to_debugging_dict(self) -> dict:
         return {
@@ -94,6 +101,7 @@ def to_debugging_dict(self) -> dict:
             "track_events": self.track_events,
             "exclude_from_summaries": self.exclude_from_summaries,
             "sampling_ratio": self.sampling_ratio,
+            "override_affected": self.override_affected,
         }
 
 
@@ -155,6 +163,7 @@ def new_eval_event(
         detail: EvaluationDetail,
         default_value: Any,
         prereq_of_flag: Optional[FeatureFlag] = None,
+        override_affected: bool = False,
     ) -> EventInputEvaluation:
         add_experiment_data = self.is_experiment(flag, detail.reason)
         return EventInputEvaluation(
@@ -168,6 +177,7 @@ def new_eval_event(
             default_value,
             prereq_of_flag,
             flag.track_events or add_experiment_data,
+            override_affected,
         )
 
     def new_default_event(