Skip to content

feat: Mark override-affected evaluations in analytics events #528

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
kinyoklion wants to merge 1 commit into
base: rlamb/overrides-python-override-store
Choose a base branch
from
Draft

feat: Mark override-affected evaluations in analytics events #528

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 22 additions & 5 deletions ldclient/async_client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,7 +486,12 @@ async def evaluate():
tracker = OpTracker(key, flag, context, detail, default_stage)
return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker})

detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE'))
# The type mismatch replaces the reason. The evaluation read the same definitions,
# so the new reason keeps the override-affected marking.
reason = error_reason('WRONG_TYPE')
if detail.reason.get('overrideAffected') is True:
reason['overrideAffected'] = True
detail = EvaluationDetail(default_stage.value, None, reason)
tracker = OpTracker(key, flag, context, detail, default_stage)
return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker})

Expand Down Expand Up @@ -546,7 +551,7 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]:
detail = result.detail
if detail.is_default_value():
detail = EvaluationDetail(default, None, detail.reason)
self._send_event(event_factory.new_eval_event(flag, context, detail, default))
self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected))
return detail, flag
except Exception as e:
log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e)))
Expand Down Expand Up @@ -630,23 +635,35 @@ async def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState
result = await self._evaluator.evaluate(flag, context, self._event_factory_default)
detail = result.detail
prerequisites = result.prerequisites
override_affected = result.override_affected
except Exception as e:
log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e)))
log.debug(traceback.format_exc())
reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}
detail = EvaluationDetail(None, None, reason)
prerequisites = []
override_affected = False
requires_experiment_data = EventFactory.is_experiment(flag, detail.reason)
track_events = flag.get('trackEvents', False) or requires_experiment_data
track_reason = requires_experiment_data
debug_events_until_date = flag.get('debugEventsUntilDate', None)
if override_affected:
# A consumer of this state sends individual events according to these fields.
# An override-affected evaluation produces no individual events, so the state
# turns them off for this flag. The flag, its value, and its reason stay.
track_events = False
track_reason = False
debug_events_until_date = None
flag_state = {
'key': flag['key'],
'value': detail.value,
'variation': detail.variation_index,
'reason': detail.reason,
'version': flag['version'],
'prerequisites': prerequisites,
'trackEvents': flag.get('trackEvents', False) or requires_experiment_data,
'trackReason': requires_experiment_data,
'debugEventsUntilDate': flag.get('debugEventsUntilDate', None),
'trackEvents': track_events,
'trackReason': track_reason,
'debugEventsUntilDate': debug_events_until_date,
}

state.add_flag(flag_state, with_reasons, details_only_if_tracked)
Expand Down
27 changes: 22 additions & 5 deletions ldclient/client.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -455,7 +455,12 @@ def evaluate():
tracker = OpTracker(key, flag, context, detail, default_stage)
return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': stage, 'tracker': tracker})

detail = EvaluationDetail(default_stage.value, None, error_reason('WRONG_TYPE'))
# The type mismatch replaces the reason. The evaluation read the same definitions,
# so the new reason keeps the override-affected marking.
reason = error_reason('WRONG_TYPE')
if detail.reason.get('overrideAffected') is True:
reason['overrideAffected'] = True
detail = EvaluationDetail(default_stage.value, None, reason)
tracker = OpTracker(key, flag, context, detail, default_stage)
return _EvaluationWithHookResult(evaluation_detail=detail, results={'default_stage': default_stage, 'tracker': tracker})

Expand Down Expand Up @@ -517,7 +522,7 @@ def not_ready() -> Tuple[EvaluationDetail, Optional[FeatureFlag]]:
detail = result.detail
if detail.is_default_value():
detail = EvaluationDetail(default, None, detail.reason)
self._send_event(event_factory.new_eval_event(flag, context, detail, default))
self._send_event(event_factory.new_eval_event(flag, context, detail, default, None, result.override_affected))
return detail, flag
except Exception as e:
log.error("Unexpected error while evaluating feature flag \"%s\": %s" % (key, repr(e)))
Expand Down Expand Up @@ -607,24 +612,36 @@ def all_flags_state(self, context: Context, **kwargs) -> FeatureFlagsState:
result = self._evaluator.evaluate(flag, context, self._event_factory_default)
detail = result.detail
prerequisites = result.prerequisites
override_affected = result.override_affected
except Exception as e:
log.error("Error evaluating flag \"%s\" in all_flags_state: %s" % (key, repr(e)))
log.debug(traceback.format_exc())
reason = {'kind': 'ERROR', 'errorKind': 'EXCEPTION'}
detail = EvaluationDetail(None, None, reason)
prerequisites = []
override_affected = False

requires_experiment_data = EventFactory.is_experiment(flag, detail.reason)
track_events = flag.get('trackEvents', False) or requires_experiment_data
track_reason = requires_experiment_data
debug_events_until_date = flag.get('debugEventsUntilDate', None)
if override_affected:
# A consumer of this state sends individual events according to these fields.
# An override-affected evaluation produces no individual events, so the state
# turns them off for this flag. The flag, its value, and its reason stay.
track_events = False
track_reason = False
debug_events_until_date = None
flag_state = {
'key': flag['key'],
'value': detail.value,
'variation': detail.variation_index,
'reason': detail.reason,
'version': flag['version'],
'prerequisites': prerequisites,
'trackEvents': flag.get('trackEvents', False) or requires_experiment_data,
'trackReason': requires_experiment_data,
'debugEventsUntilDate': flag.get('debugEventsUntilDate', None),
'trackEvents': track_events,
'trackReason': track_reason,
'debugEventsUntilDate': debug_events_until_date,
}

state.add_flag(flag_state, with_reasons, details_only_if_tracked)
Expand Down
2 changes: 1 addition & 1 deletion ldclient/impl/async_evaluator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ async def _check_prerequisites(self, flag: FeatureFlag, context: Context, state:
# off variation was. But we still need to evaluate it in order to generate an event.
if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation:
failed_prereq = prereq
event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag)
event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected)
state.add_event(event)
if failed_prereq:
return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key}
Expand Down
2 changes: 1 addition & 1 deletion ldclient/impl/evaluator.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,7 +146,7 @@ def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: EvalR
# off variation was. But we still need to evaluate it in order to generate an event.
if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation:
failed_prereq = prereq
event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag)
event = event_factory.new_eval_event(prereq_flag, context, prereq_res, None, flag, prereq_override_affected)
state.add_event(event)
if failed_prereq:
return {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': failed_prereq.key}
Expand Down
11 changes: 8 additions & 3 deletions ldclient/impl/events/event_processor_common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -181,14 +181,17 @@ def make_summary_event(self, summary: EventSummary):
}
counters = []
for ckey, cval in flag_data.counters.items():
variation, version = ckey
variation, version, override_affected = ckey
counter = {'count': cval.count, 'value': cval.value}
if variation is not None:
counter['variation'] = variation
if version is None:
counter['unknown'] = True
else:
counter['version'] = version
# The marker is present only when true, like the unknown marker.
if override_affected:
counter['overrideAffected'] = True
counters.append(counter)
flag_data_out['counters'] = counters
flags_out[key] = flag_data_out
Expand Down Expand Up @@ -266,9 +269,11 @@ def _process_event(self, event: EventInput):
context = event.context
if not event.exclude_from_summaries:
self._outbox.add_to_summary(event)
if event.track_events:
# An override-affected evaluation appears only in the summary counters. It produces
# no individual feature event and no debug event, even when the flag requests them.
if event.track_events and not event.override_affected:
full_event = event
if self._should_debug_event(event):
if not event.override_affected and self._should_debug_event(event):
debug_event = DebugEvent(event)
elif isinstance(event, EventInputIdentify):
if self._omit_anonymous_contexts:
Expand Down
10 changes: 8 additions & 2 deletions ldclient/impl/events/event_summarizer.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,16 @@ def __repr__(self) -> str: # used only in test debugging
return "EventSummaryCounter(%d, %s)" % (self.count, self.value)


# A counter is keyed by variation index, flag version, and whether the evaluations it counts
# were override-affected. The marker is part of the key, so override-affected and ordinary
# evaluations of the same flag, variation, and version accumulate into separate counters.
CounterKey = Tuple[Optional[int], Optional[int], bool]


class EventSummaryFlag:
__slots__ = ['context_kinds', 'default', 'counters']

def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[Tuple[Optional[int], Optional[int]], EventSummaryCounter]):
def __init__(self, context_kinds: Set[str], default: Any, counters: Dict[CounterKey, EventSummaryCounter]):
self.context_kinds = context_kinds
self.counters = counters
self.default = default
Expand Down Expand Up @@ -71,7 +77,7 @@ def summarize_event(self, event: EventInputEvaluation):
if c is not None:
flag_data.context_kinds.add(c.kind)

counter_key = (event.variation, None if event.flag is None else event.flag.version)
counter_key = (event.variation, None if event.flag is None else event.flag.version, event.override_affected)
counter = flag_data.counters.get(counter_key)
if counter is None:
counter = EventSummaryCounter(1, event.value)
Expand Down
10 changes: 10 additions & 0 deletions ldclient/impl/events/types.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@ class EventInputEvaluation(EventInput):
"track_events",
"sampling_ratio",
"exclude_from_summaries",
"override_affected",
]

def __init__(
Expand All @@ -66,6 +67,7 @@ def __init__(
default_value: Any,
prereq_of: Optional[FeatureFlag] = None,
track_events: bool = False,
override_affected: bool = False,
):
super().__init__(timestamp, context, 1 if flag is None else flag.sampling_ratio)
self.key = key
Expand All @@ -79,6 +81,11 @@ def __init__(
self.exclude_from_summaries = (
False if flag is None else flag.exclude_from_summaries
)
# True when a flag override affected this evaluation, directly or through a
# prerequisite or segment. The event processor keys on this scalar alone, never on the
# reason: such an evaluation produces no individual feature or debug event and is
# counted in a summary counter that carries the override-affected marker.
self.override_affected = override_affected

def to_debugging_dict(self) -> dict:
return {
Expand All @@ -94,6 +101,7 @@ def to_debugging_dict(self) -> dict:
"track_events": self.track_events,
"exclude_from_summaries": self.exclude_from_summaries,
"sampling_ratio": self.sampling_ratio,
"override_affected": self.override_affected,
}


Expand Down Expand Up @@ -155,6 +163,7 @@ def new_eval_event(
detail: EvaluationDetail,
default_value: Any,
prereq_of_flag: Optional[FeatureFlag] = None,
override_affected: bool = False,
) -> EventInputEvaluation:
add_experiment_data = self.is_experiment(flag, detail.reason)
return EventInputEvaluation(
Expand All @@ -168,6 +177,7 @@ def new_eval_event(
default_value,
prereq_of_flag,
flag.track_events or add_experiment_data,
override_affected,
)

def new_default_event(
Expand Down
Loading
Loading