From 0013c0e90a0a77272b14c7750f982e87cf42631d Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 25 Sep 2026 14:54:36 -0700 Subject: [PATCH] feat: Add the override marker to the models and mark evaluations Adds an override marker to the FeatureFlag and Segment models. The marker is an attribute of the model type, never part of the JSON representation, so serializing an entity or writing it to a persistent store never carries it. A marked shallow copy is produced with with_override_marker(), which leaves the source entity unmarked and shares its definition. The evaluator marks an evaluation as override-affected when any definition it reads carries the marker: the evaluated flag, a prerequisite at any depth, or a segment consulted during rule matching, including a segment that does not match and a segment referenced by another segment. The marking propagates upward only. A prerequisite's own record reflects the definitions its subtree read, so an unaffected prerequisite inside a marked evaluation is not marked and a marked prerequisite does not mark a sibling. An evaluation that fails with an error reason is still marked when it read an override definition. The marking appears on the evaluation reason as the overrideAffected indicator, present only when true, and on EvalResult.override_affected for the client and event processing. The async evaluator mirrors the sync one. --- ldclient/evaluation.py | 6 + ldclient/impl/async_evaluator.py | 38 ++- ldclient/impl/evaluator.py | 38 ++- ldclient/impl/evaluator_common.py | 16 +- ldclient/impl/model/feature_flag.py | 24 ++ ldclient/impl/model/segment.py | 24 ++ .../impl/test_async_evaluator_overrides.py | 154 +++++++++ .../testing/impl/test_evaluator_overrides.py | 316 ++++++++++++++++++ .../impl/test_model_override_marker.py | 76 +++++ 9 files changed, 681 insertions(+), 11 deletions(-) create mode 100644 ldclient/testing/impl/test_async_evaluator_overrides.py create mode 100644 ldclient/testing/impl/test_evaluator_overrides.py create mode 100644 ldclient/testing/impl/test_model_override_marker.py diff --git a/ldclient/evaluation.py b/ldclient/evaluation.py index 18aa9e29..a65b9c83 100644 --- a/ldclient/evaluation.py +++ b/ldclient/evaluation.py @@ -57,6 +57,12 @@ def reason(self) -> dict: * ``inExperiment``: This is true if the evaluation resulted in an experiment rollout and served one of the variations in the experiment. Otherwise it is false. + * ``overrideAffected``: This is true if a flag override affected the evaluation, directly + or indirectly. It is present only when true. An evaluation is affected when the flag + itself, a prerequisite flag at any depth, or a segment consulted during rule matching + came from the SDK's override store. Flag overrides are currently experimental and + subject to change. + * ``bigSegmentsStatus``: describes the validity of Big Segment information, if and only if the flag evaluation required querying at least one Big Segment; otherwise it returns None. Allowable values are defined in :class:`BigSegmentsStatus`. For more information, read the diff --git a/ldclient/impl/async_evaluator.py b/ldclient/impl/async_evaluator.py index ead01e08..935fc84d 100644 --- a/ldclient/impl/async_evaluator.py +++ b/ldclient/impl/async_evaluator.py @@ -15,7 +15,8 @@ _match_clause_by_kind, _match_single_context_value, _maybe_negate, - check_targets + check_targets, + mark_override_affected ) from ldclient.impl.events.types import EventFactory from ldclient.impl.model import * @@ -56,15 +57,23 @@ def __init__( async def evaluate(self, flag: FeatureFlag, context: Context, event_factory: EventFactory) -> EvalResult: state = EvalResult() state.original_flag_key = flag.key + # Reading the flag's own definition is the first read of the evaluation, so the marking + # starts from the flag's override marker. + state.override_affected = flag.is_override try: state.detail = await self._evaluate(flag, context, state, event_factory) except EvaluationException as e: if self.__logger is not None: self.__logger.error('Could not evaluate flag "%s": %s' % (flag.key, e.message)) state.detail = EvaluationDetail(None, None, {'kind': 'ERROR', 'errorKind': e.error_kind}) + # An evaluation that fails is still marked when it read an override definition. + if state.override_affected: + mark_override_affected(state.detail) return state if state.big_segments_status is not None: state.detail.reason['bigSegmentsStatus'] = state.big_segments_status + if state.override_affected: + mark_override_affected(state.detail) return state async def _evaluate(self, flag: FeatureFlag, context: Context, state: EvalResult, event_factory: EventFactory) -> EvaluationDetail: @@ -120,9 +129,21 @@ async def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: log.warning("Missing prereq flag: " + prereq_key) failed_prereq = prereq else: - state.depth += 1 - prereq_res = await self._evaluate(prereq_flag, context, state, event_factory) - state.depth -= 1 + # The prerequisite is an evaluation in its own right. Its marking starts from + # its own definition and covers only the definitions its subtree reads. The + # parent's marking is restored and merged afterward, so the marking propagates + # upward only. A failure inside the subtree still merges what was read. + parent_override_affected = state.override_affected + state.override_affected = prereq_flag.is_override + try: + state.depth += 1 + prereq_res = await self._evaluate(prereq_flag, context, state, event_factory) + state.depth -= 1 + finally: + prereq_override_affected = state.override_affected + state.override_affected = parent_override_affected or prereq_override_affected + if prereq_override_affected: + mark_override_affected(prereq_res) # Note that if the prerequisite flag is off, we don't consider it a match no matter what its # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: @@ -146,7 +167,14 @@ async def _clause_matches_context(self, clause: Clause, context: Context, state: if clause.op == 'segmentMatch': for seg_key in clause.values: segment = await self.__get_segment(seg_key) - if segment is not None and await self._segment_matches_context(segment, context, state): + if segment is None: + continue + # The segment definition was read at this point, so an override segment marks the + # evaluation whether or not it matches. A segment that does not match still shapes + # the outcome, for example through a negated clause. + if segment.is_override: + state.override_affected = True + if await self._segment_matches_context(segment, context, state): return _maybe_negate(clause, True) return _maybe_negate(clause, False) diff --git a/ldclient/impl/evaluator.py b/ldclient/impl/evaluator.py index 77a811d4..f6a14a25 100644 --- a/ldclient/impl/evaluator.py +++ b/ldclient/impl/evaluator.py @@ -17,7 +17,8 @@ _maybe_negate, _variation_index_for_context, check_targets, - error_reason + error_reason, + mark_override_affected ) from ldclient.impl.events.types import EventFactory from ldclient.impl.model import * @@ -57,15 +58,23 @@ def __init__( def evaluate(self, flag: FeatureFlag, context: Context, event_factory: EventFactory) -> EvalResult: state = EvalResult() state.original_flag_key = flag.key + # Reading the flag's own definition is the first read of the evaluation, so the marking + # starts from the flag's override marker. + state.override_affected = flag.is_override try: state.detail = self._evaluate(flag, context, state, event_factory) except EvaluationException as e: if self.__logger is not None: self.__logger.error('Could not evaluate flag "%s": %s' % (flag.key, e.message)) state.detail = EvaluationDetail(None, None, {'kind': 'ERROR', 'errorKind': e.error_kind}) + # An evaluation that fails is still marked when it read an override definition. + if state.override_affected: + mark_override_affected(state.detail) return state if state.big_segments_status is not None: state.detail.reason['bigSegmentsStatus'] = state.big_segments_status + if state.override_affected: + mark_override_affected(state.detail) return state def _evaluate(self, flag: FeatureFlag, context: Context, state: EvalResult, event_factory: EventFactory) -> EvaluationDetail: @@ -118,9 +127,21 @@ def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: EvalR log.warning("Missing prereq flag: " + prereq_key) failed_prereq = prereq else: - state.depth += 1 - prereq_res = self._evaluate(prereq_flag, context, state, event_factory) - state.depth -= 1 + # The prerequisite is an evaluation in its own right. Its marking starts from + # its own definition and covers only the definitions its subtree reads. The + # parent's marking is restored and merged afterward, so the marking propagates + # upward only. A failure inside the subtree still merges what was read. + parent_override_affected = state.override_affected + state.override_affected = prereq_flag.is_override + try: + state.depth += 1 + prereq_res = self._evaluate(prereq_flag, context, state, event_factory) + state.depth -= 1 + finally: + prereq_override_affected = state.override_affected + state.override_affected = parent_override_affected or prereq_override_affected + if prereq_override_affected: + mark_override_affected(prereq_res) # Note that if the prerequisite flag is off, we don't consider it a match no matter what its # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: @@ -144,7 +165,14 @@ def _clause_matches_context(self, clause: Clause, context: Context, state: EvalR if clause.op == 'segmentMatch': for seg_key in clause.values: segment = self.__get_segment(seg_key) - if segment is not None and self._segment_matches_context(segment, context, state): + if segment is None: + continue + # The segment definition was read at this point, so an override segment marks the + # evaluation whether or not it matches. A segment that does not match still shapes + # the outcome, for example through a negated clause. + if segment.is_override: + state.override_affected = True + if self._segment_matches_context(segment, context, state): return _maybe_negate(clause, True) return _maybe_negate(clause, False) diff --git a/ldclient/impl/evaluator_common.py b/ldclient/impl/evaluator_common.py index 04dbebfd..6e0f6a0c 100644 --- a/ldclient/impl/evaluator_common.py +++ b/ldclient/impl/evaluator_common.py @@ -25,7 +25,7 @@ # prerequisite evaluations, and the cached state of any Big Segments query that we may have # ended up having to do for the context. class EvalResult: - __slots__ = ['detail', 'events', 'big_segments_status', 'big_segments_membership', 'original_flag_key', 'prereq_stack', 'segment_stack', 'depth', 'prerequisites'] + __slots__ = ['detail', 'events', 'big_segments_status', 'big_segments_membership', 'original_flag_key', 'prereq_stack', 'segment_stack', 'depth', 'prerequisites', 'override_affected'] def __init__(self): self.detail = None @@ -37,6 +37,11 @@ def __init__(self): self.segment_stack = None # type: Optional[List[str]] self.depth = 0 self.prerequisites = [] # type: List[str] + # True when the evaluation in progress has read a flag or segment definition that carries + # the override marker. While a prerequisite is evaluated, this holds the marking of the + # prerequisite's own subtree. The evaluator merges it into the parent's marking afterward, + # so the marking propagates upward only. + self.override_affected = False def record_prerequisite(self, key: str): if self.depth == 0: @@ -255,3 +260,12 @@ def check_targets(flag: FeatureFlag, context: Context) -> Optional[EvaluationDet def error_reason(error_kind: str) -> dict: return {'kind': 'ERROR', 'errorKind': error_kind} + + +def mark_override_affected(detail: EvaluationDetail) -> None: + """ + Sets the override-affected indicator on the evaluation reason. The indicator is present + only when true, so a reason that was not affected by an override is left untouched and + serializes exactly as before. + """ + detail.reason['overrideAffected'] = True diff --git a/ldclient/impl/model/feature_flag.py b/ldclient/impl/model/feature_flag.py index ae0b6f75..f7c03f4d 100644 --- a/ldclient/impl/model/feature_flag.py +++ b/ldclient/impl/model/feature_flag.py @@ -1,3 +1,4 @@ +import copy from typing import Any, Dict, List, Optional, Set, Union from ldclient.impl.model.clause import Clause @@ -96,10 +97,12 @@ class FeatureFlag(ModelEntity): '_salt', '_track_events', '_debug_events_until_date', + '_is_override', ] def __init__(self, data: dict): super().__init__(data) + self._is_override = False # In the following logic, we're being somewhat lenient in terms of allowing most properties to # be absent even if they are really required in the schema. That's for backward compatibility # with test logic that constructed incomplete JSON, and also with the file data source which @@ -202,3 +205,24 @@ def exclude_from_summaries(self) -> bool: @property def sampling_ratio(self) -> Optional[int]: return self._sampling_ratio + + @property + def is_override(self) -> bool: + """ + True if this flag definition was supplied by an SDK override source rather than by + LaunchDarkly data. The marker is never part of the JSON representation. Only the SDK + components that manage override entries set it. Components that read a definition + through the store may treat a marked definition the same as any other. + + Flag overrides are currently experimental and subject to change. + """ + return self._is_override + + def with_override_marker(self) -> 'FeatureFlag': + """ + Returns a shallow copy of this flag with the override marker set. The copy shares the + underlying data with this instance, which is left unmarked. + """ + marked = copy.copy(self) + marked._is_override = True + return marked diff --git a/ldclient/impl/model/segment.py b/ldclient/impl/model/segment.py index 6ed1b6a8..709518a2 100644 --- a/ldclient/impl/model/segment.py +++ b/ldclient/impl/model/segment.py @@ -1,3 +1,4 @@ +import copy from typing import Any, List, Optional, Set from ldclient.impl.model.attribute_ref import ( @@ -65,10 +66,12 @@ class Segment(ModelEntity): '_unbounded', '_unbounded_context_kind', '_generation', + '_is_override', ] def __init__(self, data: dict): super().__init__(data) + self._is_override = False # In the following logic, we're being somewhat lenient in terms of allowing most properties to # be absent even if they are really required in the schema. That's for backward compatibility # with test logic that constructed incomplete JSON, and also with the file data source which @@ -137,3 +140,24 @@ def unbounded_context_kind(self) -> Optional[str]: @property def generation(self) -> Optional[int]: return self._generation + + @property + def is_override(self) -> bool: + """ + True if this segment definition was supplied by an SDK override source rather than by + LaunchDarkly data. The marker is never part of the JSON representation. Only the SDK + components that manage override entries set it. Components that read a definition + through the store may treat a marked definition the same as any other. + + Flag overrides are currently experimental and subject to change. + """ + return self._is_override + + def with_override_marker(self) -> 'Segment': + """ + Returns a shallow copy of this segment with the override marker set. The copy shares the + underlying data with this instance, which is left unmarked. + """ + marked = copy.copy(self) + marked._is_override = True + return marked diff --git a/ldclient/testing/impl/test_async_evaluator_overrides.py b/ldclient/testing/impl/test_async_evaluator_overrides.py new file mode 100644 index 00000000..e193d3bf --- /dev/null +++ b/ldclient/testing/impl/test_async_evaluator_overrides.py @@ -0,0 +1,154 @@ +""" +Tests for the override marking performed by the async evaluator. These mirror the key +scenarios of the sync evaluator override tests. +""" +import pytest + +from ldclient import Context +from ldclient.impl.events.types import EventFactory +from ldclient.testing.builders import * +from ldclient.testing.impl.test_async_evaluator import AsyncEvaluatorBuilder +from ldclient.testing.impl.test_evaluator_overrides import ( + assert_override_affected, + assert_reason_override_affected, + make_override_test_flag, + prereq_records, + require_prereq_record +) + +override_test_context = Context.create('userkey') +event_factory_with_reasons = EventFactory(True, lambda: 0) + + +@pytest.mark.asyncio +async def test_override_flag_marks_evaluation(): + flag = make_override_test_flag('feature').with_override_marker() + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_plain_flag_is_not_marked(): + flag = make_override_test_flag('feature') + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert_override_affected(False, result) + + +@pytest.mark.asyncio +async def test_malformed_override_flag_error_result_is_marked(): + flag = FlagBuilder('feature').on(True).fallthrough_variation(99).variations('off', 'on').build().with_override_marker() + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_prerequisite_cycle_through_an_override_flag_is_marked(): + prereq = make_override_test_flag('prereq', 'feature').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = AsyncEvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + assert prereq_records(result) == [] + + +@pytest.mark.asyncio +async def test_prerequisite_cycle_below_an_override_flag_keeps_the_flag_marking(): + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_override_prerequisite_marks_prerequisite_record_and_top_level(): + prereq = make_override_test_flag('prereq').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(True, require_prereq_record(result, 'prereq').reason) + + +@pytest.mark.asyncio +async def test_override_flag_does_not_mark_unaffected_prerequisite_record(): + prereq = make_override_test_flag('prereq') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +@pytest.mark.asyncio +async def test_unaffected_sibling_prerequisite_record_stays_unmarked(): + s1 = SegmentBuilder('s1').included(override_test_context.key).build() + d = make_override_test_flag('d').with_override_marker() + b = make_override_test_flag('b', 'd') + c = FlagBuilder('c').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(s1.key)).build() + ).build() + a = make_override_test_flag('a', 'b', 'c') + evaluator = AsyncEvaluatorBuilder().with_flag(b).with_flag(c).with_flag(d).with_segment(s1).build() + result = await evaluator.evaluate(a, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert len(prereq_records(result)) == 3 + assert_reason_override_affected(True, require_prereq_record(result, 'd').reason) + assert_reason_override_affected(True, require_prereq_record(result, 'b').reason) + assert_reason_override_affected(False, require_prereq_record(result, 'c').reason) + + +@pytest.mark.asyncio +async def test_override_segment_read_without_matching_marks_evaluation(): + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = AsyncEvaluatorBuilder().with_segment(segment).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_override_segment_referenced_by_segment_rule_marks_evaluation(): + nested = SegmentBuilder('nested-segment').included(override_test_context.key).build().with_override_marker() + outer = SegmentBuilder('outer-segment').rules(SegmentRuleBuilder().clauses(make_clause_matching_segment_key(nested.key)).build()).build() + flag = make_boolean_flag_matching_segment(outer) + evaluator = AsyncEvaluatorBuilder().with_segment(outer).with_segment(nested).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.value is True + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_missing_definitions_do_not_mark_evaluation(): + unrelated = make_override_test_flag('unrelated').with_override_marker() + flag = make_override_test_flag('feature', 'missing') + evaluator = AsyncEvaluatorBuilder().with_flag(unrelated).with_unknown_flag('missing').build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': 'missing'} + assert_override_affected(False, result) + + segment_flag = make_boolean_flag_with_clauses(make_clause_matching_segment_key('missing')) + evaluator = AsyncEvaluatorBuilder().with_unknown_segment('missing').build() + result = await evaluator.evaluate(segment_flag, override_test_context, event_factory_with_reasons) + assert_override_affected(False, result) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("flag_override,prereq_override", [(False, False), (True, False), (False, True), (True, True)]) +async def test_result_override_affected_matches_reason_indicator(flag_override: bool, prereq_override: bool): + prereq = make_override_test_flag('prereq') + if prereq_override: + prereq = prereq.with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + if flag_override: + flag = flag.with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(flag_override or prereq_override, result) + assert_reason_override_affected(prereq_override, require_prereq_record(result, 'prereq').reason) diff --git a/ldclient/testing/impl/test_evaluator_overrides.py b/ldclient/testing/impl/test_evaluator_overrides.py new file mode 100644 index 00000000..f69bc612 --- /dev/null +++ b/ldclient/testing/impl/test_evaluator_overrides.py @@ -0,0 +1,316 @@ +""" +Tests for the override marking performed by the evaluator. An evaluation is override-affected +when any flag or segment definition it reads carries the override marker. The marking +propagates upward from prerequisites to the flags that depend on them and never sideways or +downward. +""" +from typing import List, Optional + +import pytest + +from ldclient import Context +from ldclient.impl.evaluator_common import EvalResult +from ldclient.impl.events.types import EventFactory, EventInputEvaluation +from ldclient.testing.builders import * +from ldclient.testing.impl.evaluator_util import EvaluatorBuilder + +override_test_context = Context.create('userkey') + +# Events carry their reasons, so a prerequisite record's marking is visible on its reason. +event_factory_with_reasons = EventFactory(True, lambda: 0) + + +def assert_override_affected(expected: bool, result: EvalResult) -> None: + """Checks the reason indicator and the result scalar together. Both report the same marking.""" + assert result.override_affected is expected, "result.override_affected" + assert_reason_override_affected(expected, result.detail.reason) + + +def assert_reason_override_affected(expected: bool, reason: Optional[dict]) -> None: + assert reason is not None + if expected: + assert reason.get('overrideAffected') is True, "reason indicator: %s" % reason + else: + assert 'overrideAffected' not in reason, "reason indicator: %s" % reason + + +def make_override_test_flag(key: str, *prereq_keys: str) -> FeatureFlag: + """A flag that is on and serves variation 1 ("on") by fallthrough, with prerequisites that must each serve variation 1.""" + builder = FlagBuilder(key).on(True).fallthrough_variation(1).off_variation(0).variations('off', 'on') + for prereq_key in prereq_keys: + builder.prerequisite(prereq_key, 1) + return builder.build() + + +def prereq_records(result: EvalResult) -> List[EventInputEvaluation]: + return result.events or [] + + +def require_prereq_record(result: EvalResult, prereq_key: str) -> EventInputEvaluation: + for event in prereq_records(result): + if event.flag is not None and event.flag.key == prereq_key: + return event + raise AssertionError("no record for prerequisite %s" % prereq_key) + + +def test_override_flag_marks_evaluation_when_off(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'OFF' + assert result.detail.value == 'off' + assert_override_affected(True, result) + + +def test_override_flag_marks_evaluation_on_fallthrough(): + flag = make_override_test_flag('feature').with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + +def test_override_flag_marks_evaluation_on_rule_match(): + rule = FlagRuleBuilder().id('rule-id').variation(1).clauses(make_clause_matching_context(override_test_context)).build() + flag = FlagBuilder('feature').on(True).fallthrough_variation(0).variations('off', 'on').rules(rule).build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.reason['ruleId'] == 'rule-id' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + +def test_plain_flag_alone_is_not_marked(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'OFF'} + assert_override_affected(False, result) + + +def test_plain_flag_with_prerequisite_and_segment_is_not_marked(): + segment = SegmentBuilder('segment').included(override_test_context.key).build() + prereq = FlagBuilder('prereq').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(segment.key)).build() + ).build() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert result.detail.value == 'on' + assert_override_affected(False, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +def test_malformed_override_flag_error_result_is_marked(): + flag = FlagBuilder('feature').on(True).fallthrough_variation(99).variations('off', 'on').build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'ERROR' + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert result.detail.value is None + assert_override_affected(True, result) + + +def test_prerequisite_cycle_through_an_override_flag_is_marked(): + # feature -> prereq -> feature; only prereq is an override + prereq = make_override_test_flag('prereq', 'feature').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'ERROR' + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + assert prereq_records(result) == [] + + +def test_prerequisite_cycle_below_an_override_flag_keeps_the_flag_marking(): + # feature -> prereq -> feature; only feature is an override. The failure happens inside the + # plain prerequisite's subtree, and the top-level marking from the flag's own read survives it. + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +def test_prerequisite_cycle_through_plain_flags_is_not_marked(): + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(False, result) + + +def test_override_prerequisite_marks_prerequisite_record_and_top_level(): + prereq = make_override_test_flag('prereq').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + record = require_prereq_record(result, 'prereq') + assert record.reason is not None + assert record.reason['kind'] == 'FALLTHROUGH' + assert_reason_override_affected(True, record.reason) + + +def test_override_flag_does_not_mark_unaffected_prerequisite_record(): + # The marking propagates upward only. The top-level flag's own marker does not leak into + # the record of a prerequisite whose subtree read no override definition. + prereq = make_override_test_flag('prereq') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +def test_override_prerequisite_at_depth_two_marks_all_affected_scopes(): + prereq2 = make_override_test_flag('prereq2').with_override_marker() + prereq1 = make_override_test_flag('prereq1', 'prereq2') + flag = make_override_test_flag('feature', 'prereq1') + evaluator = EvaluatorBuilder().with_flag(prereq1).with_flag(prereq2).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + # The nested record is produced first, during the evaluation of prereq1. + records = prereq_records(result) + assert [r.flag.key for r in records] == ['prereq2', 'prereq1'] + assert_reason_override_affected(True, records[0].reason) + assert_reason_override_affected(True, records[1].reason) + + +def test_unaffected_sibling_prerequisite_record_stays_unmarked(): + # Flag a has prerequisites b and c. Only d, a prerequisite of b, is an override. The marking + # reaches a, b, and d. It does not reach the sibling c, and the plain segments s1 and s2 + # mark nothing. + s1 = SegmentBuilder('s1').included(override_test_context.key).build() + s2 = SegmentBuilder('s2').included(override_test_context.key).build() + d = make_override_test_flag('d').with_override_marker() + b = make_override_test_flag('b', 'd') + c = FlagBuilder('c').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(s1.key)).build() + ).build() + a = FlagBuilder('a').on(True).fallthrough_variation(0).off_variation(0).prerequisite('b', 1).prerequisite('c', 1).rules( + FlagRuleBuilder().id('rule-s2').variation(1).clauses(make_clause_matching_segment_key(s2.key)).build() + ).variations('off', 'on').build() + + evaluator = EvaluatorBuilder().with_flag(b).with_flag(c).with_flag(d).with_segment(s1).with_segment(s2).build() + result = evaluator.evaluate(a, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + assert len(prereq_records(result)) == 3 + assert_reason_override_affected(True, require_prereq_record(result, 'd').reason) + assert_reason_override_affected(True, require_prereq_record(result, 'b').reason) + assert_reason_override_affected(False, require_prereq_record(result, 'c').reason) + + +def test_override_segment_referenced_by_flag_rule_marks_evaluation(): + segment = SegmentBuilder('segment').included(override_test_context.key).build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_read_without_matching_marks_evaluation(): + # A read is enough to mark the evaluation. The segment does not need to match. + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert result.detail.value is False + assert_override_affected(True, result) + + +def test_override_segment_read_through_negated_clause_marks_evaluation(): + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_with_clauses(negate_clause(make_clause_matching_segment_key(segment.key))) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_referenced_by_segment_rule_marks_evaluation(): + # The outer segment is plain. A rule of the outer segment reads a nested override segment. + nested = SegmentBuilder('nested-segment').included(override_test_context.key).build().with_override_marker() + outer = SegmentBuilder('outer-segment').rules(SegmentRuleBuilder().clauses(make_clause_matching_segment_key(nested.key)).build()).build() + flag = make_boolean_flag_matching_segment(outer) + evaluator = EvaluatorBuilder().with_segment(outer).with_segment(nested).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_referenced_by_prerequisite_marks_prerequisite_record_and_top_level(): + segment = SegmentBuilder('segment').included(override_test_context.key).build().with_override_marker() + prereq = FlagBuilder('prereq').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(segment.key)).build() + ).build() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + assert_reason_override_affected(True, require_prereq_record(result, 'prereq').reason) + + +def test_missing_prerequisite_does_not_mark_evaluation(): + # A definition that cannot be resolved contributes nothing, because nothing was read. The + # store holds an unrelated override definition to show that only reads count. + unrelated = make_override_test_flag('unrelated').with_override_marker() + flag = make_override_test_flag('feature', 'missing') + evaluator = EvaluatorBuilder().with_flag(unrelated).with_unknown_flag('missing').build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': 'missing'} + assert result.detail.value == 'off' + assert_override_affected(False, result) + assert prereq_records(result) == [] + + +def test_missing_segment_does_not_mark_evaluation(): + unrelated = SegmentBuilder('unrelated').included(override_test_context.key).build().with_override_marker() + flag = make_boolean_flag_with_clauses(make_clause_matching_segment_key('missing')) + evaluator = EvaluatorBuilder().with_segment(unrelated).with_unknown_segment('missing').build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert result.detail.value is False + assert_override_affected(False, result) + + +@pytest.mark.parametrize("flag_override,prereq_override", [(False, False), (True, False), (False, True), (True, True)]) +def test_result_override_affected_matches_reason_indicator(flag_override: bool, prereq_override: bool): + prereq = make_override_test_flag('prereq') + if prereq_override: + prereq = prereq.with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + if flag_override: + flag = flag.with_override_marker() + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(flag_override or prereq_override, result) + assert_reason_override_affected(prereq_override, require_prereq_record(result, 'prereq').reason) + + +def test_marking_does_not_disturb_other_reason_properties(): + segment = SegmentBuilder('segment').unbounded(True).generation(1).build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).with_no_big_segments_for_key(override_test_context.key).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH', 'bigSegmentsStatus': 'HEALTHY', 'overrideAffected': True} + + +def test_reason_indicator_is_absent_when_events_do_not_carry_reasons(): + # The marking does not depend on the event factory. The reason on the result is marked either way. + flag = make_override_test_flag('feature').with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, EventFactory(False, lambda: 0)) + assert_override_affected(True, result) diff --git a/ldclient/testing/impl/test_model_override_marker.py b/ldclient/testing/impl/test_model_override_marker.py new file mode 100644 index 00000000..203e0fc5 --- /dev/null +++ b/ldclient/testing/impl/test_model_override_marker.py @@ -0,0 +1,76 @@ +""" +Tests for the override marker carried on the flag and segment models. The marker lives on the +model type and never in the JSON representation. +""" +import json + +import pytest + +from ldclient.impl.model import FeatureFlag, ModelEncoder, Segment +from ldclient.testing.builders import FlagBuilder, SegmentBuilder +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + +flag_data = {'key': 'flag1', 'version': 2, 'on': True, 'variations': [True, False], 'fallthrough': {'variation': 0}, 'salt': 'x'} +segment_data = {'key': 'seg1', 'version': 3, 'included': ['user1'], 'salt': 'y'} + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marker_defaults_to_false(kind, data): + entity = kind.decode(dict(data)) + assert entity.is_override is False + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marked_copy_is_marked_and_source_is_not(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.is_override is True + assert entity.is_override is False + assert type(marked) is type(entity) + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marker_is_never_serialized(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.to_json_dict() == data + assert kind.encode(marked) == data + assert json.loads(ModelEncoder().encode(marked)) == data + assert 'is_override' not in marked.to_json_dict() + assert 'override' not in json.dumps(marked.to_json_dict()).lower() + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marked_copy_shares_the_definition(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.key == entity.key + assert marked.version == entity.version + assert marked.to_json_dict() is entity.to_json_dict() + assert marked == entity + + +def test_marked_flag_keeps_its_parsed_properties(): + flag = FlagBuilder('flag1').version(7).on(True).variations('a', 'b').fallthrough_variation(1).prerequisite('p', 0).track_events(True).build() + marked = flag.with_override_marker() + assert marked.on is True + assert marked.variations == ['a', 'b'] + assert marked.fallthrough.variation == 1 + assert [p.key for p in marked.prerequisites] == ['p'] + assert marked.track_events is True + assert marked.version == 7 + + +def test_marked_segment_keeps_its_parsed_properties(): + segment = SegmentBuilder('seg1').version(4).included('a').excluded('b').build() + marked = segment.with_override_marker() + assert marked.included == {'a'} + assert marked.excluded == {'b'} + assert marked.version == 4 + + +def test_marking_a_marked_copy_is_a_marked_copy(): + flag = FeatureFlag(dict(flag_data)).with_override_marker() + again = flag.with_override_marker() + assert again.is_override is True + assert again is not flag