diff --git a/ldclient/evaluation.py b/ldclient/evaluation.py index 18aa9e29..a65b9c83 100644 --- a/ldclient/evaluation.py +++ b/ldclient/evaluation.py @@ -57,6 +57,12 @@ def reason(self) -> dict: * ``inExperiment``: This is true if the evaluation resulted in an experiment rollout and served one of the variations in the experiment. Otherwise it is false. + * ``overrideAffected``: This is true if a flag override affected the evaluation, directly + or indirectly. It is present only when true. An evaluation is affected when the flag + itself, a prerequisite flag at any depth, or a segment consulted during rule matching + came from the SDK's override store. Flag overrides are currently experimental and + subject to change. + * ``bigSegmentsStatus``: describes the validity of Big Segment information, if and only if the flag evaluation required querying at least one Big Segment; otherwise it returns None. Allowable values are defined in :class:`BigSegmentsStatus`. For more information, read the diff --git a/ldclient/impl/async_evaluator.py b/ldclient/impl/async_evaluator.py index ead01e08..935fc84d 100644 --- a/ldclient/impl/async_evaluator.py +++ b/ldclient/impl/async_evaluator.py @@ -15,7 +15,8 @@ _match_clause_by_kind, _match_single_context_value, _maybe_negate, - check_targets + check_targets, + mark_override_affected ) from ldclient.impl.events.types import EventFactory from ldclient.impl.model import * @@ -56,15 +57,23 @@ def __init__( async def evaluate(self, flag: FeatureFlag, context: Context, event_factory: EventFactory) -> EvalResult: state = EvalResult() state.original_flag_key = flag.key + # Reading the flag's own definition is the first read of the evaluation, so the marking + # starts from the flag's override marker. + state.override_affected = flag.is_override try: state.detail = await self._evaluate(flag, context, state, event_factory) except EvaluationException as e: if self.__logger is not None: self.__logger.error('Could not evaluate flag "%s": %s' % (flag.key, e.message)) state.detail = EvaluationDetail(None, None, {'kind': 'ERROR', 'errorKind': e.error_kind}) + # An evaluation that fails is still marked when it read an override definition. + if state.override_affected: + mark_override_affected(state.detail) return state if state.big_segments_status is not None: state.detail.reason['bigSegmentsStatus'] = state.big_segments_status + if state.override_affected: + mark_override_affected(state.detail) return state async def _evaluate(self, flag: FeatureFlag, context: Context, state: EvalResult, event_factory: EventFactory) -> EvaluationDetail: @@ -120,9 +129,21 @@ async def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: log.warning("Missing prereq flag: " + prereq_key) failed_prereq = prereq else: - state.depth += 1 - prereq_res = await self._evaluate(prereq_flag, context, state, event_factory) - state.depth -= 1 + # The prerequisite is an evaluation in its own right. Its marking starts from + # its own definition and covers only the definitions its subtree reads. The + # parent's marking is restored and merged afterward, so the marking propagates + # upward only. A failure inside the subtree still merges what was read. + parent_override_affected = state.override_affected + state.override_affected = prereq_flag.is_override + try: + state.depth += 1 + prereq_res = await self._evaluate(prereq_flag, context, state, event_factory) + state.depth -= 1 + finally: + prereq_override_affected = state.override_affected + state.override_affected = parent_override_affected or prereq_override_affected + if prereq_override_affected: + mark_override_affected(prereq_res) # Note that if the prerequisite flag is off, we don't consider it a match no matter what its # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: @@ -146,7 +167,14 @@ async def _clause_matches_context(self, clause: Clause, context: Context, state: if clause.op == 'segmentMatch': for seg_key in clause.values: segment = await self.__get_segment(seg_key) - if segment is not None and await self._segment_matches_context(segment, context, state): + if segment is None: + continue + # The segment definition was read at this point, so an override segment marks the + # evaluation whether or not it matches. A segment that does not match still shapes + # the outcome, for example through a negated clause. + if segment.is_override: + state.override_affected = True + if await self._segment_matches_context(segment, context, state): return _maybe_negate(clause, True) return _maybe_negate(clause, False) diff --git a/ldclient/impl/evaluator.py b/ldclient/impl/evaluator.py index 77a811d4..f6a14a25 100644 --- a/ldclient/impl/evaluator.py +++ b/ldclient/impl/evaluator.py @@ -17,7 +17,8 @@ _maybe_negate, _variation_index_for_context, check_targets, - error_reason + error_reason, + mark_override_affected ) from ldclient.impl.events.types import EventFactory from ldclient.impl.model import * @@ -57,15 +58,23 @@ def __init__( def evaluate(self, flag: FeatureFlag, context: Context, event_factory: EventFactory) -> EvalResult: state = EvalResult() state.original_flag_key = flag.key + # Reading the flag's own definition is the first read of the evaluation, so the marking + # starts from the flag's override marker. + state.override_affected = flag.is_override try: state.detail = self._evaluate(flag, context, state, event_factory) except EvaluationException as e: if self.__logger is not None: self.__logger.error('Could not evaluate flag "%s": %s' % (flag.key, e.message)) state.detail = EvaluationDetail(None, None, {'kind': 'ERROR', 'errorKind': e.error_kind}) + # An evaluation that fails is still marked when it read an override definition. + if state.override_affected: + mark_override_affected(state.detail) return state if state.big_segments_status is not None: state.detail.reason['bigSegmentsStatus'] = state.big_segments_status + if state.override_affected: + mark_override_affected(state.detail) return state def _evaluate(self, flag: FeatureFlag, context: Context, state: EvalResult, event_factory: EventFactory) -> EvaluationDetail: @@ -118,9 +127,21 @@ def _check_prerequisites(self, flag: FeatureFlag, context: Context, state: EvalR log.warning("Missing prereq flag: " + prereq_key) failed_prereq = prereq else: - state.depth += 1 - prereq_res = self._evaluate(prereq_flag, context, state, event_factory) - state.depth -= 1 + # The prerequisite is an evaluation in its own right. Its marking starts from + # its own definition and covers only the definitions its subtree reads. The + # parent's marking is restored and merged afterward, so the marking propagates + # upward only. A failure inside the subtree still merges what was read. + parent_override_affected = state.override_affected + state.override_affected = prereq_flag.is_override + try: + state.depth += 1 + prereq_res = self._evaluate(prereq_flag, context, state, event_factory) + state.depth -= 1 + finally: + prereq_override_affected = state.override_affected + state.override_affected = parent_override_affected or prereq_override_affected + if prereq_override_affected: + mark_override_affected(prereq_res) # Note that if the prerequisite flag is off, we don't consider it a match no matter what its # off variation was. But we still need to evaluate it in order to generate an event. if (not prereq_flag.on) or prereq_res.variation_index != prereq.variation: @@ -144,7 +165,14 @@ def _clause_matches_context(self, clause: Clause, context: Context, state: EvalR if clause.op == 'segmentMatch': for seg_key in clause.values: segment = self.__get_segment(seg_key) - if segment is not None and self._segment_matches_context(segment, context, state): + if segment is None: + continue + # The segment definition was read at this point, so an override segment marks the + # evaluation whether or not it matches. A segment that does not match still shapes + # the outcome, for example through a negated clause. + if segment.is_override: + state.override_affected = True + if self._segment_matches_context(segment, context, state): return _maybe_negate(clause, True) return _maybe_negate(clause, False) diff --git a/ldclient/impl/evaluator_common.py b/ldclient/impl/evaluator_common.py index 04dbebfd..6e0f6a0c 100644 --- a/ldclient/impl/evaluator_common.py +++ b/ldclient/impl/evaluator_common.py @@ -25,7 +25,7 @@ # prerequisite evaluations, and the cached state of any Big Segments query that we may have # ended up having to do for the context. class EvalResult: - __slots__ = ['detail', 'events', 'big_segments_status', 'big_segments_membership', 'original_flag_key', 'prereq_stack', 'segment_stack', 'depth', 'prerequisites'] + __slots__ = ['detail', 'events', 'big_segments_status', 'big_segments_membership', 'original_flag_key', 'prereq_stack', 'segment_stack', 'depth', 'prerequisites', 'override_affected'] def __init__(self): self.detail = None @@ -37,6 +37,11 @@ def __init__(self): self.segment_stack = None # type: Optional[List[str]] self.depth = 0 self.prerequisites = [] # type: List[str] + # True when the evaluation in progress has read a flag or segment definition that carries + # the override marker. While a prerequisite is evaluated, this holds the marking of the + # prerequisite's own subtree. The evaluator merges it into the parent's marking afterward, + # so the marking propagates upward only. + self.override_affected = False def record_prerequisite(self, key: str): if self.depth == 0: @@ -255,3 +260,12 @@ def check_targets(flag: FeatureFlag, context: Context) -> Optional[EvaluationDet def error_reason(error_kind: str) -> dict: return {'kind': 'ERROR', 'errorKind': error_kind} + + +def mark_override_affected(detail: EvaluationDetail) -> None: + """ + Sets the override-affected indicator on the evaluation reason. The indicator is present + only when true, so a reason that was not affected by an override is left untouched and + serializes exactly as before. + """ + detail.reason['overrideAffected'] = True diff --git a/ldclient/impl/model/feature_flag.py b/ldclient/impl/model/feature_flag.py index ae0b6f75..f7c03f4d 100644 --- a/ldclient/impl/model/feature_flag.py +++ b/ldclient/impl/model/feature_flag.py @@ -1,3 +1,4 @@ +import copy from typing import Any, Dict, List, Optional, Set, Union from ldclient.impl.model.clause import Clause @@ -96,10 +97,12 @@ class FeatureFlag(ModelEntity): '_salt', '_track_events', '_debug_events_until_date', + '_is_override', ] def __init__(self, data: dict): super().__init__(data) + self._is_override = False # In the following logic, we're being somewhat lenient in terms of allowing most properties to # be absent even if they are really required in the schema. That's for backward compatibility # with test logic that constructed incomplete JSON, and also with the file data source which @@ -202,3 +205,24 @@ def exclude_from_summaries(self) -> bool: @property def sampling_ratio(self) -> Optional[int]: return self._sampling_ratio + + @property + def is_override(self) -> bool: + """ + True if this flag definition was supplied by an SDK override source rather than by + LaunchDarkly data. The marker is never part of the JSON representation. Only the SDK + components that manage override entries set it. Components that read a definition + through the store may treat a marked definition the same as any other. + + Flag overrides are currently experimental and subject to change. + """ + return self._is_override + + def with_override_marker(self) -> 'FeatureFlag': + """ + Returns a shallow copy of this flag with the override marker set. The copy shares the + underlying data with this instance, which is left unmarked. + """ + marked = copy.copy(self) + marked._is_override = True + return marked diff --git a/ldclient/impl/model/segment.py b/ldclient/impl/model/segment.py index 6ed1b6a8..709518a2 100644 --- a/ldclient/impl/model/segment.py +++ b/ldclient/impl/model/segment.py @@ -1,3 +1,4 @@ +import copy from typing import Any, List, Optional, Set from ldclient.impl.model.attribute_ref import ( @@ -65,10 +66,12 @@ class Segment(ModelEntity): '_unbounded', '_unbounded_context_kind', '_generation', + '_is_override', ] def __init__(self, data: dict): super().__init__(data) + self._is_override = False # In the following logic, we're being somewhat lenient in terms of allowing most properties to # be absent even if they are really required in the schema. That's for backward compatibility # with test logic that constructed incomplete JSON, and also with the file data source which @@ -137,3 +140,24 @@ def unbounded_context_kind(self) -> Optional[str]: @property def generation(self) -> Optional[int]: return self._generation + + @property + def is_override(self) -> bool: + """ + True if this segment definition was supplied by an SDK override source rather than by + LaunchDarkly data. The marker is never part of the JSON representation. Only the SDK + components that manage override entries set it. Components that read a definition + through the store may treat a marked definition the same as any other. + + Flag overrides are currently experimental and subject to change. + """ + return self._is_override + + def with_override_marker(self) -> 'Segment': + """ + Returns a shallow copy of this segment with the override marker set. The copy shares the + underlying data with this instance, which is left unmarked. + """ + marked = copy.copy(self) + marked._is_override = True + return marked diff --git a/ldclient/testing/impl/test_async_evaluator_overrides.py b/ldclient/testing/impl/test_async_evaluator_overrides.py new file mode 100644 index 00000000..e193d3bf --- /dev/null +++ b/ldclient/testing/impl/test_async_evaluator_overrides.py @@ -0,0 +1,154 @@ +""" +Tests for the override marking performed by the async evaluator. These mirror the key +scenarios of the sync evaluator override tests. +""" +import pytest + +from ldclient import Context +from ldclient.impl.events.types import EventFactory +from ldclient.testing.builders import * +from ldclient.testing.impl.test_async_evaluator import AsyncEvaluatorBuilder +from ldclient.testing.impl.test_evaluator_overrides import ( + assert_override_affected, + assert_reason_override_affected, + make_override_test_flag, + prereq_records, + require_prereq_record +) + +override_test_context = Context.create('userkey') +event_factory_with_reasons = EventFactory(True, lambda: 0) + + +@pytest.mark.asyncio +async def test_override_flag_marks_evaluation(): + flag = make_override_test_flag('feature').with_override_marker() + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_plain_flag_is_not_marked(): + flag = make_override_test_flag('feature') + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert_override_affected(False, result) + + +@pytest.mark.asyncio +async def test_malformed_override_flag_error_result_is_marked(): + flag = FlagBuilder('feature').on(True).fallthrough_variation(99).variations('off', 'on').build().with_override_marker() + result = await AsyncEvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_prerequisite_cycle_through_an_override_flag_is_marked(): + prereq = make_override_test_flag('prereq', 'feature').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = AsyncEvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + assert prereq_records(result) == [] + + +@pytest.mark.asyncio +async def test_prerequisite_cycle_below_an_override_flag_keeps_the_flag_marking(): + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_override_prerequisite_marks_prerequisite_record_and_top_level(): + prereq = make_override_test_flag('prereq').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(True, require_prereq_record(result, 'prereq').reason) + + +@pytest.mark.asyncio +async def test_override_flag_does_not_mark_unaffected_prerequisite_record(): + prereq = make_override_test_flag('prereq') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +@pytest.mark.asyncio +async def test_unaffected_sibling_prerequisite_record_stays_unmarked(): + s1 = SegmentBuilder('s1').included(override_test_context.key).build() + d = make_override_test_flag('d').with_override_marker() + b = make_override_test_flag('b', 'd') + c = FlagBuilder('c').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(s1.key)).build() + ).build() + a = make_override_test_flag('a', 'b', 'c') + evaluator = AsyncEvaluatorBuilder().with_flag(b).with_flag(c).with_flag(d).with_segment(s1).build() + result = await evaluator.evaluate(a, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert len(prereq_records(result)) == 3 + assert_reason_override_affected(True, require_prereq_record(result, 'd').reason) + assert_reason_override_affected(True, require_prereq_record(result, 'b').reason) + assert_reason_override_affected(False, require_prereq_record(result, 'c').reason) + + +@pytest.mark.asyncio +async def test_override_segment_read_without_matching_marks_evaluation(): + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = AsyncEvaluatorBuilder().with_segment(segment).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_override_segment_referenced_by_segment_rule_marks_evaluation(): + nested = SegmentBuilder('nested-segment').included(override_test_context.key).build().with_override_marker() + outer = SegmentBuilder('outer-segment').rules(SegmentRuleBuilder().clauses(make_clause_matching_segment_key(nested.key)).build()).build() + flag = make_boolean_flag_matching_segment(outer) + evaluator = AsyncEvaluatorBuilder().with_segment(outer).with_segment(nested).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.value is True + assert_override_affected(True, result) + + +@pytest.mark.asyncio +async def test_missing_definitions_do_not_mark_evaluation(): + unrelated = make_override_test_flag('unrelated').with_override_marker() + flag = make_override_test_flag('feature', 'missing') + evaluator = AsyncEvaluatorBuilder().with_flag(unrelated).with_unknown_flag('missing').build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': 'missing'} + assert_override_affected(False, result) + + segment_flag = make_boolean_flag_with_clauses(make_clause_matching_segment_key('missing')) + evaluator = AsyncEvaluatorBuilder().with_unknown_segment('missing').build() + result = await evaluator.evaluate(segment_flag, override_test_context, event_factory_with_reasons) + assert_override_affected(False, result) + + +@pytest.mark.asyncio +@pytest.mark.parametrize("flag_override,prereq_override", [(False, False), (True, False), (False, True), (True, True)]) +async def test_result_override_affected_matches_reason_indicator(flag_override: bool, prereq_override: bool): + prereq = make_override_test_flag('prereq') + if prereq_override: + prereq = prereq.with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + if flag_override: + flag = flag.with_override_marker() + evaluator = AsyncEvaluatorBuilder().with_flag(prereq).build() + result = await evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(flag_override or prereq_override, result) + assert_reason_override_affected(prereq_override, require_prereq_record(result, 'prereq').reason) diff --git a/ldclient/testing/impl/test_evaluator_overrides.py b/ldclient/testing/impl/test_evaluator_overrides.py new file mode 100644 index 00000000..f69bc612 --- /dev/null +++ b/ldclient/testing/impl/test_evaluator_overrides.py @@ -0,0 +1,316 @@ +""" +Tests for the override marking performed by the evaluator. An evaluation is override-affected +when any flag or segment definition it reads carries the override marker. The marking +propagates upward from prerequisites to the flags that depend on them and never sideways or +downward. +""" +from typing import List, Optional + +import pytest + +from ldclient import Context +from ldclient.impl.evaluator_common import EvalResult +from ldclient.impl.events.types import EventFactory, EventInputEvaluation +from ldclient.testing.builders import * +from ldclient.testing.impl.evaluator_util import EvaluatorBuilder + +override_test_context = Context.create('userkey') + +# Events carry their reasons, so a prerequisite record's marking is visible on its reason. +event_factory_with_reasons = EventFactory(True, lambda: 0) + + +def assert_override_affected(expected: bool, result: EvalResult) -> None: + """Checks the reason indicator and the result scalar together. Both report the same marking.""" + assert result.override_affected is expected, "result.override_affected" + assert_reason_override_affected(expected, result.detail.reason) + + +def assert_reason_override_affected(expected: bool, reason: Optional[dict]) -> None: + assert reason is not None + if expected: + assert reason.get('overrideAffected') is True, "reason indicator: %s" % reason + else: + assert 'overrideAffected' not in reason, "reason indicator: %s" % reason + + +def make_override_test_flag(key: str, *prereq_keys: str) -> FeatureFlag: + """A flag that is on and serves variation 1 ("on") by fallthrough, with prerequisites that must each serve variation 1.""" + builder = FlagBuilder(key).on(True).fallthrough_variation(1).off_variation(0).variations('off', 'on') + for prereq_key in prereq_keys: + builder.prerequisite(prereq_key, 1) + return builder.build() + + +def prereq_records(result: EvalResult) -> List[EventInputEvaluation]: + return result.events or [] + + +def require_prereq_record(result: EvalResult, prereq_key: str) -> EventInputEvaluation: + for event in prereq_records(result): + if event.flag is not None and event.flag.key == prereq_key: + return event + raise AssertionError("no record for prerequisite %s" % prereq_key) + + +def test_override_flag_marks_evaluation_when_off(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'OFF' + assert result.detail.value == 'off' + assert_override_affected(True, result) + + +def test_override_flag_marks_evaluation_on_fallthrough(): + flag = make_override_test_flag('feature').with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + +def test_override_flag_marks_evaluation_on_rule_match(): + rule = FlagRuleBuilder().id('rule-id').variation(1).clauses(make_clause_matching_context(override_test_context)).build() + flag = FlagBuilder('feature').on(True).fallthrough_variation(0).variations('off', 'on').rules(rule).build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.reason['ruleId'] == 'rule-id' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + +def test_plain_flag_alone_is_not_marked(): + flag = FlagBuilder('feature').on(False).off_variation(0).variations('off', 'on').build() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'OFF'} + assert_override_affected(False, result) + + +def test_plain_flag_with_prerequisite_and_segment_is_not_marked(): + segment = SegmentBuilder('segment').included(override_test_context.key).build() + prereq = FlagBuilder('prereq').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(segment.key)).build() + ).build() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert result.detail.value == 'on' + assert_override_affected(False, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +def test_malformed_override_flag_error_result_is_marked(): + flag = FlagBuilder('feature').on(True).fallthrough_variation(99).variations('off', 'on').build().with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'ERROR' + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert result.detail.value is None + assert_override_affected(True, result) + + +def test_prerequisite_cycle_through_an_override_flag_is_marked(): + # feature -> prereq -> feature; only prereq is an override + prereq = make_override_test_flag('prereq', 'feature').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'ERROR' + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + assert prereq_records(result) == [] + + +def test_prerequisite_cycle_below_an_override_flag_keeps_the_flag_marking(): + # feature -> prereq -> feature; only feature is an override. The failure happens inside the + # plain prerequisite's subtree, and the top-level marking from the flag's own read survives it. + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(True, result) + + +def test_prerequisite_cycle_through_plain_flags_is_not_marked(): + prereq = make_override_test_flag('prereq', 'feature') + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(flag).with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['errorKind'] == 'MALFORMED_FLAG' + assert_override_affected(False, result) + + +def test_override_prerequisite_marks_prerequisite_record_and_top_level(): + prereq = make_override_test_flag('prereq').with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + record = require_prereq_record(result, 'prereq') + assert record.reason is not None + assert record.reason['kind'] == 'FALLTHROUGH' + assert_reason_override_affected(True, record.reason) + + +def test_override_flag_does_not_mark_unaffected_prerequisite_record(): + # The marking propagates upward only. The top-level flag's own marker does not leak into + # the record of a prerequisite whose subtree read no override definition. + prereq = make_override_test_flag('prereq') + flag = make_override_test_flag('feature', 'prereq').with_override_marker() + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(True, result) + assert_reason_override_affected(False, require_prereq_record(result, 'prereq').reason) + + +def test_override_prerequisite_at_depth_two_marks_all_affected_scopes(): + prereq2 = make_override_test_flag('prereq2').with_override_marker() + prereq1 = make_override_test_flag('prereq1', 'prereq2') + flag = make_override_test_flag('feature', 'prereq1') + evaluator = EvaluatorBuilder().with_flag(prereq1).with_flag(prereq2).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + + # The nested record is produced first, during the evaluation of prereq1. + records = prereq_records(result) + assert [r.flag.key for r in records] == ['prereq2', 'prereq1'] + assert_reason_override_affected(True, records[0].reason) + assert_reason_override_affected(True, records[1].reason) + + +def test_unaffected_sibling_prerequisite_record_stays_unmarked(): + # Flag a has prerequisites b and c. Only d, a prerequisite of b, is an override. The marking + # reaches a, b, and d. It does not reach the sibling c, and the plain segments s1 and s2 + # mark nothing. + s1 = SegmentBuilder('s1').included(override_test_context.key).build() + s2 = SegmentBuilder('s2').included(override_test_context.key).build() + d = make_override_test_flag('d').with_override_marker() + b = make_override_test_flag('b', 'd') + c = FlagBuilder('c').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(s1.key)).build() + ).build() + a = FlagBuilder('a').on(True).fallthrough_variation(0).off_variation(0).prerequisite('b', 1).prerequisite('c', 1).rules( + FlagRuleBuilder().id('rule-s2').variation(1).clauses(make_clause_matching_segment_key(s2.key)).build() + ).variations('off', 'on').build() + + evaluator = EvaluatorBuilder().with_flag(b).with_flag(c).with_flag(d).with_segment(s1).with_segment(s2).build() + result = evaluator.evaluate(a, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value == 'on' + assert_override_affected(True, result) + + assert len(prereq_records(result)) == 3 + assert_reason_override_affected(True, require_prereq_record(result, 'd').reason) + assert_reason_override_affected(True, require_prereq_record(result, 'b').reason) + assert_reason_override_affected(False, require_prereq_record(result, 'c').reason) + + +def test_override_segment_referenced_by_flag_rule_marks_evaluation(): + segment = SegmentBuilder('segment').included(override_test_context.key).build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_read_without_matching_marks_evaluation(): + # A read is enough to mark the evaluation. The segment does not need to match. + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert result.detail.value is False + assert_override_affected(True, result) + + +def test_override_segment_read_through_negated_clause_marks_evaluation(): + segment = SegmentBuilder('segment').included('someone-else').build().with_override_marker() + flag = make_boolean_flag_with_clauses(negate_clause(make_clause_matching_segment_key(segment.key))) + evaluator = EvaluatorBuilder().with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_referenced_by_segment_rule_marks_evaluation(): + # The outer segment is plain. A rule of the outer segment reads a nested override segment. + nested = SegmentBuilder('nested-segment').included(override_test_context.key).build().with_override_marker() + outer = SegmentBuilder('outer-segment').rules(SegmentRuleBuilder().clauses(make_clause_matching_segment_key(nested.key)).build()).build() + flag = make_boolean_flag_matching_segment(outer) + evaluator = EvaluatorBuilder().with_segment(outer).with_segment(nested).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'RULE_MATCH' + assert result.detail.value is True + assert_override_affected(True, result) + + +def test_override_segment_referenced_by_prerequisite_marks_prerequisite_record_and_top_level(): + segment = SegmentBuilder('segment').included(override_test_context.key).build().with_override_marker() + prereq = FlagBuilder('prereq').on(True).variations('off', 'on').fallthrough_variation(0).rules( + FlagRuleBuilder().variation(1).clauses(make_clause_matching_segment_key(segment.key)).build() + ).build() + flag = make_override_test_flag('feature', 'prereq') + evaluator = EvaluatorBuilder().with_flag(prereq).with_segment(segment).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason['kind'] == 'FALLTHROUGH' + assert_override_affected(True, result) + assert_reason_override_affected(True, require_prereq_record(result, 'prereq').reason) + + +def test_missing_prerequisite_does_not_mark_evaluation(): + # A definition that cannot be resolved contributes nothing, because nothing was read. The + # store holds an unrelated override definition to show that only reads count. + unrelated = make_override_test_flag('unrelated').with_override_marker() + flag = make_override_test_flag('feature', 'missing') + evaluator = EvaluatorBuilder().with_flag(unrelated).with_unknown_flag('missing').build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'PREREQUISITE_FAILED', 'prerequisiteKey': 'missing'} + assert result.detail.value == 'off' + assert_override_affected(False, result) + assert prereq_records(result) == [] + + +def test_missing_segment_does_not_mark_evaluation(): + unrelated = SegmentBuilder('unrelated').included(override_test_context.key).build().with_override_marker() + flag = make_boolean_flag_with_clauses(make_clause_matching_segment_key('missing')) + evaluator = EvaluatorBuilder().with_segment(unrelated).with_unknown_segment('missing').build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH'} + assert result.detail.value is False + assert_override_affected(False, result) + + +@pytest.mark.parametrize("flag_override,prereq_override", [(False, False), (True, False), (False, True), (True, True)]) +def test_result_override_affected_matches_reason_indicator(flag_override: bool, prereq_override: bool): + prereq = make_override_test_flag('prereq') + if prereq_override: + prereq = prereq.with_override_marker() + flag = make_override_test_flag('feature', 'prereq') + if flag_override: + flag = flag.with_override_marker() + evaluator = EvaluatorBuilder().with_flag(prereq).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert_override_affected(flag_override or prereq_override, result) + assert_reason_override_affected(prereq_override, require_prereq_record(result, 'prereq').reason) + + +def test_marking_does_not_disturb_other_reason_properties(): + segment = SegmentBuilder('segment').unbounded(True).generation(1).build().with_override_marker() + flag = make_boolean_flag_matching_segment(segment) + evaluator = EvaluatorBuilder().with_segment(segment).with_no_big_segments_for_key(override_test_context.key).build() + result = evaluator.evaluate(flag, override_test_context, event_factory_with_reasons) + assert result.detail.reason == {'kind': 'FALLTHROUGH', 'bigSegmentsStatus': 'HEALTHY', 'overrideAffected': True} + + +def test_reason_indicator_is_absent_when_events_do_not_carry_reasons(): + # The marking does not depend on the event factory. The reason on the result is marked either way. + flag = make_override_test_flag('feature').with_override_marker() + result = EvaluatorBuilder().build().evaluate(flag, override_test_context, EventFactory(False, lambda: 0)) + assert_override_affected(True, result) diff --git a/ldclient/testing/impl/test_model_override_marker.py b/ldclient/testing/impl/test_model_override_marker.py new file mode 100644 index 00000000..203e0fc5 --- /dev/null +++ b/ldclient/testing/impl/test_model_override_marker.py @@ -0,0 +1,76 @@ +""" +Tests for the override marker carried on the flag and segment models. The marker lives on the +model type and never in the JSON representation. +""" +import json + +import pytest + +from ldclient.impl.model import FeatureFlag, ModelEncoder, Segment +from ldclient.testing.builders import FlagBuilder, SegmentBuilder +from ldclient.versioned_data_kind import FEATURES, SEGMENTS + +flag_data = {'key': 'flag1', 'version': 2, 'on': True, 'variations': [True, False], 'fallthrough': {'variation': 0}, 'salt': 'x'} +segment_data = {'key': 'seg1', 'version': 3, 'included': ['user1'], 'salt': 'y'} + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marker_defaults_to_false(kind, data): + entity = kind.decode(dict(data)) + assert entity.is_override is False + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marked_copy_is_marked_and_source_is_not(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.is_override is True + assert entity.is_override is False + assert type(marked) is type(entity) + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marker_is_never_serialized(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.to_json_dict() == data + assert kind.encode(marked) == data + assert json.loads(ModelEncoder().encode(marked)) == data + assert 'is_override' not in marked.to_json_dict() + assert 'override' not in json.dumps(marked.to_json_dict()).lower() + + +@pytest.mark.parametrize("kind,data", [(FEATURES, flag_data), (SEGMENTS, segment_data)]) +def test_marked_copy_shares_the_definition(kind, data): + entity = kind.decode(dict(data)) + marked = entity.with_override_marker() + assert marked.key == entity.key + assert marked.version == entity.version + assert marked.to_json_dict() is entity.to_json_dict() + assert marked == entity + + +def test_marked_flag_keeps_its_parsed_properties(): + flag = FlagBuilder('flag1').version(7).on(True).variations('a', 'b').fallthrough_variation(1).prerequisite('p', 0).track_events(True).build() + marked = flag.with_override_marker() + assert marked.on is True + assert marked.variations == ['a', 'b'] + assert marked.fallthrough.variation == 1 + assert [p.key for p in marked.prerequisites] == ['p'] + assert marked.track_events is True + assert marked.version == 7 + + +def test_marked_segment_keeps_its_parsed_properties(): + segment = SegmentBuilder('seg1').version(4).included('a').excluded('b').build() + marked = segment.with_override_marker() + assert marked.included == {'a'} + assert marked.excluded == {'b'} + assert marked.version == 4 + + +def test_marking_a_marked_copy_is_a_marked_copy(): + flag = FeatureFlag(dict(flag_data)).with_override_marker() + again = flag.with_override_marker() + assert again.is_override is True + assert again is not flag