From 6ff51f47113938aae515c1023ca8fb3d15122a78 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 25 Sep 2026 15:30:35 -0700 Subject: [PATCH] feat: Split summary counters and suppress individual events for override-affected evaluations Carries the override-affected marking from evaluation into analytics events, as the OVERRIDE specification requires. - `EvalEvent` and `record_eval_event` carry an `override_affected` flag. The client sets it from the evaluation result for the evaluated flag, from each prerequisite record's own marking for prerequisite evaluations, and from the flag's own marker when an evaluation raises. An unknown flag is never marked. - The event dispatcher produces no individual feature event and no debug event for a marked evaluation, whatever the flag's configuration requests. Marked evaluations appear in summary events only. - The summarizer keys counters by version, variation, and the marker, so override-affected and other evaluations of the same flag, variation, and version accumulate into separate counters. The summary event writes `overrideAffected: true` on the marked counters only, like the existing `unknown` marker. - `all_flags_state` presents an override-affected flag with `trackEvents` false, `trackReason` false, and no `debugEventsUntilDate`. The flag, its value, its version, and its reason stay in the state. The specification vectors now also check the summary marker the client hands to the event processor for each evaluation. Flag overrides are currently experimental and subject to change. --- lib/ldclient-rb/events.rb | 40 ++-- lib/ldclient-rb/impl/event_summarizer.rb | 15 +- lib/ldclient-rb/impl/event_types.rb | 12 +- lib/ldclient-rb/ldclient.rb | 45 ++-- spec/events_spec.rb | 77 +++++++ spec/impl/event_summarizer_spec.rb | 32 ++- spec/ldclient_events_spec.rb | 12 +- spec/ldclient_override_events_spec.rb | 250 +++++++++++++++++++++++ spec/ldclient_override_vectors_spec.rb | 15 +- spec/override_test_components.rb | 26 +++ 10 files changed, 480 insertions(+), 44 deletions(-) create mode 100644 spec/ldclient_override_events_spec.rb diff --git a/lib/ldclient-rb/events.rb b/lib/ldclient-rb/events.rb index f056a9aa..9e0aec48 100644 --- a/lib/ldclient-rb/events.rb +++ b/lib/ldclient-rb/events.rb @@ -44,7 +44,8 @@ def record_eval_event( debug_until = nil, prereq_of = nil, sampling_ratio = nil, - exclude_from_summaries = false + exclude_from_summaries = false, + override_affected = false ) end @@ -176,10 +177,11 @@ def record_eval_event( debug_until = nil, prereq_of = nil, sampling_ratio = nil, - exclude_from_summaries = false + exclude_from_summaries = false, + override_affected = false ) post_to_inbox(LaunchDarkly::Impl::EvalEvent.new(timestamp, context, key, version, variation, value, reason, - default, track_events, debug_until, prereq_of, sampling_ratio, exclude_from_summaries)) + default, track_events, debug_until, prereq_of, sampling_ratio, exclude_from_summaries, override_affected)) end def record_identify_event(context) @@ -328,8 +330,10 @@ def dispatch_event(event, outbox) will_add_full_event = false debug_event = nil if event.is_a?(LaunchDarkly::Impl::EvalEvent) - will_add_full_event = event.track_events - if should_debug_event(event) + # An override-affected evaluation appears only in the summary counters. It produces no + # individual feature event and no debug event, whatever the flag's configuration requests. + will_add_full_event = event.track_events && !event.override_affected + if !event.override_affected && should_debug_event(event) debug_event = LaunchDarkly::Impl::DebugEvent.new(event) end else @@ -630,18 +634,22 @@ def make_output_events(events, summary) summary.counters.each do |flagKey, flagInfo| counters = [] flagInfo.versions.each do |version, variations| - variations.each do |variation, counter| - c = { - value: counter.value, - count: counter.count, - } - c[:variation] = variation unless variation.nil? - if version.nil? - c[:unknown] = true - else - c[:version] = version + variations.each do |variation, counters_by_marker| + counters_by_marker.each do |override_affected, counter| + c = { + value: counter.value, + count: counter.count, + } + c[:variation] = variation unless variation.nil? + if version.nil? + c[:unknown] = true + else + c[:version] = version + end + # The marker is present only for override-affected counters, like the unknown marker. + c[:overrideAffected] = true if override_affected + counters.push(c) end - counters.push(c) end end flags[flagKey] = { default: flagInfo.default, counters: counters, contextKinds: flagInfo.context_kinds.to_a } diff --git a/lib/ldclient-rb/impl/event_summarizer.rb b/lib/ldclient-rb/impl/event_summarizer.rb index 4109f80a..a16da506 100644 --- a/lib/ldclient-rb/impl/event_summarizer.rb +++ b/lib/ldclient-rb/impl/event_summarizer.rb @@ -5,6 +5,10 @@ module LaunchDarkly module Impl EventSummary = Struct.new(:start_date, :end_date, :counters) + # The `versions` member is a hash of hashes: flag version, then variation index, then the + # override-affected marker (a boolean), to an EventSummaryFlagVariationCounter. The marker + # takes part in the key, so override-affected and other evaluations of the same flag, + # variation, and version accumulate into separate counters. EventSummaryFlagInfo = Struct.new(:default, :versions, :context_kinds) EventSummaryFlagVariationCounter = Struct.new(:value, :count) @@ -39,9 +43,16 @@ def summarize_event(event) counters_for_flag.context_kinds.merge(event.context.kinds) - variation_counter = counters_for_flag_version[event.variation] + counters_for_variation = counters_for_flag_version[event.variation] + if counters_for_variation.nil? + counters_for_variation = Hash.new + counters_for_flag_version[event.variation] = counters_for_variation + end + + override_affected = event.override_affected + variation_counter = counters_for_variation[override_affected] if variation_counter.nil? - counters_for_flag_version[event.variation] = EventSummaryFlagVariationCounter.new(event.value, 1) + counters_for_variation[override_affected] = EventSummaryFlagVariationCounter.new(event.value, 1) else variation_counter.count = variation_counter.count + 1 end diff --git a/lib/ldclient-rb/impl/event_types.rb b/lib/ldclient-rb/impl/event_types.rb index 9745373b..172c21d8 100644 --- a/lib/ldclient-rb/impl/event_types.rb +++ b/lib/ldclient-rb/impl/event_types.rb @@ -26,7 +26,8 @@ def initialize(timestamp, context, sampling_ratio = nil, exclude_from_summaries class EvalEvent < Event def initialize(timestamp, context, key, version = nil, variation = nil, value = nil, reason = nil, default = nil, - track_events = false, debug_until = nil, prereq_of = nil, sampling_ratio = nil, exclude_from_summaries = false) + track_events = false, debug_until = nil, prereq_of = nil, sampling_ratio = nil, exclude_from_summaries = false, + override_affected = false) super(timestamp, context, sampling_ratio, exclude_from_summaries) @key = key @version = version @@ -38,6 +39,7 @@ def initialize(timestamp, context, key, version = nil, variation = nil, value = @track_events = track_events if track_events @debug_until = debug_until if debug_until @prereq_of = prereq_of if prereq_of + @override_affected = true if override_affected end attr_reader :key @@ -49,6 +51,14 @@ def initialize(timestamp, context, key, version = nil, variation = nil, value = attr_reader :track_events attr_reader :debug_until attr_reader :prereq_of + + # True if an override affected the evaluation, directly or through a prerequisite or segment. + # Such an evaluation appears only in the summary counters, under a counter that carries the + # override-affected marker. It produces no individual feature event and no debug event. + # @return [Boolean] + def override_affected + @override_affected ? true : false + end end class MigrationOpEvent < Event diff --git a/lib/ldclient-rb/ldclient.rb b/lib/ldclient-rb/ldclient.rb index 1aa0b799..b728f011 100644 --- a/lib/ldclient-rb/ldclient.rb +++ b/lib/ldclient-rb/ldclient.rb @@ -697,6 +697,17 @@ def all_flags_state(context, options={}) end requires_experiment_data = experiment?(f, detail.reason) + track_events = f[:trackEvents] || requires_experiment_data + track_reason = requires_experiment_data + debug_events_until_date = f[:debugEventsUntilDate] + if eval_result&.override_affected + # A consumer of this state sends individual events according to these fields. An + # override-affected evaluation produces no individual events, so the state turns them off + # for this flag. The flag, its value, and its reason stay. + track_events = false + track_reason = false + debug_events_until_date = nil + end flag_state = { key: f[:key], value: detail.value, @@ -704,9 +715,9 @@ def all_flags_state(context, options={}) reason: detail.reason, prerequisites: eval_state.prerequisites, version: f[:version], - trackEvents: f[:trackEvents] || requires_experiment_data, - trackReason: requires_experiment_data, - debugEventsUntilDate: f[:debugEventsUntilDate], + trackEvents: track_events, + trackReason: track_reason, + debugEventsUntilDate: debug_events_until_date, } state.add_flag(flag_state, with_reasons, details_only_if_tracked) @@ -835,25 +846,29 @@ def flag_tracker begin (res, _) = @evaluator.evaluate(feature, context) unless res.prereq_evals.nil? + # Each prerequisite record carries the marking of the prerequisite's own evaluation, not + # the marking of the evaluation that requested it. res.prereq_evals.each do |prereq_eval| - record_prereq_flag_eval(prereq_eval.prereq_flag, prereq_eval.prereq_of_flag, context, prereq_eval.detail, with_reasons) + record_prereq_flag_eval(prereq_eval.prereq_flag, prereq_eval.prereq_of_flag, context, prereq_eval.detail, + with_reasons, prereq_eval.override_affected) end end detail = res.detail if detail.default_value? detail = EvaluationDetail.new(default, nil, detail.reason) end - record_flag_eval(feature, context, detail, default, with_reasons) + record_flag_eval(feature, context, detail, default, with_reasons, res.override_affected) [detail, feature, nil] rescue => exn Impl::Util.log_exception(@config.logger, "Error evaluating feature flag \"#{key}\"", exn) detail = Evaluator.error_result(EvaluationReason::ERROR_EXCEPTION, default) - record_flag_eval_error(feature, context, default, detail.reason, with_reasons) + # The flag definition was read before the failure, so its own marker decides the marking. + record_flag_eval_error(feature, context, default, detail.reason, with_reasons, feature.override?) [detail, feature, exn.to_s] end end - private def record_flag_eval(flag, context, detail, default, with_reasons) + private def record_flag_eval(flag, context, detail, default, with_reasons, override_affected) add_experiment_data = experiment?(flag, detail.reason) @event_processor.record_eval_event( context, @@ -867,11 +882,12 @@ def flag_tracker flag[:debugEventsUntilDate], nil, flag[:samplingRatio], - !!flag[:excludeFromSummaries] + !!flag[:excludeFromSummaries], + override_affected ) end - private def record_prereq_flag_eval(prereq_flag, prereq_of_flag, context, detail, with_reasons) + private def record_prereq_flag_eval(prereq_flag, prereq_of_flag, context, detail, with_reasons, override_affected) add_experiment_data = experiment?(prereq_flag, detail.reason) @event_processor.record_eval_event( context, @@ -885,13 +901,15 @@ def flag_tracker prereq_flag[:debugEventsUntilDate], prereq_of_flag[:key], prereq_flag[:samplingRatio], - !!prereq_flag[:excludeFromSummaries] + !!prereq_flag[:excludeFromSummaries], + override_affected ) end - private def record_flag_eval_error(flag, context, default, reason, with_reasons) + private def record_flag_eval_error(flag, context, default, reason, with_reasons, override_affected) @event_processor.record_eval_event(context, flag[:key], flag[:version], nil, default, with_reasons ? reason : nil, default, - flag[:trackEvents], flag[:debugEventsUntilDate], nil, flag[:samplingRatio], !!flag[:excludeFromSummaries]) + flag[:trackEvents], flag[:debugEventsUntilDate], nil, flag[:samplingRatio], !!flag[:excludeFromSummaries], + override_affected) end # @@ -902,8 +920,9 @@ def flag_tracker # @param with_reasons [Boolean] # private def record_unknown_flag_eval(flag_key, context, default, reason, with_reasons) + # Nothing was read from the override store, so the evaluation is not marked. @event_processor.record_eval_event(context, flag_key, nil, nil, default, with_reasons ? reason : nil, default, - false, nil, nil, 1, false) + false, nil, nil, 1, false, false) end private def experiment?(flag, reason) diff --git a/spec/events_spec.rb b/spec/events_spec.rb index d5c50661..5a92627f 100644 --- a/spec/events_spec.rb +++ b/spec/events_spec.rb @@ -384,6 +384,83 @@ module LaunchDarkly end end + describe "override-affected evaluations" do + let(:future_time) { (Time.now.to_f * 1000).to_i + 1000000 } + + it "does not produce a feature event even when the flag tracks events" do + with_processor_and_sender(default_config, starting_timestamp) do |ep, sender| + ep.record_eval_event(context, 'flagkey', 11, 1, 'value', nil, 'default', true, nil, nil, nil, false, true) + + output = flush_and_get_events(ep, sender) + expect(output).to contain_exactly( + eq(index_event(default_config, context)), + include(:kind => "summary") + ) + end + end + + it "does not produce a debug event even when the flag is in debug mode" do + with_processor_and_sender(default_config, starting_timestamp) do |ep, sender| + ep.record_eval_event(context, 'flagkey', 11, 1, 'value', nil, 'default', true, future_time, nil, nil, false, true) + + output = flush_and_get_events(ep, sender) + expect(output).to contain_exactly( + eq(index_event(default_config, context)), + include(:kind => "summary") + ) + end + end + + it "is counted in a summary counter that carries the override-affected marker" do + with_processor_and_sender(default_config, starting_timestamp) do |ep, sender| + ep.record_eval_event(context, 'flagkey', 11, 1, 'value', nil, 'default', true, nil, nil, nil, false, true) + ep.record_eval_event(context, 'flagkey', 11, 1, 'value', nil, 'default', true, nil, nil, nil, false, true) + + output = flush_and_get_events(ep, sender) + summary = output.detect { |e| e[:kind] == "summary" } + expect(summary[:features][:flagkey][:counters]).to contain_exactly( + { version: 11, variation: 1, value: "value", count: 2, overrideAffected: true } + ) + end + end + + it "is counted separately from other evaluations of the same flag, version, and variation" do + with_processor_and_sender(default_config, starting_timestamp) do |ep, sender| + ep.record_eval_event(context, 'flagkey', 11, 1, 'value', nil, 'default', false, nil, nil, nil, false, true) + ep.record_eval_event(context, 'flagkey', 11, 1, 'value', nil, 'default', false) + + output = flush_and_get_events(ep, sender) + summary = output.detect { |e| e[:kind] == "summary" } + expect(summary[:features][:flagkey][:counters]).to contain_exactly( + { version: 11, variation: 1, value: "value", count: 1, overrideAffected: true }, + { version: 11, variation: 1, value: "value", count: 1 } + ) + end + end + + it "does not add the marker to counters for other evaluations" do + with_processor_and_sender(default_config, starting_timestamp) do |ep, sender| + ep.record_eval_event(context, 'flagkey', 11, 1, 'value', nil, 'default', false) + + output = flush_and_get_events(ep, sender) + summary = output.detect { |e| e[:kind] == "summary" } + expect(summary[:features][:flagkey][:counters]).to eq([{ version: 11, variation: 1, value: "value", count: 1 }]) + end + end + + it "keeps a marked prerequisite record out of the individual events" do + with_processor_and_sender(default_config, starting_timestamp) do |ep, sender| + ep.record_eval_event(context, 'prereq', 11, 1, 'value', nil, nil, true, nil, 'top', nil, false, true) + + output = flush_and_get_events(ep, sender) + expect(output).to contain_exactly( + eq(index_event(default_config, context)), + include(:kind => "summary") + ) + end + end + end + it "queues custom event with context" do with_processor_and_sender(default_config, starting_timestamp) do |ep, sender| ep.record_custom_event(context, 'eventkey', { thing: 'stuff' }, 1.5) diff --git a/spec/impl/event_summarizer_spec.rb b/spec/impl/event_summarizer_spec.rb index 1aedb757..3ff82e09 100644 --- a/spec/impl/event_summarizer_spec.rb +++ b/spec/impl/event_summarizer_spec.rb @@ -55,8 +55,8 @@ module Impl 'key1' => EventSummaryFlagInfo.new( 'default1', { 11 => { - 1 => EventSummaryFlagVariationCounter.new('value1', 2), - 2 => EventSummaryFlagVariationCounter.new('value2', 1), + 1 => { false => EventSummaryFlagVariationCounter.new('value1', 2) }, + 2 => { false => EventSummaryFlagVariationCounter.new('value2', 1) }, }, }, Set.new(["user"]) @@ -64,7 +64,7 @@ module Impl 'key2' => EventSummaryFlagInfo.new( 'default2', { 22 => { - 1 => EventSummaryFlagVariationCounter.new('value99', 1), + 1 => { false => EventSummaryFlagVariationCounter.new('value99', 1) }, }, }, Set.new(["user"]) @@ -72,7 +72,31 @@ module Impl 'badkey' => EventSummaryFlagInfo.new( 'default3', { nil => { - nil => EventSummaryFlagVariationCounter.new('default3', 1), + nil => { false => EventSummaryFlagVariationCounter.new('default3', 1) }, + }, + }, + Set.new(["user"]) + ), + } + expect(data.counters).to eq expected_counters + end + + it "counts override-affected evaluations separately from other evaluations of the same flag, version, and variation" do + es = subject.new + plain = make_eval_event(0, context, 'key1', 11, 1, 'value1', nil, 'default1') + marked = LaunchDarkly::Impl::EvalEvent.new(0, context, 'key1', 11, 1, 'value1', nil, 'default1', + false, nil, nil, nil, false, true) + [plain, marked, marked].each { |e| es.summarize_event(e) } + data = es.snapshot + + expected_counters = { + 'key1' => EventSummaryFlagInfo.new( + 'default1', { + 11 => { + 1 => { + false => EventSummaryFlagVariationCounter.new('value1', 1), + true => EventSummaryFlagVariationCounter.new('value1', 2), + }, }, }, Set.new(["user"]) diff --git a/spec/ldclient_events_spec.rb b/spec/ldclient_events_spec.rb index 9c1d7f07..1a78fb1f 100644 --- a/spec/ldclient_events_spec.rb +++ b/spec/ldclient_events_spec.rb @@ -22,7 +22,7 @@ def event_processor(client) with_client(test_config) do |client| context = basic_context expect(event_processor(client)).to receive(:record_eval_event).with( - context, 'badkey', nil, nil, 'default', nil, 'default', false, nil, nil, 1, false + context, 'badkey', nil, nil, 'default', nil, 'default', false, nil, nil, 1, false, false ) client.variation("badkey", context, "default") end @@ -35,7 +35,7 @@ def event_processor(client) context = basic_context with_client(test_config(data_source: td)) do |client| expect(event_processor(client)).to receive(:record_eval_event).with( - context, 'flagkey', 1, 0, 'value', nil, 'default', false, nil, nil, nil, false + context, 'flagkey', 1, 0, 'value', nil, 'default', false, nil, nil, nil, false, false ) client.variation("flagkey", context, "default") end @@ -81,7 +81,7 @@ def event_processor(client) with_client(test_config(data_source: td)) do |client| expect(event_processor(client)).to receive(:record_eval_event).with( context, 'flagkey', 100, 0, 'value', LaunchDarkly::EvaluationReason::rule_match(0, 'id'), - 'default', true, nil, nil, nil, false + 'default', true, nil, nil, nil, false, false ) client.variation("flagkey", context, "default") end @@ -98,7 +98,7 @@ def event_processor(client) with_client(test_config(data_source: td)) do |client| expect(event_processor(client)).to receive(:record_eval_event).with( context, 'flagkey', 100, 0, 'value', LaunchDarkly::EvaluationReason::fallthrough, - 'default', true, nil, nil, nil, false + 'default', true, nil, nil, nil, false, false ) client.variation("flagkey", context, "default") end @@ -112,7 +112,7 @@ def event_processor(client) expect(event_processor(client)).to receive(:record_eval_event).with( context, 'badkey', nil, nil, 'default', LaunchDarkly::EvaluationReason::error(LaunchDarkly::EvaluationReason::ERROR_FLAG_NOT_FOUND), - 'default', false, nil, nil, 1, false + 'default', false, nil, nil, 1, false, false ) client.variation_detail("badkey", context, "default") end @@ -126,7 +126,7 @@ def event_processor(client) with_client(test_config(data_source: td)) do |client| expect(event_processor(client)).to receive(:record_eval_event).with( context, 'flagkey', 1, 0, 'value', LaunchDarkly::EvaluationReason::off, - 'default', false, nil, nil, nil, false + 'default', false, nil, nil, nil, false, false ) client.variation_detail("flagkey", context, "default") end diff --git a/spec/ldclient_override_events_spec.rb b/spec/ldclient_override_events_spec.rb new file mode 100644 index 00000000..36c56559 --- /dev/null +++ b/spec/ldclient_override_events_spec.rb @@ -0,0 +1,250 @@ +require "spec_helper" +require "events_test_util" +require "model_builders" +require "override_test_components" + +module LaunchDarkly + describe LDClient, "events for override-affected evaluations" do + let(:context) { LDContext.create({ key: "user-key", kind: "user" }) } + let(:debug_until) { (Time.now.to_f * 1000).to_i + 1000000 } + + def flag(key, value, version: 100, prereqs: [], segment: nil, track: true) + data = { + key: key, version: version, on: true, offVariation: 0, fallthrough: { variation: 1 }, + variations: ["#{value}-off", value], + prerequisites: prereqs.map { |p| { key: p, variation: 1 } }, + trackEvents: track + } + data[:debugEventsUntilDate] = debug_until if track + if segment + data[:rules] = [{ id: "segment-rule", variation: 1, clauses: [{ attribute: "", op: "segmentMatch", values: [segment] }] }] + data[:fallthrough] = { variation: 0 } + end + data + end + + def segment(key, *included) + { key: key, version: 100, included: included } + end + + def data_system(overrides:, flags: {}, segments: {}) + DataSystem.custom + .initializers([TestDataInitializer.new(flags: flags, segments: segments)]) + .overrides(overrides) + .build + end + + def with_recording_client(data_system_config) + config = Config.new(data_system_config: data_system_config, send_events: false, logger: $null_log) + client = LDClient.new("sdk-key", config, 5) + events = RecordingEventProcessor.new + client.instance_variable_set(:@event_processor, events) + begin + yield client, events + ensure + client.close + end + end + + describe "records handed to the event processor" do + it "carries the marking of a directly overridden flag" do + source = TestOverrideSource.new([flag("flag", "override")]) + with_recording_client(data_system(overrides: source, flags: { flag: flag("flag", "ld") })) do |client, events| + client.variation("flag", context, "default") + + record = events.records_for("flag")[0] + expect(record.override_affected).to be true + expect(record.track_events).to be true + expect(record.debug_until).to eq debug_until + expect(record.value).to eq "override" + end + end + + it "does not mark an unaffected flag" do + source = TestOverrideSource.new([flag("flag", "override")]) + with_recording_client(data_system(overrides: source, flags: { normal: flag("normal", "n") })) do |client, events| + client.variation("normal", context, "default") + + expect(events.records_for("normal")[0].override_affected).to be false + end + end + + it "marks a flag affected through an overridden segment" do + source = TestOverrideSource.new([], [segment("seg", context.key)]) + with_recording_client(data_system(overrides: source, flags: { flag: flag("flag", "included", segment: "seg") }, + segments: { seg: segment("seg") })) do |client, events| + expect(client.variation("flag", context, "default")).to eq "included" + + expect(events.records_for("flag")[0].override_affected).to be true + end + end + + it "marks the prerequisite record by the prerequisite's own reads" do + source = TestOverrideSource.new([flag("overridden-prereq", "op")]) + flags = { + "overridden-prereq": flag("overridden-prereq", "ld-op", version: 50), + "plain-prereq": flag("plain-prereq", "pp"), + top: flag("top", "t", prereqs: ["overridden-prereq", "plain-prereq"]), + } + with_recording_client(data_system(overrides: source, flags: flags)) do |client, events| + expect(client.variation("top", context, "default")).to eq "t" + + top = events.records_for("top")[0] + expect(top.override_affected).to be true + overridden = events.records_for("overridden-prereq")[0] + expect(overridden.override_affected).to be true + expect(overridden.version).to eq 100 + expect(overridden.prereq_of).to eq "top" + plain = events.records_for("plain-prereq")[0] + expect(plain.override_affected).to be false + expect(plain.prereq_of).to eq "top" + end + end + + it "does not mark an unknown flag" do + source = TestOverrideSource.new([flag("flag", "override")]) + with_recording_client(data_system(overrides: source)) do |client, events| + client.variation("unknown", context, "default") + + expect(events.records_for("unknown")[0].override_affected).to be false + end + end + + it "marks an error record by the flag's own marker when the evaluation raises" do + source = TestOverrideSource.new([flag("flag", "override")]) + with_recording_client(data_system(overrides: source)) do |client, events| + evaluator = client.instance_variable_get(:@evaluator) + allow(evaluator).to receive(:evaluate).and_raise(RuntimeError, "boom") + + expect(client.variation("flag", context, "default")).to eq "default" + + record = events.records_for("flag")[0] + expect(record.override_affected).to be true + expect(record.variation).to be_nil + end + end + end + + describe "all_flags_state" do + it "turns off event tracking for override-affected flags and keeps it for others" do + source = TestOverrideSource.new([flag("direct", "override")], [segment("seg", context.key)]) + flags = { + direct: flag("direct", "ld"), + transitive: flag("transitive", "included", segment: "seg"), + normal: flag("normal", "n"), + } + with_recording_client(data_system(overrides: source, flags: flags, segments: { seg: segment("seg") })) do |client, _| + state = client.all_flags_state(context, with_reasons: true) + meta = state.as_json["$flagsState"] + + expect(state.values_map).to eq({ "direct" => "override", "transitive" => "included", "normal" => "n" }) + %w[direct transitive].each do |key| + expect(meta[key]).not_to have_key(:trackEvents) + expect(meta[key]).not_to have_key(:trackReason) + expect(meta[key]).not_to have_key(:debugEventsUntilDate) + expect(meta[key][:version]).to eq 100 + expect(meta[key][:reason].override_affected).to be true + end + expect(meta["normal"][:trackEvents]).to be true + expect(meta["normal"][:debugEventsUntilDate]).to eq debug_until + expect(meta["normal"][:reason].override_affected).to be false + end + end + + it "turns off tracking that an experiment would otherwise require" do + experiment = flag("experiment", "e", track: false).merge(trackEventsFallthrough: true) + source = TestOverrideSource.new([experiment]) + with_recording_client(data_system(overrides: source, flags: { experiment: experiment })) do |client, _| + meta = client.all_flags_state(context).as_json["$flagsState"] + + expect(meta["experiment"]).not_to have_key(:trackEvents) + expect(meta["experiment"]).not_to have_key(:trackReason) + end + end + + it "omits details for an override-affected flag when details are only wanted for tracked flags" do + source = TestOverrideSource.new([flag("direct", "override")]) + with_recording_client(data_system(overrides: source, flags: { normal: flag("normal", "n") })) do |client, _| + meta = client.all_flags_state(context, details_only_for_tracked_flags: true).as_json["$flagsState"] + + expect(meta["direct"]).not_to have_key(:version) + expect(meta["normal"][:version]).to eq 100 + end + end + end + + describe "delivered payload" do + def with_sending_client(data_system_config) + config = Config.new(data_system_config: data_system_config, logger: $null_log, diagnostic_opt_out: true) + client = LDClient.new("sdk-key", config, 5) + sender = FakeEventSender.new + ep = EventProcessor.new("sdk-key", config, nil, nil, { event_sender: sender }) + client.instance_variable_set(:@event_processor, ep) + begin + yield client, ep, sender + ensure + ep.stop + client.close + end + end + + def payload(ep, sender) + ep.flush + ep.wait_until_inactive + sender.analytics_payloads.pop(timeout: 5) + end + + it "sends only an index event and a marked summary counter for an overridden flag that tracks events" do + source = TestOverrideSource.new([flag("flag", "override", version: 300)]) + with_sending_client(data_system(overrides: source, flags: { normal: flag("normal", "n", track: false) })) do |client, ep, sender| + 2.times { client.variation("flag", context, "default1") } + client.variation("normal", context, "default2") + + events = payload(ep, sender) + expect(events.map { |e| e[:kind] }.sort).to eq %w[index summary] + summary = events.detect { |e| e[:kind] == "summary" } + expect(summary[:features][:flag][:default]).to eq "default1" + expected_counters = [{ version: 300, variation: 1, value: "override", count: 2, overrideAffected: true }] + expect(summary[:features][:flag][:counters]).to eq(expected_counters) + expect(summary[:features][:normal][:counters]).to eq([{ version: 100, variation: 1, value: "n", count: 1 }]) + end + end + + it "sends the individual event for an unaffected prerequisite inside a marked evaluation and none for the marked ones" do + source = TestOverrideSource.new([flag("overridden-prereq", "op", version: 200)]) + flags = { + "overridden-prereq": flag("overridden-prereq", "ld-op"), + "plain-prereq": flag("plain-prereq", "pp"), + mixed: flag("mixed", "m", prereqs: ["overridden-prereq", "plain-prereq"]), + } + with_sending_client(data_system(overrides: source, flags: flags)) do |client, ep, sender| + expect(client.variation("mixed", context, "default")).to eq "m" + + events = payload(ep, sender) + feature_events = events.select { |e| e[:kind] == "feature" } + expect(feature_events.map { |e| e[:key] }).to eq ["plain-prereq"] + expect(feature_events[0][:prereqOf]).to eq "mixed" + # The unaffected prerequisite is in debug mode too, so it alone produces a debug event. + expect(events.select { |e| e[:kind] == "debug" }.map { |e| e[:key] }).to eq ["plain-prereq"] + + summary = events.detect { |e| e[:kind] == "summary" } + expect(summary[:features][:mixed][:counters]).to eq([{ version: 100, variation: 1, value: "m", count: 1, overrideAffected: true }]) + expect(summary[:features][:"overridden-prereq"][:counters]).to eq([{ version: 200, variation: 1, value: "op", count: 1, overrideAffected: true }]) + expect(summary[:features][:"plain-prereq"][:counters]).to eq([{ version: 100, variation: 1, value: "pp", count: 1 }]) + end + end + + it "sends the usual feature and debug events when nothing is overridden" do + source = TestOverrideSource.new([]) + with_sending_client(data_system(overrides: source, flags: { flag: flag("flag", "ld") })) do |client, ep, sender| + client.variation("flag", context, "default") + + events = payload(ep, sender) + expect(events.map { |e| e[:kind] }.sort).to eq %w[debug feature index summary] + summary = events.detect { |e| e[:kind] == "summary" } + expect(summary[:features][:flag][:counters]).to eq([{ version: 100, variation: 1, value: "ld", count: 1 }]) + end + end + end + end +end diff --git a/spec/ldclient_override_vectors_spec.rb b/spec/ldclient_override_vectors_spec.rb index 54b13fec..c7da3817 100644 --- a/spec/ldclient_override_vectors_spec.rb +++ b/spec/ldclient_override_vectors_spec.rb @@ -49,7 +49,12 @@ def build_client(vector) end config = Config.new(data_system_config: builder.build, send_events: false, logger: $null_log) - LDClient.new("sdk-key", config, wait) + client = LDClient.new("sdk-key", config, wait) + # The summary marker is the marking the client hands to the event processor for this + # evaluation. Event handling keys on that scalar, not on the reason. + events = RecordingEventProcessor.new + client.instance_variable_set(:@event_processor, events) + [client, events] end # Compares the actual reason against only the fields present in the expected reason. An expected @@ -68,7 +73,7 @@ def expect_reason(expected, actual) vector_file[:vectors].each do |vector| it "#{vector[:group]}: #{vector[:description]}" do - client = build_client(vector) + client, events = build_client(vector) begin expect(client.initialized?).to eq(vector[:launchDarklyData][:initialized]) @@ -84,6 +89,12 @@ def expect_reason(expected, actual) expect(detail.variation_index).to eq(expected[:variationIndex]) end expect_reason(expected[:reason], detail.reason) + + unless expected[:summaryOverrideAffected].nil? + records = events.records_for(evaluate[:flagKey]) + expect(records.length).to eq(1), "expected exactly one evaluation record for the flag" + expect(records[0].override_affected).to eq(expected[:summaryOverrideAffected]), "summaryOverrideAffected" + end ensure client.close end diff --git a/spec/override_test_components.rb b/spec/override_test_components.rb index bcfb0e62..e16e6dfd 100644 --- a/spec/override_test_components.rb +++ b/spec/override_test_components.rb @@ -117,6 +117,32 @@ def stop end end + # + # An event processor for tests that records the evaluation records the client hands to it. + # + class RecordingEventProcessor + include EventProcessorMethods + + EvalRecord = Struct.new(:context, :key, :version, :variation, :value, :reason, :default, :track_events, + :debug_until, :prereq_of, :sampling_ratio, :exclude_from_summaries, :override_affected) + + attr_reader :records + + def initialize + @records = [] + @lock = Mutex.new + end + + def record_eval_event(*args) + @lock.synchronize { @records << EvalRecord.new(*args) } + end + + # Returns the records for a flag key, in order. + def records_for(key) + @lock.synchronize { @records.select { |r| r.key == key } } + end + end + # # A flag change listener for tests that collects the changed keys. #