Skip to content

feat: Add the file-based override source #453

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
kinyoklion wants to merge 1 commit into
base: rlamb/overrides-ruby-events
Choose a base branch
from
Draft

feat: Add the file-based override source #453

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 10 additions & 7 deletions lib/ldclient-rb/impl/file_data/reloader.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,15 +51,18 @@ class Reloader
# automatically. If zero or negative, there is no automatic retry.
# @param skip_unchanged [Boolean] if true, `apply` is not invoked when the files' raw
# contents are byte-identical to the last successfully applied contents.
# @param log_prefix [String] the prefix of every log line this reloader writes
#
def initialize(paths:, logger:, apply:, on_error: nil,
duplicate_keys_handling: DuplicateKeysHandling::FAIL,
skip_missing_paths: false,
debounce_delay: DEFAULT_DEBOUNCE_DELAY,
retry_delay: DEFAULT_RETRY_DELAY,
skip_unchanged: false)
skip_unchanged: false,
log_prefix: "[LDClient]")
@paths = paths
@logger = logger
@log_prefix = log_prefix
@apply = apply
@on_error = on_error
@duplicate_keys_handling = duplicate_keys_handling
Expand Down Expand Up @@ -149,9 +152,9 @@ def stop
break if action == :stop

if action == :reload
@logger.info { "[LDClient] Reloading flag data after detecting a change" }
@logger.info { "#{@log_prefix} Reloading flag data after detecting a change" }
else
@logger.debug { "[LDClient] Retrying flag data load after earlier failure" }
@logger.debug { "#{@log_prefix} Retrying flag data load after earlier failure" }
end
ok = reload(retrying: action == :retry)
@mutex.synchronize do
Expand All @@ -161,7 +164,7 @@ def stop
end
end
rescue => e
Util.log_exception(@logger, "Unexpected error in file data reloader", e)
Util.log_exception(@logger, "#{@log_prefix} Unexpected error in file data reloader", e)
end

#
Expand Down Expand Up @@ -223,7 +226,7 @@ def stop
content = FileData.read_file(path)
rescue ReadError => e
if e.missing && @skip_missing_paths
@logger.debug { "[LDClient] File #{path} does not exist; it contributes no data" }
@logger.debug { "#{@log_prefix} File #{path} does not exist; it contributes no data" }
files << FileSummary.new(path, false, 0, 0)
next
end
Expand Down Expand Up @@ -285,12 +288,12 @@ def stop
# level and do not re-invoke on_error.
message = error.message
if message == @last_error_message
@logger.debug { "[LDClient] Unable to load flags: #{message}" }
@logger.debug { "#{@log_prefix} Unable to load flags: #{message}" }
return false
end

@last_error_message = message
@logger.error { "[LDClient] Unable to load flags: #{message}" }
@logger.error { "#{@log_prefix} Unable to load flags: #{message}" }
@on_error&.call(error)
false
end
Expand Down
151 changes: 151 additions & 0 deletions lib/ldclient-rb/impl/integrations/file_override_source.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
# frozen_string_literal: true

require "ldclient-rb/impl/file_data"
require "ldclient-rb/interfaces/overrides"

module LaunchDarkly
module Impl
module Integrations
#
# The file-based override source. It reads flag and segment overrides from one or more local
# files and reloads them as the files change. See
# {LaunchDarkly::Integrations::FileData.override_source} for the public API and its options.
#
# Flag overrides are currently experimental and subject to change.
#
# @private
#
class FileOverrideSource
include LaunchDarkly::Interfaces::Overrides::OverrideSource

LOG_PREFIX = "[LDClient] FileOverrideSource:"

#
# @param paths [Array<String>] absolute file paths, in precedence order
# @param duplicate_keys_handling [Symbol] `:fail` or `:ignore`
# @param change_detection [Symbol] `:polling` or `:watching`
# @param poll_interval [Numeric] seconds between examinations in polling mode
# @param logger [Logger]
#
def initialize(paths:, duplicate_keys_handling:, change_detection:, poll_interval:, logger:)
@paths = paths
@duplicate_keys_handling = duplicate_keys_handling
@change_detection = change_detection
@poll_interval = poll_interval
@logger = logger
@lock = Mutex.new
@reloader = nil
@change_detector = nil
@stopped = false
end

# @return [Array<String>]
attr_reader :paths

# @return [Symbol]
attr_reader :duplicate_keys_handling

# @return [Symbol]
attr_reader :change_detection

# @return [Numeric]
attr_reader :poll_interval

#
# Performs the initial load synchronously, so overrides present in the files are in effect
# when this method returns, then starts change detection. A file that does not exist yet
# contributes no overrides. A file that cannot be read or parsed is not fatal: the client
# runs with the last good overrides, the failure is logged, and the retry plus the change
# signal recover once the file is readable.
#
# (see LaunchDarkly::Interfaces::Overrides::OverrideSource#start)
#
def start(sink)
@lock.synchronize do
return if @stopped

@reloader = FileData::Reloader.new(
paths: @paths,
logger: @logger,
log_prefix: LOG_PREFIX,
duplicate_keys_handling: @duplicate_keys_handling,
skip_missing_paths: true,
skip_unchanged: true,
apply: lambda do |merged|
sink.set_overrides(merged.flags.values, merged.segments.values)
log_overrides_in_effect(merged)
end
)
end

@reloader.reload_now

@lock.synchronize do
return if @stopped

trigger = @reloader.method(:trigger)
@change_detector =
if @change_detection == :watching
FileData::Watcher.new(@paths, trigger, @logger)
else
FileData::Poller.new(@paths, @poll_interval, trigger, @logger)
end
end
end

# (see LaunchDarkly::Interfaces::Overrides::OverrideSource#stop)
def stop
reloader = nil
change_detector = nil
@lock.synchronize do
@stopped = true
reloader = @reloader
change_detector = @change_detector
@reloader = nil
@change_detector = nil
end
change_detector&.stop
reloader&.stop
end

#
# Reports the overrides now in effect and the file each came from. The reloader applies a
# snapshot only when the content changed, so this logs each change once.
#
# @param merged [LaunchDarkly::Impl::FileData::MergeResult]
#
private def log_overrides_in_effect(merged)
details = merged.files.map do |file|
if !file.present
"#{file.path}: absent"
elsif file.flags.zero? && file.segments.zero?
"#{file.path}: no entries"
else
"#{file.path}: #{counts_text(file.flags, file.segments)}"
end
end.join("; ")

if merged.empty?
@logger.info { "#{LOG_PREFIX} Flag overrides: none in effect (#{details})" }
else
@logger.info do
"#{LOG_PREFIX} Flag overrides in effect: #{counts_text(merged.flags.length, merged.segments.length)} (#{details})"
end
end
end

# Formats flag and segment counts, for example "2 flags, 1 segment".
private def counts_text(flags, segments)
parts = []
parts << pluralize(flags, "flag") if flags > 0
parts << pluralize(segments, "segment") if segments > 0
parts.join(", ")
end

private def pluralize(count, noun)
count == 1 ? "1 #{noun}" : "#{count} #{noun}s"
end
end
end
end
end
140 changes: 140 additions & 0 deletions lib/ldclient-rb/integrations/file_data.rb
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,7 @@
require 'ldclient-rb/impl/file_data'
require 'ldclient-rb/impl/integrations/file_data_source'
require 'ldclient-rb/impl/integrations/file_data_source_v2'
require 'ldclient-rb/impl/integrations/file_override_source'

module LaunchDarkly
module Integrations
Expand Down Expand Up @@ -81,6 +83,11 @@ module Integrations
# If the data source encounters any error in any file-- malformed content, a missing file, or a
# duplicate key-- it will not load flags from any of the files.
#
# The same file format serves the file-based override source, {FileData.override_source}, which
# supplies flag and segment overrides that take precedence over LaunchDarkly data instead of
# replacing the connection to LaunchDarkly. Flag overrides are currently experimental and subject
# to change.
#
module FileData
#
# Returns a factory for the file data source component.
Expand Down Expand Up @@ -149,6 +156,139 @@ def self.data_source_v2(options = {})

FileDataSourceV2Builder.new(paths, poll_interval)
end

#
# Returns a builder for a file-based override source. Flag overrides are currently experimental
# and subject to change.
#
# Overrides are flag and segment definitions that take precedence over data received from
# LaunchDarkly at evaluation time, on a per-key basis. They exist for resilience during an
# incident. An operator can force one or more flags to a known state on a running application,
# whether or not the application can reach LaunchDarkly, by writing them to a file. The override
# stays in effect until it is removed from the file. Flags not present in the files are
# completely unaffected.
#
# The files use the same document format as the file data source: a JSON or YAML document with
# optional `flags`, `flagValues`, and `segments` members. A `flagValues` entry expands into a
# flag that serves the given value for every context. When several files are configured, their
# entries are combined in the configured order, and the duplicate keys handling decides what
# happens when the same key appears in more than one file.
#
# A reload replaces the entire set of overrides, so removing an entry from a file removes the
# override. A configured file that does not exist contributes no overrides. It can be created
# later, and deleting a file removes its overrides. A file that exists but cannot be read or
# parsed fails that whole reload. The previously loaded overrides stay in effect, the failure is
# logged, and the source retries after a short delay and recovers on its own once the file is
# readable. Whenever the set of overrides in effect changes, including at startup, the source
# logs the overrides in effect and what each file supplied, at Info level.
#
# An evaluation that an override affects, directly or through a prerequisite or segment, is
# marked: its reason reports {EvaluationReason#override_affected}, it produces no individual
# analytics event, and it is counted under a separate summary counter.
#
# Pass the returned builder to {LaunchDarkly::DataSystem::ConfigBuilder#overrides}. The
# configuration is validated when the client is created, and an invalid configuration raises
# `ArgumentError` from `LDClient.new`.
#
# @example
# overrides = LaunchDarkly::Integrations::FileData.override_source(paths: ["/etc/launchdarkly/overrides.json"])
# config = LaunchDarkly::Config.new(data_system: LaunchDarkly::DataSystem.default.overrides(overrides))
# client = LaunchDarkly::LDClient.new(sdk_key, config)
#
# @param options [Hash] the configuration options
# @option options [Array<String>, String] :paths One or more files, in precedence order. Required.
# Paths may be absolute or relative to the current working directory.
# @option options [Symbol] :duplicate_keys_handling What to do when the same key appears in more
# than one file. `:fail` (the default) treats the reload as failed and keeps the previous
# overrides. `:ignore` keeps the entry from the first configured file that defines the key.
# @option options [Symbol] :change_detection How the source detects file changes. `:polling` (the
# default) examines the files on an interval and works on every file system, including network
# mounts and directories whose contents are swapped through symbolic links. `:watching` uses file
# system change notifications through the `listen` gem, which the host application must provide.
# @option options [Numeric] :poll_interval Seconds between examinations of the files in polling
# mode. The default is 1. An interval below 1 is raised to 1 with a warning.
# @return [FileOverrideSourceBuilder] a builder for {LaunchDarkly::DataSystem::ConfigBuilder#overrides}
#
def self.override_source(options = {})
FileOverrideSourceBuilder.new(options)
end
end

#
# Builder for the file-based override source. Create it with {FileData.override_source}.
#
# Flag overrides are currently experimental and subject to change.
#
class FileOverrideSourceBuilder
# The default interval, in seconds, between examinations of the files in polling mode.
DEFAULT_POLL_INTERVAL = 1

# The shortest allowed polling interval, in seconds. A configured interval below this is raised
# to it. The minimum exists only to prevent a tight loop over the file system.
MINIMUM_POLL_INTERVAL = 1

DUPLICATE_KEYS_HANDLING_VALUES = [:fail, :ignore].freeze
CHANGE_DETECTION_VALUES = [:polling, :watching].freeze
OPTION_KEYS = [:paths, :duplicate_keys_handling, :change_detection, :poll_interval].freeze
private_constant :DUPLICATE_KEYS_HANDLING_VALUES, :CHANGE_DETECTION_VALUES, :OPTION_KEYS

#
# @param options [Hash] see {FileData.override_source}
#
def initialize(options)
raise ArgumentError, "options for the file-based override source must be a Hash" unless options.is_a?(Hash)

@options = options
end

#
# Builds the override source. Called by the SDK when the client is created.
#
# @param sdk_key [String] unused
# @param config [LaunchDarkly::Config] the client configuration, for its logger
# @return [LaunchDarkly::Interfaces::Overrides::OverrideSource]
# @raise [ArgumentError] if the options are invalid
#
def build(sdk_key, config)
unknown = @options.keys - OPTION_KEYS
raise ArgumentError, "unknown options for the file-based override source: #{unknown.join(', ')}" unless unknown.empty?

paths = Impl::FileData.absolute_paths(@options[:paths] || [])
raise ArgumentError, "no file paths were specified for the file-based override source" if paths.empty?

duplicate_keys_handling = @options.fetch(:duplicate_keys_handling, :fail)
unless DUPLICATE_KEYS_HANDLING_VALUES.include?(duplicate_keys_handling)
raise ArgumentError,
"unrecognized duplicate keys handling #{duplicate_keys_handling.inspect} for the file-based override source"
end

change_detection = @options.fetch(:change_detection, :polling)
unless CHANGE_DETECTION_VALUES.include?(change_detection)
raise ArgumentError, "unrecognized change detection mode #{change_detection.inspect} for the file-based override source"
end
if change_detection == :watching && !Impl::FileData::Watcher.available?
raise ArgumentError, "change detection mode :watching for the file-based override source requires the listen gem"
end

poll_interval = @options.fetch(:poll_interval, DEFAULT_POLL_INTERVAL)
unless poll_interval.is_a?(Numeric)
raise ArgumentError, "poll interval #{poll_interval.inspect} for the file-based override source must be a number"
end
if change_detection == :polling && poll_interval < MINIMUM_POLL_INTERVAL
config.logger.warn do
"#{Impl::Integrations::FileOverrideSource::LOG_PREFIX} Poll interval #{poll_interval}s is below the minimum; using #{MINIMUM_POLL_INTERVAL}s"
end
poll_interval = MINIMUM_POLL_INTERVAL
end

Impl::Integrations::FileOverrideSource.new(
paths: paths,
duplicate_keys_handling: duplicate_keys_handling,
change_detection: change_detection,
poll_interval: poll_interval,
logger: config.logger
)
end
end

#
Expand Down
Loading
Loading