From 657a598e31e62ad34162fe11cbe3bd93e6e3d3a0 Mon Sep 17 00:00:00 2001 From: Ryan Lamb <4955475+kinyoklion@users.noreply.github.com> Date: Fri, 25 Sep 2026 16:07:01 -0700 Subject: [PATCH] feat: Add the file-based override source The OVERRIDE specification defines a file-based override source that reads override entries from one or more local files and reloads them when the files change. FileOverrides.source() provides it, configured through DataSystemBuilder.overrides(...). The source accepts the file data source document format (flags, flagValues, segments) in JSON or YAML, merges the files in the configured order with the configured duplicate keys handling (fail by default, or ignore all but the first), and keeps the versions that the documents specify. A flagValues entry becomes a flag that is off and serves the value as its single variation. Change detection is polling (the default, one second by default and at minimum) or watching. A configured file that does not exist contributes no overrides. A file that exists but cannot be read or parsed fails that reload, keeps the last good overrides, logs the failure, and is retried. The initial load completes during client construction. Every applied change is logged at Info with the overrides in effect and what each file supplied. Missing file paths, a null change detection mode, or a null poll interval fail client construction. Flag overrides are currently experimental and subject to change. --- .../integrations/DataSystemBuilder.java | 1 + .../FileOverrideSourceBuilder.java | 147 ++++++ .../integrations/FileOverrideSourceImpl.java | 169 +++++++ .../server/integrations/FileOverrides.java | 106 ++++ .../sdk/server/subsystems/OverrideSource.java | 2 + .../integrations/FileOverrideSourceTest.java | 475 ++++++++++++++++++ 6 files changed, 900 insertions(+) create mode 100644 lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceBuilder.java create mode 100644 lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceImpl.java create mode 100644 lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrides.java create mode 100644 lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceTest.java diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java index b07cc037..24e63aa7 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java @@ -134,6 +134,7 @@ public DataSystemBuilder persistentStore(ComponentConfigurer persiste * * @param overrideSource the override source configuration, or null for none * @return a reference to the builder + * @see FileOverrides * @since 7.18.0 */ public DataSystemBuilder overrides(ComponentConfigurer overrideSource) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceBuilder.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceBuilder.java new file mode 100644 index 00000000..ea19763e --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceBuilder.java @@ -0,0 +1,147 @@ +package com.launchdarkly.sdk.server.integrations; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.sdk.server.integrations.FileOverrides.ChangeDetection; +import com.launchdarkly.sdk.server.subsystems.ClientContext; +import com.launchdarkly.sdk.server.subsystems.ComponentConfigurer; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; + +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +/** + * Configures the file-based override source. Obtain an instance with {@link FileOverrides#source()}, + * call {@link #filePaths(String...)} or {@link #filePaths(Path...)} to specify the files, adjust + * any other options, and pass the builder to + * {@link DataSystemBuilder#overrides(ComponentConfigurer)}. + *

+ * For more details, see {@link FileOverrides}. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @since 7.18.0 + */ +public final class FileOverrideSourceBuilder implements ComponentConfigurer { + private final List filePaths = new ArrayList<>(); + private FileData.DuplicateKeysHandling duplicateKeysHandling = FileData.DuplicateKeysHandling.FAIL; + private ChangeDetection changeDetection = ChangeDetection.POLLING; + private Duration pollInterval = FileOverrides.DEFAULT_POLL_INTERVAL; + + FileOverrideSourceBuilder() {} + + /** + * Adds any number of files to load overrides from, specifying each file path as a string. The + * order is significant: it determines which file wins under the duplicate keys handling when the + * same key appears in more than one file. + *

+ * Files are parsed as JSON if their first non-whitespace character is '{'. Otherwise, they are + * parsed as YAML. + * + * @param filePaths path(s) to the file(s); may be absolute or relative to the current working directory + * @return the same builder + * @throws InvalidPathException if one of the parameters is not a valid file path + */ + public FileOverrideSourceBuilder filePaths(String... filePaths) throws InvalidPathException { + for (String p : filePaths) { + this.filePaths.add(Paths.get(p)); + } + return this; + } + + /** + * Adds any number of files to load overrides from, specifying each file path as a Path. The + * order is significant: it determines which file wins under the duplicate keys handling when the + * same key appears in more than one file. + * + * @param filePaths path(s) to the file(s); may be absolute or relative to the current working directory + * @return the same builder + */ + public FileOverrideSourceBuilder filePaths(Path... filePaths) { + for (Path p : filePaths) { + this.filePaths.add(p); + } + return this; + } + + /** + * Specifies how to handle the same key appearing in more than one file. + *

+ * With {@link FileData.DuplicateKeysHandling#FAIL}, the default, a reload fails if the same flag or + * segment key appears in more than one file, and the previously loaded overrides stay in effect. + * With {@link FileData.DuplicateKeysHandling#IGNORE}, the entry from the first configured file that + * defines the key is kept and the others are discarded. A null value selects the default. + * + * @param duplicateKeysHandling specifies how to handle duplicate keys + * @return the same builder + */ + public FileOverrideSourceBuilder duplicateKeysHandling(FileData.DuplicateKeysHandling duplicateKeysHandling) { + this.duplicateKeysHandling = duplicateKeysHandling == null + ? FileData.DuplicateKeysHandling.FAIL : duplicateKeysHandling; + return this; + } + + /** + * Selects how the source detects file changes. The default is {@link ChangeDetection#POLLING}. + * The two modes are alternatives, so setting one replaces the other. + * + * @param changeDetection the change detection mode + * @return the same builder + */ + public FileOverrideSourceBuilder changeDetection(ChangeDetection changeDetection) { + this.changeDetection = changeDetection; + return this; + } + + /** + * Sets the interval between examinations of the files in {@link ChangeDetection#POLLING} mode. + * {@link ChangeDetection#WATCHING} mode ignores it. The default is + * {@link FileOverrides#DEFAULT_POLL_INTERVAL}. An interval below + * {@link FileOverrides#MINIMUM_POLL_INTERVAL} is raised to the minimum. + * + * @param pollInterval the polling interval + * @return the same builder + */ + public FileOverrideSourceBuilder pollInterval(Duration pollInterval) { + this.pollInterval = pollInterval; + return this; + } + + /** + * Called internally by the SDK to create the override source. + * + * @param context the client context + * @return the override source + * @throws IllegalArgumentException if no file paths were specified, or the change detection mode + * or the poll interval is null + */ + @Override + public OverrideSource build(ClientContext context) { + if (filePaths.isEmpty()) { + throw new IllegalArgumentException("no file paths were specified for the file-based override source"); + } + if (changeDetection == null) { + throw new IllegalArgumentException("a change detection mode is required for the file-based override source"); + } + if (pollInterval == null) { + throw new IllegalArgumentException("a poll interval is required for the file-based override source"); + } + List absolutePaths = new ArrayList<>(filePaths.size()); + for (Path p : filePaths) { + absolutePaths.add(p.toAbsolutePath().normalize()); + } + + LDLogger logger = context.getBaseLogger().subLogger("FileOverrideSource"); + + Duration effectivePollInterval = pollInterval; + if (changeDetection == ChangeDetection.POLLING && pollInterval.compareTo(FileOverrides.MINIMUM_POLL_INTERVAL) < 0) { + logger.warn("Poll interval {} is below the minimum; using {}", pollInterval, FileOverrides.MINIMUM_POLL_INTERVAL); + effectivePollInterval = FileOverrides.MINIMUM_POLL_INTERVAL; + } + + return new FileOverrideSourceImpl(absolutePaths, duplicateKeysHandling, changeDetection, effectivePollInterval, logger); + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceImpl.java new file mode 100644 index 00000000..22d719c4 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceImpl.java @@ -0,0 +1,169 @@ +package com.launchdarkly.sdk.server.integrations; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.sdk.server.integrations.FileDataSourceParsing.FileDataException; +import com.launchdarkly.sdk.server.integrations.FileOverrides.ChangeDetection; +import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.FileSummary; +import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.LoadResult; +import com.launchdarkly.sdk.server.subsystems.OverrideSink; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * The file-based override source. It reads one or more files in the file data source document + * format, merges them in the configured order, and supplies the result to the override sink as + * one snapshot. It reloads when the files change, by polling or by watching. + *

+ * A configured file that does not exist contributes no overrides. A file that exists but cannot + * be read or parsed fails that reload: the sink is not called, so the last good overrides stay in + * effect, and the reload is retried. + */ +final class FileOverrideSourceImpl implements OverrideSource { + private final List paths; + private final ChangeDetection changeDetection; + private final Duration pollInterval; + private final LDLogger logger; + private final OverrideFileLoader loader; + + private volatile FileDataReloader reloader; + private volatile FileDataPoller poller; + private volatile FileDataWatcher watcher; + private final AtomicBoolean closed = new AtomicBoolean(false); + + FileOverrideSourceImpl( + List paths, + FileData.DuplicateKeysHandling duplicateKeysHandling, + ChangeDetection changeDetection, + Duration pollInterval, + LDLogger logger + ) { + this.paths = new ArrayList<>(paths); + this.changeDetection = changeDetection; + this.pollInterval = pollInterval; + this.logger = logger; + this.loader = new OverrideFileLoader(paths, duplicateKeysHandling); + } + + /** + * Performs the initial load synchronously, so overrides present in the files are in effect when + * the client constructor returns, then starts change detection. A file that does not exist yet + * contributes no overrides. A file that cannot be read or parsed is not fatal: the client runs + * with the last good overrides, the failure is logged, and the retry, plus the change signal, + * recovers once the file is readable. + */ + @Override + public void start(OverrideSink sink) { + reloader = new FileDataReloader( + loader::load, + new FileDataReloader.Handler() { + @Override + public void apply(LoadResult result) { + sink.setOverrides(result.getData()); + logOverridesInEffect(result); + } + + @Override + public void onError(FileDataException e) { + // The reloader has logged the failure. The last good overrides stay in effect. + } + }, + logger, + FileDataReloader.DEFAULT_DEBOUNCE_DELAY, + FileDataReloader.DEFAULT_RETRY_DELAY, + true + ); + reloader.reloadNow(); + + if (closed.get()) { + return; + } + switch (changeDetection) { + case WATCHING: + try { + FileDataWatcher w = FileDataWatcher.create(paths, logger); + watcher = w; + w.start(reloader::trigger); + } catch (IOException e) { + // COVERAGE: constructing a watcher only fails under unusual OS conditions + logger.error("Unable to watch override files: {}", e.toString()); + } + break; + case POLLING: + default: + poller = new FileDataPoller(paths, pollInterval, reloader::trigger); + break; + } + } + + /** + * Returns the interval between file examinations in polling mode. Visible for tests. + * + * @return the effective poll interval + */ + Duration getPollInterval() { + return pollInterval; + } + + // Reports the overrides now in effect and the file each came from. The reloader applies a + // snapshot only when the content changed, so this logs each change once. + private void logOverridesInEffect(LoadResult result) { + List details = new ArrayList<>(result.getFiles().size()); + for (FileSummary file : result.getFiles()) { + if (!file.isPresent()) { + details.add(file.getPath() + ": absent"); + } else if (file.getFlags() == 0 && file.getSegments() == 0) { + details.add(file.getPath() + ": no entries"); + } else { + details.add(file.getPath() + ": " + countsText(file.getFlags(), file.getSegments())); + } + } + String fileDetails = String.join("; ", details); + if (result.getFlagCount() == 0 && result.getSegmentCount() == 0) { + logger.info("Flag overrides: none in effect ({})", fileDetails); + return; + } + logger.info("Flag overrides in effect: {} ({})", countsText(result.getFlagCount(), result.getSegmentCount()), + fileDetails); + } + + // Formats flag and segment counts, for example "2 flags, 1 segment". + static String countsText(int flags, int segments) { + List parts = new ArrayList<>(2); + if (flags > 0) { + parts.add(pluralize(flags, "flag")); + } + if (segments > 0) { + parts.add(pluralize(segments, "segment")); + } + return String.join(", ", parts); + } + + private static String pluralize(int count, String noun) { + return count == 1 ? "1 " + noun : count + " " + noun + "s"; + } + + @Override + public void close() { + if (closed.getAndSet(true)) { + return; + } + FileDataWatcher w = watcher; + if (w != null) { + w.close(); + } + FileDataPoller p = poller; + if (p != null) { + p.close(); + } + FileDataReloader r = reloader; + if (r != null) { + r.close(); + } + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrides.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrides.java new file mode 100644 index 00000000..34342505 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrides.java @@ -0,0 +1,106 @@ +package com.launchdarkly.sdk.server.integrations; + +import java.time.Duration; + +/** + * Integration between the LaunchDarkly SDK and file-based flag overrides. + *

+ * Flag overrides are currently experimental and subject to change. + *

+ * Overrides are flag and segment definitions that take precedence over data received from + * LaunchDarkly at evaluation time, on a per-key basis. They exist for resilience during an + * incident. An operator can force one or more flags to a known state on a running application, + * whether or not the application can reach LaunchDarkly. The override stays in effect until the + * operator removes it. Flags not present in the override data are completely unaffected. + *

+ * This class provides one override source: {@link #source()}, which reads overrides from local + * files and reloads them as the files change. Configure it with the data system builder: + * 


+ *     LDConfig config = new LDConfig.Builder()
+ *         .dataSystem(Components.dataSystem().defaultMode()
+ *             .overrides(FileOverrides.source().filePaths("/etc/launchdarkly/overrides.json")))
+ *         .build();
+ *
+ *

+ * An evaluation that an override affects is marked. The marking is direct or transitive: it + * applies when the evaluated flag, a prerequisite at any depth, or a segment read during the + * evaluation came from the override layer. {@link com.launchdarkly.sdk.EvaluationReason#isOverrideAffected()} + * reports the marking. Marked evaluations appear in analytics summary events only, under separate + * counters, so LaunchDarkly can distinguish them. They produce no individual evaluation events. + * + * @see FileOverrideSourceBuilder + * @see DataSystemBuilder#overrides(com.launchdarkly.sdk.server.subsystems.ComponentConfigurer) + * @since 7.18.0 + */ +public abstract class FileOverrides { + /** + * Selects how the file-based override source learns that a file changed. The two modes are + * alternatives. Flag overrides are currently experimental and subject to change. + * + * @see FileOverrideSourceBuilder#changeDetection(ChangeDetection) + */ + public enum ChangeDetection { + /** + * The source examines the files on a fixed interval and reloads when the modification time or + * the size of a file changes. Polling works on every file system, including network mounts and + * directories whose contents are swapped through symbolic links, as Kubernetes does for mounted + * ConfigMaps. It is the default. + */ + POLLING, + + /** + * The source reloads in response to file system change notifications. It reacts faster than + * polling. It depends on notifications, which some file systems do not deliver reliably. + */ + WATCHING + } + + /** + * The interval at which the file source examines the files for changes in + * {@link ChangeDetection#POLLING} mode when no interval was specified. Because the source reads + * local files rather than contacting a service, a short interval keeps an override responsive + * during an incident at negligible cost. + */ + public static final Duration DEFAULT_POLL_INTERVAL = Duration.ofSeconds(1); + + /** + * The shortest allowed polling interval. A configured interval below this is raised to it. The + * minimum exists only to prevent a pathological tight loop over the file system. + */ + public static final Duration MINIMUM_POLL_INTERVAL = Duration.ofSeconds(1); + + private FileOverrides() {} + + /** + * Creates a {@link FileOverrideSourceBuilder} for a file-based override source. The source reads + * flag and segment overrides from one or more local files and reloads them as the files change. + *

+ * The files use the same document format as the file data source (see {@link FileData}): each + * file is a JSON or YAML document with optional {@code "flags"}, {@code "flagValues"}, and + * {@code "segments"} members. A {@code "flagValues"} entry is expanded into a full flag + * definition that is off and serves the given value as its single variation for every context. + * When multiple files are configured, their entries are combined in the configured order, and + * the duplicate keys handling decides which file wins for a key that appears more than once. + *

+ * A reload replaces the entire override set, so removing an entry from the files removes the + * override. A configured file that does not exist contributes no overrides. Deleting a file + * therefore removes its overrides, and deleting every file removes them all. A file that exists + * but cannot be read or parsed makes that whole reload fail: the previously loaded overrides + * stay in effect, the source logs the failure, retries after a short delay, and recovers on its + * own once the file is readable again. + *

+ * Whenever the set of overrides in effect changes, including at startup, the source logs the + * overrides in effect and what each configured file supplied, at Info level. + *

+ * By default the source polls the files for changes once per second. See + * {@link FileOverrideSourceBuilder#changeDetection(ChangeDetection)} and + * {@link FileOverrideSourceBuilder#pollInterval(Duration)}. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @return a builder for the file-based override source + */ + public static FileOverrideSourceBuilder source() { + return new FileOverrideSourceBuilder(); + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSource.java index 0e83f01e..94665d33 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSource.java @@ -14,6 +14,8 @@ *

* To configure an override source, use * {@link com.launchdarkly.sdk.server.integrations.DataSystemBuilder#overrides(ComponentConfigurer)}. + * The SDK provides a file-based source; see + * {@link com.launchdarkly.sdk.server.integrations.FileOverrides}. *

* Flag overrides are currently experimental and subject to change. * diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceTest.java new file mode 100644 index 00000000..51d1f942 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceTest.java @@ -0,0 +1,475 @@ +package com.launchdarkly.sdk.server.integrations; + +import com.launchdarkly.logging.LDLogLevel; +import com.launchdarkly.logging.LogCapture; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.server.Components; +import com.launchdarkly.sdk.server.LDConfig; +import com.launchdarkly.sdk.server.TestComponents; +import com.launchdarkly.sdk.server.integrations.FileOverrides.ChangeDetection; +import com.launchdarkly.sdk.server.subsystems.ClientContext; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; +import com.launchdarkly.sdk.server.subsystems.OverrideSink; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; +import com.launchdarkly.testhelpers.TempDir; + +import org.junit.After; +import org.junit.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static com.launchdarkly.sdk.server.DataModel.FEATURES; +import static com.launchdarkly.sdk.server.DataModel.SEGMENTS; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@SuppressWarnings("javadoc") +public class FileOverrideSourceTest { + private static final long WAIT_MILLIS = 10000; + + private final LogCapture logCapture = Logs.capture(); + private final ClientContext context = TestComponents.clientContext("sdk-key", + new LDConfig.Builder().logging(Components.logging(logCapture).level(LDLogLevel.DEBUG)).build()); + private final List sources = new ArrayList<>(); + + @After + public void closeSources() throws Exception { + for (OverrideSource s : sources) { + s.close(); + } + } + + /** + * Records every snapshot the source supplies, as a map of kind name to key to item. + */ + static final class RecordingSink implements OverrideSink { + final BlockingQueue>> snapshots = new LinkedBlockingQueue<>(); + + @Override + public void setOverrides(Iterable>> data) { + Map> snapshot = new HashMap<>(); + snapshot.put(FEATURES.getName(), new HashMap<>()); + snapshot.put(SEGMENTS.getName(), new HashMap<>()); + for (Map.Entry> kind : data) { + Map items = snapshot.computeIfAbsent(kind.getKey().getName(), k -> new HashMap<>()); + for (Map.Entry item : kind.getValue().getItems()) { + items.put(item.getKey(), item.getValue()); + } + } + snapshots.add(snapshot); + } + + Map> await() throws InterruptedException { + Map> s = snapshots.poll(WAIT_MILLIS, TimeUnit.MILLISECONDS); + if (s == null) { + throw new AssertionError("timed out waiting for an override snapshot"); + } + return s; + } + + void assertNoSnapshot(long millis) throws InterruptedException { + assertNull(snapshots.poll(millis, TimeUnit.MILLISECONDS)); + } + } + + private OverrideSource start(FileOverrideSourceBuilder builder, RecordingSink sink) { + OverrideSource source = builder.build(context); + sources.add(source); + source.start(sink); + return source; + } + + private static String flagValue(Map> snapshot, String key) { + ItemDescriptor item = snapshot.get(FEATURES.getName()).get(key); + if (item == null) { + return null; + } + LDValue json = LDValue.parse(FEATURES.serialize(item)); + return json.get("variations").get(0).stringValue(); + } + + private static Path write(TempDir dir, String name, String contents) throws Exception { + Path p = dir.getPath().resolve(name); + Files.write(p, contents.getBytes()); + return p; + } + + private static String docWith(String key, String value) { + return "{\"flagValues\":{\"" + key + "\":\"" + value + "\"}}"; + } + + @Test + public void buildRequiresFilePaths() { + try { + FileOverrides.source().build(context); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("no file paths")); + } + } + + @Test + public void buildRequiresChangeDetectionMode() { + try { + FileOverrides.source().filePaths("x.json").changeDetection(null).build(context); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("change detection")); + } + } + + @Test + public void buildRequiresPollInterval() { + try { + FileOverrides.source().filePaths("x.json").pollInterval(null).build(context); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("poll interval")); + } + } + + @Test + public void nullDuplicateKeysHandlingSelectsFail() throws Exception { + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.json", docWith("flag", "a")); + Path b = write(dir, "b.json", docWith("flag", "b")); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(a, b).duplicateKeysHandling(null), sink); + sink.assertNoSnapshot(200); + assertThat(logCapture.getMessageStrings(), hasItem(allOf(startsWith("ERROR:Unable to load flags: "), + containsString("in features, key \"flag\" was already defined"), containsString(b.toString())))); + } + } + + @Test + public void initialLoadCompletesBeforeStartReturns() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", + "{\"flags\":{\"full\":{\"key\":\"full\",\"version\":7,\"on\":true,\"variations\":[\"x\"],\"fallthrough\":{\"variation\":0}}}," + + "\"flagValues\":{\"simple\":\"value\"}," + + "\"segments\":{\"seg\":{\"key\":\"seg\",\"version\":3,\"included\":[\"u\"]}}}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + + Map> snapshot = sink.snapshots.poll(); + assertNotNull("the initial snapshot must be supplied before start returns", snapshot); + assertEquals(7, snapshot.get(FEATURES.getName()).get("full").getVersion()); + assertEquals("value", flagValue(snapshot, "simple")); + // A value-only entry is an off flag with version 0. + ItemDescriptor simple = snapshot.get(FEATURES.getName()).get("simple"); + assertEquals(0, simple.getVersion()); + LDValue simpleJson = LDValue.parse(FEATURES.serialize(simple)); + assertEquals(LDValue.of(false), simpleJson.get("on")); + assertEquals(LDValue.of(0), simpleJson.get("offVariation")); + assertEquals(3, snapshot.get(SEGMENTS.getName()).get("seg").getVersion()); + assertThat(logCapture.getMessageStrings(), hasItem( + "INFO:Flag overrides in effect: 2 flags, 1 segment (" + file + ": 2 flags, 1 segment)")); + } + } + + @Test + public void loadsYamlDocument() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", "flagValues:\n yaml-flag: \"override-value\"\n"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + assertEquals("override-value", flagValue(sink.await(), "yaml-flag")); + } + } + + @Test + public void mergesFilesInConfiguredOrderWithIgnoreHandling() throws Exception { + try (TempDir dir = TempDir.create()) { + Path first = write(dir, "first.json", docWith("shared", "first")); + Path second = write(dir, "second.json", "{\"flagValues\":{\"shared\":\"second\",\"only-second\":\"x\"}}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(first, second) + .duplicateKeysHandling(FileData.DuplicateKeysHandling.IGNORE), sink); + + Map> snapshot = sink.await(); + assertEquals("first", flagValue(snapshot, "shared")); + assertEquals("x", flagValue(snapshot, "only-second")); + assertThat(logCapture.getMessageStrings(), hasItem( + "INFO:Flag overrides in effect: 2 flags (" + first + ": 1 flag; " + second + ": 1 flag)")); + } + } + + @Test + public void duplicateKeysFailTheLoadByDefault() throws Exception { + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.json", docWith("flag", "a")); + Path b = write(dir, "b.json", docWith("flag", "b")); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(a, b), sink); + sink.assertNoSnapshot(200); + } + } + + @Test + public void missingFileContributesNoEntriesAndIsNotAnError() throws Exception { + try (TempDir dir = TempDir.create()) { + Path present = write(dir, "present.json", docWith("flag", "value")); + Path missing = dir.getPath().resolve("missing.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(present, missing), sink); + + Map> snapshot = sink.await(); + assertEquals("value", flagValue(snapshot, "flag")); + assertEquals(1, snapshot.get(FEATURES.getName()).size()); + for (String message : logCapture.getMessageStrings()) { + assertTrue("unexpected error log: " + message, !message.startsWith("ERROR:")); + } + assertThat(logCapture.getMessageStrings(), hasItem( + "INFO:Flag overrides in effect: 1 flag (" + present + ": 1 flag; " + missing + ": absent)")); + } + } + + @Test + public void startsWithNoFilesAndLogsNoneInEffect() throws Exception { + try (TempDir dir = TempDir.create()) { + Path missing = dir.getPath().resolve("missing.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(missing), sink); + + Map> snapshot = sink.await(); + assertTrue(snapshot.get(FEATURES.getName()).isEmpty()); + assertThat(logCapture.getMessageStrings(), hasItem("INFO:Flag overrides: none in effect (" + missing + ": absent)")); + } + } + + @Test + public void emptyDocumentLogsNoEntries() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "empty.json", "{}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + sink.await(); + assertThat(logCapture.getMessageStrings(), hasItem("INFO:Flag overrides: none in effect (" + file + ": no entries)")); + } + } + + @Test + public void malformedFileAtStartupSuppliesNothingAndLogsError() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "bad.json", "{\"flagValues\""); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + sink.assertNoSnapshot(200); + assertThat(logCapture.getMessageStrings(), hasItem(startsWith("ERROR:Unable to load flags:"))); + } + } + + private void reloadsOnChange(ChangeDetection mode) throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", docWith("flag", "first")); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(mode).pollInterval(Duration.ofSeconds(1)), sink); + assertEquals("first", flagValue(sink.await(), "flag")); + + // A changed file is reloaded. + Files.write(file, docWith("flag", "second").getBytes()); + assertEquals("second", flagValue(sink.await(), "flag")); + + // A malformed edit keeps the last good overrides and supplies nothing. + Files.write(file, "{\"flagValues\"".getBytes()); + sink.assertNoSnapshot(1500); + assertThat(logCapture.getMessageStrings(), hasItem(startsWith("ERROR:Unable to load flags:"))); + + // A later good edit is applied. + Files.write(file, docWith("flag", "third").getBytes()); + assertEquals("third", flagValue(sink.await(), "flag")); + + // Deleting the file removes its overrides. + Files.delete(file); + assertTrue(sink.await().get(FEATURES.getName()).isEmpty()); + + // Recreating the file brings them back. + Files.write(file, docWith("flag", "fourth").getBytes()); + assertEquals("fourth", flagValue(sink.await(), "flag")); + } + } + + @Test + public void pollingModeReloadsOnChange() throws Exception { + reloadsOnChange(ChangeDetection.POLLING); + } + + @Test + public void watchingModeReloadsOnChange() throws Exception { + reloadsOnChange(ChangeDetection.WATCHING); + } + + @Test + public void fileThatDoesNotExistYetTakesEffectWhenItAppears() throws Exception { + for (ChangeDetection mode : ChangeDetection.values()) { + try (TempDir dir = TempDir.create()) { + Path file = dir.getPath().resolve("later.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(mode), sink); + assertTrue(sink.await().get(FEATURES.getName()).isEmpty()); + + Files.write(file, docWith("flag", "appeared").getBytes()); + assertEquals(mode.toString(), "appeared", flagValue(sink.await(), "flag")); + } + } + } + + @Test + public void pollIntervalBelowMinimumIsRaisedWithWarning() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + RecordingSink sink = new RecordingSink(); + OverrideSource source = start(FileOverrides.source().filePaths(file).pollInterval(Duration.ofMillis(10)), sink); + assertEquals(FileOverrides.MINIMUM_POLL_INTERVAL, ((FileOverrideSourceImpl) source).getPollInterval()); + assertThat(logCapture.getMessageStrings(), hasItem( + "WARN:Poll interval PT0.01S is below the minimum; using PT1S")); + } + } + + @Test + public void pollIntervalAtOrAboveMinimumIsKept() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + OverrideSource source = start(FileOverrides.source().filePaths(file).pollInterval(Duration.ofSeconds(3)), + new RecordingSink()); + assertEquals(Duration.ofSeconds(3), ((FileOverrideSourceImpl) source).getPollInterval()); + OverrideSource defaulted = start(FileOverrides.source().filePaths(file), new RecordingSink()); + assertEquals(FileOverrides.DEFAULT_POLL_INTERVAL, ((FileOverrideSourceImpl) defaulted).getPollInterval()); + } + } + + @Test + public void failedLoadIsRetriedWithoutAFileChange() throws Exception { + // Watching mode delivers no notification while the file is untouched, so only the source's own + // retry can attempt the load again. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "bad.json", "{\"flagValues\""); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(ChangeDetection.WATCHING), sink); + assertThat(logCapture.getMessageStrings(), hasItem(startsWith("ERROR:Unable to load flags:"))); + + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (!logCapture.getMessageStrings().contains("DEBUG:Retrying flag data load after earlier failure")) { + if (System.currentTimeMillis() > deadline) { + fail("the failed load was not retried"); + } + Thread.sleep(50); + } + sink.assertNoSnapshot(100); + } + } + + @Test + public void watchingModeIgnoresPollIntervalMinimum() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(ChangeDetection.WATCHING) + .pollInterval(Duration.ofMillis(10)), sink); + for (String message : logCapture.getMessageStrings()) { + assertTrue(message, !message.startsWith("WARN:Poll interval")); + } + } + } + + private static int liveThreadsNamed(String prefix) { + int count = 0; + for (Thread t : Thread.getAllStackTraces().keySet()) { + if (t.isAlive() && t.getName().startsWith(prefix)) { + count++; + } + } + return count; + } + + private static void awaitLiveThreadsNamed(String prefix, int expected) throws InterruptedException { + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (liveThreadsNamed(prefix) != expected) { + if (System.currentTimeMillis() > deadline) { + fail("expected " + expected + " live threads named " + prefix + " but found " + liveThreadsNamed(prefix)); + } + Thread.sleep(20); + } + } + + private void closeReleasesThreads(ChangeDetection mode, String threadPrefix) throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", docWith("flag", "first")); + int baseline = liveThreadsNamed(threadPrefix); + RecordingSink sink = new RecordingSink(); + OverrideSource source = start(FileOverrides.source().filePaths(file).changeDetection(mode) + .pollInterval(Duration.ofSeconds(1)), sink); + sink.await(); + awaitLiveThreadsNamed(threadPrefix, baseline + 1); + + source.close(); + source.close(); + + awaitLiveThreadsNamed(threadPrefix, baseline); + Files.write(file, docWith("flag", "second").getBytes()); + sink.assertNoSnapshot(1500); + } + } + + @Test + public void closeIsIdempotentAndStopsThePoller() throws Exception { + closeReleasesThreads(ChangeDetection.POLLING, "LaunchDarkly-FileDataPoller"); + } + + @Test + public void closeIsIdempotentAndStopsTheWatcher() throws Exception { + closeReleasesThreads(ChangeDetection.WATCHING, "LaunchDarkly-FileDataWatcher"); + } + + @Test + public void countsTextFormatsCounts() { + assertEquals("1 flag", FileOverrideSourceImpl.countsText(1, 0)); + assertEquals("2 flags, 1 segment", FileOverrideSourceImpl.countsText(2, 1)); + assertEquals("3 segments", FileOverrideSourceImpl.countsText(0, 3)); + assertEquals("", FileOverrideSourceImpl.countsText(0, 0)); + } + + @Test + public void relativePathIsResolvedAgainstWorkingDirectory() throws Exception { + Path relative = java.nio.file.Paths.get("no-such-dir-for-override-test", "overrides.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(relative.toString()), sink); + assertTrue(sink.await().get(FEATURES.getName()).isEmpty()); + assertThat(logCapture.getMessageStrings(), hasItem(containsString(relative.toAbsolutePath().normalize().toString()))); + } + + @Test + public void loggerUsesSubLoggerName() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + start(FileOverrides.source().filePaths(file), new RecordingSink()); + boolean found = false; + for (LogCapture.Message m : logCapture.getMessages()) { + if (m.getLevel() == LDLogLevel.INFO && m.getLoggerName().endsWith("FileOverrideSource")) { + found = true; + } + } + assertTrue(found); + } + } +}