diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java index b07cc037..24e63aa7 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/DataSystemBuilder.java @@ -134,6 +134,7 @@ public DataSystemBuilder persistentStore(ComponentConfigurer persiste * * @param overrideSource the override source configuration, or null for none * @return a reference to the builder + * @see FileOverrides * @since 7.18.0 */ public DataSystemBuilder overrides(ComponentConfigurer overrideSource) { diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceBuilder.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceBuilder.java new file mode 100644 index 00000000..ea19763e --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceBuilder.java @@ -0,0 +1,147 @@ +package com.launchdarkly.sdk.server.integrations; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.sdk.server.integrations.FileOverrides.ChangeDetection; +import com.launchdarkly.sdk.server.subsystems.ClientContext; +import com.launchdarkly.sdk.server.subsystems.ComponentConfigurer; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; + +import java.nio.file.InvalidPathException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; + +/** + * Configures the file-based override source. Obtain an instance with {@link FileOverrides#source()}, + * call {@link #filePaths(String...)} or {@link #filePaths(Path...)} to specify the files, adjust + * any other options, and pass the builder to + * {@link DataSystemBuilder#overrides(ComponentConfigurer)}. + *

+ * For more details, see {@link FileOverrides}. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @since 7.18.0 + */ +public final class FileOverrideSourceBuilder implements ComponentConfigurer { + private final List filePaths = new ArrayList<>(); + private FileData.DuplicateKeysHandling duplicateKeysHandling = FileData.DuplicateKeysHandling.FAIL; + private ChangeDetection changeDetection = ChangeDetection.POLLING; + private Duration pollInterval = FileOverrides.DEFAULT_POLL_INTERVAL; + + FileOverrideSourceBuilder() {} + + /** + * Adds any number of files to load overrides from, specifying each file path as a string. The + * order is significant: it determines which file wins under the duplicate keys handling when the + * same key appears in more than one file. + *

+ * Files are parsed as JSON if their first non-whitespace character is '{'. Otherwise, they are + * parsed as YAML. + * + * @param filePaths path(s) to the file(s); may be absolute or relative to the current working directory + * @return the same builder + * @throws InvalidPathException if one of the parameters is not a valid file path + */ + public FileOverrideSourceBuilder filePaths(String... filePaths) throws InvalidPathException { + for (String p : filePaths) { + this.filePaths.add(Paths.get(p)); + } + return this; + } + + /** + * Adds any number of files to load overrides from, specifying each file path as a Path. The + * order is significant: it determines which file wins under the duplicate keys handling when the + * same key appears in more than one file. + * + * @param filePaths path(s) to the file(s); may be absolute or relative to the current working directory + * @return the same builder + */ + public FileOverrideSourceBuilder filePaths(Path... filePaths) { + for (Path p : filePaths) { + this.filePaths.add(p); + } + return this; + } + + /** + * Specifies how to handle the same key appearing in more than one file. + *

+ * With {@link FileData.DuplicateKeysHandling#FAIL}, the default, a reload fails if the same flag or + * segment key appears in more than one file, and the previously loaded overrides stay in effect. + * With {@link FileData.DuplicateKeysHandling#IGNORE}, the entry from the first configured file that + * defines the key is kept and the others are discarded. A null value selects the default. + * + * @param duplicateKeysHandling specifies how to handle duplicate keys + * @return the same builder + */ + public FileOverrideSourceBuilder duplicateKeysHandling(FileData.DuplicateKeysHandling duplicateKeysHandling) { + this.duplicateKeysHandling = duplicateKeysHandling == null + ? FileData.DuplicateKeysHandling.FAIL : duplicateKeysHandling; + return this; + } + + /** + * Selects how the source detects file changes. The default is {@link ChangeDetection#POLLING}. + * The two modes are alternatives, so setting one replaces the other. + * + * @param changeDetection the change detection mode + * @return the same builder + */ + public FileOverrideSourceBuilder changeDetection(ChangeDetection changeDetection) { + this.changeDetection = changeDetection; + return this; + } + + /** + * Sets the interval between examinations of the files in {@link ChangeDetection#POLLING} mode. + * {@link ChangeDetection#WATCHING} mode ignores it. The default is + * {@link FileOverrides#DEFAULT_POLL_INTERVAL}. An interval below + * {@link FileOverrides#MINIMUM_POLL_INTERVAL} is raised to the minimum. + * + * @param pollInterval the polling interval + * @return the same builder + */ + public FileOverrideSourceBuilder pollInterval(Duration pollInterval) { + this.pollInterval = pollInterval; + return this; + } + + /** + * Called internally by the SDK to create the override source. + * + * @param context the client context + * @return the override source + * @throws IllegalArgumentException if no file paths were specified, or the change detection mode + * or the poll interval is null + */ + @Override + public OverrideSource build(ClientContext context) { + if (filePaths.isEmpty()) { + throw new IllegalArgumentException("no file paths were specified for the file-based override source"); + } + if (changeDetection == null) { + throw new IllegalArgumentException("a change detection mode is required for the file-based override source"); + } + if (pollInterval == null) { + throw new IllegalArgumentException("a poll interval is required for the file-based override source"); + } + List absolutePaths = new ArrayList<>(filePaths.size()); + for (Path p : filePaths) { + absolutePaths.add(p.toAbsolutePath().normalize()); + } + + LDLogger logger = context.getBaseLogger().subLogger("FileOverrideSource"); + + Duration effectivePollInterval = pollInterval; + if (changeDetection == ChangeDetection.POLLING && pollInterval.compareTo(FileOverrides.MINIMUM_POLL_INTERVAL) < 0) { + logger.warn("Poll interval {} is below the minimum; using {}", pollInterval, FileOverrides.MINIMUM_POLL_INTERVAL); + effectivePollInterval = FileOverrides.MINIMUM_POLL_INTERVAL; + } + + return new FileOverrideSourceImpl(absolutePaths, duplicateKeysHandling, changeDetection, effectivePollInterval, logger); + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceImpl.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceImpl.java new file mode 100644 index 00000000..22d719c4 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceImpl.java @@ -0,0 +1,169 @@ +package com.launchdarkly.sdk.server.integrations; + +import com.launchdarkly.logging.LDLogger; +import com.launchdarkly.sdk.server.integrations.FileDataSourceParsing.FileDataException; +import com.launchdarkly.sdk.server.integrations.FileOverrides.ChangeDetection; +import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.FileSummary; +import com.launchdarkly.sdk.server.integrations.OverrideFileLoader.LoadResult; +import com.launchdarkly.sdk.server.subsystems.OverrideSink; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; + +import java.io.IOException; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/** + * The file-based override source. It reads one or more files in the file data source document + * format, merges them in the configured order, and supplies the result to the override sink as + * one snapshot. It reloads when the files change, by polling or by watching. + *

+ * A configured file that does not exist contributes no overrides. A file that exists but cannot + * be read or parsed fails that reload: the sink is not called, so the last good overrides stay in + * effect, and the reload is retried. + */ +final class FileOverrideSourceImpl implements OverrideSource { + private final List paths; + private final ChangeDetection changeDetection; + private final Duration pollInterval; + private final LDLogger logger; + private final OverrideFileLoader loader; + + private volatile FileDataReloader reloader; + private volatile FileDataPoller poller; + private volatile FileDataWatcher watcher; + private final AtomicBoolean closed = new AtomicBoolean(false); + + FileOverrideSourceImpl( + List paths, + FileData.DuplicateKeysHandling duplicateKeysHandling, + ChangeDetection changeDetection, + Duration pollInterval, + LDLogger logger + ) { + this.paths = new ArrayList<>(paths); + this.changeDetection = changeDetection; + this.pollInterval = pollInterval; + this.logger = logger; + this.loader = new OverrideFileLoader(paths, duplicateKeysHandling); + } + + /** + * Performs the initial load synchronously, so overrides present in the files are in effect when + * the client constructor returns, then starts change detection. A file that does not exist yet + * contributes no overrides. A file that cannot be read or parsed is not fatal: the client runs + * with the last good overrides, the failure is logged, and the retry, plus the change signal, + * recovers once the file is readable. + */ + @Override + public void start(OverrideSink sink) { + reloader = new FileDataReloader( + loader::load, + new FileDataReloader.Handler() { + @Override + public void apply(LoadResult result) { + sink.setOverrides(result.getData()); + logOverridesInEffect(result); + } + + @Override + public void onError(FileDataException e) { + // The reloader has logged the failure. The last good overrides stay in effect. + } + }, + logger, + FileDataReloader.DEFAULT_DEBOUNCE_DELAY, + FileDataReloader.DEFAULT_RETRY_DELAY, + true + ); + reloader.reloadNow(); + + if (closed.get()) { + return; + } + switch (changeDetection) { + case WATCHING: + try { + FileDataWatcher w = FileDataWatcher.create(paths, logger); + watcher = w; + w.start(reloader::trigger); + } catch (IOException e) { + // COVERAGE: constructing a watcher only fails under unusual OS conditions + logger.error("Unable to watch override files: {}", e.toString()); + } + break; + case POLLING: + default: + poller = new FileDataPoller(paths, pollInterval, reloader::trigger); + break; + } + } + + /** + * Returns the interval between file examinations in polling mode. Visible for tests. + * + * @return the effective poll interval + */ + Duration getPollInterval() { + return pollInterval; + } + + // Reports the overrides now in effect and the file each came from. The reloader applies a + // snapshot only when the content changed, so this logs each change once. + private void logOverridesInEffect(LoadResult result) { + List details = new ArrayList<>(result.getFiles().size()); + for (FileSummary file : result.getFiles()) { + if (!file.isPresent()) { + details.add(file.getPath() + ": absent"); + } else if (file.getFlags() == 0 && file.getSegments() == 0) { + details.add(file.getPath() + ": no entries"); + } else { + details.add(file.getPath() + ": " + countsText(file.getFlags(), file.getSegments())); + } + } + String fileDetails = String.join("; ", details); + if (result.getFlagCount() == 0 && result.getSegmentCount() == 0) { + logger.info("Flag overrides: none in effect ({})", fileDetails); + return; + } + logger.info("Flag overrides in effect: {} ({})", countsText(result.getFlagCount(), result.getSegmentCount()), + fileDetails); + } + + // Formats flag and segment counts, for example "2 flags, 1 segment". + static String countsText(int flags, int segments) { + List parts = new ArrayList<>(2); + if (flags > 0) { + parts.add(pluralize(flags, "flag")); + } + if (segments > 0) { + parts.add(pluralize(segments, "segment")); + } + return String.join(", ", parts); + } + + private static String pluralize(int count, String noun) { + return count == 1 ? "1 " + noun : count + " " + noun + "s"; + } + + @Override + public void close() { + if (closed.getAndSet(true)) { + return; + } + FileDataWatcher w = watcher; + if (w != null) { + w.close(); + } + FileDataPoller p = poller; + if (p != null) { + p.close(); + } + FileDataReloader r = reloader; + if (r != null) { + r.close(); + } + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrides.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrides.java new file mode 100644 index 00000000..34342505 --- /dev/null +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/integrations/FileOverrides.java @@ -0,0 +1,106 @@ +package com.launchdarkly.sdk.server.integrations; + +import java.time.Duration; + +/** + * Integration between the LaunchDarkly SDK and file-based flag overrides. + *

+ * Flag overrides are currently experimental and subject to change. + *

+ * Overrides are flag and segment definitions that take precedence over data received from + * LaunchDarkly at evaluation time, on a per-key basis. They exist for resilience during an + * incident. An operator can force one or more flags to a known state on a running application, + * whether or not the application can reach LaunchDarkly. The override stays in effect until the + * operator removes it. Flags not present in the override data are completely unaffected. + *

+ * This class provides one override source: {@link #source()}, which reads overrides from local + * files and reloads them as the files change. Configure it with the data system builder: + * 


+ *     LDConfig config = new LDConfig.Builder()
+ *         .dataSystem(Components.dataSystem().defaultMode()
+ *             .overrides(FileOverrides.source().filePaths("/etc/launchdarkly/overrides.json")))
+ *         .build();
+ *
+ *

+ * An evaluation that an override affects is marked. The marking is direct or transitive: it + * applies when the evaluated flag, a prerequisite at any depth, or a segment read during the + * evaluation came from the override layer. {@link com.launchdarkly.sdk.EvaluationReason#isOverrideAffected()} + * reports the marking. Marked evaluations appear in analytics summary events only, under separate + * counters, so LaunchDarkly can distinguish them. They produce no individual evaluation events. + * + * @see FileOverrideSourceBuilder + * @see DataSystemBuilder#overrides(com.launchdarkly.sdk.server.subsystems.ComponentConfigurer) + * @since 7.18.0 + */ +public abstract class FileOverrides { + /** + * Selects how the file-based override source learns that a file changed. The two modes are + * alternatives. Flag overrides are currently experimental and subject to change. + * + * @see FileOverrideSourceBuilder#changeDetection(ChangeDetection) + */ + public enum ChangeDetection { + /** + * The source examines the files on a fixed interval and reloads when the modification time or + * the size of a file changes. Polling works on every file system, including network mounts and + * directories whose contents are swapped through symbolic links, as Kubernetes does for mounted + * ConfigMaps. It is the default. + */ + POLLING, + + /** + * The source reloads in response to file system change notifications. It reacts faster than + * polling. It depends on notifications, which some file systems do not deliver reliably. + */ + WATCHING + } + + /** + * The interval at which the file source examines the files for changes in + * {@link ChangeDetection#POLLING} mode when no interval was specified. Because the source reads + * local files rather than contacting a service, a short interval keeps an override responsive + * during an incident at negligible cost. + */ + public static final Duration DEFAULT_POLL_INTERVAL = Duration.ofSeconds(1); + + /** + * The shortest allowed polling interval. A configured interval below this is raised to it. The + * minimum exists only to prevent a pathological tight loop over the file system. + */ + public static final Duration MINIMUM_POLL_INTERVAL = Duration.ofSeconds(1); + + private FileOverrides() {} + + /** + * Creates a {@link FileOverrideSourceBuilder} for a file-based override source. The source reads + * flag and segment overrides from one or more local files and reloads them as the files change. + *

+ * The files use the same document format as the file data source (see {@link FileData}): each + * file is a JSON or YAML document with optional {@code "flags"}, {@code "flagValues"}, and + * {@code "segments"} members. A {@code "flagValues"} entry is expanded into a full flag + * definition that is off and serves the given value as its single variation for every context. + * When multiple files are configured, their entries are combined in the configured order, and + * the duplicate keys handling decides which file wins for a key that appears more than once. + *

+ * A reload replaces the entire override set, so removing an entry from the files removes the + * override. A configured file that does not exist contributes no overrides. Deleting a file + * therefore removes its overrides, and deleting every file removes them all. A file that exists + * but cannot be read or parsed makes that whole reload fail: the previously loaded overrides + * stay in effect, the source logs the failure, retries after a short delay, and recovers on its + * own once the file is readable again. + *

+ * Whenever the set of overrides in effect changes, including at startup, the source logs the + * overrides in effect and what each configured file supplied, at Info level. + *

+ * By default the source polls the files for changes once per second. See + * {@link FileOverrideSourceBuilder#changeDetection(ChangeDetection)} and + * {@link FileOverrideSourceBuilder#pollInterval(Duration)}. + *

+ * Flag overrides are currently experimental and subject to change. + * + * @return a builder for the file-based override source + */ + public static FileOverrideSourceBuilder source() { + return new FileOverrideSourceBuilder(); + } +} diff --git a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSource.java b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSource.java index 0e83f01e..94665d33 100644 --- a/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSource.java +++ b/lib/sdk/server/src/main/java/com/launchdarkly/sdk/server/subsystems/OverrideSource.java @@ -14,6 +14,8 @@ *

* To configure an override source, use * {@link com.launchdarkly.sdk.server.integrations.DataSystemBuilder#overrides(ComponentConfigurer)}. + * The SDK provides a file-based source; see + * {@link com.launchdarkly.sdk.server.integrations.FileOverrides}. *

* Flag overrides are currently experimental and subject to change. * diff --git a/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceTest.java b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceTest.java new file mode 100644 index 00000000..51d1f942 --- /dev/null +++ b/lib/sdk/server/src/test/java/com/launchdarkly/sdk/server/integrations/FileOverrideSourceTest.java @@ -0,0 +1,475 @@ +package com.launchdarkly.sdk.server.integrations; + +import com.launchdarkly.logging.LDLogLevel; +import com.launchdarkly.logging.LogCapture; +import com.launchdarkly.logging.Logs; +import com.launchdarkly.sdk.LDValue; +import com.launchdarkly.sdk.server.Components; +import com.launchdarkly.sdk.server.LDConfig; +import com.launchdarkly.sdk.server.TestComponents; +import com.launchdarkly.sdk.server.integrations.FileOverrides.ChangeDetection; +import com.launchdarkly.sdk.server.subsystems.ClientContext; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.DataKind; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.ItemDescriptor; +import com.launchdarkly.sdk.server.subsystems.DataStoreTypes.KeyedItems; +import com.launchdarkly.sdk.server.subsystems.OverrideSink; +import com.launchdarkly.sdk.server.subsystems.OverrideSource; +import com.launchdarkly.testhelpers.TempDir; + +import org.junit.After; +import org.junit.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.Duration; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.BlockingQueue; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.TimeUnit; + +import static com.launchdarkly.sdk.server.DataModel.FEATURES; +import static com.launchdarkly.sdk.server.DataModel.SEGMENTS; +import static org.hamcrest.MatcherAssert.assertThat; +import static org.hamcrest.Matchers.allOf; +import static org.hamcrest.Matchers.containsString; +import static org.hamcrest.Matchers.hasItem; +import static org.hamcrest.Matchers.startsWith; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + +@SuppressWarnings("javadoc") +public class FileOverrideSourceTest { + private static final long WAIT_MILLIS = 10000; + + private final LogCapture logCapture = Logs.capture(); + private final ClientContext context = TestComponents.clientContext("sdk-key", + new LDConfig.Builder().logging(Components.logging(logCapture).level(LDLogLevel.DEBUG)).build()); + private final List sources = new ArrayList<>(); + + @After + public void closeSources() throws Exception { + for (OverrideSource s : sources) { + s.close(); + } + } + + /** + * Records every snapshot the source supplies, as a map of kind name to key to item. + */ + static final class RecordingSink implements OverrideSink { + final BlockingQueue>> snapshots = new LinkedBlockingQueue<>(); + + @Override + public void setOverrides(Iterable>> data) { + Map> snapshot = new HashMap<>(); + snapshot.put(FEATURES.getName(), new HashMap<>()); + snapshot.put(SEGMENTS.getName(), new HashMap<>()); + for (Map.Entry> kind : data) { + Map items = snapshot.computeIfAbsent(kind.getKey().getName(), k -> new HashMap<>()); + for (Map.Entry item : kind.getValue().getItems()) { + items.put(item.getKey(), item.getValue()); + } + } + snapshots.add(snapshot); + } + + Map> await() throws InterruptedException { + Map> s = snapshots.poll(WAIT_MILLIS, TimeUnit.MILLISECONDS); + if (s == null) { + throw new AssertionError("timed out waiting for an override snapshot"); + } + return s; + } + + void assertNoSnapshot(long millis) throws InterruptedException { + assertNull(snapshots.poll(millis, TimeUnit.MILLISECONDS)); + } + } + + private OverrideSource start(FileOverrideSourceBuilder builder, RecordingSink sink) { + OverrideSource source = builder.build(context); + sources.add(source); + source.start(sink); + return source; + } + + private static String flagValue(Map> snapshot, String key) { + ItemDescriptor item = snapshot.get(FEATURES.getName()).get(key); + if (item == null) { + return null; + } + LDValue json = LDValue.parse(FEATURES.serialize(item)); + return json.get("variations").get(0).stringValue(); + } + + private static Path write(TempDir dir, String name, String contents) throws Exception { + Path p = dir.getPath().resolve(name); + Files.write(p, contents.getBytes()); + return p; + } + + private static String docWith(String key, String value) { + return "{\"flagValues\":{\"" + key + "\":\"" + value + "\"}}"; + } + + @Test + public void buildRequiresFilePaths() { + try { + FileOverrides.source().build(context); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("no file paths")); + } + } + + @Test + public void buildRequiresChangeDetectionMode() { + try { + FileOverrides.source().filePaths("x.json").changeDetection(null).build(context); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("change detection")); + } + } + + @Test + public void buildRequiresPollInterval() { + try { + FileOverrides.source().filePaths("x.json").pollInterval(null).build(context); + fail("expected exception"); + } catch (IllegalArgumentException e) { + assertThat(e.getMessage(), containsString("poll interval")); + } + } + + @Test + public void nullDuplicateKeysHandlingSelectsFail() throws Exception { + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.json", docWith("flag", "a")); + Path b = write(dir, "b.json", docWith("flag", "b")); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(a, b).duplicateKeysHandling(null), sink); + sink.assertNoSnapshot(200); + assertThat(logCapture.getMessageStrings(), hasItem(allOf(startsWith("ERROR:Unable to load flags: "), + containsString("in features, key \"flag\" was already defined"), containsString(b.toString())))); + } + } + + @Test + public void initialLoadCompletesBeforeStartReturns() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", + "{\"flags\":{\"full\":{\"key\":\"full\",\"version\":7,\"on\":true,\"variations\":[\"x\"],\"fallthrough\":{\"variation\":0}}}," + + "\"flagValues\":{\"simple\":\"value\"}," + + "\"segments\":{\"seg\":{\"key\":\"seg\",\"version\":3,\"included\":[\"u\"]}}}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + + Map> snapshot = sink.snapshots.poll(); + assertNotNull("the initial snapshot must be supplied before start returns", snapshot); + assertEquals(7, snapshot.get(FEATURES.getName()).get("full").getVersion()); + assertEquals("value", flagValue(snapshot, "simple")); + // A value-only entry is an off flag with version 0. + ItemDescriptor simple = snapshot.get(FEATURES.getName()).get("simple"); + assertEquals(0, simple.getVersion()); + LDValue simpleJson = LDValue.parse(FEATURES.serialize(simple)); + assertEquals(LDValue.of(false), simpleJson.get("on")); + assertEquals(LDValue.of(0), simpleJson.get("offVariation")); + assertEquals(3, snapshot.get(SEGMENTS.getName()).get("seg").getVersion()); + assertThat(logCapture.getMessageStrings(), hasItem( + "INFO:Flag overrides in effect: 2 flags, 1 segment (" + file + ": 2 flags, 1 segment)")); + } + } + + @Test + public void loadsYamlDocument() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.yaml", "flagValues:\n yaml-flag: \"override-value\"\n"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + assertEquals("override-value", flagValue(sink.await(), "yaml-flag")); + } + } + + @Test + public void mergesFilesInConfiguredOrderWithIgnoreHandling() throws Exception { + try (TempDir dir = TempDir.create()) { + Path first = write(dir, "first.json", docWith("shared", "first")); + Path second = write(dir, "second.json", "{\"flagValues\":{\"shared\":\"second\",\"only-second\":\"x\"}}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(first, second) + .duplicateKeysHandling(FileData.DuplicateKeysHandling.IGNORE), sink); + + Map> snapshot = sink.await(); + assertEquals("first", flagValue(snapshot, "shared")); + assertEquals("x", flagValue(snapshot, "only-second")); + assertThat(logCapture.getMessageStrings(), hasItem( + "INFO:Flag overrides in effect: 2 flags (" + first + ": 1 flag; " + second + ": 1 flag)")); + } + } + + @Test + public void duplicateKeysFailTheLoadByDefault() throws Exception { + try (TempDir dir = TempDir.create()) { + Path a = write(dir, "a.json", docWith("flag", "a")); + Path b = write(dir, "b.json", docWith("flag", "b")); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(a, b), sink); + sink.assertNoSnapshot(200); + } + } + + @Test + public void missingFileContributesNoEntriesAndIsNotAnError() throws Exception { + try (TempDir dir = TempDir.create()) { + Path present = write(dir, "present.json", docWith("flag", "value")); + Path missing = dir.getPath().resolve("missing.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(present, missing), sink); + + Map> snapshot = sink.await(); + assertEquals("value", flagValue(snapshot, "flag")); + assertEquals(1, snapshot.get(FEATURES.getName()).size()); + for (String message : logCapture.getMessageStrings()) { + assertTrue("unexpected error log: " + message, !message.startsWith("ERROR:")); + } + assertThat(logCapture.getMessageStrings(), hasItem( + "INFO:Flag overrides in effect: 1 flag (" + present + ": 1 flag; " + missing + ": absent)")); + } + } + + @Test + public void startsWithNoFilesAndLogsNoneInEffect() throws Exception { + try (TempDir dir = TempDir.create()) { + Path missing = dir.getPath().resolve("missing.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(missing), sink); + + Map> snapshot = sink.await(); + assertTrue(snapshot.get(FEATURES.getName()).isEmpty()); + assertThat(logCapture.getMessageStrings(), hasItem("INFO:Flag overrides: none in effect (" + missing + ": absent)")); + } + } + + @Test + public void emptyDocumentLogsNoEntries() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "empty.json", "{}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + sink.await(); + assertThat(logCapture.getMessageStrings(), hasItem("INFO:Flag overrides: none in effect (" + file + ": no entries)")); + } + } + + @Test + public void malformedFileAtStartupSuppliesNothingAndLogsError() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "bad.json", "{\"flagValues\""); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file), sink); + sink.assertNoSnapshot(200); + assertThat(logCapture.getMessageStrings(), hasItem(startsWith("ERROR:Unable to load flags:"))); + } + } + + private void reloadsOnChange(ChangeDetection mode) throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", docWith("flag", "first")); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(mode).pollInterval(Duration.ofSeconds(1)), sink); + assertEquals("first", flagValue(sink.await(), "flag")); + + // A changed file is reloaded. + Files.write(file, docWith("flag", "second").getBytes()); + assertEquals("second", flagValue(sink.await(), "flag")); + + // A malformed edit keeps the last good overrides and supplies nothing. + Files.write(file, "{\"flagValues\"".getBytes()); + sink.assertNoSnapshot(1500); + assertThat(logCapture.getMessageStrings(), hasItem(startsWith("ERROR:Unable to load flags:"))); + + // A later good edit is applied. + Files.write(file, docWith("flag", "third").getBytes()); + assertEquals("third", flagValue(sink.await(), "flag")); + + // Deleting the file removes its overrides. + Files.delete(file); + assertTrue(sink.await().get(FEATURES.getName()).isEmpty()); + + // Recreating the file brings them back. + Files.write(file, docWith("flag", "fourth").getBytes()); + assertEquals("fourth", flagValue(sink.await(), "flag")); + } + } + + @Test + public void pollingModeReloadsOnChange() throws Exception { + reloadsOnChange(ChangeDetection.POLLING); + } + + @Test + public void watchingModeReloadsOnChange() throws Exception { + reloadsOnChange(ChangeDetection.WATCHING); + } + + @Test + public void fileThatDoesNotExistYetTakesEffectWhenItAppears() throws Exception { + for (ChangeDetection mode : ChangeDetection.values()) { + try (TempDir dir = TempDir.create()) { + Path file = dir.getPath().resolve("later.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(mode), sink); + assertTrue(sink.await().get(FEATURES.getName()).isEmpty()); + + Files.write(file, docWith("flag", "appeared").getBytes()); + assertEquals(mode.toString(), "appeared", flagValue(sink.await(), "flag")); + } + } + } + + @Test + public void pollIntervalBelowMinimumIsRaisedWithWarning() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + RecordingSink sink = new RecordingSink(); + OverrideSource source = start(FileOverrides.source().filePaths(file).pollInterval(Duration.ofMillis(10)), sink); + assertEquals(FileOverrides.MINIMUM_POLL_INTERVAL, ((FileOverrideSourceImpl) source).getPollInterval()); + assertThat(logCapture.getMessageStrings(), hasItem( + "WARN:Poll interval PT0.01S is below the minimum; using PT1S")); + } + } + + @Test + public void pollIntervalAtOrAboveMinimumIsKept() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + OverrideSource source = start(FileOverrides.source().filePaths(file).pollInterval(Duration.ofSeconds(3)), + new RecordingSink()); + assertEquals(Duration.ofSeconds(3), ((FileOverrideSourceImpl) source).getPollInterval()); + OverrideSource defaulted = start(FileOverrides.source().filePaths(file), new RecordingSink()); + assertEquals(FileOverrides.DEFAULT_POLL_INTERVAL, ((FileOverrideSourceImpl) defaulted).getPollInterval()); + } + } + + @Test + public void failedLoadIsRetriedWithoutAFileChange() throws Exception { + // Watching mode delivers no notification while the file is untouched, so only the source's own + // retry can attempt the load again. + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "bad.json", "{\"flagValues\""); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(ChangeDetection.WATCHING), sink); + assertThat(logCapture.getMessageStrings(), hasItem(startsWith("ERROR:Unable to load flags:"))); + + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (!logCapture.getMessageStrings().contains("DEBUG:Retrying flag data load after earlier failure")) { + if (System.currentTimeMillis() > deadline) { + fail("the failed load was not retried"); + } + Thread.sleep(50); + } + sink.assertNoSnapshot(100); + } + } + + @Test + public void watchingModeIgnoresPollIntervalMinimum() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(file).changeDetection(ChangeDetection.WATCHING) + .pollInterval(Duration.ofMillis(10)), sink); + for (String message : logCapture.getMessageStrings()) { + assertTrue(message, !message.startsWith("WARN:Poll interval")); + } + } + } + + private static int liveThreadsNamed(String prefix) { + int count = 0; + for (Thread t : Thread.getAllStackTraces().keySet()) { + if (t.isAlive() && t.getName().startsWith(prefix)) { + count++; + } + } + return count; + } + + private static void awaitLiveThreadsNamed(String prefix, int expected) throws InterruptedException { + long deadline = System.currentTimeMillis() + WAIT_MILLIS; + while (liveThreadsNamed(prefix) != expected) { + if (System.currentTimeMillis() > deadline) { + fail("expected " + expected + " live threads named " + prefix + " but found " + liveThreadsNamed(prefix)); + } + Thread.sleep(20); + } + } + + private void closeReleasesThreads(ChangeDetection mode, String threadPrefix) throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", docWith("flag", "first")); + int baseline = liveThreadsNamed(threadPrefix); + RecordingSink sink = new RecordingSink(); + OverrideSource source = start(FileOverrides.source().filePaths(file).changeDetection(mode) + .pollInterval(Duration.ofSeconds(1)), sink); + sink.await(); + awaitLiveThreadsNamed(threadPrefix, baseline + 1); + + source.close(); + source.close(); + + awaitLiveThreadsNamed(threadPrefix, baseline); + Files.write(file, docWith("flag", "second").getBytes()); + sink.assertNoSnapshot(1500); + } + } + + @Test + public void closeIsIdempotentAndStopsThePoller() throws Exception { + closeReleasesThreads(ChangeDetection.POLLING, "LaunchDarkly-FileDataPoller"); + } + + @Test + public void closeIsIdempotentAndStopsTheWatcher() throws Exception { + closeReleasesThreads(ChangeDetection.WATCHING, "LaunchDarkly-FileDataWatcher"); + } + + @Test + public void countsTextFormatsCounts() { + assertEquals("1 flag", FileOverrideSourceImpl.countsText(1, 0)); + assertEquals("2 flags, 1 segment", FileOverrideSourceImpl.countsText(2, 1)); + assertEquals("3 segments", FileOverrideSourceImpl.countsText(0, 3)); + assertEquals("", FileOverrideSourceImpl.countsText(0, 0)); + } + + @Test + public void relativePathIsResolvedAgainstWorkingDirectory() throws Exception { + Path relative = java.nio.file.Paths.get("no-such-dir-for-override-test", "overrides.json"); + RecordingSink sink = new RecordingSink(); + start(FileOverrides.source().filePaths(relative.toString()), sink); + assertTrue(sink.await().get(FEATURES.getName()).isEmpty()); + assertThat(logCapture.getMessageStrings(), hasItem(containsString(relative.toAbsolutePath().normalize().toString()))); + } + + @Test + public void loggerUsesSubLoggerName() throws Exception { + try (TempDir dir = TempDir.create()) { + Path file = write(dir, "overrides.json", "{}"); + start(FileOverrides.source().filePaths(file), new RecordingSink()); + boolean found = false; + for (LogCapture.Message m : logCapture.getMessages()) { + if (m.getLevel() == LDLogLevel.INFO && m.getLoggerName().endsWith("FileOverrideSource")) { + found = true; + } + } + assertTrue(found); + } + } +}