diff --git a/packages/common/lib/launchdarkly_dart_common.dart b/packages/common/lib/launchdarkly_dart_common.dart index bfb3ec59..ae22f497 100644 --- a/packages/common/lib/launchdarkly_dart_common.dart +++ b/packages/common/lib/launchdarkly_dart_common.dart @@ -69,10 +69,14 @@ export 'src/network/utils.dart' show Headers, appendPath, + classifyHttpStatus, + credentialHint, isHttpGloballyRecoverable, isHttpLocallyRecoverable, urlSafeSha256Hash; +export 'src/retry/retry_state.dart' show RetryState, FailureKind; + export 'src/async/async_single_queue.dart' show AsyncSingleQueue, TaskComplete, TaskShed, TaskError, TaskResult; export 'src/config/validations.dart' diff --git a/packages/common/lib/src/network/utils.dart b/packages/common/lib/src/network/utils.dart index 240df556..b16f7194 100644 --- a/packages/common/lib/src/network/utils.dart +++ b/packages/common/lib/src/network/utils.dart @@ -79,6 +79,19 @@ bool isHttpLocallyRecoverable(num status) { return isHttpGloballyRecoverable(status); } +/// Classifies a failed HTTP response by its [status]. +FailureKind classifyHttpStatus(num status) => isHttpGloballyRecoverable(status) + ? FailureKind.normal + : FailureKind.unexpected; + +/// Returns a sentence to add to the log message for a failed request with +/// [status], or an empty string. A 401 or a 403 usually means that the SDK +/// credential is not correct. +String credentialHint(num status) => status == 401 || status == 403 + ? ' Check that the SDK credential (mobile key or client-side ID) is ' + 'correct.' + : ''; + /// Hashes the input, base64 encodes result, then sanitizes it for URL usage. String urlSafeSha256Hash(String input) { return _encodeAndSanitize(sha256.convert(utf8.encode(input)).bytes); diff --git a/packages/common/lib/src/retry/retry_state.dart b/packages/common/lib/src/retry/retry_state.dart new file mode 100644 index 00000000..e158554d --- /dev/null +++ b/packages/common/lib/src/retry/retry_state.dart @@ -0,0 +1,163 @@ +import 'dart:math' as math; + +/// The classification of a failure. It controls how long a long-running +/// component waits before it tries again. +enum FailureKind { + /// A failure that is likely to clear quickly. Examples are a 5xx status, + /// a network error, a timeout and a bad payload. + normal, + + /// A failure that is not likely to clear quickly. Examples are a 401 or a + /// 403 status. + unexpected, +} + +const _streamingInitialDelay = Duration(seconds: 1); +const _streamingCeiling = Duration(seconds: 30); +const _minimumExtendedInitialDelay = Duration(minutes: 5); +const _minimumExtendedCeiling = Duration(hours: 1); +const _streamingResetThreshold = Duration(seconds: 60); +const _pollingResetThreshold = 2; + +// Every ceiling is much less than 2^16 times its initial delay, so the cap +// does not change a result. It keeps the product small enough to not +// overflow, also on the web where integers are doubles. +const _maxExponent = 16; + +Duration Function() _monotonicClock() { + final stopwatch = Stopwatch()..start(); + return () => stopwatch.elapsed; +} + +Duration _max(Duration a, Duration b) => a > b ? a : b; + +/// Computes the wait before a long-running component tries again after a +/// failure. +/// +/// The wait uses exponential backoff with jitter. A normal failure backs +/// off from the normal initial delay up to the normal ceiling. An +/// unexpected failure moves the state to the extended regime, which backs +/// off from the extended initial delay up to the extended ceiling. The +/// state stays in the extended regime until a reset. +/// +/// This class is not thread safe. +final class RetryState { + final Duration _normalInitialDelay; + final Duration _normalCeiling; + final Duration _extendedInitialDelay; + final Duration _extendedCeiling; + + /// No wait is shorter than this. + final Duration _minimumWait; + + /// The duration of healthy operation that resets the state, if the reset + /// uses a duration. + final Duration? _healthyResetThreshold; + + /// The number of consecutive successes that resets the state, if the + /// reset uses a count. + final int? _successResetThreshold; + + final math.Random _random; + final Duration Function() _clock; + + int _attempts = 0; + bool _extended = false; + Duration? _healthySince; + int _consecutiveSuccesses = 0; + + /// The retry state for a streaming data source. + /// + /// The normal regime backs off from 1 second up to 30 seconds. The + /// extended regime backs off from 5 minutes up to 1 hour. The state resets + /// when a failure comes after 60 seconds of healthy operation. + /// + /// The [clock] must be monotonic. The [random] and [clock] parameters are + /// for tests. + RetryState.streaming({math.Random? random, Duration Function()? clock}) + : _normalInitialDelay = _streamingInitialDelay, + _normalCeiling = _streamingCeiling, + _extendedInitialDelay = _minimumExtendedInitialDelay, + _extendedCeiling = _minimumExtendedCeiling, + _minimumWait = Duration.zero, + _healthyResetThreshold = _streamingResetThreshold, + _successResetThreshold = null, + _random = random ?? math.Random(), + _clock = clock ?? _monotonicClock(); + + /// The retry state for a polling data source. + /// + /// A normal failure waits the [pollInterval]. The extended regime backs + /// off from 5 minutes up to 1 hour, but these values are never less than + /// the [pollInterval]. No wait is shorter than the [pollInterval]. The + /// state resets after 2 successful polls in a row. + /// + /// The [random] parameter is for tests. + RetryState.polling(Duration pollInterval, {math.Random? random}) + : _normalInitialDelay = pollInterval, + _normalCeiling = pollInterval, + _extendedInitialDelay = + _max(_minimumExtendedInitialDelay, pollInterval), + _extendedCeiling = _max(_minimumExtendedCeiling, pollInterval), + _minimumWait = pollInterval, + _healthyResetThreshold = null, + _successResetThreshold = _pollingResetThreshold, + _random = random ?? math.Random(), + _clock = _monotonicClock(); + + /// Records healthy operation. + /// + /// For streaming, call this for each payload. The first call after a + /// failure starts the healthy period. For polling, call this for each + /// successful poll. + void recordSuccess() { + if (_healthyResetThreshold != null) { + _healthySince ??= _clock(); + } + if (_successResetThreshold case final threshold?) { + _consecutiveSuccesses++; + if (_consecutiveSuccesses >= threshold) { + _reset(); + } + } + } + + /// Records a failure of the given [kind] and returns the wait before the + /// next attempt. + Duration recordFailure(FailureKind kind) { + if ((_healthyResetThreshold, _healthySince) + case (final threshold?, final healthySince?) + when _clock() - healthySince >= threshold) { + _reset(); + } + _healthySince = null; + _consecutiveSuccesses = 0; + + switch (kind) { + case FailureKind.unexpected when !_extended: + // Start the extended regime at its initial delay. + _extended = true; + _attempts = 1; + case FailureKind.unexpected: + case FailureKind.normal: + _attempts++; + } + return _nextDelay(); + } + + void _reset() { + _attempts = 0; + _extended = false; + _consecutiveSuccesses = 0; + } + + Duration _nextDelay() { + final initial = _extended ? _extendedInitialDelay : _normalInitialDelay; + final ceiling = _extended ? _extendedCeiling : _normalCeiling; + final exponent = math.min(_attempts - 1, _maxExponent); + final base = initial * math.pow(2, exponent); + final clamped = base > ceiling ? ceiling : base; + final jitter = clamped * (_random.nextDouble() / 2); + return _max(_minimumWait, clamped - jitter); + } +} diff --git a/packages/common/test/network/utils_test.dart b/packages/common/test/network/utils_test.dart index 6d62bc8f..a4a76e89 100644 --- a/packages/common/test/network/utils_test.dart +++ b/packages/common/test/network/utils_test.dart @@ -57,4 +57,32 @@ void main() { expect(result['X-LaunchDarkly-Tags'], 'application-id/id application-name/name application-version-name/versionName'); }); + + group('classifyHttpStatus', () { + for (final status in [400, 408, 429, 500, 503, 599, 302, 100]) { + test('$status is normal', () { + expect(classifyHttpStatus(status), FailureKind.normal); + }); + } + + for (final status in [401, 403, 404, 405, 413, 499]) { + test('$status is unexpected', () { + expect(classifyHttpStatus(status), FailureKind.unexpected); + }); + } + }); + + group('credentialHint', () { + for (final status in [401, 403]) { + test('$status returns the hint', () { + expect(credentialHint(status), contains('SDK credential')); + }); + } + + for (final status in [404, 500]) { + test('$status returns an empty string', () { + expect(credentialHint(status), isEmpty); + }); + } + }); } diff --git a/packages/common/test/retry/retry_state_test.dart b/packages/common/test/retry/retry_state_test.dart new file mode 100644 index 00000000..14f56ef3 --- /dev/null +++ b/packages/common/test/retry/retry_state_test.dart @@ -0,0 +1,235 @@ +import 'dart:math' as math; + +import 'package:launchdarkly_dart_common/launchdarkly_dart_common.dart'; +import 'package:test/test.dart'; + +/// A random source that always returns the same fraction. +final class _FixedRandom implements math.Random { + final double fraction; + + _FixedRandom(this.fraction); + + @override + double nextDouble() => fraction; + + @override + bool nextBool() => fraction >= 0.5; + + @override + int nextInt(int max) => (fraction * max).floor(); +} + +/// A monotonic clock that tests move by hand. +final class _ManualClock { + Duration now = Duration.zero; + + Duration call() => now; +} + +void main() { + group('streaming', () { + RetryState noJitter([_ManualClock? clock]) => RetryState.streaming( + random: _FixedRandom(0), clock: (clock ?? _ManualClock()).call); + + test('normal failures double from 1 second up to 30 seconds', () { + final state = noJitter(); + final delays = [ + for (var i = 0; i < 7; i++) state.recordFailure(FailureKind.normal) + ]; + expect(delays, const [ + Duration(seconds: 1), + Duration(seconds: 2), + Duration(seconds: 4), + Duration(seconds: 8), + Duration(seconds: 16), + Duration(seconds: 30), + Duration(seconds: 30), + ]); + }); + + test('unexpected failures double from 5 minutes up to 1 hour', () { + final state = noJitter(); + final delays = [ + for (var i = 0; i < 6; i++) state.recordFailure(FailureKind.unexpected) + ]; + expect(delays, const [ + Duration(minutes: 5), + Duration(minutes: 10), + Duration(minutes: 20), + Duration(minutes: 40), + Duration(hours: 1), + Duration(hours: 1), + ]); + }); + + test('an unexpected failure after normal failures starts at 5 minutes', () { + final state = noJitter(); + state.recordFailure(FailureKind.normal); + state.recordFailure(FailureKind.normal); + state.recordFailure(FailureKind.normal); + expect(state.recordFailure(FailureKind.unexpected), + const Duration(minutes: 5)); + }); + + test('a normal failure does not leave the extended regime', () { + final state = noJitter(); + state.recordFailure(FailureKind.unexpected); + expect( + state.recordFailure(FailureKind.normal), const Duration(minutes: 10)); + expect( + state.recordFailure(FailureKind.normal), const Duration(minutes: 20)); + }); + + test('60 seconds of healthy operation resets the state', () { + final clock = _ManualClock(); + final state = noJitter(clock); + state.recordFailure(FailureKind.unexpected); + state.recordFailure(FailureKind.unexpected); + + state.recordSuccess(); + clock.now += const Duration(seconds: 60); + + expect( + state.recordFailure(FailureKind.normal), const Duration(seconds: 1)); + }); + + test('less than 60 seconds of healthy operation keeps the state', () { + final clock = _ManualClock(); + final state = noJitter(clock); + state.recordFailure(FailureKind.unexpected); + + state.recordSuccess(); + clock.now += const Duration(seconds: 59); + + expect( + state.recordFailure(FailureKind.normal), const Duration(minutes: 10)); + }); + + test('healthy time is measured from the first success after a failure', () { + final clock = _ManualClock(); + final state = noJitter(clock); + state.recordFailure(FailureKind.unexpected); + + state.recordSuccess(); + clock.now += const Duration(seconds: 50); + state.recordSuccess(); + clock.now += const Duration(seconds: 10); + + expect( + state.recordFailure(FailureKind.normal), const Duration(seconds: 1)); + }); + + test('a failure restarts the healthy measurement', () { + final clock = _ManualClock(); + final state = noJitter(clock); + state.recordFailure(FailureKind.unexpected); + + state.recordSuccess(); + clock.now += const Duration(seconds: 30); + state.recordFailure(FailureKind.normal); + clock.now += const Duration(seconds: 40); + + expect( + state.recordFailure(FailureKind.normal), const Duration(minutes: 20)); + }); + + test('many failures stay at the ceiling', () { + final state = noJitter(); + Duration delay = Duration.zero; + for (var i = 0; i < 10000; i++) { + delay = state.recordFailure(FailureKind.unexpected); + } + expect(delay, const Duration(hours: 1)); + }); + + test('the jitter removes up to half of the delay', () { + final state = RetryState.streaming(random: _FixedRandom(0.999999)); + state.recordFailure(FailureKind.normal); + state.recordFailure(FailureKind.normal); + final delay = state.recordFailure(FailureKind.normal); + expect(delay, greaterThanOrEqualTo(const Duration(seconds: 2))); + expect(delay, lessThan(const Duration(milliseconds: 2010))); + }); + + test('the jitter stays in range with a real random source', () { + final state = RetryState.streaming(random: math.Random(1)); + // The first four delays are below the ceiling. + for (var i = 0; i < 4; i++) { + state.recordFailure(FailureKind.unexpected); + } + for (var i = 0; i < 200; i++) { + final delay = state.recordFailure(FailureKind.unexpected); + expect(delay, greaterThanOrEqualTo(const Duration(minutes: 30))); + expect(delay, lessThanOrEqualTo(const Duration(hours: 1))); + } + }); + }); + + group('polling', () { + const interval = Duration(seconds: 30); + + test('a normal failure waits the poll interval', () { + final state = + RetryState.polling(interval, random: _FixedRandom(0.999999)); + for (var i = 0; i < 5; i++) { + expect(state.recordFailure(FailureKind.normal), interval); + } + }); + + test('unexpected failures double from 5 minutes up to 1 hour', () { + final state = RetryState.polling(interval, random: _FixedRandom(0)); + final delays = [ + for (var i = 0; i < 6; i++) state.recordFailure(FailureKind.unexpected) + ]; + expect(delays, const [ + Duration(minutes: 5), + Duration(minutes: 10), + Duration(minutes: 20), + Duration(minutes: 40), + Duration(hours: 1), + Duration(hours: 1), + ]); + }); + + test('a poll interval above the extended values raises them', () { + const longInterval = Duration(hours: 2); + final state = RetryState.polling(longInterval, random: _FixedRandom(0)); + expect(state.recordFailure(FailureKind.unexpected), longInterval); + expect(state.recordFailure(FailureKind.unexpected), longInterval); + }); + + test('no wait is shorter than the poll interval', () { + const longInterval = Duration(minutes: 8); + final state = + RetryState.polling(longInterval, random: _FixedRandom(0.999999)); + // The base delay is 8 minutes and the jitter can remove up to half. + expect(state.recordFailure(FailureKind.unexpected), longInterval); + }); + + test('two successful polls in a row reset the state', () { + final state = RetryState.polling(interval, random: _FixedRandom(0)); + state.recordFailure(FailureKind.unexpected); + state.recordSuccess(); + state.recordSuccess(); + expect(state.recordFailure(FailureKind.normal), interval); + }); + + test('one successful poll does not reset the state', () { + final state = RetryState.polling(interval, random: _FixedRandom(0)); + state.recordFailure(FailureKind.unexpected); + state.recordSuccess(); + expect( + state.recordFailure(FailureKind.normal), const Duration(minutes: 10)); + }); + + test('a failure between successes restarts the count', () { + final state = RetryState.polling(interval, random: _FixedRandom(0)); + state.recordFailure(FailureKind.unexpected); + state.recordSuccess(); + state.recordFailure(FailureKind.normal); + state.recordSuccess(); + expect( + state.recordFailure(FailureKind.normal), const Duration(minutes: 20)); + }); + }); +}